@turns/dev 0.0.28 → 0.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-anchor.js +38 -12
- package/dist/codex-async-question-recovery.js +18 -1
- package/dist/codex-update.js +55 -3
- package/dist/compressed-transcripts.js +99 -0
- package/dist/harness-exit.js +44 -0
- package/dist/index.js +18 -3
- package/dist/local-session-sources.js +36 -6
- package/dist/queue-worker.js +22 -0
- package/dist/upload-sessions.js +22 -0
- package/package.json +1 -1
package/dist/agent-anchor.js
CHANGED
|
@@ -13,7 +13,9 @@ import { ACP_PROFILES, lettaHeadlessEnvironment, lettaSessionIdentity, PROCESS_H
|
|
|
13
13
|
import { asyncQuestionInteractions, asyncQuestionReplyText, isAsyncQuestionInteraction, parseAsyncQuestionReplies, } from "./codex-async-questions.js";
|
|
14
14
|
import { recordLettaSessionContext } from "./letta-session-context.js";
|
|
15
15
|
import { recoverAsyncQuestion } from "./codex-async-question-recovery.js";
|
|
16
|
+
import { existingTranscriptPath } from "./compressed-transcripts.js";
|
|
16
17
|
import { BackgroundTaskState } from "./background-task-state.js";
|
|
18
|
+
import { harnessExitError, StderrTail } from "./harness-exit.js";
|
|
17
19
|
const socketIndex = process.argv.indexOf('--socket');
|
|
18
20
|
const socketPath = socketIndex >= 0 ? path.resolve(process.argv[socketIndex + 1] || '') : '';
|
|
19
21
|
if (!socketPath)
|
|
@@ -72,7 +74,7 @@ function writeLine(socket, value) {
|
|
|
72
74
|
if (!socket.destroyed && socket.writable)
|
|
73
75
|
socket.write(`${JSON.stringify(value)}\n`);
|
|
74
76
|
}
|
|
75
|
-
function childResult(child, timeoutMs) {
|
|
77
|
+
function childResult(child, timeoutMs, harness, stderrTail) {
|
|
76
78
|
const result = deferred();
|
|
77
79
|
let settled = false;
|
|
78
80
|
const finish = (value) => {
|
|
@@ -83,7 +85,17 @@ function childResult(child, timeoutMs) {
|
|
|
83
85
|
}
|
|
84
86
|
};
|
|
85
87
|
child.once('error', (error) => finish({ exitCode: null, signal: null, timedOut: false, error: { message: errorText(error) } }));
|
|
86
|
-
child.once('close', (code, signal) =>
|
|
88
|
+
child.once('close', (code, signal) => {
|
|
89
|
+
if (!code)
|
|
90
|
+
return finish({ exitCode: code, signal, timedOut: false, error: null });
|
|
91
|
+
const error = harnessExitError(harness, code, signal, stderrTail.lines());
|
|
92
|
+
finish({
|
|
93
|
+
exitCode: code,
|
|
94
|
+
signal,
|
|
95
|
+
timedOut: false,
|
|
96
|
+
error: { message: error.message, harnessExit: error.harnessExit },
|
|
97
|
+
});
|
|
98
|
+
});
|
|
87
99
|
const timer = setTimeout(() => {
|
|
88
100
|
try {
|
|
89
101
|
process.kill(-child.pid, 'SIGTERM');
|
|
@@ -107,6 +119,7 @@ class LineTransport {
|
|
|
107
119
|
pending;
|
|
108
120
|
lastEventAt;
|
|
109
121
|
child;
|
|
122
|
+
stderrTail = new StderrTail();
|
|
110
123
|
constructor(anchor, harness, command, args, cwd, env = {}) {
|
|
111
124
|
this.anchor = anchor;
|
|
112
125
|
this.harness = harness;
|
|
@@ -128,14 +141,17 @@ class LineTransport {
|
|
|
128
141
|
});
|
|
129
142
|
anchor.child = this.child;
|
|
130
143
|
readline.createInterface({ input: this.child.stdout }).on('line', (line) => this.receive(line));
|
|
131
|
-
readline.createInterface({ input: this.child.stderr }).on('line', (line) =>
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
144
|
+
readline.createInterface({ input: this.child.stderr }).on('line', (line) => {
|
|
145
|
+
this.stderrTail.observe(line);
|
|
146
|
+
anchor.emitLive({
|
|
147
|
+
event: 'process.line',
|
|
148
|
+
source: `${harness}.stderr`,
|
|
149
|
+
raw: line,
|
|
150
|
+
phase: 'queued',
|
|
151
|
+
});
|
|
152
|
+
});
|
|
137
153
|
this.child.once('close', (code, signal) => {
|
|
138
|
-
const error =
|
|
154
|
+
const error = harnessExitError(anchor.source, code, signal, this.stderrTail.lines());
|
|
139
155
|
for (const waiter of this.pending.values())
|
|
140
156
|
waiter.reject(error);
|
|
141
157
|
this.pending.clear();
|
|
@@ -747,7 +763,9 @@ class CodexAppServerTransport extends LineTransport {
|
|
|
747
763
|
: 'workspace-write',
|
|
748
764
|
approvalPolicy: this.anchor.permissionMode === 'bypassPermissions' ? 'never' : 'on-request',
|
|
749
765
|
};
|
|
750
|
-
|
|
766
|
+
// Codex may have compressed or restored the rollout since discovery; it
|
|
767
|
+
// resumes either spelling and writes the plain `.jsonl` back.
|
|
768
|
+
const resumePath = createSession ? '' : existingTranscriptPath(this.anchor.transcriptPath);
|
|
751
769
|
const opened = createSession
|
|
752
770
|
? await this.request('thread/start', {
|
|
753
771
|
...configuration,
|
|
@@ -1769,7 +1787,12 @@ class Anchor {
|
|
|
1769
1787
|
result: null,
|
|
1770
1788
|
};
|
|
1771
1789
|
this.setStatus('running');
|
|
1772
|
-
void this.run(Boolean(request.createSession), this.turnTimeoutMs).then((result) => this.finish(result), (error) => this.finish({
|
|
1790
|
+
void this.run(Boolean(request.createSession), this.turnTimeoutMs).then((result) => this.finish(result), (error) => this.finish({
|
|
1791
|
+
exitCode: 1,
|
|
1792
|
+
signal: null,
|
|
1793
|
+
timedOut: false,
|
|
1794
|
+
error: { message: errorText(error), ...(error?.harnessExit ? { harnessExit: error.harnessExit } : {}) },
|
|
1795
|
+
}));
|
|
1773
1796
|
return { accepted: true, messageId: this.currentTurn.messageId, pid: process.pid };
|
|
1774
1797
|
}
|
|
1775
1798
|
async run(createSession, timeoutMs) {
|
|
@@ -1868,6 +1891,7 @@ class Anchor {
|
|
|
1868
1891
|
this.child = child;
|
|
1869
1892
|
if (invocation.stdin)
|
|
1870
1893
|
child.stdin.end(`${invocation.stdin}\n`);
|
|
1894
|
+
const stderrTail = new StderrTail();
|
|
1871
1895
|
for (const [stream, source] of [
|
|
1872
1896
|
[child.stdout, `${this.source}.stdout`],
|
|
1873
1897
|
[child.stderr, `${this.source}.stderr`],
|
|
@@ -1881,9 +1905,11 @@ class Anchor {
|
|
|
1881
1905
|
const processSessionId = processEventSessionId(this.source, json);
|
|
1882
1906
|
if (processSessionId && processSessionId !== this.agentSessionId)
|
|
1883
1907
|
this.updateIdentity(processSessionId);
|
|
1908
|
+
if (stream === child.stderr)
|
|
1909
|
+
stderrTail.observe(line);
|
|
1884
1910
|
this.emitLive({ event: 'process.line', source, raw: line, json, phase: 'queued' });
|
|
1885
1911
|
});
|
|
1886
|
-
return childResult(child, timeoutMs);
|
|
1912
|
+
return childResult(child, timeoutMs, this.source, stderrTail);
|
|
1887
1913
|
}
|
|
1888
1914
|
if (ACP_PROFILES[this.source]) {
|
|
1889
1915
|
if (!(this.transport instanceof AcpTransport) ||
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import readline from 'node:readline';
|
|
3
|
+
import { Readable } from 'node:stream';
|
|
3
4
|
import { ASYNC_QUESTION_TOOL, asyncQuestionInteractions, parseAsyncQuestionReplies } from "./codex-async-questions.js";
|
|
5
|
+
import { existingTranscriptPath, isCompressedTranscript, readTranscriptSync } from "./compressed-transcripts.js";
|
|
4
6
|
export function isAsyncQuestionId(value) {
|
|
5
7
|
if (typeof value !== 'string')
|
|
6
8
|
return false;
|
|
@@ -30,7 +32,22 @@ export async function recoverAsyncQuestion(transcriptPath, interactionId) {
|
|
|
30
32
|
const [, callId] = JSON.parse(interactionId);
|
|
31
33
|
let candidate = null;
|
|
32
34
|
let delivered = false;
|
|
33
|
-
|
|
35
|
+
// Codex compresses idle rollouts and restores them on resume, so read
|
|
36
|
+
// whichever spelling of this anchor's rollout exists now.
|
|
37
|
+
const file = existingTranscriptPath(transcriptPath);
|
|
38
|
+
if (!file)
|
|
39
|
+
return null;
|
|
40
|
+
let stream;
|
|
41
|
+
try {
|
|
42
|
+
stream = isCompressedTranscript(file)
|
|
43
|
+
? Readable.from([readTranscriptSync(file).toString('utf8')])
|
|
44
|
+
: fs.createReadStream(file, { encoding: 'utf8' });
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (error?.code === 'ENOENT')
|
|
48
|
+
return null;
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
34
51
|
const lines = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
35
52
|
try {
|
|
36
53
|
for await (const line of lines) {
|
package/dist/codex-update.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
/** Machine-local Codex CLI self-update, callable by the Turns daemon. */
|
|
2
2
|
import { spawn } from 'node:child_process';
|
|
3
|
+
import fs from 'node:fs';
|
|
3
4
|
import os from 'node:os';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { executablePath } from "./executable-path.js";
|
|
4
7
|
import { codexCommandIsAppManaged, resolveHarnessCommand } from "./harness-registry.js";
|
|
5
8
|
const VERSION_TIMEOUT_MS = 15_000;
|
|
6
9
|
const UPDATE_TIMEOUT_MS = 5 * 60_000;
|
|
@@ -11,7 +14,9 @@ function runCommand(command, args, timeoutMs) {
|
|
|
11
14
|
return new Promise((resolve) => {
|
|
12
15
|
const child = spawn(command, args, {
|
|
13
16
|
cwd: os.homedir(),
|
|
14
|
-
env
|
|
17
|
+
// npm and Codex's npm launcher are `#!/usr/bin/env node` scripts; a
|
|
18
|
+
// service manager's PATH may not include the daemon's own Node.
|
|
19
|
+
env: { ...process.env, PATH: executablePath() },
|
|
15
20
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
16
21
|
});
|
|
17
22
|
let output = '';
|
|
@@ -56,10 +61,54 @@ function commandFailure(result, fallback) {
|
|
|
56
61
|
.filter(Boolean);
|
|
57
62
|
return new Error(lines.at(-1) || fallback);
|
|
58
63
|
}
|
|
64
|
+
function realpathOrSelf(file) {
|
|
65
|
+
try {
|
|
66
|
+
return fs.realpathSync(file);
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return file;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
function findExecutable(name, preferred) {
|
|
73
|
+
for (const directory of [...preferred, ...executablePath().split(path.delimiter)]) {
|
|
74
|
+
const candidate = path.join(directory, name);
|
|
75
|
+
try {
|
|
76
|
+
fs.accessSync(candidate, fs.constants.X_OK);
|
|
77
|
+
return candidate;
|
|
78
|
+
}
|
|
79
|
+
catch { }
|
|
80
|
+
}
|
|
81
|
+
return '';
|
|
82
|
+
}
|
|
83
|
+
export function codexUpgradeCommand(command, lookup = {}) {
|
|
84
|
+
const resolved = (lookup.realpath || realpathOrSelf)(command).replaceAll('\\', '/');
|
|
85
|
+
const find = lookup.findExecutable || findExecutable;
|
|
86
|
+
// `npm i -g @openai/codex` links <prefix>/bin/codex to the package's launcher.
|
|
87
|
+
const npm = resolved.match(/^(.*?)\/(?:lib\/)?node_modules\/@openai\/codex\//u);
|
|
88
|
+
// Windows npm is a .cmd shim, which cannot be spawned without a shell.
|
|
89
|
+
if (npm && process.platform !== 'win32') {
|
|
90
|
+
const prefix = npm[1];
|
|
91
|
+
const binary = find('npm', [path.join(prefix, 'bin')]);
|
|
92
|
+
if (binary)
|
|
93
|
+
return { manager: 'npm', command: binary, args: ['install', '-g', '--prefix', prefix, '@openai/codex@latest'] };
|
|
94
|
+
}
|
|
95
|
+
const brew = resolved.match(/^(.*?)\/(Cellar|Caskroom)\/codex\//u);
|
|
96
|
+
if (brew) {
|
|
97
|
+
const binary = find('brew', [path.join(brew[1], 'bin')]);
|
|
98
|
+
if (binary)
|
|
99
|
+
return {
|
|
100
|
+
manager: 'brew',
|
|
101
|
+
command: binary,
|
|
102
|
+
args: ['upgrade', ...(brew[2] === 'Caskroom' ? ['--cask'] : []), 'codex'],
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
return { manager: 'codex', command, args: ['update'] };
|
|
106
|
+
}
|
|
59
107
|
export function createCodexUpdateManager(dependencies = {}) {
|
|
60
108
|
const resolveCommand = dependencies.resolveCommand || (() => resolveHarnessCommand('codex'));
|
|
61
109
|
const isAppManaged = dependencies.isAppManaged || codexCommandIsAppManaged;
|
|
62
110
|
const run = dependencies.run || runCommand;
|
|
111
|
+
const upgradeCommand = dependencies.upgradeCommand || ((command) => codexUpgradeCommand(command));
|
|
63
112
|
let active = null;
|
|
64
113
|
const updateCommand = () => {
|
|
65
114
|
const command = resolveCommand();
|
|
@@ -74,9 +123,12 @@ export function createCodexUpdateManager(dependencies = {}) {
|
|
|
74
123
|
throw commandFailure(beforeRun, 'Codex is not installed or its version could not be read.');
|
|
75
124
|
}
|
|
76
125
|
const previousVersion = parseCodexVersion(beforeRun.output);
|
|
77
|
-
const
|
|
126
|
+
const upgrade = upgradeCommand(updateCommand());
|
|
127
|
+
const updateRun = await run(upgrade.command, upgrade.args, UPDATE_TIMEOUT_MS);
|
|
78
128
|
if (updateRun.failed || updateRun.timedOut || updateRun.exitCode !== 0) {
|
|
79
|
-
throw commandFailure(updateRun,
|
|
129
|
+
throw commandFailure(updateRun, upgrade.manager === 'codex'
|
|
130
|
+
? 'Codex could not update itself. Run `codex update` in a terminal on the machine.'
|
|
131
|
+
: `${upgrade.manager} could not upgrade Codex. Run \`${[path.basename(upgrade.command), ...upgrade.args].join(' ')}\` in a terminal on the machine.`);
|
|
80
132
|
}
|
|
81
133
|
// Resolve again because the updater may replace a shim or install path.
|
|
82
134
|
const afterRun = await run(updateCommand(), ['--version'], VERSION_TIMEOUT_MS);
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import { zstdDecompressSync } from 'node:zlib';
|
|
3
|
+
/**
|
|
4
|
+
* Codex compresses idle rollouts in place (`rollout-*.jsonl` becomes
|
|
5
|
+
* `rollout-*.jsonl.zst`) and restores the plain file when the thread resumes.
|
|
6
|
+
* Both spellings are one logical transcript: descriptors keep the `.jsonl`
|
|
7
|
+
* source path so the turns.dev session ID is unchanged across that rename, and
|
|
8
|
+
* only the physical `file` names the compressed copy.
|
|
9
|
+
*/
|
|
10
|
+
const COMPRESSED_SUFFIX = '.zst';
|
|
11
|
+
const PREFIX_CACHE_BYTES = 32 * 1024 * 1024;
|
|
12
|
+
const MAX_PREFIX_BYTES = 1024 * 1024;
|
|
13
|
+
export function isCompressedTranscript(file) {
|
|
14
|
+
return /\.jsonl\.zst$/iu.test(String(file || ''));
|
|
15
|
+
}
|
|
16
|
+
/** The plain `.jsonl` path a compressed rollout was made from. */
|
|
17
|
+
export function logicalTranscriptPath(file) {
|
|
18
|
+
return isCompressedTranscript(file) ? file.slice(0, -COMPRESSED_SUFFIX.length) : file;
|
|
19
|
+
}
|
|
20
|
+
function isFile(file) {
|
|
21
|
+
try {
|
|
22
|
+
return fs.statSync(file).isFile();
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/** True when a compressed rollout should yield to its live plain sibling. */
|
|
29
|
+
export function compressedTranscriptIsShadowed(file) {
|
|
30
|
+
return isCompressedTranscript(file) && isFile(logicalTranscriptPath(file));
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* The on-disk spelling of a logical transcript right now. Codex may compress
|
|
34
|
+
* or restore the file between discovery and use, so callers holding either
|
|
35
|
+
* path get whichever one exists, preferring the plain file.
|
|
36
|
+
*/
|
|
37
|
+
export function existingTranscriptPath(file) {
|
|
38
|
+
const plain = logicalTranscriptPath(file);
|
|
39
|
+
if (isFile(plain))
|
|
40
|
+
return plain;
|
|
41
|
+
if (/\.jsonl$/iu.test(plain) && isFile(`${plain}${COMPRESSED_SUFFIX}`))
|
|
42
|
+
return `${plain}${COMPRESSED_SUFFIX}`;
|
|
43
|
+
return isFile(file) ? file : '';
|
|
44
|
+
}
|
|
45
|
+
function statFingerprint(file) {
|
|
46
|
+
const stat = fs.statSync(file);
|
|
47
|
+
return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}`;
|
|
48
|
+
}
|
|
49
|
+
// Sync zstd decoding cannot stop early, so every read decompresses the whole
|
|
50
|
+
// frame. A compressed rollout is idle by construction, so metadata probes
|
|
51
|
+
// (working directory, subagent markers, lineage) share a fingerprinted cache
|
|
52
|
+
// of bounded prefixes instead of re-decoding on every inventory pass. Whole
|
|
53
|
+
// bodies are never cached: a long rollout can decode to hundreds of MB.
|
|
54
|
+
const prefixes = new Map();
|
|
55
|
+
let prefixBytes = 0;
|
|
56
|
+
/** Whole decoded contents of a transcript file, compressed or not. */
|
|
57
|
+
export function readTranscriptSync(file) {
|
|
58
|
+
const contents = fs.readFileSync(file);
|
|
59
|
+
return isCompressedTranscript(file) ? zstdDecompressSync(contents) : contents;
|
|
60
|
+
}
|
|
61
|
+
function plainPrefix(file, maxBytes) {
|
|
62
|
+
const handle = fs.openSync(file, 'r');
|
|
63
|
+
try {
|
|
64
|
+
const buffer = Buffer.alloc(Math.max(0, Math.min(fs.fstatSync(handle).size, maxBytes)));
|
|
65
|
+
const bytesRead = fs.readSync(handle, buffer, 0, buffer.length, 0);
|
|
66
|
+
return buffer.subarray(0, bytesRead);
|
|
67
|
+
}
|
|
68
|
+
finally {
|
|
69
|
+
fs.closeSync(handle);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/** At most `maxBytes` decoded bytes from the start of a transcript file. */
|
|
73
|
+
export function readTranscriptPrefixSync(file, maxBytes) {
|
|
74
|
+
if (!isCompressedTranscript(file))
|
|
75
|
+
return plainPrefix(file, maxBytes);
|
|
76
|
+
const fingerprint = statFingerprint(file);
|
|
77
|
+
const cached = prefixes.get(file);
|
|
78
|
+
if (cached?.fingerprint === fingerprint &&
|
|
79
|
+
(cached.prefix.length >= maxBytes || cached.prefix.length < MAX_PREFIX_BYTES)) {
|
|
80
|
+
prefixes.delete(file);
|
|
81
|
+
prefixes.set(file, cached);
|
|
82
|
+
return cached.prefix.subarray(0, maxBytes);
|
|
83
|
+
}
|
|
84
|
+
const contents = readTranscriptSync(file);
|
|
85
|
+
if (cached) {
|
|
86
|
+
prefixes.delete(file);
|
|
87
|
+
prefixBytes -= cached.prefix.length;
|
|
88
|
+
}
|
|
89
|
+
// Copy so the cache never pins a whole decoded rollout.
|
|
90
|
+
const prefix = Buffer.from(contents.subarray(0, MAX_PREFIX_BYTES));
|
|
91
|
+
prefixes.set(file, { fingerprint, prefix });
|
|
92
|
+
prefixBytes += prefix.length;
|
|
93
|
+
while (prefixBytes > PREFIX_CACHE_BYTES && prefixes.size > 1) {
|
|
94
|
+
const [oldest, entry] = prefixes.entries().next().value;
|
|
95
|
+
prefixes.delete(oldest);
|
|
96
|
+
prefixBytes -= entry.prefix.length;
|
|
97
|
+
}
|
|
98
|
+
return contents.subarray(0, maxBytes);
|
|
99
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/** Failure details for a harness process that exits underneath Turns. */
|
|
2
|
+
const STDERR_TAIL_LINES = 12;
|
|
3
|
+
const STDERR_TAIL_CHARS = 4_000;
|
|
4
|
+
/** The last few stderr lines of a harness process. When the process exits on
|
|
5
|
+
* its own, they are usually the only explanation (for example, an outdated CLI
|
|
6
|
+
* rejecting an argument), and the turn's failure must carry them. */
|
|
7
|
+
export class StderrTail {
|
|
8
|
+
recent = [];
|
|
9
|
+
observe(line) {
|
|
10
|
+
const text = String(line)
|
|
11
|
+
.replace(/\u001b\[[0-9;?]*[A-Za-z]/gu, '')
|
|
12
|
+
.trimEnd();
|
|
13
|
+
if (!text.trim())
|
|
14
|
+
return;
|
|
15
|
+
this.recent.push(text.slice(0, STDERR_TAIL_CHARS));
|
|
16
|
+
if (this.recent.length > STDERR_TAIL_LINES)
|
|
17
|
+
this.recent.shift();
|
|
18
|
+
}
|
|
19
|
+
lines() {
|
|
20
|
+
const kept = [];
|
|
21
|
+
let size = 0;
|
|
22
|
+
for (const line of [...this.recent].reverse()) {
|
|
23
|
+
size += line.length + 1;
|
|
24
|
+
if (size > STDERR_TAIL_CHARS)
|
|
25
|
+
break;
|
|
26
|
+
kept.unshift(line);
|
|
27
|
+
}
|
|
28
|
+
return kept;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/** A harness process that ended underneath Turns rather than reporting a
|
|
32
|
+
* failure through its own protocol. */
|
|
33
|
+
export function harnessExitError(harness, code, signal, stderr) {
|
|
34
|
+
const label = harnessLabel(harness);
|
|
35
|
+
const how = signal ? `was stopped by ${signal}` : `exited with code ${code}`;
|
|
36
|
+
const message = [`${label} ${how} before the turn finished.`, ...(stderr.length ? ['', ...stderr] : [])].join('\n');
|
|
37
|
+
return Object.assign(new Error(message), {
|
|
38
|
+
harnessExit: { harness: String(harness), code: code ?? null, signal: signal ?? null, stderr },
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
function harnessLabel(harness) {
|
|
42
|
+
const name = String(harness || 'agent');
|
|
43
|
+
return name.charAt(0).toUpperCase() + name.slice(1);
|
|
44
|
+
}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
3
4
|
import path from 'node:path';
|
|
4
5
|
import process from 'node:process';
|
|
5
6
|
import { fileURLToPath } from 'node:url';
|
|
@@ -225,6 +226,22 @@ async function runUpload(credentials, { paths, watch = false, all = false, inter
|
|
|
225
226
|
provider?.close();
|
|
226
227
|
}
|
|
227
228
|
}
|
|
229
|
+
/** First lines of `turns daemon run`, printed before authentication so a
|
|
230
|
+
* report pasted from a terminal or log always says which build and runtime
|
|
231
|
+
* produced it (`npx` can silently run an older cached release). */
|
|
232
|
+
export function daemonStartupBanner({ version, nodeVersion = process.version, platform = process.platform, arch = process.arch, osName = `${os.type()} ${os.release()}`, hostname = os.hostname(), pid = process.pid, packageDirectory = CLI_DIRECTORY, paths, allowedDirectories = [], }) {
|
|
233
|
+
return [
|
|
234
|
+
`Turns daemon ${version}`,
|
|
235
|
+
` Node ${nodeVersion} on ${platform}-${arch} (${osName}), host ${hostname}, pid ${pid}`,
|
|
236
|
+
` Package ${packageDirectory}`,
|
|
237
|
+
` Config ${paths.configDirectory}`,
|
|
238
|
+
` State ${paths.stateDirectory}`,
|
|
239
|
+
allowedDirectories.length
|
|
240
|
+
? ` Directories:\n${allowedDirectories.map((value) => ` ${value}`).join('\n')}`
|
|
241
|
+
: ' Directories: all',
|
|
242
|
+
'',
|
|
243
|
+
].join('\n');
|
|
244
|
+
}
|
|
228
245
|
async function main(rawArguments = process.argv.slice(2)) {
|
|
229
246
|
const parsed = parseGlobalArguments(rawArguments);
|
|
230
247
|
const argv = parsed.argv;
|
|
@@ -274,6 +291,7 @@ async function main(rawArguments = process.argv.slice(2)) {
|
|
|
274
291
|
throw new Error('--once cannot be combined with `turns daemon run`.');
|
|
275
292
|
if (parsed.devUser && !parsed.origin)
|
|
276
293
|
throw new Error('--dev-user requires an explicit --server URL.');
|
|
294
|
+
logInfo(daemonStartupBanner({ version: pkg.version, paths, allowedDirectories }));
|
|
277
295
|
let credentials;
|
|
278
296
|
if (parsed.devUser) {
|
|
279
297
|
credentials = { origin: normalizeOrigin(parsed.origin) };
|
|
@@ -282,9 +300,6 @@ async function main(rawArguments = process.argv.slice(2)) {
|
|
|
282
300
|
await migrateLegacyCredentials(paths, parsed.origin);
|
|
283
301
|
credentials = await authenticate({ origin: parsed.origin, paths, interactive: false });
|
|
284
302
|
}
|
|
285
|
-
if (allowedDirectories.length) {
|
|
286
|
-
logInfo(`Daemon directory allowlist:\n${allowedDirectories.map((value) => ` ${value}`).join('\n')}\n`);
|
|
287
|
-
}
|
|
288
303
|
await runUpload(credentials, {
|
|
289
304
|
paths,
|
|
290
305
|
watch: true,
|
|
@@ -4,6 +4,7 @@ import os from 'node:os';
|
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import { DatabaseSync } from 'node:sqlite';
|
|
6
6
|
import { StringDecoder } from 'node:string_decoder';
|
|
7
|
+
import { compressedTranscriptIsShadowed, existingTranscriptPath, isCompressedTranscript, logicalTranscriptPath, readTranscriptPrefixSync, } from "./compressed-transcripts.js";
|
|
7
8
|
import { readDeepseekTrajectoryHeader } from "./deepseek-trajectories.js";
|
|
8
9
|
import { lettaSessionIdentity } from "./harness-registry.js";
|
|
9
10
|
import { readLettaSessionContext } from "./letta-session-context.js";
|
|
@@ -38,7 +39,8 @@ const EXTRA_FILE_STORES = {
|
|
|
38
39
|
},
|
|
39
40
|
codex: {
|
|
40
41
|
root: path.join(process.env.CODEX_HOME || path.join(HOME, '.codex'), 'sessions'),
|
|
41
|
-
|
|
42
|
+
// Codex zstd-compresses idle rollouts in place; see compressed-transcripts.ts.
|
|
43
|
+
include: (file) => /[\\/]rollout-[^\\/]+\.jsonl(?:\.zst)?$/i.test(file),
|
|
42
44
|
},
|
|
43
45
|
grok: { root: path.join(HOME, '.grok', 'sessions'), include: (file) => path.basename(file) === 'chat_history.jsonl' },
|
|
44
46
|
kimi: {
|
|
@@ -281,6 +283,10 @@ function descriptorNativeSessionId(descriptor) {
|
|
|
281
283
|
nativeSessionIdFromPath(String(descriptor?.harness || ''), descriptor?.sourcePath || descriptor?.file));
|
|
282
284
|
}
|
|
283
285
|
function fileDescriptor(harness, file) {
|
|
286
|
+
// While Codex restores or recompresses a rollout both spellings can exist
|
|
287
|
+
// briefly; the plain file is the one a live writer appends to.
|
|
288
|
+
if (compressedTranscriptIsShadowed(file))
|
|
289
|
+
return null;
|
|
284
290
|
const stat = safeStat(file);
|
|
285
291
|
if (!stat?.isFile() || stat.size === 0)
|
|
286
292
|
return null;
|
|
@@ -288,7 +294,7 @@ function fileDescriptor(harness, file) {
|
|
|
288
294
|
harness,
|
|
289
295
|
agent: harness,
|
|
290
296
|
file,
|
|
291
|
-
sourcePath: displayPath(file),
|
|
297
|
+
sourcePath: displayPath(logicalTranscriptPath(file)),
|
|
292
298
|
size: stat.size,
|
|
293
299
|
mtimeMs: stat.mtimeMs,
|
|
294
300
|
createdAt: (stat.birthtimeMs > 0 ? stat.birthtime : stat.mtime).toISOString(),
|
|
@@ -301,14 +307,16 @@ export function nativeDescriptorForSource(harness, sourcePath) {
|
|
|
301
307
|
const store = Object.hasOwn(EXTRA_FILE_STORES, harness) ? EXTRA_FILE_STORES[harness] : null;
|
|
302
308
|
if (!store || !sourcePath || sourcePath.includes('\0'))
|
|
303
309
|
return null;
|
|
304
|
-
const
|
|
305
|
-
if (!path.isAbsolute(
|
|
310
|
+
const logical = sourcePath.startsWith('~/') ? path.join(HOME, sourcePath.slice(2)) : sourcePath;
|
|
311
|
+
if (!path.isAbsolute(logical) || !store.include(logical))
|
|
306
312
|
return null;
|
|
313
|
+
// A source path always names the plain transcript; follow it to the
|
|
314
|
+
// compressed copy when that is what exists on disk now.
|
|
315
|
+
const physical = existingTranscriptPath(logical);
|
|
316
|
+
const file = physical && store.include(physical) ? physical : logical;
|
|
307
317
|
const relative = path.relative(store.root, file);
|
|
308
318
|
if (!relative || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative))
|
|
309
319
|
return null;
|
|
310
|
-
if (!store.include(file))
|
|
311
|
-
return null;
|
|
312
320
|
// Discovery does not traverse symlinked entries. Keep this direct lookup
|
|
313
321
|
// from admitting a transcript that an inventory walk would never discover.
|
|
314
322
|
try {
|
|
@@ -383,6 +391,17 @@ function firstJsonDocument(file) {
|
|
|
383
391
|
const stat = safeStat(file);
|
|
384
392
|
if (!stat?.isFile())
|
|
385
393
|
return null;
|
|
394
|
+
if (isCompressedTranscript(file)) {
|
|
395
|
+
try {
|
|
396
|
+
const source = readTranscriptPrefixSync(file, 1024 * 1024)
|
|
397
|
+
.toString('utf8')
|
|
398
|
+
.replace(/^\uFEFF/u, '');
|
|
399
|
+
return JSON.parse(source.split(/\r?\n/u, 1)[0]);
|
|
400
|
+
}
|
|
401
|
+
catch {
|
|
402
|
+
return null;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
386
405
|
let descriptor;
|
|
387
406
|
try {
|
|
388
407
|
descriptor = fs.openSync(file, 'r');
|
|
@@ -896,6 +915,17 @@ function filePrefix(file, maxBytes) {
|
|
|
896
915
|
}
|
|
897
916
|
}
|
|
898
917
|
function recordPrefix(file, limit, maxBytes) {
|
|
918
|
+
if (isCompressedTranscript(file)) {
|
|
919
|
+
try {
|
|
920
|
+
// Callers split and cap the record count themselves.
|
|
921
|
+
return readTranscriptPrefixSync(file, maxBytes)
|
|
922
|
+
.toString('utf8')
|
|
923
|
+
.replace(/^\uFEFF/u, '');
|
|
924
|
+
}
|
|
925
|
+
catch {
|
|
926
|
+
return '';
|
|
927
|
+
}
|
|
928
|
+
}
|
|
899
929
|
let handle;
|
|
900
930
|
try {
|
|
901
931
|
const stat = fs.statSync(file);
|
package/dist/queue-worker.js
CHANGED
|
@@ -301,6 +301,28 @@ export function runtimeEvent(anchorName, messageId, event, redactor = DEFAULT_RE
|
|
|
301
301
|
value: redactor.redactJsonLine(JSON.stringify(event)),
|
|
302
302
|
};
|
|
303
303
|
}
|
|
304
|
+
// A harness that exits underneath its turn often reports the reason only on
|
|
305
|
+
// stderr, before any protocol frame the reducer could show. Record that
|
|
306
|
+
// ending explicitly so the transcript can explain the failed turn.
|
|
307
|
+
if (event?.event === 'turn.complete' && event.result?.error?.harnessExit && !event.result.interrupted) {
|
|
308
|
+
const exit = event.result.error.harnessExit;
|
|
309
|
+
return {
|
|
310
|
+
eventKey: key,
|
|
311
|
+
kind: 'session.harness_exit',
|
|
312
|
+
phase: 'queued',
|
|
313
|
+
ts,
|
|
314
|
+
source: 'agent-anchor',
|
|
315
|
+
value: redactor.redactJsonLine(JSON.stringify({
|
|
316
|
+
type: 'turns.session.harness_exit',
|
|
317
|
+
messageId: String(event.messageId || messageId || ''),
|
|
318
|
+
harness: String(exit.harness || event.agent || ''),
|
|
319
|
+
code: typeof exit.code === 'number' ? exit.code : null,
|
|
320
|
+
signal: exit.signal ? String(exit.signal) : null,
|
|
321
|
+
message: String(event.result.error.message || ''),
|
|
322
|
+
stderr: Array.isArray(exit.stderr) ? exit.stderr.map(String) : [],
|
|
323
|
+
})),
|
|
324
|
+
};
|
|
325
|
+
}
|
|
304
326
|
if (event?.event === 'turn.steer') {
|
|
305
327
|
return {
|
|
306
328
|
eventKey: key,
|
package/dist/upload-sessions.js
CHANGED
|
@@ -9,6 +9,7 @@ import WebSocket from 'ws';
|
|
|
9
9
|
import { logError, logInfo } from "./logging.js";
|
|
10
10
|
import { NativeInventoryWorker } from "./native-inventory-worker.js";
|
|
11
11
|
import { QueueDispatchLifecycle } from "./queue-lifecycle.js";
|
|
12
|
+
import { isCompressedTranscript, readTranscriptSync } from "./compressed-transcripts.js";
|
|
12
13
|
import { streamDeepseekTrajectoryLines } from "./deepseek-trajectories.js";
|
|
13
14
|
import { normalizeAllowedDirectories, workingDirectoryIsAllowed } from "./directory-policy.js";
|
|
14
15
|
import { discoverGitAssociation } from "./git-association.js";
|
|
@@ -481,6 +482,22 @@ async function* completeNativeFileLines(file, startOffset, endOffset) {
|
|
|
481
482
|
input.destroy();
|
|
482
483
|
}
|
|
483
484
|
}
|
|
485
|
+
/** Complete lines of a compressed rollout. Offsets are decoded-byte offsets,
|
|
486
|
+
* which never compare with the compressed file size, so these files always
|
|
487
|
+
* rescan from the start (see canResumeNativeCursor). */
|
|
488
|
+
function* completeDecodedLines(contents) {
|
|
489
|
+
let lineStart = 0;
|
|
490
|
+
for (;;) {
|
|
491
|
+
const newline = contents.indexOf(0x0a, lineStart);
|
|
492
|
+
if (newline < 0)
|
|
493
|
+
return;
|
|
494
|
+
let line = contents.subarray(lineStart, newline);
|
|
495
|
+
if (line.at(-1) === 0x0d)
|
|
496
|
+
line = line.subarray(0, -1);
|
|
497
|
+
yield { raw: line.toString('utf8'), endOffset: newline + 1 };
|
|
498
|
+
lineStart = newline + 1;
|
|
499
|
+
}
|
|
500
|
+
}
|
|
484
501
|
async function* nativeSourceRecords(descriptor, cursor) {
|
|
485
502
|
if (descriptor.harness === 'deepseek') {
|
|
486
503
|
for await (const raw of streamDeepseekTrajectoryLines(descriptor.file)) {
|
|
@@ -502,6 +519,10 @@ async function* nativeSourceRecords(descriptor, cursor) {
|
|
|
502
519
|
}
|
|
503
520
|
return;
|
|
504
521
|
}
|
|
522
|
+
if (isCompressedTranscript(descriptor.file)) {
|
|
523
|
+
yield* completeDecodedLines(readTranscriptSync(descriptor.file));
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
505
526
|
yield* completeNativeFileLines(descriptor.file, Number(cursor.offset) || 0, Math.max(-1, (Number(descriptor.size) || 0) - 1));
|
|
506
527
|
}
|
|
507
528
|
/** Stream canonical raw events while retaining only one WebSocket batch. The
|
|
@@ -962,6 +983,7 @@ function nativeSourceIdentity(file) {
|
|
|
962
983
|
function canResumeNativeCursor(descriptor, cursor, nextRow) {
|
|
963
984
|
return Boolean(cursor &&
|
|
964
985
|
!descriptor.file.endsWith('.json') &&
|
|
986
|
+
!isCompressedTranscript(descriptor.file) &&
|
|
965
987
|
descriptor.harness !== 'deepseek' &&
|
|
966
988
|
!descriptor.sourcePath.includes('.db#') &&
|
|
967
989
|
cursor.file === descriptor.file &&
|