iteration-cli 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/dist/claude-transcript.d.ts +6 -0
- package/dist/claude-transcript.js +255 -0
- package/dist/clerk-device-auth.d.ts +2 -1
- package/dist/clerk-device-auth.js +6 -4
- package/dist/cli.js +104 -61
- package/dist/collector.d.ts +1 -1
- package/dist/collector.js +32 -14
- package/dist/history-import.d.ts +1 -1
- package/dist/history-import.js +18 -15
- package/dist/launch-agent.d.ts +5 -4
- package/dist/launch-agent.js +26 -23
- package/dist/rollout.js +122 -135
- package/dist/service.d.ts +2 -1
- package/dist/service.js +14 -5
- package/dist/status-view.d.ts +3 -2
- package/dist/status-view.js +43 -36
- package/dist/storage.d.ts +21 -10
- package/dist/storage.js +143 -51
- package/package.json +2 -2
package/dist/collector.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { promises as fs } from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { parseTraceEventAcknowledgement, } from "iteration-trace-contracts";
|
|
4
|
+
import { parseClaudeLine } from "./claude-transcript.js";
|
|
4
5
|
import { parseRolloutLine } from "./rollout.js";
|
|
5
|
-
import { acknowledgeEvents, appendEvents, loadState, nextBatch, outboxSize, pendingEventIds,
|
|
6
|
+
import { acknowledgeEvents, appendEvents, collectorSource, collectorSourceDirectory, loadState, nextBatch, outboxSize, pendingEventIds, sourceFiles, saveState, } from "./storage.js";
|
|
6
7
|
const MAX_SCAN_BYTES_PER_FILE = 16 * 1024 * 1024;
|
|
7
8
|
const MAX_SCAN_BYTES_PER_TICK = 16 * 1024 * 1024;
|
|
8
9
|
const MAX_RECORD_BYTES = 3 * 1024 * 1024;
|
|
@@ -29,16 +30,31 @@ function logGap(filename, offset, reason) {
|
|
|
29
30
|
reason,
|
|
30
31
|
}));
|
|
31
32
|
}
|
|
32
|
-
|
|
33
|
+
function parseSourceRecord(source, line, filename, byteOffset) {
|
|
34
|
+
if (source === "claude")
|
|
35
|
+
return parseClaudeLine(line, filename, byteOffset);
|
|
36
|
+
if (source === "codex") {
|
|
37
|
+
const parsed = parseRolloutLine(line, filename, byteOffset);
|
|
38
|
+
return parsed.kind === "event"
|
|
39
|
+
? { events: [parsed.event], gaps: [] }
|
|
40
|
+
: parsed.kind === "gap"
|
|
41
|
+
? { events: [], gaps: [parsed.reason] }
|
|
42
|
+
: { events: [], gaps: [] };
|
|
43
|
+
}
|
|
44
|
+
const unhandled = source;
|
|
45
|
+
throw new Error(`Unsupported collector source: ${unhandled}`);
|
|
46
|
+
}
|
|
47
|
+
export async function scanSourceOnce(config, paths, signal) {
|
|
33
48
|
signal?.throwIfAborted();
|
|
49
|
+
const source = collectorSource(config);
|
|
50
|
+
const sourceDirectory = path.resolve(collectorSourceDirectory(config));
|
|
34
51
|
const state = await loadState(paths);
|
|
35
|
-
if (!state.initialized ||
|
|
36
|
-
|
|
37
|
-
throw new Error("Collector state does not match CODEX_HOME; run iteration connect codex again.");
|
|
52
|
+
if (!state.initialized || state.sourceDirectory !== sourceDirectory) {
|
|
53
|
+
throw new Error(`Collector state does not match the ${source} directory; run iteration connect ${source} again.`);
|
|
38
54
|
}
|
|
39
55
|
const cursors = { ...state.cursors };
|
|
40
56
|
const skipping = new Set(state.skipping);
|
|
41
|
-
const files = await
|
|
57
|
+
const files = await sourceFiles(source, sourceDirectory);
|
|
42
58
|
const queuedIds = await pendingEventIds(paths);
|
|
43
59
|
const events = [];
|
|
44
60
|
let remainingBudget = MAX_SCAN_BYTES_PER_TICK;
|
|
@@ -108,17 +124,19 @@ export async function scanRolloutsOnce(config, paths, signal) {
|
|
|
108
124
|
lineStart = index + 1;
|
|
109
125
|
continue;
|
|
110
126
|
}
|
|
111
|
-
const parsed =
|
|
112
|
-
|
|
113
|
-
logGap(filename, byteOffset,
|
|
114
|
-
|
|
115
|
-
|
|
127
|
+
const parsed = parseSourceRecord(source, line, filename, byteOffset);
|
|
128
|
+
for (const reason of parsed.gaps)
|
|
129
|
+
logGap(filename, byteOffset, reason);
|
|
130
|
+
for (const parsedEvent of parsed.events) {
|
|
131
|
+
if (queuedIds.has(parsedEvent.eventId))
|
|
132
|
+
continue;
|
|
133
|
+
const eventBytes = Buffer.byteLength(JSON.stringify(parsedEvent), "utf8");
|
|
116
134
|
if (eventBytes > MAX_EVENT_BYTES) {
|
|
117
135
|
logGap(filename, byteOffset, "normalized_event_exceeds_upload_limit");
|
|
118
136
|
}
|
|
119
137
|
else {
|
|
120
|
-
events.push(
|
|
121
|
-
queuedIds.add(
|
|
138
|
+
events.push(parsedEvent);
|
|
139
|
+
queuedIds.add(parsedEvent.eventId);
|
|
122
140
|
}
|
|
123
141
|
}
|
|
124
142
|
lineStart = index + 1;
|
|
@@ -192,7 +210,7 @@ async function uploadPendingOnce(config, paths, fetcher = fetch, signal, onAckno
|
|
|
192
210
|
}
|
|
193
211
|
export async function collectorTick(config, paths, fetcher = fetch, signal, onAcknowledged) {
|
|
194
212
|
let uploaded = await uploadPendingOnce(config, paths, fetcher, signal, onAcknowledged);
|
|
195
|
-
const scan = await
|
|
213
|
+
const scan = await scanSourceOnce(config, paths, signal);
|
|
196
214
|
uploaded += await uploadPendingOnce(config, paths, fetcher, signal, onAcknowledged);
|
|
197
215
|
return {
|
|
198
216
|
discovered: scan.discovered,
|
package/dist/history-import.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export type HistoryImportControl = Readonly<{
|
|
|
9
9
|
start: (directory: string) => Promise<unknown>;
|
|
10
10
|
acquireLock: (paths: CollectorPaths) => Promise<() => Promise<void>>;
|
|
11
11
|
}>;
|
|
12
|
-
export declare function
|
|
12
|
+
export declare function importSourceHistory(config: CollectorConfig, paths: CollectorPaths, confirmed: boolean, control?: HistoryImportControl): Promise<Readonly<{
|
|
13
13
|
imported: boolean;
|
|
14
14
|
size: HistoryImportSize;
|
|
15
15
|
backgroundRunning: boolean | null;
|
package/dist/history-import.js
CHANGED
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { installLaunchAgent, launchAgentStatus, uninstallLaunchAgent, } from "./launch-agent.js";
|
|
3
3
|
import { waitForCollectorLock } from "./service.js";
|
|
4
|
-
import { historyImportSize, loadState, resetHistoryCursors, } from "./storage.js";
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
4
|
+
import { collectorSource, collectorSourceDirectory, historyImportSize, loadState, resetHistoryCursors, } from "./storage.js";
|
|
5
|
+
function macControl(source) {
|
|
6
|
+
return {
|
|
7
|
+
isRunning: () => launchAgentStatus(source),
|
|
8
|
+
stop: () => uninstallLaunchAgent(source),
|
|
9
|
+
start: (directory) => installLaunchAgent(directory, source),
|
|
10
|
+
acquireLock: waitForCollectorLock,
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
export async function importSourceHistory(config, paths, confirmed, control = macControl(collectorSource(config))) {
|
|
14
|
+
const source = collectorSource(config);
|
|
15
|
+
const directory = collectorSourceDirectory(config);
|
|
12
16
|
const state = await loadState(paths);
|
|
13
|
-
if (!state.initialized ||
|
|
14
|
-
|
|
15
|
-
throw new Error("Connect the Codex source before importing its history.");
|
|
17
|
+
if (!state.initialized || state.sourceDirectory !== path.resolve(directory)) {
|
|
18
|
+
throw new Error(`Connect the ${source} source before importing its history.`);
|
|
16
19
|
}
|
|
17
|
-
const estimate = await historyImportSize(
|
|
20
|
+
const estimate = await historyImportSize(directory, source);
|
|
18
21
|
if (!confirmed)
|
|
19
22
|
return { imported: false, size: estimate, backgroundRunning: null };
|
|
20
23
|
const wasRunning = await control.isRunning();
|
|
@@ -29,7 +32,7 @@ export async function importCodexHistory(config, paths, confirmed, control = mac
|
|
|
29
32
|
}
|
|
30
33
|
const releaseLock = await control.acquireLock(paths);
|
|
31
34
|
try {
|
|
32
|
-
size = await resetHistoryCursors(
|
|
35
|
+
size = await resetHistoryCursors(directory, paths, source);
|
|
33
36
|
rewound = true;
|
|
34
37
|
}
|
|
35
38
|
finally {
|
|
@@ -45,8 +48,8 @@ export async function importCodexHistory(config, paths, confirmed, control = mac
|
|
|
45
48
|
}
|
|
46
49
|
catch (error) {
|
|
47
50
|
throw new Error(rewound
|
|
48
|
-
?
|
|
49
|
-
:
|
|
51
|
+
? `History checkpoints were rewound, but the collector could not restart. Run iteration start ${source} to resume uploads.`
|
|
52
|
+
: `The collector could not be restored after a failed history import. Run iteration start ${source} to resume uploads.`, {
|
|
50
53
|
cause: failure ? new AggregateError([failure.error, error]) : error,
|
|
51
54
|
});
|
|
52
55
|
}
|
package/dist/launch-agent.d.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
import type { CollectorSource } from "iteration-trace-contracts";
|
|
1
2
|
export type LaunchAgentRestart = {
|
|
2
3
|
isLoaded(timeoutMs: number): Promise<boolean>;
|
|
3
4
|
bootstrap(timeoutMs: number): Promise<void>;
|
|
4
5
|
wait(delayMs: number): Promise<void>;
|
|
5
6
|
now(): number;
|
|
6
7
|
};
|
|
7
|
-
export declare function launchAgentPlist(cliPath?: string, nodePath?: string, logDirectory?: string, dataDirectory?: string): string;
|
|
8
|
+
export declare function launchAgentPlist(cliPath?: string, nodePath?: string, logDirectory?: string, dataDirectory?: string, source?: CollectorSource): string;
|
|
8
9
|
export declare function bootstrapAfterBootout(wasLoaded: boolean, control: LaunchAgentRestart): Promise<void>;
|
|
9
|
-
export declare function installLaunchAgent(dataDirectory?: string): Promise<string>;
|
|
10
|
-
export declare function uninstallLaunchAgent(): Promise<void>;
|
|
11
|
-
export declare function launchAgentStatus(): Promise<boolean>;
|
|
10
|
+
export declare function installLaunchAgent(dataDirectory?: string, source?: CollectorSource): Promise<string>;
|
|
11
|
+
export declare function uninstallLaunchAgent(source?: CollectorSource): Promise<void>;
|
|
12
|
+
export declare function launchAgentStatus(source?: CollectorSource): Promise<boolean>;
|
|
12
13
|
export declare function launchAgentRunning(output: string): boolean;
|
package/dist/launch-agent.js
CHANGED
|
@@ -6,7 +6,9 @@ import path from "node:path";
|
|
|
6
6
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
8
|
const execFile = promisify(execFileCallback);
|
|
9
|
-
|
|
9
|
+
function label(source) {
|
|
10
|
+
return `com.iteration.${source}-collector`;
|
|
11
|
+
}
|
|
10
12
|
const LAUNCHCTL_TIMEOUT_MS = 5_000;
|
|
11
13
|
const RESTART_TIMEOUT_MS = 10_000;
|
|
12
14
|
const RESTART_POLL_MS = 200;
|
|
@@ -18,14 +20,15 @@ function xml(value) {
|
|
|
18
20
|
.replaceAll('"', """)
|
|
19
21
|
.replaceAll("'", "'");
|
|
20
22
|
}
|
|
21
|
-
function plistPath(homeDirectory = os.homedir()) {
|
|
22
|
-
return path.join(homeDirectory, "Library", "LaunchAgents", `${
|
|
23
|
+
function plistPath(source, homeDirectory = os.homedir()) {
|
|
24
|
+
return path.join(homeDirectory, "Library", "LaunchAgents", `${label(source)}.plist`);
|
|
23
25
|
}
|
|
24
|
-
export function launchAgentPlist(cliPath = fileURLToPath(new URL("./cli.js", import.meta.url)), nodePath = process.execPath, logDirectory = path.join(os.homedir(), "Library", "Logs"), dataDirectory = path.join(os.homedir(), ".config", "iteration", "codex-collector")) {
|
|
26
|
+
export function launchAgentPlist(cliPath = fileURLToPath(new URL("./cli.js", import.meta.url)), nodePath = process.execPath, logDirectory = path.join(os.homedir(), "Library", "Logs"), dataDirectory = path.join(os.homedir(), ".config", "iteration", "codex-collector"), source = "codex") {
|
|
25
27
|
const argumentsList = [
|
|
26
28
|
nodePath,
|
|
27
29
|
cliPath,
|
|
28
30
|
"run",
|
|
31
|
+
source,
|
|
29
32
|
"--data-directory",
|
|
30
33
|
dataDirectory,
|
|
31
34
|
];
|
|
@@ -36,13 +39,13 @@ export function launchAgentPlist(cliPath = fileURLToPath(new URL("./cli.js", imp
|
|
|
36
39
|
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
37
40
|
'<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">',
|
|
38
41
|
'<plist version="1.0"><dict>',
|
|
39
|
-
`<key>Label</key><string>${
|
|
42
|
+
`<key>Label</key><string>${label(source)}</string>`,
|
|
40
43
|
`<key>ProgramArguments</key><array>${argumentsXml}</array>`,
|
|
41
44
|
"<key>RunAtLoad</key><true/>",
|
|
42
45
|
"<key>KeepAlive</key><true/>",
|
|
43
46
|
"<key>ThrottleInterval</key><integer>10</integer>",
|
|
44
|
-
`<key>StandardOutPath</key><string>${xml(path.join(logDirectory,
|
|
45
|
-
`<key>StandardErrorPath</key><string>${xml(path.join(logDirectory,
|
|
47
|
+
`<key>StandardOutPath</key><string>${xml(path.join(logDirectory, `iteration-${source}-collector.log`))}</string>`,
|
|
48
|
+
`<key>StandardErrorPath</key><string>${xml(path.join(logDirectory, `iteration-${source}-collector.error.log`))}</string>`,
|
|
46
49
|
"</dict></plist>",
|
|
47
50
|
].join("");
|
|
48
51
|
}
|
|
@@ -61,9 +64,9 @@ function serviceMissing(error) {
|
|
|
61
64
|
return (String(details.code) === "113" ||
|
|
62
65
|
details.stderr?.includes("Could not find service") === true);
|
|
63
66
|
}
|
|
64
|
-
async function serviceLoaded(target, timeoutMs = LAUNCHCTL_TIMEOUT_MS) {
|
|
67
|
+
async function serviceLoaded(target, source, timeoutMs = LAUNCHCTL_TIMEOUT_MS) {
|
|
65
68
|
try {
|
|
66
|
-
await execFile("launchctl", ["print", `${target}/${
|
|
69
|
+
await execFile("launchctl", ["print", `${target}/${label(source)}`], {
|
|
67
70
|
timeout: timeoutMs,
|
|
68
71
|
});
|
|
69
72
|
return true;
|
|
@@ -74,10 +77,10 @@ async function serviceLoaded(target, timeoutMs = LAUNCHCTL_TIMEOUT_MS) {
|
|
|
74
77
|
throw error;
|
|
75
78
|
}
|
|
76
79
|
}
|
|
77
|
-
async function bootoutIfLoaded(target) {
|
|
78
|
-
if (!(await serviceLoaded(target)))
|
|
80
|
+
async function bootoutIfLoaded(target, source) {
|
|
81
|
+
if (!(await serviceLoaded(target, source)))
|
|
79
82
|
return false;
|
|
80
|
-
await execFile("launchctl", ["bootout", `${target}/${
|
|
83
|
+
await execFile("launchctl", ["bootout", `${target}/${label(source)}`], {
|
|
81
84
|
timeout: LAUNCHCTL_TIMEOUT_MS,
|
|
82
85
|
});
|
|
83
86
|
return true;
|
|
@@ -124,21 +127,21 @@ export async function bootstrapAfterBootout(wasLoaded, control) {
|
|
|
124
127
|
}
|
|
125
128
|
}
|
|
126
129
|
}
|
|
127
|
-
export async function installLaunchAgent(dataDirectory) {
|
|
130
|
+
export async function installLaunchAgent(dataDirectory, source = "codex") {
|
|
128
131
|
if (process.platform !== "darwin") {
|
|
129
132
|
throw new Error("Automatic startup is currently supported on macOS only.");
|
|
130
133
|
}
|
|
131
|
-
const file = plistPath();
|
|
134
|
+
const file = plistPath(source);
|
|
132
135
|
const launchAgents = path.dirname(file);
|
|
133
136
|
const logs = path.join(os.homedir(), "Library", "Logs");
|
|
134
137
|
await fs.mkdir(launchAgents, { recursive: true, mode: 0o755 });
|
|
135
138
|
await fs.mkdir(logs, { recursive: true, mode: 0o755 });
|
|
136
139
|
await fs.writeFile(file, launchAgentPlist(undefined, undefined, undefined, dataDirectory ??
|
|
137
|
-
path.join(os.homedir(), ".config", "iteration",
|
|
140
|
+
path.join(os.homedir(), ".config", "iteration", `${source}-collector`), source), { encoding: "utf8", mode: 0o644 });
|
|
138
141
|
const target = launchctlTarget();
|
|
139
|
-
const wasLoaded = await bootoutIfLoaded(target);
|
|
142
|
+
const wasLoaded = await bootoutIfLoaded(target, source);
|
|
140
143
|
await bootstrapAfterBootout(wasLoaded, {
|
|
141
|
-
isLoaded: (timeoutMs) => serviceLoaded(target, timeoutMs),
|
|
144
|
+
isLoaded: (timeoutMs) => serviceLoaded(target, source, timeoutMs),
|
|
142
145
|
bootstrap: async (timeoutMs) => {
|
|
143
146
|
await execFile("launchctl", ["bootstrap", target, file], {
|
|
144
147
|
timeout: timeoutMs,
|
|
@@ -147,26 +150,26 @@ export async function installLaunchAgent(dataDirectory) {
|
|
|
147
150
|
wait: sleep,
|
|
148
151
|
now: Date.now,
|
|
149
152
|
});
|
|
150
|
-
await execFile("launchctl", ["kickstart", "-p", `${target}/${
|
|
153
|
+
await execFile("launchctl", ["kickstart", "-p", `${target}/${label(source)}`], {
|
|
151
154
|
timeout: LAUNCHCTL_TIMEOUT_MS,
|
|
152
155
|
});
|
|
153
156
|
return file;
|
|
154
157
|
}
|
|
155
|
-
export async function uninstallLaunchAgent() {
|
|
158
|
+
export async function uninstallLaunchAgent(source = "codex") {
|
|
156
159
|
if (process.platform !== "darwin") {
|
|
157
160
|
throw new Error("Automatic startup is currently supported on macOS only.");
|
|
158
161
|
}
|
|
159
|
-
const file = plistPath();
|
|
160
|
-
await bootoutIfLoaded(launchctlTarget());
|
|
162
|
+
const file = plistPath(source);
|
|
163
|
+
await bootoutIfLoaded(launchctlTarget(), source);
|
|
161
164
|
await fs.rm(file, { force: true });
|
|
162
165
|
}
|
|
163
|
-
export async function launchAgentStatus() {
|
|
166
|
+
export async function launchAgentStatus(source = "codex") {
|
|
164
167
|
if (process.platform !== "darwin")
|
|
165
168
|
return false;
|
|
166
169
|
try {
|
|
167
170
|
const { stdout } = await execFile("launchctl", [
|
|
168
171
|
"print",
|
|
169
|
-
`${launchctlTarget()}/${
|
|
172
|
+
`${launchctlTarget()}/${label(source)}`,
|
|
170
173
|
]);
|
|
171
174
|
return launchAgentRunning(stdout);
|
|
172
175
|
}
|
package/dist/rollout.js
CHANGED
|
@@ -3,6 +3,40 @@ import path from "node:path";
|
|
|
3
3
|
import { parseTracePayload, } from "iteration-trace-contracts";
|
|
4
4
|
import * as z from "zod";
|
|
5
5
|
const recordSchema = z.record(z.string(), z.unknown());
|
|
6
|
+
const CONTEXT_RECORDS = [
|
|
7
|
+
"session_meta",
|
|
8
|
+
"turn_context",
|
|
9
|
+
"compacted",
|
|
10
|
+
"token_usage_record",
|
|
11
|
+
"world_state",
|
|
12
|
+
"retained_context",
|
|
13
|
+
"security_risk_score",
|
|
14
|
+
];
|
|
15
|
+
const RESPONSE_CONTEXT_TYPES = [
|
|
16
|
+
"reasoning",
|
|
17
|
+
"todo_list",
|
|
18
|
+
"compaction",
|
|
19
|
+
"context_compaction",
|
|
20
|
+
"configuration_update",
|
|
21
|
+
"additional_tools",
|
|
22
|
+
];
|
|
23
|
+
const EVENT_CONTEXT_TYPES = [
|
|
24
|
+
"task_complete",
|
|
25
|
+
"task_started",
|
|
26
|
+
"turn_complete",
|
|
27
|
+
"token_count",
|
|
28
|
+
"thread_goal_updated",
|
|
29
|
+
"thread_rolled_back",
|
|
30
|
+
"turn_aborted",
|
|
31
|
+
"thread_settings_applied",
|
|
32
|
+
"agent_reasoning",
|
|
33
|
+
"agent_reasoning_raw_content",
|
|
34
|
+
];
|
|
35
|
+
const COMPLETED_CONTEXT_TYPES = [
|
|
36
|
+
"reasoning",
|
|
37
|
+
"todo_list",
|
|
38
|
+
];
|
|
39
|
+
const IGNORED_EVENT_MESSAGES = ["turn_started"];
|
|
6
40
|
function parseJsonRecord(value) {
|
|
7
41
|
const parsed = recordSchema.safeParse(value);
|
|
8
42
|
return parsed.success ? parsed.data : null;
|
|
@@ -11,6 +45,18 @@ function parseJsonString(value) {
|
|
|
11
45
|
const parsed = z.string().safeParse(value);
|
|
12
46
|
return parsed.success ? parsed.data : null;
|
|
13
47
|
}
|
|
48
|
+
function oneOf(value, allowed) {
|
|
49
|
+
return value !== null && allowed.includes(value);
|
|
50
|
+
}
|
|
51
|
+
const sourceOrdinalSchema = z
|
|
52
|
+
.number()
|
|
53
|
+
.int()
|
|
54
|
+
.min(0)
|
|
55
|
+
.max(Number.MAX_SAFE_INTEGER);
|
|
56
|
+
function sourceOrdinal(value) {
|
|
57
|
+
const parsed = sourceOrdinalSchema.safeParse(value);
|
|
58
|
+
return parsed.success ? parsed.data : null;
|
|
59
|
+
}
|
|
14
60
|
function sessionFromPath(filename) {
|
|
15
61
|
const basename = path.basename(filename, ".jsonl");
|
|
16
62
|
return basename.startsWith("rollout-")
|
|
@@ -33,6 +79,32 @@ function safePayload(value) {
|
|
|
33
79
|
return null;
|
|
34
80
|
}
|
|
35
81
|
}
|
|
82
|
+
function eventTimestamp(record) {
|
|
83
|
+
const timestamp = parseJsonString(record.timestamp);
|
|
84
|
+
return timestamp !== null && !Number.isNaN(Date.parse(timestamp))
|
|
85
|
+
? timestamp
|
|
86
|
+
: null;
|
|
87
|
+
}
|
|
88
|
+
function capturedEvent(record, filename, byteOffset, eventType, payload, callId, run) {
|
|
89
|
+
const eventPayload = safePayload(payload);
|
|
90
|
+
if (eventPayload === null)
|
|
91
|
+
return { kind: "gap", reason: `invalid_${eventType}_payload` };
|
|
92
|
+
const sessionId = sessionFromPath(filename);
|
|
93
|
+
return {
|
|
94
|
+
kind: "event",
|
|
95
|
+
event: {
|
|
96
|
+
eventId: sha256(`${sessionId}:${byteOffset}`),
|
|
97
|
+
sessionId,
|
|
98
|
+
sequence: byteOffset,
|
|
99
|
+
callId,
|
|
100
|
+
projectId: null,
|
|
101
|
+
occurredAt: eventTimestamp(record),
|
|
102
|
+
eventType,
|
|
103
|
+
payload: eventPayload,
|
|
104
|
+
...(run === undefined ? {} : { run }),
|
|
105
|
+
},
|
|
106
|
+
};
|
|
107
|
+
}
|
|
36
108
|
function mapResponseItem(payload) {
|
|
37
109
|
const type = parseJsonString(payload.type);
|
|
38
110
|
if (type === "message") {
|
|
@@ -46,9 +118,10 @@ function mapResponseItem(payload) {
|
|
|
46
118
|
if (type === "agent_message") {
|
|
47
119
|
return { eventType: "assistant_message", callId: null };
|
|
48
120
|
}
|
|
49
|
-
if (
|
|
50
|
-
return null;
|
|
51
|
-
|
|
121
|
+
if (oneOf(type, RESPONSE_CONTEXT_TYPES)) {
|
|
122
|
+
return { eventType: type, callId: null };
|
|
123
|
+
}
|
|
124
|
+
if (oneOf(type, [
|
|
52
125
|
"function_call",
|
|
53
126
|
"custom_tool_call",
|
|
54
127
|
"mcp_tool_call",
|
|
@@ -56,42 +129,39 @@ function mapResponseItem(payload) {
|
|
|
56
129
|
"tool_search_call",
|
|
57
130
|
"image_generation_call",
|
|
58
131
|
"local_shell_call",
|
|
59
|
-
]
|
|
132
|
+
])) {
|
|
60
133
|
return { eventType: "tool_call", callId: callIdFrom(payload) };
|
|
61
134
|
}
|
|
62
|
-
if ([
|
|
135
|
+
if (oneOf(type, [
|
|
63
136
|
"function_call_output",
|
|
64
137
|
"custom_tool_call_output",
|
|
65
138
|
"mcp_tool_call_output",
|
|
66
139
|
"web_search_output",
|
|
67
140
|
"tool_search_output",
|
|
68
|
-
]
|
|
141
|
+
])) {
|
|
69
142
|
return { eventType: "tool_result", callId: callIdFrom(payload) };
|
|
70
143
|
}
|
|
71
144
|
return null;
|
|
72
145
|
}
|
|
73
146
|
function mapCompletedItem(item) {
|
|
74
147
|
const type = parseJsonString(item.type);
|
|
75
|
-
if ([
|
|
148
|
+
if (oneOf(type, [
|
|
76
149
|
"command_execution",
|
|
77
150
|
"mcp_tool_call",
|
|
78
151
|
"collab_tool_call",
|
|
79
152
|
"web_search",
|
|
80
153
|
"file_change",
|
|
81
|
-
]
|
|
154
|
+
])) {
|
|
82
155
|
return { eventType: "tool_result", callId: callIdFrom(item) };
|
|
83
156
|
}
|
|
84
157
|
if (type === "agent_message") {
|
|
85
158
|
return { eventType: "assistant_message", callId: null };
|
|
86
159
|
}
|
|
160
|
+
if (oneOf(type, COMPLETED_CONTEXT_TYPES)) {
|
|
161
|
+
return { eventType: type, callId: null };
|
|
162
|
+
}
|
|
87
163
|
return null;
|
|
88
164
|
}
|
|
89
|
-
function eventTimestamp(record) {
|
|
90
|
-
const timestamp = parseJsonString(record.timestamp);
|
|
91
|
-
return timestamp !== null && !Number.isNaN(Date.parse(timestamp))
|
|
92
|
-
? timestamp
|
|
93
|
-
: null;
|
|
94
|
-
}
|
|
95
165
|
export function parseRolloutLine(line, filename, byteOffset) {
|
|
96
166
|
let decoded;
|
|
97
167
|
try {
|
|
@@ -105,131 +175,66 @@ export function parseRolloutLine(line, filename, byteOffset) {
|
|
|
105
175
|
return { kind: "gap", reason: "record_is_not_an_object" };
|
|
106
176
|
const recordType = parseJsonString(record.type);
|
|
107
177
|
const payload = parseJsonRecord(record.payload);
|
|
108
|
-
if (
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
"
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
178
|
+
if (oneOf(recordType, CONTEXT_RECORDS)) {
|
|
179
|
+
if (payload === null)
|
|
180
|
+
return { kind: "gap", reason: `${recordType}_without_payload` };
|
|
181
|
+
let run;
|
|
182
|
+
if (recordType === "session_meta" &&
|
|
183
|
+
parseJsonString(payload.id) === sessionFromPath(filename).slice(-36)) {
|
|
184
|
+
const source = parseJsonRecord(payload.source);
|
|
185
|
+
const subagent = source === null ? null : parseJsonRecord(source.subagent);
|
|
186
|
+
const spawn = subagent === null ? null : parseJsonRecord(subagent.thread_spawn);
|
|
187
|
+
const parentId = parseJsonString(payload.parent_thread_id) ??
|
|
188
|
+
(spawn === null ? null : parseJsonString(spawn.parent_thread_id));
|
|
189
|
+
const startOrdinal = sourceOrdinal(payload.subagent_history_start_ordinal);
|
|
190
|
+
run = {
|
|
191
|
+
kind: "identity",
|
|
192
|
+
id: sessionFromPath(filename).slice(-36),
|
|
193
|
+
parentId,
|
|
194
|
+
ownStart: parentId === null,
|
|
195
|
+
startOrdinal,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
return capturedEvent(record, filename, byteOffset, recordType, payload, null, run);
|
|
118
199
|
}
|
|
119
200
|
if (recordType === "response_item" && payload !== null) {
|
|
120
201
|
const mapped = mapResponseItem(payload);
|
|
121
202
|
if (mapped === null) {
|
|
122
203
|
const payloadType = parseJsonString(payload.type) ?? "unknown";
|
|
123
|
-
if ([
|
|
124
|
-
"reasoning",
|
|
125
|
-
"compaction",
|
|
126
|
-
"context_compaction",
|
|
127
|
-
"configuration_update",
|
|
128
|
-
"additional_tools",
|
|
129
|
-
].includes(payloadType)) {
|
|
130
|
-
return { kind: "ignored" };
|
|
131
|
-
}
|
|
132
204
|
return {
|
|
133
205
|
kind: "gap",
|
|
134
206
|
reason: `unsupported_response_item:${payloadType}`,
|
|
135
207
|
};
|
|
136
208
|
}
|
|
137
|
-
|
|
138
|
-
if (eventPayload === null)
|
|
139
|
-
return { kind: "gap", reason: "invalid_response_item_payload" };
|
|
140
|
-
const sessionId = sessionFromPath(filename);
|
|
141
|
-
return {
|
|
142
|
-
kind: "event",
|
|
143
|
-
event: {
|
|
144
|
-
eventId: sha256(`${sessionId}:${byteOffset}`),
|
|
145
|
-
sessionId,
|
|
146
|
-
sequence: byteOffset,
|
|
147
|
-
callId: mapped.callId,
|
|
148
|
-
projectId: null,
|
|
149
|
-
occurredAt: eventTimestamp(record),
|
|
150
|
-
eventType: mapped.eventType,
|
|
151
|
-
payload: eventPayload,
|
|
152
|
-
},
|
|
153
|
-
};
|
|
209
|
+
return capturedEvent(record, filename, byteOffset, mapped.eventType, payload, mapped.callId);
|
|
154
210
|
}
|
|
155
211
|
if (recordType === "event_msg" && payload !== null) {
|
|
156
212
|
const messageType = parseJsonString(payload.type);
|
|
157
213
|
if (messageType === "user_message") {
|
|
158
|
-
|
|
159
|
-
if (eventPayload === null)
|
|
160
|
-
return { kind: "gap", reason: "invalid_user_message_payload" };
|
|
161
|
-
const sessionId = sessionFromPath(filename);
|
|
162
|
-
return {
|
|
163
|
-
kind: "event",
|
|
164
|
-
event: {
|
|
165
|
-
eventId: sha256(`${sessionId}:${byteOffset}`),
|
|
166
|
-
sessionId,
|
|
167
|
-
sequence: byteOffset,
|
|
168
|
-
callId: null,
|
|
169
|
-
projectId: null,
|
|
170
|
-
occurredAt: eventTimestamp(record),
|
|
171
|
-
eventType: "user_message",
|
|
172
|
-
payload: eventPayload,
|
|
173
|
-
},
|
|
174
|
-
};
|
|
214
|
+
return capturedEvent(record, filename, byteOffset, "user_message", payload, null);
|
|
175
215
|
}
|
|
176
216
|
if (messageType === "agent_message") {
|
|
177
|
-
|
|
178
|
-
if (eventPayload === null)
|
|
179
|
-
return { kind: "gap", reason: "invalid_agent_message_payload" };
|
|
180
|
-
const sessionId = sessionFromPath(filename);
|
|
181
|
-
return {
|
|
182
|
-
kind: "event",
|
|
183
|
-
event: {
|
|
184
|
-
eventId: sha256(`${sessionId}:${byteOffset}`),
|
|
185
|
-
sessionId,
|
|
186
|
-
sequence: byteOffset,
|
|
187
|
-
callId: null,
|
|
188
|
-
projectId: null,
|
|
189
|
-
occurredAt: eventTimestamp(record),
|
|
190
|
-
eventType: "assistant_message",
|
|
191
|
-
payload: eventPayload,
|
|
192
|
-
},
|
|
193
|
-
};
|
|
217
|
+
return capturedEvent(record, filename, byteOffset, "assistant_message", payload, null);
|
|
194
218
|
}
|
|
195
|
-
if ([
|
|
219
|
+
if (oneOf(messageType, [
|
|
196
220
|
"mcp_tool_call_end",
|
|
197
221
|
"web_search_end",
|
|
198
222
|
"image_generation_end",
|
|
199
223
|
"patch_apply_end",
|
|
200
|
-
]
|
|
201
|
-
|
|
202
|
-
if (eventPayload === null)
|
|
203
|
-
return { kind: "gap", reason: "invalid_tool_result_payload" };
|
|
204
|
-
const sessionId = sessionFromPath(filename);
|
|
205
|
-
return {
|
|
206
|
-
kind: "event",
|
|
207
|
-
event: {
|
|
208
|
-
eventId: sha256(`${sessionId}:${byteOffset}`),
|
|
209
|
-
sessionId,
|
|
210
|
-
sequence: byteOffset,
|
|
211
|
-
callId: callIdFrom(payload),
|
|
212
|
-
projectId: null,
|
|
213
|
-
occurredAt: eventTimestamp(record),
|
|
214
|
-
eventType: "tool_result",
|
|
215
|
-
payload: eventPayload,
|
|
216
|
-
},
|
|
217
|
-
};
|
|
224
|
+
])) {
|
|
225
|
+
return capturedEvent(record, filename, byteOffset, "tool_result", payload, callIdFrom(payload));
|
|
218
226
|
}
|
|
219
|
-
if (
|
|
220
|
-
"task_started",
|
|
221
|
-
"turn_started",
|
|
222
|
-
"task_complete",
|
|
223
|
-
"turn_complete",
|
|
224
|
-
"token_count",
|
|
225
|
-
"thread_goal_updated",
|
|
226
|
-
"thread_rolled_back",
|
|
227
|
-
"turn_aborted",
|
|
228
|
-
"thread_settings_applied",
|
|
229
|
-
"agent_reasoning",
|
|
230
|
-
"agent_reasoning_raw_content",
|
|
231
|
-
].includes(messageType ?? "")) {
|
|
227
|
+
if (oneOf(messageType, IGNORED_EVENT_MESSAGES))
|
|
232
228
|
return { kind: "ignored" };
|
|
229
|
+
if (oneOf(messageType, EVENT_CONTEXT_TYPES)) {
|
|
230
|
+
const ordinal = sourceOrdinal(record.ordinal);
|
|
231
|
+
const run = messageType === "thread_settings_applied" &&
|
|
232
|
+
parseJsonString(payload.thread_id) ===
|
|
233
|
+
sessionFromPath(filename).slice(-36) &&
|
|
234
|
+
ordinal !== null
|
|
235
|
+
? { kind: "start", sourceOrdinal: ordinal }
|
|
236
|
+
: undefined;
|
|
237
|
+
return capturedEvent(record, filename, byteOffset, messageType, payload, null, run);
|
|
233
238
|
}
|
|
234
239
|
if (messageType !== "item_completed") {
|
|
235
240
|
return {
|
|
@@ -243,27 +248,9 @@ export function parseRolloutLine(line, filename, byteOffset) {
|
|
|
243
248
|
const mapped = mapCompletedItem(item);
|
|
244
249
|
if (mapped === null) {
|
|
245
250
|
const itemType = parseJsonString(item.type) ?? "unknown";
|
|
246
|
-
if (["reasoning", "todo_list"].includes(itemType))
|
|
247
|
-
return { kind: "ignored" };
|
|
248
251
|
return { kind: "gap", reason: `unsupported_completed_item:${itemType}` };
|
|
249
252
|
}
|
|
250
|
-
|
|
251
|
-
if (eventPayload === null)
|
|
252
|
-
return { kind: "gap", reason: "invalid_completed_item_payload" };
|
|
253
|
-
const sessionId = sessionFromPath(filename);
|
|
254
|
-
return {
|
|
255
|
-
kind: "event",
|
|
256
|
-
event: {
|
|
257
|
-
eventId: sha256(`${sessionId}:${byteOffset}`),
|
|
258
|
-
sessionId,
|
|
259
|
-
sequence: byteOffset,
|
|
260
|
-
callId: mapped.callId,
|
|
261
|
-
projectId: null,
|
|
262
|
-
occurredAt: eventTimestamp(record),
|
|
263
|
-
eventType: mapped.eventType,
|
|
264
|
-
payload: eventPayload,
|
|
265
|
-
},
|
|
266
|
-
};
|
|
253
|
+
return capturedEvent(record, filename, byteOffset, mapped.eventType, item, mapped.callId);
|
|
267
254
|
}
|
|
268
255
|
return {
|
|
269
256
|
kind: "gap",
|
package/dist/service.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
+
import type { CollectorSource } from "iteration-trace-contracts";
|
|
1
2
|
import { type CollectorPaths } from "./storage.js";
|
|
2
3
|
export declare function acquireCollectorLock(paths: CollectorPaths): Promise<() => Promise<void>>;
|
|
3
4
|
export declare function waitForCollectorLock(paths: CollectorPaths): Promise<() => Promise<void>>;
|
|
4
|
-
export declare function runCollector(directory?: string): Promise<void>;
|
|
5
|
+
export declare function runCollector(directory?: string, expectedSource?: CollectorSource): Promise<void>;
|