iteration-cli 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/dist/claude-transcript.d.ts +6 -0
- package/dist/claude-transcript.js +255 -0
- package/dist/clerk-device-auth.d.ts +2 -1
- package/dist/clerk-device-auth.js +6 -4
- package/dist/cli.js +135 -77
- package/dist/collector.d.ts +11 -2
- package/dist/collector.js +86 -38
- package/dist/history-import.d.ts +1 -1
- package/dist/history-import.js +18 -15
- package/dist/launch-agent.d.ts +5 -4
- package/dist/launch-agent.js +26 -23
- package/dist/rollout.js +122 -135
- package/dist/service.d.ts +2 -1
- package/dist/service.js +64 -7
- package/dist/status-view.d.ts +30 -0
- package/dist/status-view.js +246 -0
- package/dist/storage.d.ts +43 -10
- package/dist/storage.js +215 -51
- package/package.json +2 -2
package/dist/cli.js
CHANGED
|
@@ -4,11 +4,13 @@ import path from "node:path";
|
|
|
4
4
|
import { promises as fs } from "node:fs";
|
|
5
5
|
import { parseArgs } from "node:util";
|
|
6
6
|
import * as z from "zod";
|
|
7
|
+
import { collectorSourceLabel, } from "iteration-trace-contracts";
|
|
7
8
|
import { authenticateAndEnrollCollector } from "./clerk-device-auth.js";
|
|
8
|
-
import {
|
|
9
|
+
import { importSourceHistory } from "./history-import.js";
|
|
9
10
|
import { runCollector } from "./service.js";
|
|
11
|
+
import { renderDetailedStatus, renderHome, } from "./status-view.js";
|
|
10
12
|
import { installLaunchAgent, launchAgentStatus, uninstallLaunchAgent, } from "./launch-agent.js";
|
|
11
|
-
import { assertFreshCollectorDirectory, collectorPaths, ensureStartableState, historyScanProgress, loadConfig, outboxSize, pendingEventIds,
|
|
13
|
+
import { assertFreshCollectorDirectory, collectorSource, collectorSourceDirectory, collectorPaths, defaultCollectorDirectory, ensureStartableState, healthSnapshot, historyScanProgress, loadConfig, outboxSize, pendingEventIds, prepareSourceInitialState, saveConfig, saveState, validateCollectorEndpoint, } from "./storage.js";
|
|
12
14
|
const DEFAULT_INGEST_ENDPOINT = "https://useiteration.com/api/traces/ingest";
|
|
13
15
|
function argumentsForCli() {
|
|
14
16
|
const parsed = parseArgs({
|
|
@@ -21,6 +23,7 @@ function argumentsForCli() {
|
|
|
21
23
|
"device-name": { type: "string" },
|
|
22
24
|
"organization-id": { type: "string" },
|
|
23
25
|
"codex-home": { type: "string" },
|
|
26
|
+
"claude-home": { type: "string" },
|
|
24
27
|
"data-directory": { type: "string" },
|
|
25
28
|
"include-history": { type: "boolean" },
|
|
26
29
|
yes: { type: "boolean" },
|
|
@@ -48,8 +51,9 @@ function optionalString(values, name) {
|
|
|
48
51
|
function usage(command) {
|
|
49
52
|
if (command === "connect") {
|
|
50
53
|
process.stdout.write([
|
|
51
|
-
"Connect
|
|
54
|
+
"Connect a coding agent to Iteration",
|
|
52
55
|
" iteration connect codex",
|
|
56
|
+
" iteration connect claude",
|
|
53
57
|
"",
|
|
54
58
|
"Existing local sessions and new activity are collected automatically.",
|
|
55
59
|
"",
|
|
@@ -57,6 +61,7 @@ function usage(command) {
|
|
|
57
61
|
" --device-name NAME Name this Mac in Iteration",
|
|
58
62
|
" --organization-id UUID Select a workspace without a prompt",
|
|
59
63
|
" --codex-home DIR Use a different Codex data directory",
|
|
64
|
+
" --claude-home DIR Use a different Claude data directory",
|
|
60
65
|
" --data-directory DIR Use a different Iteration state directory",
|
|
61
66
|
" --endpoint URL Use a different Iteration server",
|
|
62
67
|
].join("\n") + "\n");
|
|
@@ -66,47 +71,55 @@ function usage(command) {
|
|
|
66
71
|
"Iteration CLI",
|
|
67
72
|
"",
|
|
68
73
|
"Get started:",
|
|
69
|
-
" iteration
|
|
74
|
+
" iteration See connected sources or connect Codex",
|
|
70
75
|
"",
|
|
71
76
|
"Commands:",
|
|
72
77
|
" iteration connect codex Connect Codex directly",
|
|
73
|
-
" iteration
|
|
74
|
-
" iteration
|
|
78
|
+
" iteration connect claude Connect Claude Code directly",
|
|
79
|
+
" iteration status See scan, upload, and error details",
|
|
80
|
+
" iteration disconnect codex|claude Stop a source",
|
|
75
81
|
"",
|
|
76
82
|
"Run iteration connect codex --help for advanced setup options.",
|
|
77
83
|
].join("\n") + "\n");
|
|
78
84
|
}
|
|
79
|
-
async function
|
|
85
|
+
async function fileExists(filename) {
|
|
80
86
|
try {
|
|
81
|
-
await fs.access(
|
|
87
|
+
await fs.access(filename);
|
|
82
88
|
return true;
|
|
83
89
|
}
|
|
84
90
|
catch (error) {
|
|
85
91
|
if (error instanceof Error && "code" in error && error.code === "ENOENT") {
|
|
86
92
|
return false;
|
|
87
93
|
}
|
|
88
|
-
throw new Error(`Could not inspect collector
|
|
94
|
+
throw new Error(`Could not inspect collector file at ${filename}.`, {
|
|
89
95
|
cause: error,
|
|
90
96
|
});
|
|
91
97
|
}
|
|
92
98
|
}
|
|
93
99
|
async function defaultCommand() {
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
-
|
|
100
|
+
const [codex, claude] = await Promise.all([
|
|
101
|
+
statusSnapshot("codex", {}),
|
|
102
|
+
statusSnapshot("claude", {}),
|
|
103
|
+
]);
|
|
104
|
+
const connected = [
|
|
105
|
+
{ source: "codex", snapshot: codex },
|
|
106
|
+
{ source: "claude", snapshot: claude },
|
|
107
|
+
].filter(({ snapshot }) => snapshot.kind !== "not_connected");
|
|
108
|
+
if (connected.length > 0) {
|
|
109
|
+
process.stdout.write(`${connected.map(({ source, snapshot }) => renderHome(snapshot, Date.now(), source)).join("\n\n")}\n`);
|
|
97
110
|
return;
|
|
98
111
|
}
|
|
99
112
|
if (process.stdin.isTTY !== true || process.stdout.isTTY !== true) {
|
|
100
|
-
process.stdout.write("Run iteration
|
|
113
|
+
process.stdout.write("Run iteration connect codex or iteration connect claude in a terminal.\n");
|
|
101
114
|
return;
|
|
102
115
|
}
|
|
103
|
-
await initialize({});
|
|
116
|
+
await initialize("codex", {});
|
|
104
117
|
}
|
|
105
|
-
function dataDirectory(values) {
|
|
118
|
+
function dataDirectory(source, values) {
|
|
106
119
|
return path.resolve(optionalString(values, "data-directory") ??
|
|
107
|
-
|
|
120
|
+
defaultCollectorDirectory(source));
|
|
108
121
|
}
|
|
109
|
-
async function initialize(values) {
|
|
122
|
+
async function initialize(source, values) {
|
|
110
123
|
if (process.platform !== "darwin") {
|
|
111
124
|
throw new Error("Automatic startup is currently supported on macOS only.");
|
|
112
125
|
}
|
|
@@ -114,118 +127,163 @@ async function initialize(values) {
|
|
|
114
127
|
const endpointUrl = validateCollectorEndpoint(endpoint);
|
|
115
128
|
const deviceName = optionalString(values, "device-name") ?? os.hostname();
|
|
116
129
|
const organizationId = optionalString(values, "organization-id");
|
|
117
|
-
const directory = dataDirectory(values);
|
|
130
|
+
const directory = dataDirectory(source, values);
|
|
118
131
|
const paths = collectorPaths(directory);
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
132
|
+
const sourceHome = path.resolve(source === "codex"
|
|
133
|
+
? (optionalString(values, "codex-home") ??
|
|
134
|
+
process.env.CODEX_HOME ??
|
|
135
|
+
path.join(os.homedir(), ".codex"))
|
|
136
|
+
: (optionalString(values, "claude-home") ??
|
|
137
|
+
process.env.CLAUDE_CONFIG_DIR ??
|
|
138
|
+
path.join(os.homedir(), ".claude")));
|
|
139
|
+
await assertFreshCollectorDirectory(paths, source);
|
|
140
|
+
const preparedState = await prepareSourceInitialState(source, sourceHome, true);
|
|
141
|
+
const enrollment = await authenticateAndEnrollCollector(endpointUrl, deviceName, organizationId, source);
|
|
142
|
+
const connection = {
|
|
126
143
|
endpoint: endpointUrl.toString(),
|
|
127
144
|
token: enrollment.token,
|
|
128
|
-
codexHome,
|
|
129
145
|
dataDirectory: directory,
|
|
146
|
+
organizationName: enrollment.organizationName,
|
|
130
147
|
};
|
|
148
|
+
const config = source === "codex"
|
|
149
|
+
? { ...connection, codexHome: sourceHome }
|
|
150
|
+
: { ...connection, source: "claude", claudeHome: sourceHome };
|
|
131
151
|
await saveConfig(config, paths);
|
|
132
152
|
try {
|
|
133
153
|
await saveState(preparedState, paths);
|
|
134
154
|
}
|
|
135
155
|
catch (error) {
|
|
136
|
-
throw new Error(
|
|
156
|
+
throw new Error(`Collector access was approved, but local setup did not finish. Run iteration start ${source} to repair it.`, { cause: error });
|
|
137
157
|
}
|
|
138
158
|
try {
|
|
139
|
-
await installLaunchAgent(directory);
|
|
159
|
+
await installLaunchAgent(directory, source);
|
|
140
160
|
}
|
|
141
161
|
catch (error) {
|
|
142
|
-
throw new Error(
|
|
162
|
+
throw new Error(`Collector access was approved, but background startup failed. Run iteration start ${source} to retry it.`, { cause: error });
|
|
143
163
|
}
|
|
144
|
-
process.stdout.write(
|
|
164
|
+
process.stdout.write(`${collectorSourceLabel(source)} is connected to ${enrollment.organizationName}. Existing sessions are importing and new activity is collected automatically. Run iteration to check progress.\n`);
|
|
145
165
|
}
|
|
146
|
-
async function importHistory(values) {
|
|
166
|
+
async function importHistory(source, values) {
|
|
147
167
|
if (process.platform !== "darwin") {
|
|
148
168
|
throw new Error("Automatic startup is currently supported on macOS only.");
|
|
149
169
|
}
|
|
150
|
-
const directory = dataDirectory(values);
|
|
170
|
+
const directory = dataDirectory(source, values);
|
|
151
171
|
const paths = collectorPaths(directory);
|
|
152
172
|
const config = await loadConfig(paths);
|
|
153
|
-
|
|
154
|
-
|
|
173
|
+
if (collectorSource(config) !== source)
|
|
174
|
+
throw new Error(`This collector directory belongs to ${collectorSource(config)}.`);
|
|
175
|
+
const estimate = (await importSourceHistory(config, paths, false)).size;
|
|
176
|
+
process.stdout.write(`Found ${estimate.files} local ${source} session files (${estimate.bytes} bytes). History may include prompts, code, and tool output that workspace members can read.\n`);
|
|
155
177
|
if (values.yes !== true) {
|
|
156
|
-
process.stdout.write(
|
|
178
|
+
process.stdout.write(`No changes made. Run iteration import-history ${source} --yes to confirm this upload.\n`);
|
|
157
179
|
return;
|
|
158
180
|
}
|
|
159
|
-
const result = await
|
|
160
|
-
process.stdout.write(`History scan scheduled for ${result.size.files} local session files (${result.size.bytes} bytes). Scanning and uploads run in bounded batches and may take time. Older sessions appear under the date Iteration receives them. Use iteration status to see local scan progress and pending uploads.${result.backgroundRunning ? "" :
|
|
181
|
+
const result = await importSourceHistory(config, paths, true);
|
|
182
|
+
process.stdout.write(`History scan scheduled for ${result.size.files} local session files (${result.size.bytes} bytes). Scanning and uploads run in bounded batches and may take time. Older sessions appear under the date Iteration receives them. Use iteration status ${source} to see local scan progress and pending uploads.${result.backgroundRunning ? "" : ` The collector was not running; run iteration start ${source} to begin scanning.`}\n`);
|
|
161
183
|
}
|
|
162
|
-
async function start(values) {
|
|
184
|
+
async function start(source, values) {
|
|
163
185
|
if (process.platform !== "darwin") {
|
|
164
186
|
throw new Error("Automatic startup is currently supported on macOS only.");
|
|
165
187
|
}
|
|
166
|
-
const directory = dataDirectory(values);
|
|
188
|
+
const directory = dataDirectory(source, values);
|
|
167
189
|
const paths = collectorPaths(directory);
|
|
168
190
|
const config = await loadConfig(paths);
|
|
191
|
+
if (collectorSource(config) !== source)
|
|
192
|
+
throw new Error(`This collector directory belongs to ${collectorSource(config)}.`);
|
|
169
193
|
const repairedState = await ensureStartableState(config, paths);
|
|
170
|
-
await installLaunchAgent(directory);
|
|
171
|
-
process.stdout.write(
|
|
194
|
+
await installLaunchAgent(directory, source);
|
|
195
|
+
process.stdout.write(`${collectorSourceLabel(source)} is collecting again. Run iteration to check progress.${repairedState ? " Incomplete setup was repaired." : ""}\n`);
|
|
172
196
|
}
|
|
173
|
-
async function
|
|
174
|
-
const paths = collectorPaths(dataDirectory(values));
|
|
175
|
-
if (!(await
|
|
176
|
-
|
|
177
|
-
|
|
197
|
+
async function statusSnapshot(source, values) {
|
|
198
|
+
const paths = collectorPaths(dataDirectory(source, values));
|
|
199
|
+
if (!(await fileExists(paths.config))) {
|
|
200
|
+
const [stateExists, pendingBytes] = await Promise.all([
|
|
201
|
+
fileExists(paths.state),
|
|
202
|
+
outboxSize(paths),
|
|
203
|
+
]);
|
|
204
|
+
if (stateExists || pendingBytes > 0) {
|
|
205
|
+
return { kind: "needs_repair", pendingBytes, configPath: paths.config };
|
|
206
|
+
}
|
|
207
|
+
return { kind: "not_connected" };
|
|
178
208
|
}
|
|
179
209
|
const config = await loadConfig(paths);
|
|
180
|
-
|
|
181
|
-
|
|
210
|
+
if (collectorSource(config) !== source)
|
|
211
|
+
throw new Error(`This collector directory belongs to ${collectorSource(config)}.`);
|
|
212
|
+
const [running, size, ids, scan, localHealth] = await Promise.all([
|
|
213
|
+
launchAgentStatus(source),
|
|
182
214
|
outboxSize(paths),
|
|
183
215
|
pendingEventIds(paths),
|
|
184
|
-
historyScanProgress(config
|
|
216
|
+
historyScanProgress(collectorSourceDirectory(config), paths, source),
|
|
217
|
+
healthSnapshot(paths),
|
|
185
218
|
]);
|
|
186
|
-
const
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
219
|
+
const pendingEvents = size === 0 ? 0 : ids.size;
|
|
220
|
+
const pendingBytes = ids.size === 0 ? 0 : size;
|
|
221
|
+
return {
|
|
222
|
+
kind: "connected",
|
|
223
|
+
workspaceName: config.organizationName ?? null,
|
|
224
|
+
running,
|
|
225
|
+
scan,
|
|
226
|
+
pendingEvents,
|
|
227
|
+
pendingBytes,
|
|
228
|
+
health: localHealth.health,
|
|
229
|
+
healthAvailable: localHealth.kind === "available",
|
|
230
|
+
tracesUrl: new URL("/app/sessions", config.endpoint).toString(),
|
|
231
|
+
logPath: path.join(os.homedir(), "Library", "Logs", `iteration-${source}-collector.error.log`),
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
async function status(source, values) {
|
|
235
|
+
if (source !== undefined) {
|
|
236
|
+
const snapshot = await statusSnapshot(source, values);
|
|
237
|
+
process.stdout.write(`${renderDetailedStatus(snapshot, Date.now(), source)}\n`);
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
const [codex, claude] = await Promise.all([
|
|
241
|
+
statusSnapshot("codex", values),
|
|
242
|
+
statusSnapshot("claude", values),
|
|
243
|
+
]);
|
|
244
|
+
process.stdout.write(`${renderDetailedStatus(codex, Date.now(), "codex")}\n\n${renderDetailedStatus(claude, Date.now(), "claude")}\n`);
|
|
198
245
|
}
|
|
199
246
|
async function main() {
|
|
200
247
|
const { command, source, values } = argumentsForCli();
|
|
248
|
+
const selectedSource = source === "codex" || source === "claude" ? source : null;
|
|
201
249
|
if (values.help === true || command === "help") {
|
|
202
250
|
return usage(command === "help" ? source : command);
|
|
203
251
|
}
|
|
204
252
|
if (values["include-history"] === true &&
|
|
205
|
-
!(command === "connect" &&
|
|
253
|
+
!(command === "connect" && selectedSource !== null)) {
|
|
206
254
|
throw new Error("Existing sessions are included automatically. Run iteration to connect.");
|
|
207
255
|
}
|
|
208
256
|
if (command === "" || (command === "connect" && source === undefined)) {
|
|
209
257
|
if (Object.keys(values).length > 0) {
|
|
210
|
-
throw new Error("Options require a source. Run iteration connect
|
|
258
|
+
throw new Error("Options require a source. Run iteration connect --help for details.");
|
|
211
259
|
}
|
|
212
260
|
return defaultCommand();
|
|
213
261
|
}
|
|
214
|
-
if (
|
|
215
|
-
|
|
262
|
+
if (selectedSource !== null) {
|
|
263
|
+
if (selectedSource === "codex" && values["claude-home"] !== undefined)
|
|
264
|
+
throw new Error("--claude-home requires the Claude source.");
|
|
265
|
+
if (selectedSource === "claude" && values["codex-home"] !== undefined)
|
|
266
|
+
throw new Error("--codex-home requires the Codex source.");
|
|
267
|
+
}
|
|
268
|
+
if (command === "connect" && selectedSource !== null)
|
|
269
|
+
return initialize(selectedSource, values);
|
|
216
270
|
if (command === "run") {
|
|
217
|
-
if (source !== undefined)
|
|
218
|
-
throw new Error("The run
|
|
219
|
-
return runCollector(optionalString(values, "data-directory") ?? undefined);
|
|
220
|
-
}
|
|
221
|
-
if (command === "status" &&
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
if (command === "
|
|
228
|
-
|
|
271
|
+
if (source !== undefined && selectedSource === null)
|
|
272
|
+
throw new Error("The run source must be codex or claude.");
|
|
273
|
+
return runCollector(optionalString(values, "data-directory") ?? undefined, selectedSource ?? undefined);
|
|
274
|
+
}
|
|
275
|
+
if (command === "status" &&
|
|
276
|
+
(source === undefined || selectedSource !== null)) {
|
|
277
|
+
if (source === undefined && values["data-directory"] !== undefined)
|
|
278
|
+
throw new Error("--data-directory requires a source with status.");
|
|
279
|
+
return status(selectedSource ?? undefined, values);
|
|
280
|
+
}
|
|
281
|
+
if (command === "import-history" && selectedSource !== null)
|
|
282
|
+
return importHistory(selectedSource, values);
|
|
283
|
+
if (command === "start" && selectedSource !== null)
|
|
284
|
+
return start(selectedSource, values);
|
|
285
|
+
if (command === "disconnect" && selectedSource !== null) {
|
|
286
|
+
await uninstallLaunchAgent(selectedSource);
|
|
229
287
|
process.stdout.write("Collector startup removed. Local pending data was retained.\n");
|
|
230
288
|
return;
|
|
231
289
|
}
|
package/dist/collector.d.ts
CHANGED
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
import { type CollectorConfig, type CollectorPaths } from "./storage.js";
|
|
2
|
-
export declare
|
|
3
|
-
|
|
2
|
+
export declare class TraceUploadError extends Error {
|
|
3
|
+
readonly kind: "http" | "network" | "response";
|
|
4
|
+
readonly status: number | null;
|
|
5
|
+
constructor(kind: "http" | "network" | "response", status: number | null, message: string, options?: ErrorOptions);
|
|
6
|
+
}
|
|
7
|
+
export declare function scanSourceOnce(config: CollectorConfig, paths: CollectorPaths, signal?: AbortSignal): Promise<Readonly<{
|
|
8
|
+
discovered: number;
|
|
9
|
+
caughtUp: boolean;
|
|
10
|
+
}>>;
|
|
11
|
+
export declare function collectorTick(config: CollectorConfig, paths: CollectorPaths, fetcher?: typeof fetch, signal?: AbortSignal, onAcknowledged?: () => void): Promise<Readonly<{
|
|
4
12
|
discovered: number;
|
|
5
13
|
uploaded: number;
|
|
6
14
|
outboxBytes: number;
|
|
15
|
+
historyCaughtUp: boolean;
|
|
7
16
|
}>>;
|
package/dist/collector.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { promises as fs } from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { parseTraceEventAcknowledgement, } from "iteration-trace-contracts";
|
|
4
|
+
import { parseClaudeLine } from "./claude-transcript.js";
|
|
4
5
|
import { parseRolloutLine } from "./rollout.js";
|
|
5
|
-
import { acknowledgeEvents, appendEvents, loadState, nextBatch, outboxSize, pendingEventIds,
|
|
6
|
+
import { acknowledgeEvents, appendEvents, collectorSource, collectorSourceDirectory, loadState, nextBatch, outboxSize, pendingEventIds, sourceFiles, saveState, } from "./storage.js";
|
|
6
7
|
const MAX_SCAN_BYTES_PER_FILE = 16 * 1024 * 1024;
|
|
7
8
|
const MAX_SCAN_BYTES_PER_TICK = 16 * 1024 * 1024;
|
|
8
9
|
const MAX_RECORD_BYTES = 3 * 1024 * 1024;
|
|
@@ -11,6 +12,16 @@ const MAX_EVENT_BYTES = MAX_UPLOAD_BYTES - 1024;
|
|
|
11
12
|
const MAX_UPLOADS_PER_TICK = 5;
|
|
12
13
|
const MAX_REQUEST_BYTES = 4 * 1024 * 1024;
|
|
13
14
|
const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true });
|
|
15
|
+
export class TraceUploadError extends Error {
|
|
16
|
+
kind;
|
|
17
|
+
status;
|
|
18
|
+
constructor(kind, status, message, options) {
|
|
19
|
+
super(message, options);
|
|
20
|
+
this.name = "TraceUploadError";
|
|
21
|
+
this.kind = kind;
|
|
22
|
+
this.status = status;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
14
25
|
function logGap(filename, offset, reason) {
|
|
15
26
|
console.error(JSON.stringify({
|
|
16
27
|
level: "capture_gap",
|
|
@@ -19,23 +30,41 @@ function logGap(filename, offset, reason) {
|
|
|
19
30
|
reason,
|
|
20
31
|
}));
|
|
21
32
|
}
|
|
22
|
-
|
|
33
|
+
function parseSourceRecord(source, line, filename, byteOffset) {
|
|
34
|
+
if (source === "claude")
|
|
35
|
+
return parseClaudeLine(line, filename, byteOffset);
|
|
36
|
+
if (source === "codex") {
|
|
37
|
+
const parsed = parseRolloutLine(line, filename, byteOffset);
|
|
38
|
+
return parsed.kind === "event"
|
|
39
|
+
? { events: [parsed.event], gaps: [] }
|
|
40
|
+
: parsed.kind === "gap"
|
|
41
|
+
? { events: [], gaps: [parsed.reason] }
|
|
42
|
+
: { events: [], gaps: [] };
|
|
43
|
+
}
|
|
44
|
+
const unhandled = source;
|
|
45
|
+
throw new Error(`Unsupported collector source: ${unhandled}`);
|
|
46
|
+
}
|
|
47
|
+
export async function scanSourceOnce(config, paths, signal) {
|
|
23
48
|
signal?.throwIfAborted();
|
|
49
|
+
const source = collectorSource(config);
|
|
50
|
+
const sourceDirectory = path.resolve(collectorSourceDirectory(config));
|
|
24
51
|
const state = await loadState(paths);
|
|
25
|
-
if (!state.initialized ||
|
|
26
|
-
|
|
27
|
-
throw new Error("Collector state does not match CODEX_HOME; run iteration connect codex again.");
|
|
52
|
+
if (!state.initialized || state.sourceDirectory !== sourceDirectory) {
|
|
53
|
+
throw new Error(`Collector state does not match the ${source} directory; run iteration connect ${source} again.`);
|
|
28
54
|
}
|
|
29
55
|
const cursors = { ...state.cursors };
|
|
30
56
|
const skipping = new Set(state.skipping);
|
|
31
|
-
const files = await
|
|
57
|
+
const files = await sourceFiles(source, sourceDirectory);
|
|
32
58
|
const queuedIds = await pendingEventIds(paths);
|
|
33
59
|
const events = [];
|
|
34
60
|
let remainingBudget = MAX_SCAN_BYTES_PER_TICK;
|
|
61
|
+
let caughtUp = true;
|
|
35
62
|
for (const filename of files) {
|
|
36
63
|
signal?.throwIfAborted();
|
|
37
|
-
if (remainingBudget === 0)
|
|
64
|
+
if (remainingBudget === 0) {
|
|
65
|
+
caughtUp = false;
|
|
38
66
|
break;
|
|
67
|
+
}
|
|
39
68
|
const stats = await fs.stat(filename);
|
|
40
69
|
let offset = cursors[filename] ?? 0;
|
|
41
70
|
if (offset > stats.size) {
|
|
@@ -50,15 +79,19 @@ export async function scanRolloutsOnce(config, paths, signal) {
|
|
|
50
79
|
}
|
|
51
80
|
const handle = await fs.open(filename, "r");
|
|
52
81
|
let buffer;
|
|
82
|
+
let readEnd = offset;
|
|
53
83
|
try {
|
|
54
84
|
buffer = Buffer.alloc(bytesToRead);
|
|
55
85
|
const result = await handle.read(buffer, 0, bytesToRead, offset);
|
|
56
86
|
buffer = buffer.subarray(0, result.bytesRead);
|
|
87
|
+
readEnd = offset + result.bytesRead;
|
|
57
88
|
remainingBudget -= result.bytesRead;
|
|
58
89
|
}
|
|
59
90
|
finally {
|
|
60
91
|
await handle.close();
|
|
61
92
|
}
|
|
93
|
+
if (readEnd < stats.size)
|
|
94
|
+
caughtUp = false;
|
|
62
95
|
if (skipping.has(filename)) {
|
|
63
96
|
const newline = buffer.indexOf(10);
|
|
64
97
|
if (newline < 0) {
|
|
@@ -91,17 +124,19 @@ export async function scanRolloutsOnce(config, paths, signal) {
|
|
|
91
124
|
lineStart = index + 1;
|
|
92
125
|
continue;
|
|
93
126
|
}
|
|
94
|
-
const parsed =
|
|
95
|
-
|
|
96
|
-
logGap(filename, byteOffset,
|
|
97
|
-
|
|
98
|
-
|
|
127
|
+
const parsed = parseSourceRecord(source, line, filename, byteOffset);
|
|
128
|
+
for (const reason of parsed.gaps)
|
|
129
|
+
logGap(filename, byteOffset, reason);
|
|
130
|
+
for (const parsedEvent of parsed.events) {
|
|
131
|
+
if (queuedIds.has(parsedEvent.eventId))
|
|
132
|
+
continue;
|
|
133
|
+
const eventBytes = Buffer.byteLength(JSON.stringify(parsedEvent), "utf8");
|
|
99
134
|
if (eventBytes > MAX_EVENT_BYTES) {
|
|
100
135
|
logGap(filename, byteOffset, "normalized_event_exceeds_upload_limit");
|
|
101
136
|
}
|
|
102
137
|
else {
|
|
103
|
-
events.push(
|
|
104
|
-
queuedIds.add(
|
|
138
|
+
events.push(parsedEvent);
|
|
139
|
+
queuedIds.add(parsedEvent.eventId);
|
|
105
140
|
}
|
|
106
141
|
}
|
|
107
142
|
lineStart = index + 1;
|
|
@@ -120,9 +155,9 @@ export async function scanRolloutsOnce(config, paths, signal) {
|
|
|
120
155
|
sourceDirectory: state.sourceDirectory,
|
|
121
156
|
initialized: true,
|
|
122
157
|
}, paths);
|
|
123
|
-
return events.length;
|
|
158
|
+
return { discovered: events.length, caughtUp };
|
|
124
159
|
}
|
|
125
|
-
async function uploadPendingOnce(config, paths, fetcher = fetch, signal) {
|
|
160
|
+
async function uploadPendingOnce(config, paths, fetcher = fetch, signal, onAcknowledged) {
|
|
126
161
|
let uploaded = 0;
|
|
127
162
|
for (let attempt = 0; attempt < MAX_UPLOADS_PER_TICK; attempt += 1) {
|
|
128
163
|
signal?.throwIfAborted();
|
|
@@ -133,41 +168,54 @@ async function uploadPendingOnce(config, paths, fetcher = fetch, signal) {
|
|
|
133
168
|
if (Buffer.byteLength(body, "utf8") > MAX_REQUEST_BYTES) {
|
|
134
169
|
throw new Error("Queued trace batch exceeds the upload request limit.");
|
|
135
170
|
}
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
:
|
|
146
|
-
|
|
171
|
+
let response;
|
|
172
|
+
try {
|
|
173
|
+
response = await fetcher(config.endpoint, {
|
|
174
|
+
method: "POST",
|
|
175
|
+
headers: {
|
|
176
|
+
authorization: `Bearer ${config.token}`,
|
|
177
|
+
"content-type": "application/json",
|
|
178
|
+
},
|
|
179
|
+
body,
|
|
180
|
+
signal: signal
|
|
181
|
+
? AbortSignal.any([signal, AbortSignal.timeout(15_000)])
|
|
182
|
+
: AbortSignal.timeout(15_000),
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
catch (error) {
|
|
186
|
+
if (signal?.aborted)
|
|
187
|
+
throw error;
|
|
188
|
+
throw new TraceUploadError("network", null, "Trace upload could not reach Iteration.", { cause: error });
|
|
189
|
+
}
|
|
147
190
|
if (response.status !== 202) {
|
|
148
|
-
throw new
|
|
191
|
+
throw new TraceUploadError("http", response.status, `Trace upload failed with HTTP ${response.status}.`);
|
|
149
192
|
}
|
|
150
193
|
let acknowledgement;
|
|
151
194
|
try {
|
|
152
195
|
acknowledgement = await response.json();
|
|
153
196
|
}
|
|
154
197
|
catch (error) {
|
|
155
|
-
throw new
|
|
156
|
-
cause: error,
|
|
157
|
-
});
|
|
198
|
+
throw new TraceUploadError("response", null, "Trace upload returned an unreadable acknowledgement.", { cause: error });
|
|
158
199
|
}
|
|
159
200
|
const expectedIds = batch.map((event) => event.eventId);
|
|
160
201
|
const parsed = parseTraceEventAcknowledgement(acknowledgement, expectedIds);
|
|
161
|
-
if (!parsed.success)
|
|
162
|
-
throw new
|
|
202
|
+
if (!parsed.success) {
|
|
203
|
+
throw new TraceUploadError("response", null, parsed.error);
|
|
204
|
+
}
|
|
163
205
|
await acknowledgeEvents(new Set(parsed.value.acceptedEventIds), paths);
|
|
206
|
+
onAcknowledged?.();
|
|
164
207
|
uploaded += batch.length;
|
|
165
208
|
}
|
|
166
209
|
return uploaded;
|
|
167
210
|
}
|
|
168
|
-
export async function collectorTick(config, paths, fetcher = fetch, signal) {
|
|
169
|
-
let uploaded = await uploadPendingOnce(config, paths, fetcher, signal);
|
|
170
|
-
const
|
|
171
|
-
uploaded += await uploadPendingOnce(config, paths, fetcher, signal);
|
|
172
|
-
return {
|
|
211
|
+
export async function collectorTick(config, paths, fetcher = fetch, signal, onAcknowledged) {
|
|
212
|
+
let uploaded = await uploadPendingOnce(config, paths, fetcher, signal, onAcknowledged);
|
|
213
|
+
const scan = await scanSourceOnce(config, paths, signal);
|
|
214
|
+
uploaded += await uploadPendingOnce(config, paths, fetcher, signal, onAcknowledged);
|
|
215
|
+
return {
|
|
216
|
+
discovered: scan.discovered,
|
|
217
|
+
uploaded,
|
|
218
|
+
outboxBytes: await outboxSize(paths),
|
|
219
|
+
historyCaughtUp: scan.caughtUp,
|
|
220
|
+
};
|
|
173
221
|
}
|
package/dist/history-import.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export type HistoryImportControl = Readonly<{
|
|
|
9
9
|
start: (directory: string) => Promise<unknown>;
|
|
10
10
|
acquireLock: (paths: CollectorPaths) => Promise<() => Promise<void>>;
|
|
11
11
|
}>;
|
|
12
|
-
export declare function
|
|
12
|
+
export declare function importSourceHistory(config: CollectorConfig, paths: CollectorPaths, confirmed: boolean, control?: HistoryImportControl): Promise<Readonly<{
|
|
13
13
|
imported: boolean;
|
|
14
14
|
size: HistoryImportSize;
|
|
15
15
|
backgroundRunning: boolean | null;
|
package/dist/history-import.js
CHANGED
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { installLaunchAgent, launchAgentStatus, uninstallLaunchAgent, } from "./launch-agent.js";
|
|
3
3
|
import { waitForCollectorLock } from "./service.js";
|
|
4
|
-
import { historyImportSize, loadState, resetHistoryCursors, } from "./storage.js";
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
4
|
+
import { collectorSource, collectorSourceDirectory, historyImportSize, loadState, resetHistoryCursors, } from "./storage.js";
|
|
5
|
+
function macControl(source) {
|
|
6
|
+
return {
|
|
7
|
+
isRunning: () => launchAgentStatus(source),
|
|
8
|
+
stop: () => uninstallLaunchAgent(source),
|
|
9
|
+
start: (directory) => installLaunchAgent(directory, source),
|
|
10
|
+
acquireLock: waitForCollectorLock,
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
export async function importSourceHistory(config, paths, confirmed, control = macControl(collectorSource(config))) {
|
|
14
|
+
const source = collectorSource(config);
|
|
15
|
+
const directory = collectorSourceDirectory(config);
|
|
12
16
|
const state = await loadState(paths);
|
|
13
|
-
if (!state.initialized ||
|
|
14
|
-
|
|
15
|
-
throw new Error("Connect the Codex source before importing its history.");
|
|
17
|
+
if (!state.initialized || state.sourceDirectory !== path.resolve(directory)) {
|
|
18
|
+
throw new Error(`Connect the ${source} source before importing its history.`);
|
|
16
19
|
}
|
|
17
|
-
const estimate = await historyImportSize(
|
|
20
|
+
const estimate = await historyImportSize(directory, source);
|
|
18
21
|
if (!confirmed)
|
|
19
22
|
return { imported: false, size: estimate, backgroundRunning: null };
|
|
20
23
|
const wasRunning = await control.isRunning();
|
|
@@ -29,7 +32,7 @@ export async function importCodexHistory(config, paths, confirmed, control = mac
|
|
|
29
32
|
}
|
|
30
33
|
const releaseLock = await control.acquireLock(paths);
|
|
31
34
|
try {
|
|
32
|
-
size = await resetHistoryCursors(
|
|
35
|
+
size = await resetHistoryCursors(directory, paths, source);
|
|
33
36
|
rewound = true;
|
|
34
37
|
}
|
|
35
38
|
finally {
|
|
@@ -45,8 +48,8 @@ export async function importCodexHistory(config, paths, confirmed, control = mac
|
|
|
45
48
|
}
|
|
46
49
|
catch (error) {
|
|
47
50
|
throw new Error(rewound
|
|
48
|
-
?
|
|
49
|
-
:
|
|
51
|
+
? `History checkpoints were rewound, but the collector could not restart. Run iteration start ${source} to resume uploads.`
|
|
52
|
+
: `The collector could not be restored after a failed history import. Run iteration start ${source} to resume uploads.`, {
|
|
50
53
|
cause: failure ? new AggregateError([failure.error, error]) : error,
|
|
51
54
|
});
|
|
52
55
|
}
|