iteration-cli 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/cli.js +18207 -429
- package/dist/cursor-hook-entry.js +11667 -139
- package/dist/launcher.js +6731 -0
- package/package.json +4 -3
- package/dist/auto-connect.d.ts +0 -32
- package/dist/auto-connect.js +0 -235
- package/dist/background-service.d.ts +0 -7
- package/dist/background-service.js +0 -37
- package/dist/claude-transcript.d.ts +0 -6
- package/dist/claude-transcript.js +0 -256
- package/dist/clerk-device-auth.d.ts +0 -7
- package/dist/clerk-device-auth.js +0 -279
- package/dist/cli.d.ts +0 -2
- package/dist/collector-fetch.d.ts +0 -9
- package/dist/collector-fetch.js +0 -57
- package/dist/collector.d.ts +0 -17
- package/dist/collector.js +0 -268
- package/dist/cursor-hook-entry.d.ts +0 -1
- package/dist/cursor-hooks.d.ts +0 -7
- package/dist/cursor-hooks.js +0 -163
- package/dist/cursor-transcript.d.ts +0 -8
- package/dist/cursor-transcript.js +0 -294
- package/dist/history-import.d.ts +0 -17
- package/dist/history-import.js +0 -63
- package/dist/launch-agent.d.ts +0 -13
- package/dist/launch-agent.js +0 -183
- package/dist/redact-trace.d.ts +0 -2
- package/dist/redact-trace.js +0 -186
- package/dist/rollout.d.ts +0 -11
- package/dist/rollout.js +0 -259
- package/dist/service.d.ts +0 -5
- package/dist/service.js +0 -249
- package/dist/status-view.d.ts +0 -33
- package/dist/status-view.js +0 -257
- package/dist/storage.d.ts +0 -94
- package/dist/storage.js +0 -809
- package/dist/systemd-user.d.ts +0 -7
- package/dist/systemd-user.js +0 -137
- package/dist/turn-summary.d.ts +0 -23
- package/dist/turn-summary.js +0 -365
|
@@ -1,279 +0,0 @@
|
|
|
1
|
-
import { execFile } from "node:child_process";
|
|
2
|
-
import { createInterface } from "node:readline/promises";
|
|
3
|
-
import { promisify } from "node:util";
|
|
4
|
-
import { parseCollectorEnrollmentResponse, } from "iteration-trace-contracts";
|
|
5
|
-
import * as z from "zod";
|
|
6
|
-
const execFileAsync = promisify(execFile);
|
|
7
|
-
const serverAuthConfigSchema = z
|
|
8
|
-
.object({
|
|
9
|
-
issuerUrl: z.url(),
|
|
10
|
-
clientId: z.string().min(1),
|
|
11
|
-
})
|
|
12
|
-
.strict();
|
|
13
|
-
const oauthMetadataSchema = z.object({
|
|
14
|
-
device_authorization_endpoint: z.url(),
|
|
15
|
-
token_endpoint: z.url(),
|
|
16
|
-
issuer: z.url().optional(),
|
|
17
|
-
});
|
|
18
|
-
const deviceCodeSchema = z.object({
|
|
19
|
-
device_code: z.string().min(1),
|
|
20
|
-
user_code: z.string().min(1),
|
|
21
|
-
verification_uri: z.url(),
|
|
22
|
-
verification_uri_complete: z.url().optional(),
|
|
23
|
-
expires_in: z.number().int().min(1).max(86_400),
|
|
24
|
-
interval: z.number().int().min(1).max(3_600),
|
|
25
|
-
});
|
|
26
|
-
const oauthTokenSchema = z.object({
|
|
27
|
-
access_token: z.string().min(1),
|
|
28
|
-
});
|
|
29
|
-
const oauthErrorSchema = z.object({
|
|
30
|
-
error: z.string().min(1),
|
|
31
|
-
error_description: z.string().optional(),
|
|
32
|
-
});
|
|
33
|
-
function apiRoute(endpoint, relativePath) {
|
|
34
|
-
return new URL(relativePath, endpoint);
|
|
35
|
-
}
|
|
36
|
-
async function responseJson(response) {
|
|
37
|
-
try {
|
|
38
|
-
return await response.json();
|
|
39
|
-
}
|
|
40
|
-
catch (error) {
|
|
41
|
-
throw new Error("The authentication server returned invalid JSON.", {
|
|
42
|
-
cause: error,
|
|
43
|
-
});
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
async function loadOAuthConfiguration(endpoint) {
|
|
47
|
-
const response = await fetch(apiRoute(endpoint, "./collector-auth/config"), {
|
|
48
|
-
signal: AbortSignal.timeout(10_000),
|
|
49
|
-
});
|
|
50
|
-
const body = await responseJson(response);
|
|
51
|
-
if (!response.ok) {
|
|
52
|
-
throw new Error(response.status === 503
|
|
53
|
-
? "Collector sign-in is not configured on the Iteration server."
|
|
54
|
-
: "Could not load collector sign-in configuration.");
|
|
55
|
-
}
|
|
56
|
-
const parsed = serverAuthConfigSchema.safeParse(body);
|
|
57
|
-
if (!parsed.success) {
|
|
58
|
-
throw new Error("Collector sign-in configuration has an invalid shape.");
|
|
59
|
-
}
|
|
60
|
-
const issuer = new URL(parsed.data.issuerUrl);
|
|
61
|
-
if (issuer.protocol !== "https:" ||
|
|
62
|
-
issuer.username !== "" ||
|
|
63
|
-
issuer.password !== "" ||
|
|
64
|
-
issuer.pathname !== "/" ||
|
|
65
|
-
issuer.search !== "" ||
|
|
66
|
-
issuer.hash !== "") {
|
|
67
|
-
throw new Error("The configured Clerk issuer URL is invalid.");
|
|
68
|
-
}
|
|
69
|
-
return { issuerUrl: issuer.origin, clientId: parsed.data.clientId };
|
|
70
|
-
}
|
|
71
|
-
async function discoverDeviceEndpoints(issuerUrl) {
|
|
72
|
-
const issuer = new URL(issuerUrl);
|
|
73
|
-
const response = await fetch(new URL(".well-known/oauth-authorization-server", issuer), { signal: AbortSignal.timeout(10_000) });
|
|
74
|
-
if (!response.ok) {
|
|
75
|
-
throw new Error("Clerk OAuth device authorization is unavailable.");
|
|
76
|
-
}
|
|
77
|
-
const parsed = oauthMetadataSchema.safeParse(await responseJson(response));
|
|
78
|
-
if (!parsed.success) {
|
|
79
|
-
throw new Error("Clerk OAuth metadata does not advertise device authorization.");
|
|
80
|
-
}
|
|
81
|
-
const deviceAuthorizationUrl = new URL(parsed.data.device_authorization_endpoint);
|
|
82
|
-
const tokenUrl = new URL(parsed.data.token_endpoint);
|
|
83
|
-
if (deviceAuthorizationUrl.protocol !== "https:" ||
|
|
84
|
-
tokenUrl.protocol !== "https:" ||
|
|
85
|
-
deviceAuthorizationUrl.origin !== issuer.origin ||
|
|
86
|
-
tokenUrl.origin !== issuer.origin) {
|
|
87
|
-
throw new Error("Clerk OAuth metadata contains an unexpected endpoint.");
|
|
88
|
-
}
|
|
89
|
-
return {
|
|
90
|
-
deviceAuthorizationUrl,
|
|
91
|
-
tokenUrl,
|
|
92
|
-
};
|
|
93
|
-
}
|
|
94
|
-
async function openBrowser(url) {
|
|
95
|
-
try {
|
|
96
|
-
await execFileAsync("open", [url], { timeout: 5000 });
|
|
97
|
-
}
|
|
98
|
-
catch {
|
|
99
|
-
process.stdout.write("Open the verification link in your browser.\n");
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
function delay(milliseconds) {
|
|
103
|
-
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
104
|
-
}
|
|
105
|
-
async function requestDeviceCode(url, clientId) {
|
|
106
|
-
const body = new URLSearchParams({
|
|
107
|
-
client_id: clientId,
|
|
108
|
-
scope: "openid profile email",
|
|
109
|
-
});
|
|
110
|
-
const response = await fetch(url, {
|
|
111
|
-
method: "POST",
|
|
112
|
-
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
113
|
-
body,
|
|
114
|
-
signal: AbortSignal.timeout(10_000),
|
|
115
|
-
});
|
|
116
|
-
const payload = await responseJson(response);
|
|
117
|
-
const parsed = deviceCodeSchema.safeParse(payload);
|
|
118
|
-
if (!response.ok || !parsed.success) {
|
|
119
|
-
const providerError = oauthErrorSchema.safeParse(payload);
|
|
120
|
-
throw new Error(providerError.success
|
|
121
|
-
? "Clerk device sign-in could not start: " +
|
|
122
|
-
(providerError.data.error_description ?? providerError.data.error)
|
|
123
|
-
: "Clerk returned an invalid device authorization response.");
|
|
124
|
-
}
|
|
125
|
-
return parsed.data;
|
|
126
|
-
}
|
|
127
|
-
async function pollForAccessToken(tokenUrl, clientId, deviceCode, expiresIn, initialInterval) {
|
|
128
|
-
const expiresAt = Date.now() + expiresIn * 1000;
|
|
129
|
-
let interval = initialInterval;
|
|
130
|
-
while (Date.now() < expiresAt) {
|
|
131
|
-
await delay(interval * 1000);
|
|
132
|
-
if (Date.now() >= expiresAt)
|
|
133
|
-
break;
|
|
134
|
-
let response;
|
|
135
|
-
try {
|
|
136
|
-
response = await fetch(tokenUrl, {
|
|
137
|
-
method: "POST",
|
|
138
|
-
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
139
|
-
body: new URLSearchParams({
|
|
140
|
-
client_id: clientId,
|
|
141
|
-
grant_type: "urn:ietf:params:oauth:grant-type:device_code",
|
|
142
|
-
device_code: deviceCode,
|
|
143
|
-
}),
|
|
144
|
-
signal: AbortSignal.timeout(10_000),
|
|
145
|
-
});
|
|
146
|
-
}
|
|
147
|
-
catch {
|
|
148
|
-
continue;
|
|
149
|
-
}
|
|
150
|
-
const payload = await responseJson(response);
|
|
151
|
-
const token = oauthTokenSchema.safeParse(payload);
|
|
152
|
-
if (response.ok && token.success)
|
|
153
|
-
return token.data.access_token;
|
|
154
|
-
const providerError = oauthErrorSchema.safeParse(payload);
|
|
155
|
-
if (!providerError.success) {
|
|
156
|
-
throw new Error("Clerk returned an invalid device token response.");
|
|
157
|
-
}
|
|
158
|
-
switch (providerError.data.error) {
|
|
159
|
-
case "authorization_pending":
|
|
160
|
-
continue;
|
|
161
|
-
case "slow_down":
|
|
162
|
-
interval += 5;
|
|
163
|
-
continue;
|
|
164
|
-
case "access_denied":
|
|
165
|
-
throw new Error("You denied Iteration collector access.");
|
|
166
|
-
case "expired_token":
|
|
167
|
-
throw new Error("The sign-in code expired. Run iteration again.");
|
|
168
|
-
default:
|
|
169
|
-
throw new Error("Clerk device sign-in failed: " +
|
|
170
|
-
(providerError.data.error_description ?? providerError.data.error));
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
throw new Error("The sign-in code expired. Run iteration again.");
|
|
174
|
-
}
|
|
175
|
-
async function clerkDeviceSignIn(configuration) {
|
|
176
|
-
const endpoints = await discoverDeviceEndpoints(configuration.issuerUrl);
|
|
177
|
-
const challenge = await requestDeviceCode(endpoints.deviceAuthorizationUrl, configuration.clientId);
|
|
178
|
-
const verificationUrl = challenge.verification_uri_complete ?? challenge.verification_uri;
|
|
179
|
-
process.stdout.write("Sign in to Iteration to authorize this collector.\n" +
|
|
180
|
-
"Code: " +
|
|
181
|
-
challenge.user_code +
|
|
182
|
-
"\n" +
|
|
183
|
-
"Link: " +
|
|
184
|
-
verificationUrl +
|
|
185
|
-
"\n");
|
|
186
|
-
await openBrowser(verificationUrl);
|
|
187
|
-
return pollForAccessToken(endpoints.tokenUrl, configuration.clientId, challenge.device_code, challenge.expires_in, challenge.interval);
|
|
188
|
-
}
|
|
189
|
-
async function enrollDevice(endpoint, accessToken, deviceName, organizationId, source) {
|
|
190
|
-
const body = { deviceName };
|
|
191
|
-
if (source !== "codex")
|
|
192
|
-
body.source = source;
|
|
193
|
-
if (organizationId !== null)
|
|
194
|
-
body.organizationId = organizationId;
|
|
195
|
-
const response = await fetch(apiRoute(endpoint, "./collector-auth/enroll"), {
|
|
196
|
-
method: "POST",
|
|
197
|
-
headers: {
|
|
198
|
-
Authorization: "Bearer " + accessToken,
|
|
199
|
-
"Content-Type": "application/json",
|
|
200
|
-
},
|
|
201
|
-
body: JSON.stringify(body),
|
|
202
|
-
signal: AbortSignal.timeout(15_000),
|
|
203
|
-
});
|
|
204
|
-
const payload = await responseJson(response);
|
|
205
|
-
const parsed = parseCollectorEnrollmentResponse(payload);
|
|
206
|
-
if (response.ok && parsed.success)
|
|
207
|
-
return parsed.value;
|
|
208
|
-
const error = z.object({ error: z.string().min(1) }).safeParse(payload);
|
|
209
|
-
if (error.success) {
|
|
210
|
-
throw new Error(error.data.error);
|
|
211
|
-
}
|
|
212
|
-
throw new Error("The Iteration server returned an invalid enrollment response.");
|
|
213
|
-
}
|
|
214
|
-
async function chooseOrganization(organizations, hasMore) {
|
|
215
|
-
process.stdout.write("Choose an Iteration workspace for this collector:\n");
|
|
216
|
-
for (const [index, organization] of organizations.entries()) {
|
|
217
|
-
process.stdout.write(index + 1 + ". " + organization.name + " (" + organization.id + ")\n");
|
|
218
|
-
}
|
|
219
|
-
if (hasMore) {
|
|
220
|
-
process.stdout.write("More workspaces exist; enter an organization ID to choose one not listed.\n");
|
|
221
|
-
}
|
|
222
|
-
if (process.stdin.isTTY !== true) {
|
|
223
|
-
throw new Error("Run iteration interactively or pass --organization-id.");
|
|
224
|
-
}
|
|
225
|
-
const prompt = createInterface({
|
|
226
|
-
input: process.stdin,
|
|
227
|
-
output: process.stdout,
|
|
228
|
-
});
|
|
229
|
-
try {
|
|
230
|
-
const answer = await prompt.question("Workspace number or organization ID: ");
|
|
231
|
-
const id = z.uuid().safeParse(answer.trim());
|
|
232
|
-
if (id.success)
|
|
233
|
-
return id.data;
|
|
234
|
-
const selection = z.coerce
|
|
235
|
-
.number()
|
|
236
|
-
.int()
|
|
237
|
-
.min(1)
|
|
238
|
-
.max(organizations.length)
|
|
239
|
-
.safeParse(answer.trim());
|
|
240
|
-
if (!selection.success) {
|
|
241
|
-
throw new Error("Choose a listed number or enter a valid organization ID.");
|
|
242
|
-
}
|
|
243
|
-
const organization = organizations[selection.data - 1];
|
|
244
|
-
if (organization === undefined) {
|
|
245
|
-
throw new Error("The selected workspace is not available.");
|
|
246
|
-
}
|
|
247
|
-
return organization.id;
|
|
248
|
-
}
|
|
249
|
-
finally {
|
|
250
|
-
prompt.close();
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
export async function authenticateAndEnrollCollector(endpoint, deviceNameValue, organizationId, source = "codex") {
|
|
254
|
-
const deviceName = z
|
|
255
|
-
.string()
|
|
256
|
-
.trim()
|
|
257
|
-
.min(1)
|
|
258
|
-
.max(160)
|
|
259
|
-
.safeParse(deviceNameValue);
|
|
260
|
-
if (!deviceName.success) {
|
|
261
|
-
throw new Error("--device-name must contain 1 to 160 characters.");
|
|
262
|
-
}
|
|
263
|
-
const configuration = await loadOAuthConfiguration(endpoint);
|
|
264
|
-
const clerkAccessToken = await clerkDeviceSignIn(configuration);
|
|
265
|
-
let result = await enrollDevice(endpoint, clerkAccessToken, deviceName.data, organizationId, source);
|
|
266
|
-
if (result.status === "choose_organization") {
|
|
267
|
-
const selectedOrganizationId = await chooseOrganization(result.organizations, result.hasMore);
|
|
268
|
-
result = await enrollDevice(endpoint, clerkAccessToken, deviceName.data, selectedOrganizationId, source);
|
|
269
|
-
}
|
|
270
|
-
if (result.status !== "enrolled") {
|
|
271
|
-
throw new Error("The collector could not be assigned to that workspace.");
|
|
272
|
-
}
|
|
273
|
-
return {
|
|
274
|
-
token: result.token,
|
|
275
|
-
deviceId: result.deviceId,
|
|
276
|
-
organizationId: result.organizationId,
|
|
277
|
-
organizationName: result.organizationName,
|
|
278
|
-
};
|
|
279
|
-
}
|
package/dist/cli.d.ts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
type CollectorRequest = Readonly<{
|
|
2
|
-
method?: "GET" | "POST";
|
|
3
|
-
headers?: Readonly<Record<string, string>>;
|
|
4
|
-
body?: string;
|
|
5
|
-
signal?: AbortSignal;
|
|
6
|
-
}>;
|
|
7
|
-
export type CollectorFetcher = (url: string | URL, init?: CollectorRequest) => Promise<Response>;
|
|
8
|
-
export declare const collectorFetch: CollectorFetcher;
|
|
9
|
-
export {};
|
package/dist/collector-fetch.js
DELETED
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
import http from "node:http";
|
|
2
|
-
import https from "node:https";
|
|
3
|
-
const MAX_RESPONSE_BYTES = 1024 * 1024;
|
|
4
|
-
const NULL_BODY_STATUSES = new Set([101, 103, 204, 205, 304]);
|
|
5
|
-
// Reused connections failed uploads in production: Node's HTTP/2 fetch kept a
|
|
6
|
-
// destroyed session, and kept-alive sockets timed out on ~40 s server waits.
|
|
7
|
-
// See docs/architecture/trace-ingestion.md.
|
|
8
|
-
const agents = {
|
|
9
|
-
"http:": new http.Agent({ keepAlive: false }),
|
|
10
|
-
"https:": new https.Agent({ keepAlive: false }),
|
|
11
|
-
};
|
|
12
|
-
export const collectorFetch = (input, init = {}) => {
|
|
13
|
-
const url = new URL(input);
|
|
14
|
-
if (url.protocol !== "https:" && url.protocol !== "http:") {
|
|
15
|
-
throw new TypeError(`Unsupported collector request protocol ${url.protocol}`);
|
|
16
|
-
}
|
|
17
|
-
const body = init.body ?? null;
|
|
18
|
-
const headers = { ...init.headers };
|
|
19
|
-
if (body !== null) {
|
|
20
|
-
headers["content-length"] = String(Buffer.byteLength(body, "utf8"));
|
|
21
|
-
}
|
|
22
|
-
const transport = url.protocol === "https:" ? https : http;
|
|
23
|
-
return new Promise((resolve, reject) => {
|
|
24
|
-
const request = transport.request(url, {
|
|
25
|
-
method: init.method ?? "GET",
|
|
26
|
-
headers,
|
|
27
|
-
agent: agents[url.protocol],
|
|
28
|
-
signal: init.signal,
|
|
29
|
-
}, (response) => {
|
|
30
|
-
const chunks = [];
|
|
31
|
-
let size = 0;
|
|
32
|
-
response.on("data", (chunk) => {
|
|
33
|
-
size += chunk.byteLength;
|
|
34
|
-
if (size > MAX_RESPONSE_BYTES) {
|
|
35
|
-
response.destroy(new Error("Iteration returned an oversized response."));
|
|
36
|
-
return;
|
|
37
|
-
}
|
|
38
|
-
chunks.push(chunk);
|
|
39
|
-
});
|
|
40
|
-
response.on("error", reject);
|
|
41
|
-
response.on("end", () => {
|
|
42
|
-
const status = response.statusCode ?? 0;
|
|
43
|
-
const responseHeaders = new Headers();
|
|
44
|
-
for (const [name, value] of Object.entries(response.headers)) {
|
|
45
|
-
if (value === undefined)
|
|
46
|
-
continue;
|
|
47
|
-
for (const item of Array.isArray(value) ? value : [value]) {
|
|
48
|
-
responseHeaders.append(name, item);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
resolve(new Response(NULL_BODY_STATUSES.has(status) ? null : Buffer.concat(chunks), { status, headers: responseHeaders }));
|
|
52
|
-
});
|
|
53
|
-
});
|
|
54
|
-
request.on("error", reject);
|
|
55
|
-
request.end(body ?? undefined);
|
|
56
|
-
});
|
|
57
|
-
};
|
package/dist/collector.d.ts
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { type CollectorFetcher } from "./collector-fetch.js";
|
|
2
|
-
import { type CollectorConfig, type CollectorPaths } from "./storage.js";
|
|
3
|
-
export declare class TraceUploadError extends Error {
|
|
4
|
-
readonly kind: "http" | "network" | "response";
|
|
5
|
-
readonly status: number | null;
|
|
6
|
-
constructor(kind: "http" | "network" | "response", status: number | null, message: string, options?: ErrorOptions);
|
|
7
|
-
}
|
|
8
|
-
export declare function scanSourceOnce(config: CollectorConfig, paths: CollectorPaths, signal?: AbortSignal): Promise<Readonly<{
|
|
9
|
-
discovered: number;
|
|
10
|
-
caughtUp: boolean;
|
|
11
|
-
}>>;
|
|
12
|
-
export declare function collectorTick(config: CollectorConfig, paths: CollectorPaths, fetcher?: CollectorFetcher, signal?: AbortSignal, onAcknowledged?: () => void): Promise<Readonly<{
|
|
13
|
-
discovered: number;
|
|
14
|
-
uploaded: number;
|
|
15
|
-
outboxBytes: number;
|
|
16
|
-
historyCaughtUp: boolean;
|
|
17
|
-
}>>;
|
package/dist/collector.js
DELETED
|
@@ -1,268 +0,0 @@
|
|
|
1
|
-
import { promises as fs } from "node:fs";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { parseTraceEventAcknowledgement, } from "iteration-trace-contracts";
|
|
4
|
-
import { collectorFetch } from "./collector-fetch.js";
|
|
5
|
-
import { parseClaudeLine } from "./claude-transcript.js";
|
|
6
|
-
import { cursorHookFiles } from "./cursor-hooks.js";
|
|
7
|
-
import { parseCursorHookLine, parseCursorTranscriptLine, } from "./cursor-transcript.js";
|
|
8
|
-
import { redactTraceEvent } from "./redact-trace.js";
|
|
9
|
-
import { parseRolloutLine } from "./rollout.js";
|
|
10
|
-
import { acknowledgeEvents, appendEvents, collectorSource, collectorSourceDirectory, loadState, nextBatch, outboxSize, pendingEventIds, redactPendingOutbox, sourceFiles, saveState, } from "./storage.js";
|
|
11
|
-
const MAX_SCAN_BYTES_PER_FILE = 16 * 1024 * 1024;
|
|
12
|
-
const MAX_SCAN_BYTES_PER_TICK = 16 * 1024 * 1024;
|
|
13
|
-
const MAX_RECORD_BYTES = 3 * 1024 * 1024;
|
|
14
|
-
const MAX_UPLOAD_BYTES = 3 * 1024 * 1024;
|
|
15
|
-
const MAX_EVENT_BYTES = MAX_UPLOAD_BYTES - 1024;
|
|
16
|
-
const MAX_UPLOADS_PER_TICK = 5;
|
|
17
|
-
const MAX_REQUEST_BYTES = 4 * 1024 * 1024;
|
|
18
|
-
// Production stores some batches in about 40 seconds. Abandoning a request
|
|
19
|
-
// earlier does not stop the server, so the retry only waits behind it and the
|
|
20
|
-
// collector reports a failure for work that succeeded.
|
|
21
|
-
const UPLOAD_TIMEOUT_MS = 90_000;
|
|
22
|
-
// A tick can queue up to MAX_SCAN_BYTES_PER_TICK of new events, so scanning
|
|
23
|
-
// pauses well below the outbox capacity and resumes once uploads drain it.
|
|
24
|
-
const SCAN_PAUSE_OUTBOX_BYTES = 32 * 1024 * 1024;
|
|
25
|
-
const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true });
|
|
26
|
-
export class TraceUploadError extends Error {
|
|
27
|
-
kind;
|
|
28
|
-
status;
|
|
29
|
-
constructor(kind, status, message, options) {
|
|
30
|
-
super(message, options);
|
|
31
|
-
this.name = "TraceUploadError";
|
|
32
|
-
this.kind = kind;
|
|
33
|
-
this.status = status;
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
function logGap(filename, offset, reason) {
|
|
37
|
-
console.error(JSON.stringify({
|
|
38
|
-
level: "capture_gap",
|
|
39
|
-
filename,
|
|
40
|
-
offset,
|
|
41
|
-
reason,
|
|
42
|
-
}));
|
|
43
|
-
}
|
|
44
|
-
function parseSourceRecord(source, line, filename, byteOffset) {
|
|
45
|
-
if (source === "claude")
|
|
46
|
-
return parseClaudeLine(line, filename, byteOffset);
|
|
47
|
-
if (source === "cursor")
|
|
48
|
-
return filename.includes(`${path.sep}cursor-hooks${path.sep}`)
|
|
49
|
-
? parseCursorHookLine(line, filename)
|
|
50
|
-
: parseCursorTranscriptLine(line, filename, byteOffset);
|
|
51
|
-
if (source === "codex") {
|
|
52
|
-
const parsed = parseRolloutLine(line, filename, byteOffset);
|
|
53
|
-
return parsed.kind === "event"
|
|
54
|
-
? { events: [parsed.event], gaps: [] }
|
|
55
|
-
: parsed.kind === "gap"
|
|
56
|
-
? { events: [], gaps: [parsed.reason] }
|
|
57
|
-
: { events: [], gaps: [] };
|
|
58
|
-
}
|
|
59
|
-
const unhandled = source;
|
|
60
|
-
throw new Error(`Unsupported collector source: ${unhandled}`);
|
|
61
|
-
}
|
|
62
|
-
export async function scanSourceOnce(config, paths, signal) {
|
|
63
|
-
signal?.throwIfAborted();
|
|
64
|
-
const source = collectorSource(config);
|
|
65
|
-
const sourceDirectory = path.resolve(collectorSourceDirectory(config));
|
|
66
|
-
const state = await loadState(paths);
|
|
67
|
-
if (!state.initialized || state.sourceDirectory !== sourceDirectory) {
|
|
68
|
-
throw new Error(`Collector state does not match the ${source} directory; run iteration connect ${source} again.`);
|
|
69
|
-
}
|
|
70
|
-
const cursors = { ...state.cursors };
|
|
71
|
-
const skipping = new Set(state.skipping);
|
|
72
|
-
const files = [
|
|
73
|
-
...(await sourceFiles(source, sourceDirectory)),
|
|
74
|
-
...(source === "cursor" ? await cursorHookFiles(config.dataDirectory) : []),
|
|
75
|
-
];
|
|
76
|
-
const completedHooks = [];
|
|
77
|
-
const queuedIds = await pendingEventIds(paths);
|
|
78
|
-
const tickIndexes = new Map();
|
|
79
|
-
const events = [];
|
|
80
|
-
let remainingBudget = MAX_SCAN_BYTES_PER_TICK;
|
|
81
|
-
let caughtUp = true;
|
|
82
|
-
for (const filename of files) {
|
|
83
|
-
signal?.throwIfAborted();
|
|
84
|
-
const recordLimit = source === "cursor" &&
|
|
85
|
-
filename.includes(`${path.sep}cursor-hooks${path.sep}`)
|
|
86
|
-
? 12 * 1024 * 1024
|
|
87
|
-
: MAX_RECORD_BYTES;
|
|
88
|
-
if (remainingBudget === 0) {
|
|
89
|
-
caughtUp = false;
|
|
90
|
-
break;
|
|
91
|
-
}
|
|
92
|
-
const stats = await fs.stat(filename);
|
|
93
|
-
let offset = cursors[filename] ?? 0;
|
|
94
|
-
if (offset > stats.size) {
|
|
95
|
-
logGap(filename, offset, "rollout_file_shrank_resetting_cursor");
|
|
96
|
-
offset = 0;
|
|
97
|
-
skipping.delete(filename);
|
|
98
|
-
}
|
|
99
|
-
const bytesToRead = Math.min(stats.size - offset, MAX_SCAN_BYTES_PER_FILE, remainingBudget);
|
|
100
|
-
if (bytesToRead === 0) {
|
|
101
|
-
cursors[filename] = offset;
|
|
102
|
-
continue;
|
|
103
|
-
}
|
|
104
|
-
const handle = await fs.open(filename, "r");
|
|
105
|
-
let buffer;
|
|
106
|
-
let readEnd = offset;
|
|
107
|
-
try {
|
|
108
|
-
buffer = Buffer.alloc(bytesToRead);
|
|
109
|
-
const result = await handle.read(buffer, 0, bytesToRead, offset);
|
|
110
|
-
buffer = buffer.subarray(0, result.bytesRead);
|
|
111
|
-
readEnd = offset + result.bytesRead;
|
|
112
|
-
remainingBudget -= result.bytesRead;
|
|
113
|
-
}
|
|
114
|
-
finally {
|
|
115
|
-
await handle.close();
|
|
116
|
-
}
|
|
117
|
-
if (readEnd < stats.size)
|
|
118
|
-
caughtUp = false;
|
|
119
|
-
if (skipping.has(filename)) {
|
|
120
|
-
const newline = buffer.indexOf(10);
|
|
121
|
-
if (newline < 0) {
|
|
122
|
-
cursors[filename] = offset + buffer.length;
|
|
123
|
-
continue;
|
|
124
|
-
}
|
|
125
|
-
offset += newline + 1;
|
|
126
|
-
buffer = buffer.subarray(newline + 1);
|
|
127
|
-
skipping.delete(filename);
|
|
128
|
-
}
|
|
129
|
-
let lineStart = 0;
|
|
130
|
-
for (let index = 0; index < buffer.length; index += 1) {
|
|
131
|
-
if (buffer[index] !== 10)
|
|
132
|
-
continue;
|
|
133
|
-
let lineBytes = buffer.subarray(lineStart, index);
|
|
134
|
-
if (lineBytes.at(-1) === 13)
|
|
135
|
-
lineBytes = lineBytes.subarray(0, -1);
|
|
136
|
-
const byteOffset = offset + lineStart;
|
|
137
|
-
if (lineBytes.length > recordLimit) {
|
|
138
|
-
logGap(filename, byteOffset, "rollout_record_exceeds_scan_limit");
|
|
139
|
-
lineStart = index + 1;
|
|
140
|
-
continue;
|
|
141
|
-
}
|
|
142
|
-
let line;
|
|
143
|
-
try {
|
|
144
|
-
line = UTF8_DECODER.decode(lineBytes);
|
|
145
|
-
}
|
|
146
|
-
catch {
|
|
147
|
-
logGap(filename, byteOffset, "rollout_record_is_not_utf8");
|
|
148
|
-
lineStart = index + 1;
|
|
149
|
-
continue;
|
|
150
|
-
}
|
|
151
|
-
const parsed = parseSourceRecord(source, line, filename, byteOffset);
|
|
152
|
-
for (const reason of parsed.gaps)
|
|
153
|
-
logGap(filename, byteOffset, reason);
|
|
154
|
-
for (const parsedEvent of parsed.events) {
|
|
155
|
-
// Claude repeats a response's usage on every line of that response,
|
|
156
|
-
// and only the last copy carries the final output count.
|
|
157
|
-
const queuedAt = tickIndexes.get(parsedEvent.eventId);
|
|
158
|
-
if (queuedAt !== undefined &&
|
|
159
|
-
parsedEvent.eventType === "token_usage_record") {
|
|
160
|
-
const event = await redactTraceEvent(parsedEvent);
|
|
161
|
-
if (Buffer.byteLength(JSON.stringify(event), "utf8") <= MAX_EVENT_BYTES)
|
|
162
|
-
events[queuedAt] = event;
|
|
163
|
-
continue;
|
|
164
|
-
}
|
|
165
|
-
if (queuedIds.has(parsedEvent.eventId))
|
|
166
|
-
continue;
|
|
167
|
-
const event = await redactTraceEvent(parsedEvent);
|
|
168
|
-
const eventBytes = Buffer.byteLength(JSON.stringify(event), "utf8");
|
|
169
|
-
if (eventBytes > MAX_EVENT_BYTES) {
|
|
170
|
-
logGap(filename, byteOffset, "normalized_event_exceeds_upload_limit");
|
|
171
|
-
}
|
|
172
|
-
else {
|
|
173
|
-
tickIndexes.set(event.eventId, events.length);
|
|
174
|
-
events.push(event);
|
|
175
|
-
queuedIds.add(event.eventId);
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
lineStart = index + 1;
|
|
179
|
-
}
|
|
180
|
-
if (buffer.length - lineStart > recordLimit) {
|
|
181
|
-
logGap(filename, offset + lineStart, "incomplete_rollout_record_exceeds_scan_limit");
|
|
182
|
-
skipping.add(filename);
|
|
183
|
-
lineStart = buffer.length;
|
|
184
|
-
}
|
|
185
|
-
cursors[filename] = offset + lineStart;
|
|
186
|
-
if (source === "cursor" &&
|
|
187
|
-
filename.includes(`${path.sep}cursor-hooks${path.sep}`) &&
|
|
188
|
-
cursors[filename] === stats.size)
|
|
189
|
-
completedHooks.push(filename);
|
|
190
|
-
}
|
|
191
|
-
await appendEvents(events, paths);
|
|
192
|
-
for (const filename of completedHooks) {
|
|
193
|
-
await fs.rm(filename);
|
|
194
|
-
delete cursors[filename];
|
|
195
|
-
}
|
|
196
|
-
await saveState({
|
|
197
|
-
cursors,
|
|
198
|
-
skipping: [...skipping],
|
|
199
|
-
sourceDirectory: state.sourceDirectory,
|
|
200
|
-
initialized: true,
|
|
201
|
-
}, paths);
|
|
202
|
-
return { discovered: events.length, caughtUp };
|
|
203
|
-
}
|
|
204
|
-
async function uploadPendingOnce(config, paths, fetcher = collectorFetch, signal, onAcknowledged) {
|
|
205
|
-
let uploaded = 0;
|
|
206
|
-
for (let attempt = 0; attempt < MAX_UPLOADS_PER_TICK; attempt += 1) {
|
|
207
|
-
signal?.throwIfAborted();
|
|
208
|
-
const batch = await nextBatch(paths, MAX_UPLOAD_BYTES);
|
|
209
|
-
if (batch.length === 0)
|
|
210
|
-
return uploaded;
|
|
211
|
-
const body = JSON.stringify({ events: batch });
|
|
212
|
-
if (Buffer.byteLength(body, "utf8") > MAX_REQUEST_BYTES) {
|
|
213
|
-
throw new Error("Queued trace batch exceeds the upload request limit.");
|
|
214
|
-
}
|
|
215
|
-
let response;
|
|
216
|
-
try {
|
|
217
|
-
response = await fetcher(config.endpoint, {
|
|
218
|
-
method: "POST",
|
|
219
|
-
headers: {
|
|
220
|
-
authorization: `Bearer ${config.token}`,
|
|
221
|
-
"content-type": "application/json",
|
|
222
|
-
},
|
|
223
|
-
body,
|
|
224
|
-
signal: signal
|
|
225
|
-
? AbortSignal.any([signal, AbortSignal.timeout(UPLOAD_TIMEOUT_MS)])
|
|
226
|
-
: AbortSignal.timeout(UPLOAD_TIMEOUT_MS),
|
|
227
|
-
});
|
|
228
|
-
}
|
|
229
|
-
catch (error) {
|
|
230
|
-
if (signal?.aborted)
|
|
231
|
-
throw error;
|
|
232
|
-
throw new TraceUploadError("network", null, "Trace upload could not reach Iteration.", { cause: error });
|
|
233
|
-
}
|
|
234
|
-
if (response.status !== 202) {
|
|
235
|
-
throw new TraceUploadError("http", response.status, `Trace upload failed with HTTP ${response.status}.`);
|
|
236
|
-
}
|
|
237
|
-
let acknowledgement;
|
|
238
|
-
try {
|
|
239
|
-
acknowledgement = await response.json();
|
|
240
|
-
}
|
|
241
|
-
catch (error) {
|
|
242
|
-
throw new TraceUploadError("response", null, "Trace upload returned an unreadable acknowledgement.", { cause: error });
|
|
243
|
-
}
|
|
244
|
-
const expectedIds = batch.map((event) => event.eventId);
|
|
245
|
-
const parsed = parseTraceEventAcknowledgement(acknowledgement, expectedIds);
|
|
246
|
-
if (!parsed.success) {
|
|
247
|
-
throw new TraceUploadError("response", null, parsed.error);
|
|
248
|
-
}
|
|
249
|
-
await acknowledgeEvents(new Set(parsed.value.acceptedEventIds), paths);
|
|
250
|
-
onAcknowledged?.();
|
|
251
|
-
uploaded += batch.length;
|
|
252
|
-
}
|
|
253
|
-
return uploaded;
|
|
254
|
-
}
|
|
255
|
-
export async function collectorTick(config, paths, fetcher = collectorFetch, signal, onAcknowledged) {
|
|
256
|
-
await redactPendingOutbox(paths);
|
|
257
|
-
let uploaded = await uploadPendingOnce(config, paths, fetcher, signal, onAcknowledged);
|
|
258
|
-
const scan = (await outboxSize(paths)) >= SCAN_PAUSE_OUTBOX_BYTES
|
|
259
|
-
? { discovered: 0, caughtUp: false }
|
|
260
|
-
: await scanSourceOnce(config, paths, signal);
|
|
261
|
-
uploaded += await uploadPendingOnce(config, paths, fetcher, signal, onAcknowledged);
|
|
262
|
-
return {
|
|
263
|
-
discovered: scan.discovered,
|
|
264
|
-
uploaded,
|
|
265
|
-
outboxBytes: await outboxSize(paths),
|
|
266
|
-
historyCaughtUp: scan.caughtUp,
|
|
267
|
-
};
|
|
268
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
package/dist/cursor-hooks.d.ts
DELETED
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
export declare const CURSOR_HOOKS: readonly ["sessionStart", "beforeSubmitPrompt", "preToolUse", "postToolUse", "postToolUseFailure", "afterAgentThought", "afterAgentResponse", "preCompact", "stop", "subagentStart", "subagentStop", "sessionEnd"];
|
|
2
|
-
export declare function cursorHookDirectory(dataDirectory: string): string;
|
|
3
|
-
export declare function cursorHookFailurePath(dataDirectory: string): string;
|
|
4
|
-
export declare function cursorHookFailureAtMs(dataDirectory: string): Promise<number | null>;
|
|
5
|
-
export declare function installCursorHooks(cursorHome: string, dataDirectory: string): Promise<void>;
|
|
6
|
-
export declare function uninstallCursorHooks(cursorHome: string, dataDirectory: string): Promise<void>;
|
|
7
|
-
export declare function cursorHookFiles(dataDirectory: string): Promise<string[]>;
|