iteration-cli 0.7.0 → 0.7.2

package/dist/auto-connect.d.ts

@@ -1,4 +1,5 @@
 import { type CollectorSource } from "iteration-trace-contracts";
+import { type CollectorFetcher } from "./collector-fetch.js";
 import { type CollectorConfig, type CollectorState } from "./storage.js";
 export type MachineConsent = Readonly<{
     version: 1;
@@ -24,7 +25,7 @@ export declare function readCollectorScope(anchor: CollectorConfig): Promise<Rea
     organizationId: string;
     organizationName: string;
 }>>;
-export declare function expandSource(anchor: CollectorConfig, source: CollectorSource, organizationId: string | null, request?: typeof fetch): Promise<SourceEnrollment>;
+export declare function expandSource(anchor: CollectorConfig, source: CollectorSource, organizationId: string | null, request?: CollectorFetcher): Promise<SourceEnrollment>;
 export declare function activateSource(source: CollectorSource, home: string, endpoint: string, enrollment: SourceEnrollment, state: CollectorState): Promise<CollectorConfig>;
 export declare function connectNewDetectedSources(anchor: CollectorConfig, consent: MachineConsent): Promise<CollectorSource[]>;
 export declare function connectedConfig(source: CollectorSource): Promise<CollectorConfig | null>;

package/dist/auto-connect.js

@@ -5,6 +5,7 @@ import path from "node:path";
 import * as z from "zod";
 import { COLLECTOR_SOURCES, parseCollectorEnrollmentResponse, } from "iteration-trace-contracts";
 import { installBackgroundService } from "./background-service.js";
+import { collectorFetch } from "./collector-fetch.js";
 import { installTurnSummary } from "./turn-summary.js";
 import { assertFreshCollectorDirectory, collectorPaths, defaultCollectorDirectory, ensurePrivateDirectory, loadConfig, prepareSourceInitialState, saveConfig, saveState, validateCollectorEndpoint, } from "./storage.js";
 const consentSchema = z
@@ -92,7 +93,7 @@ export async function saveMachineConsent(consent) {
 }
 export async function readCollectorScope(anchor) {
     const endpoint = validateCollectorEndpoint(anchor.endpoint);
-    const response = await fetch(new URL("./collector-auth/expand", endpoint), {
+    const response = await collectorFetch(new URL("./collector-auth/expand", endpoint), {
         headers: { Authorization: `Bearer ${anchor.token}` },
         signal: AbortSignal.timeout(10_000),
     });
@@ -109,7 +110,7 @@ export async function readCollectorScope(anchor) {
         throw new Error("Iteration returned an invalid collector workspace.");
     return parsed.data;
 }
-export async function expandSource(anchor, source, organizationId, request = fetch) {
+export async function expandSource(anchor, source, organizationId, request = collectorFetch) {
     const endpoint = validateCollectorEndpoint(anchor.endpoint);
     const token = createHmac("sha256", anchor.token)
         .update(`iteration-source:${source}`, "utf8")

package/dist/collector-fetch.d.ts

@@ -0,0 +1,9 @@
+type CollectorRequest = Readonly<{
+    method?: "GET" | "POST";
+    headers?: Readonly<Record<string, string>>;
+    body?: string;
+    signal?: AbortSignal;
+}>;
+export type CollectorFetcher = (url: string | URL, init?: CollectorRequest) => Promise<Response>;
+export declare const collectorFetch: CollectorFetcher;
+export {};

package/dist/collector-fetch.js

@@ -0,0 +1,58 @@
+import http from "node:http";
+import https from "node:https";
+const MAX_RESPONSE_BYTES = 1024 * 1024;
+const NULL_BODY_STATUSES = new Set([101, 103, 204, 205, 304]);
+// Node's fetch can negotiate HTTP/2 and, after a timed-out request breaks the
+// TLS connection, keep handing every later request the destroyed session until
+// the process restarts. The collector runs for days, so it uses HTTP/1.1
+// connections that are discarded when they fail.
+const agents = {
+    "http:": new http.Agent({ keepAlive: true, maxSockets: 2 }),
+    "https:": new https.Agent({ keepAlive: true, maxSockets: 2 }),
+};
+export const collectorFetch = (input, init = {}) => {
+    const url = new URL(input);
+    if (url.protocol !== "https:" && url.protocol !== "http:") {
+        throw new TypeError(`Unsupported collector request protocol ${url.protocol}`);
+    }
+    const body = init.body ?? null;
+    const headers = { ...init.headers };
+    if (body !== null) {
+        headers["content-length"] = String(Buffer.byteLength(body, "utf8"));
+    }
+    const transport = url.protocol === "https:" ? https : http;
+    return new Promise((resolve, reject) => {
+        const request = transport.request(url, {
+            method: init.method ?? "GET",
+            headers,
+            agent: agents[url.protocol],
+            signal: init.signal,
+        }, (response) => {
+            const chunks = [];
+            let size = 0;
+            response.on("data", (chunk) => {
+                size += chunk.byteLength;
+                if (size > MAX_RESPONSE_BYTES) {
+                    response.destroy(new Error("Iteration returned an oversized response."));
+                    return;
+                }
+                chunks.push(chunk);
+            });
+            response.on("error", reject);
+            response.on("end", () => {
+                const status = response.statusCode ?? 0;
+                const responseHeaders = new Headers();
+                for (const [name, value] of Object.entries(response.headers)) {
+                    if (value === undefined)
+                        continue;
+                    for (const item of Array.isArray(value) ? value : [value]) {
+                        responseHeaders.append(name, item);
+                    }
+                }
+                resolve(new Response(NULL_BODY_STATUSES.has(status) ? null : Buffer.concat(chunks), { status, headers: responseHeaders }));
+            });
+        });
+        request.on("error", reject);
+        request.end(body ?? undefined);
+    });
+};

package/dist/collector.d.ts

@@ -1,3 +1,4 @@
+import { type CollectorFetcher } from "./collector-fetch.js";
 import { type CollectorConfig, type CollectorPaths } from "./storage.js";
 export declare class TraceUploadError extends Error {
     readonly kind: "http" | "network" | "response";
@@ -8,7 +9,7 @@ export declare function scanSourceOnce(config: CollectorConfig, paths: Collector
     discovered: number;
     caughtUp: boolean;
 }>>;
-export declare function collectorTick(config: CollectorConfig, paths: CollectorPaths, fetcher?: typeof fetch, signal?: AbortSignal, onAcknowledged?: () => void): Promise<Readonly<{
+export declare function collectorTick(config: CollectorConfig, paths: CollectorPaths, fetcher?: CollectorFetcher, signal?: AbortSignal, onAcknowledged?: () => void): Promise<Readonly<{
     discovered: number;
     uploaded: number;
     outboxBytes: number;

package/dist/collector.js

@@ -1,6 +1,7 @@
 import { promises as fs } from "node:fs";
 import path from "node:path";
 import { parseTraceEventAcknowledgement, } from "iteration-trace-contracts";
+import { collectorFetch } from "./collector-fetch.js";
 import { parseClaudeLine } from "./claude-transcript.js";
 import { parseRolloutLine } from "./rollout.js";
 import { acknowledgeEvents, appendEvents, collectorSource, collectorSourceDirectory, loadState, nextBatch, outboxSize, pendingEventIds, sourceFiles, saveState, } from "./storage.js";
@@ -11,6 +12,13 @@ const MAX_UPLOAD_BYTES = 3 * 1024 * 1024;
 const MAX_EVENT_BYTES = MAX_UPLOAD_BYTES - 1024;
 const MAX_UPLOADS_PER_TICK = 5;
 const MAX_REQUEST_BYTES = 4 * 1024 * 1024;
+// Production stores some batches in about 40 seconds. Abandoning a request
+// earlier does not stop the server, so the retry only waits behind it and the
+// collector reports a failure for work that succeeded.
+const UPLOAD_TIMEOUT_MS = 90_000;
+// A tick can queue up to MAX_SCAN_BYTES_PER_TICK of new events, so scanning
+// pauses well below the outbox capacity and resumes once uploads drain it.
+const SCAN_PAUSE_OUTBOX_BYTES = 32 * 1024 * 1024;
 const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true });
 export class TraceUploadError extends Error {
     kind;
@@ -157,7 +165,7 @@ export async function scanSourceOnce(config, paths, signal) {
     }, paths);
     return { discovered: events.length, caughtUp };
 }
-async function uploadPendingOnce(config, paths, fetcher = fetch, signal, onAcknowledged) {
+async function uploadPendingOnce(config, paths, fetcher = collectorFetch, signal, onAcknowledged) {
     let uploaded = 0;
     for (let attempt = 0; attempt < MAX_UPLOADS_PER_TICK; attempt += 1) {
         signal?.throwIfAborted();
@@ -178,8 +186,8 @@ async function uploadPendingOnce(config, paths, fetcher = fetch, signal, onAckno
                 },
                 body,
                 signal: signal
-                    ? AbortSignal.any([signal, AbortSignal.timeout(15_000)])
-                    : AbortSignal.timeout(15_000),
+                    ? AbortSignal.any([signal, AbortSignal.timeout(UPLOAD_TIMEOUT_MS)])
+                    : AbortSignal.timeout(UPLOAD_TIMEOUT_MS),
             });
         }
         catch (error) {
@@ -208,9 +216,11 @@ async function uploadPendingOnce(config, paths, fetcher = fetch, signal, onAckno
     }
     return uploaded;
 }
-export async function collectorTick(config, paths, fetcher = fetch, signal, onAcknowledged) {
+export async function collectorTick(config, paths, fetcher = collectorFetch, signal, onAcknowledged) {
     let uploaded = await uploadPendingOnce(config, paths, fetcher, signal, onAcknowledged);
-    const scan = await scanSourceOnce(config, paths, signal);
+    const scan = (await outboxSize(paths)) >= SCAN_PAUSE_OUTBOX_BYTES
+        ? { discovered: 0, caughtUp: false }
+        : await scanSourceOnce(config, paths, signal);
     uploaded += await uploadPendingOnce(config, paths, fetcher, signal, onAcknowledged);
     return {
         discovered: scan.discovered,

package/dist/service.js

@@ -2,6 +2,7 @@ import { promises as fs } from "node:fs";
 import path from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
 import { connectNewDetectedSources, loadMachineConsent, } from "./auto-connect.js";
+import { collectorFetch } from "./collector-fetch.js";
 import { collectorTick, TraceUploadError } from "./collector.js";
 import { refreshTurnSummaries } from "./turn-summary.js";
 import { collectorSource, collectorPaths, ensureClaudeHistoryBackfill, ensureHistoryBackfill, ensurePrivateDirectory, healthSnapshot, loadConfig, saveHealth, } from "./storage.js";
@@ -12,6 +13,21 @@ function errorCode(error) {
 }
 class CollectorLockBusyError extends Error {
 }
+// Fetch reports network failures as a generic TypeError; the actionable
+// reason (DNS, TLS, reset, timeout) is only in its cause chain.
+function errorCauses(error) {
+    const causes = [];
+    let current = error instanceof Error ? error.cause : undefined;
+    while (current instanceof Error && causes.length < 4) {
+        causes.push({
+            name: current.name,
+            code: errorCode(current),
+            message: current.message.slice(0, 300),
+        });
+        current = current.cause;
+    }
+    return causes;
+}
 function collectorFailure(error) {
     return {
         atMs: Date.now(),
@@ -143,7 +159,7 @@ export async function runCollector(directory, expectedSource) {
             }
             let acknowledgedAtMs = null;
             try {
-                const result = await collectorTick(config, paths, fetch, controller.signal, () => {
+                const result = await collectorTick(config, paths, collectorFetch, controller.signal, () => {
                     acknowledgedAtMs = Date.now();
                 });
                 if (acknowledgedAtMs !== null ||
@@ -160,6 +176,7 @@ export async function runCollector(directory, expectedSource) {
                 if (result.discovered > 0 || result.uploaded > 0) {
                     console.info(JSON.stringify({
                         level: "info",
+                        at: new Date().toISOString(),
                         source,
                         discovered: result.discovered,
                         uploaded: result.uploaded,
@@ -178,9 +195,11 @@ export async function runCollector(directory, expectedSource) {
                 };
                 console.error(JSON.stringify({
                     level: "error",
+                    at: new Date().toISOString(),
                     source,
                     operation: "collector_tick",
                     errorName: error instanceof Error ? error.name : "UnknownError",
+                    causes: errorCauses(error),
                     message: error instanceof Error
                         ? error.message
                         : "Unknown collector failure.",
@@ -189,16 +208,18 @@ export async function runCollector(directory, expectedSource) {
                 retryDelayMs = Math.min(retryDelayMs * 2, 60_000);
             }
             try {
-                await refreshTurnSummaries(config, summariesRefreshedAt, Date.now(), fetch, controller.signal);
+                await refreshTurnSummaries(config, summariesRefreshedAt, Date.now(), collectorFetch, controller.signal);
             }
             catch (error) {
                 if (controller.signal.aborted)
                     break;
                 console.error(JSON.stringify({
                     level: "warning",
+                    at: new Date().toISOString(),
                     source,
                     operation: "refresh_turn_summaries",
                     errorName: error instanceof Error ? error.name : "UnknownError",
+                    causes: errorCauses(error),
                     message: error instanceof Error
                         ? error.message
                         : "Unknown turn summary failure.",

package/dist/turn-summary.d.ts

@@ -1,4 +1,5 @@
 import { type CollectorFrictionSummary } from "iteration-trace-contracts";
+import { type CollectorFetcher } from "./collector-fetch.js";
 import { type CollectorConfig } from "./storage.js";
 export type TurnSummaryPaths = Readonly<{
     directory: string;
@@ -17,6 +18,6 @@ export declare class TurnSummaryRefreshError extends Error {
 export declare function refreshTurnSummaries(config: CollectorConfig, refreshedAt: Map<string, Readonly<{
     directory: string;
     atMs: number;
-}>>, nowMs: number, fetcher?: typeof fetch, signal?: AbortSignal): Promise<number>;
+}>>, nowMs: number, fetcher?: CollectorFetcher, signal?: AbortSignal): Promise<number>;
 export type TurnSummaryApproval = "approved" | "needs_approval" | "not_installed";
 export declare function codexTurnSummaryApproval(config: CollectorConfig): Promise<TurnSummaryApproval>;

package/dist/turn-summary.js

@@ -3,6 +3,7 @@ import { promises as fs } from "node:fs";
 import path from "node:path";
 import { isLaunchDirectory, parseCollectorFrictionSummary, } from "iteration-trace-contracts";
 import * as z from "zod";
+import { collectorFetch } from "./collector-fetch.js";
 import { ensurePrivateDirectory } from "./storage.js";
 const REFRESH_INTERVAL_MS = 5 * 60 * 1000;
 const REQUEST_RETENTION_MS = 14 * 24 * 60 * 60 * 1000;
@@ -249,7 +250,7 @@ async function forget(paths, key) {
 // Refreshes the summary line for each repository the hook asked about. The
 // hook reads the result on a later turn, so a slow or failed request never
 // delays the agent.
-export async function refreshTurnSummaries(config, refreshedAt, nowMs, fetcher = fetch, signal) {
+export async function refreshTurnSummaries(config, refreshedAt, nowMs, fetcher = collectorFetch, signal) {
     const paths = turnSummaryPaths(config.dataDirectory);
     let keys;
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "iteration-cli",
-  "version": "0.7.0",
+  "version": "0.7.2",
   "type": "module",
   "repository": {
     "type": "git",