iteration-cli 0.4.1 → 0.4.3

package/dist/cli.js

@@ -8,7 +8,7 @@ import { authenticateAndEnrollCollector } from "./clerk-device-auth.js";
 import { importCodexHistory } from "./history-import.js";
 import { runCollector } from "./service.js";
 import { installLaunchAgent, launchAgentStatus, uninstallLaunchAgent, } from "./launch-agent.js";
-import { assertFreshCollectorDirectory, collectorPaths, ensureStartableState, loadConfig, outboxSize, pendingEventIds, prepareInitialState, saveConfig, saveState, validateCollectorEndpoint, } from "./storage.js";
+import { assertFreshCollectorDirectory, collectorPaths, ensureStartableState, historyScanProgress, loadConfig, outboxSize, pendingEventIds, prepareInitialState, saveConfig, saveState, validateCollectorEndpoint, } from "./storage.js";
 const DEFAULT_INGEST_ENDPOINT = "https://useiteration.com/api/traces/ingest";
 function argumentsForCli() {
     const parsed = parseArgs({
@@ -22,6 +22,7 @@ function argumentsForCli() {
             "organization-id": { type: "string" },
             "codex-home": { type: "string" },
             "data-directory": { type: "string" },
+            "include-history": { type: "boolean" },
             yes: { type: "boolean" },
         },
     });
@@ -156,7 +157,7 @@ async function importHistory(values) {
         return;
     }
     const result = await importCodexHistory(config, paths, true);
-    process.stdout.write(`History scan scheduled for ${result.size.files} local session files (${result.size.bytes} bytes). Scanning and uploads run in bounded batches and may take time. Older sessions appear under the date Iteration receives them. iteration status codex shows events waiting to upload, not total scan progress.${result.backgroundRunning ? "" : " The collector was not running; run iteration start codex to begin scanning."}\n`);
+    process.stdout.write(`History scan scheduled for ${result.size.files} local session files (${result.size.bytes} bytes). Scanning and uploads run in bounded batches and may take time. Older sessions appear under the date Iteration receives them. Use iteration status to see local scan progress and pending uploads.${result.backgroundRunning ? "" : " The collector was not running; run iteration start codex to begin scanning."}\n`);
 }
 async function start(values) {
     if (process.platform !== "darwin") {
@@ -175,15 +176,22 @@ async function status(values) {
         process.stdout.write("Installed: false\nRunning: false\nPending events: 0\n");
         return;
     }
-    await loadConfig(paths);
+    const config = await loadConfig(paths);
     const running = await launchAgentStatus();
-    const [size, ids] = await Promise.all([
+    const [size, ids, scan] = await Promise.all([
         outboxSize(paths),
         pendingEventIds(paths),
+        historyScanProgress(config.codexHome, paths),
     ]);
+    const scanLabel = scan.kind === "waiting"
+        ? "Local history: waiting for collector start"
+        : scan.totalBytes === 0
+            ? "Local scan: no session data found"
+            : `Local scan: ${scan.scannedBytes} of ${scan.totalBytes} bytes processed`;
     process.stdout.write([
         "Installed: true",
         `Running: ${running}`,
+        scanLabel,
         `Pending events: ${ids.size}`,
         `Pending bytes: ${size}`,
     ].join("\n") + "\n");
@@ -193,6 +201,10 @@ async function main() {
     if (values.help === true || command === "help") {
         return usage(command === "help" ? source : command);
     }
+    if (values["include-history"] === true &&
+        !(command === "connect" && source === "codex")) {
+        throw new Error("Existing sessions are included automatically. Run iteration to connect.");
+    }
     if (command === "" || (command === "connect" && source === undefined)) {
         if (Object.keys(values).length > 0) {
             throw new Error("Options require a source. Run iteration connect codex --help for details.");

package/dist/launch-agent.d.ts

@@ -1,4 +1,12 @@
+export type LaunchAgentRestart = {
+    isLoaded(timeoutMs: number): Promise<boolean>;
+    bootstrap(timeoutMs: number): Promise<void>;
+    wait(delayMs: number): Promise<void>;
+    now(): number;
+};
 export declare function launchAgentPlist(cliPath?: string, nodePath?: string, logDirectory?: string, dataDirectory?: string): string;
+export declare function bootstrapAfterBootout(wasLoaded: boolean, control: LaunchAgentRestart): Promise<void>;
 export declare function installLaunchAgent(dataDirectory?: string): Promise<string>;
 export declare function uninstallLaunchAgent(): Promise<void>;
 export declare function launchAgentStatus(): Promise<boolean>;
+export declare function launchAgentRunning(output: string): boolean;

package/dist/launch-agent.js

@@ -3,9 +3,13 @@ import { execFile as execFileCallback } from "node:child_process";
 import { promises as fs } from "node:fs";
 import os from "node:os";
 import path from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
 import { fileURLToPath } from "node:url";
 const execFile = promisify(execFileCallback);
 const LABEL = "com.iteration.codex-collector";
+const LAUNCHCTL_TIMEOUT_MS = 5_000;
+const RESTART_TIMEOUT_MS = 10_000;
+const RESTART_POLL_MS = 200;
 function xml(value) {
     return value
         .replaceAll("&", "&")
@@ -52,18 +56,73 @@ function launchctlTarget() {
         throw new Error("Could not determine the current macOS user id.");
     return `gui/${userId}`;
 }
-async function bootoutIfLoaded(target) {
+function serviceMissing(error) {
+    const details = error;
+    return (String(details.code) === "113" ||
+        details.stderr?.includes("Could not find service") === true);
+}
+async function serviceLoaded(target, timeoutMs = LAUNCHCTL_TIMEOUT_MS) {
     try {
-        await execFile("launchctl", ["print", `${target}/${LABEL}`]);
+        await execFile("launchctl", ["print", `${target}/${LABEL}`], {
+            timeout: timeoutMs,
+        });
+        return true;
     }
     catch (error) {
-        const details = error;
-        if (String(details.code) === "113" ||
-            details.stderr?.includes("Could not find service"))
-            return;
+        if (serviceMissing(error))
+            return false;
         throw error;
     }
-    await execFile("launchctl", ["bootout", `${target}/${LABEL}`]);
+}
+async function bootoutIfLoaded(target) {
+    if (!(await serviceLoaded(target)))
+        return false;
+    await execFile("launchctl", ["bootout", `${target}/${LABEL}`], {
+        timeout: LAUNCHCTL_TIMEOUT_MS,
+    });
+    return true;
+}
+function transientBootstrapFailure(error) {
+    const details = error;
+    return (String(details.code) === "5" ||
+        /^Bootstrap failed:\s*5\b/m.test(details.stderr ?? ""));
+}
+function restartTimeoutError(lastBootstrapFailure) {
+    return new Error(lastBootstrapFailure === undefined
+        ? "The previous Codex collector did not stop within 10 seconds."
+        : "launchctl bootstrap still failed with error 5 after 10 seconds.", { cause: lastBootstrapFailure });
+}
+export async function bootstrapAfterBootout(wasLoaded, control) {
+    if (!wasLoaded) {
+        await control.bootstrap(LAUNCHCTL_TIMEOUT_MS);
+        return;
+    }
+    const deadline = control.now() + RESTART_TIMEOUT_MS;
+    let lastBootstrapFailure;
+    while (true) {
+        let remaining = deadline - control.now();
+        if (remaining <= 0) {
+            throw restartTimeoutError(lastBootstrapFailure);
+        }
+        if (!(await control.isLoaded(Math.min(remaining, LAUNCHCTL_TIMEOUT_MS)))) {
+            remaining = deadline - control.now();
+            if (remaining <= 0)
+                continue;
+            try {
+                await control.bootstrap(Math.min(remaining, LAUNCHCTL_TIMEOUT_MS));
+                return;
+            }
+            catch (error) {
+                if (!transientBootstrapFailure(error))
+                    throw error;
+                lastBootstrapFailure = error;
+            }
+        }
+        remaining = deadline - control.now();
+        if (remaining > 0) {
+            await control.wait(Math.min(remaining, RESTART_POLL_MS));
+        }
+    }
 }
 export async function installLaunchAgent(dataDirectory) {
     if (process.platform !== "darwin") {
@@ -77,8 +136,20 @@ export async function installLaunchAgent(dataDirectory) {
     await fs.writeFile(file, launchAgentPlist(undefined, undefined, undefined, dataDirectory ??
         path.join(os.homedir(), ".config", "iteration", "codex-collector")), { encoding: "utf8", mode: 0o644 });
     const target = launchctlTarget();
-    await bootoutIfLoaded(target);
-    await execFile("launchctl", ["bootstrap", target, file]);
+    const wasLoaded = await bootoutIfLoaded(target);
+    await bootstrapAfterBootout(wasLoaded, {
+        isLoaded: (timeoutMs) => serviceLoaded(target, timeoutMs),
+        bootstrap: async (timeoutMs) => {
+            await execFile("launchctl", ["bootstrap", target, file], {
+                timeout: timeoutMs,
+            });
+        },
+        wait: sleep,
+        now: Date.now,
+    });
+    await execFile("launchctl", ["kickstart", "-p", `${target}/${LABEL}`], {
+        timeout: LAUNCHCTL_TIMEOUT_MS,
+    });
     return file;
 }
 export async function uninstallLaunchAgent() {
@@ -93,10 +164,17 @@ export async function launchAgentStatus() {
     if (process.platform !== "darwin")
         return false;
     try {
-        await execFile("launchctl", ["print", `${launchctlTarget()}/${LABEL}`]);
-        return true;
+        const { stdout } = await execFile("launchctl", [
+            "print",
+            `${launchctlTarget()}/${LABEL}`,
+        ]);
+        return launchAgentRunning(stdout);
     }
     catch {
         return false;
     }
 }
+export function launchAgentRunning(output) {
+    const state = output.match(/^\s*state = ([^\r\n]+)$/m);
+    return state?.[1]?.trim() === "running";
+}

package/dist/storage.d.ts

@@ -35,6 +35,13 @@ export declare function initializeState(codexHome: string, paths: CollectorPaths
 export declare function prepareInitialState(codexHome: string, includeHistory: boolean): Promise<CollectorState>;
 export declare function ensureStartableState(config: CollectorConfig, paths: CollectorPaths): Promise<boolean>;
 export declare function ensureHistoryBackfill(codexHome: string, paths: CollectorPaths): Promise<boolean>;
+export declare function historyScanProgress(codexHome: string, paths: CollectorPaths): Promise<Readonly<{
+    kind: "waiting";
+} | {
+    kind: "scanning";
+    scannedBytes: number;
+    totalBytes: number;
+}>>;
 export declare function resetHistoryCursors(codexHome: string, paths: CollectorPaths): Promise<Readonly<{
     files: number;
     bytes: number;

package/dist/storage.js

@@ -322,9 +322,10 @@ export async function initializeState(codexHome, paths, includeHistory) {
 export async function prepareInitialState(codexHome, includeHistory) {
     const sourceDirectory = path.resolve(codexHome);
     const sessionsRoot = path.join(sourceDirectory, "sessions");
+    const files = await rolloutFiles(sessionsRoot);
     const cursors = {};
     if (!includeHistory) {
-        for (const filename of await rolloutFiles(sessionsRoot)) {
+        for (const filename of files) {
             cursors[filename] = await lastCompleteRecordOffset(filename, (await fs.stat(filename)).size);
         }
     }
@@ -349,7 +350,7 @@ export async function ensureStartableState(config, paths) {
     await initializeState(config.codexHome, paths, true);
     return true;
 }
-export async function ensureHistoryBackfill(codexHome, paths) {
+async function hasHistoryBackfillMarker(paths) {
     let marker;
     try {
         marker = await fs.readFile(paths.historyBackfill, "utf8");
@@ -363,16 +364,63 @@ export async function ensureHistoryBackfill(codexHome, paths) {
         if (marker !== '{"version":1}\n') {
             throw new Error("Collector history backfill marker has an invalid shape.");
         }
-        return false;
+        return true;
     }
+    return false;
+}
+export async function ensureHistoryBackfill(codexHome, paths) {
+    if (await hasHistoryBackfillMarker(paths))
+        return false;
     const state = await loadState(paths);
     if (!state.initialized || state.sourceDirectory !== path.resolve(codexHome)) {
         throw new Error("Collector state does not match CODEX_HOME; reconnect Codex before uploading.");
     }
+    await rolloutFiles(path.join(codexHome, "sessions"));
     await saveState({ ...state, cursors: {}, skipping: [] }, paths);
     await writeJsonAtomic(paths.historyBackfill, { version: 1 });
     return true;
 }
+export async function historyScanProgress(codexHome, paths) {
+    if (!(await hasHistoryBackfillMarker(paths)))
+        return { kind: "waiting" };
+    const state = await loadState(paths);
+    if (!state.initialized || state.sourceDirectory !== path.resolve(codexHome)) {
+        throw new Error("Collector state does not match CODEX_HOME.");
+    }
+    const files = await rolloutFiles(path.join(codexHome, "sessions"));
+    let scannedBytes = 0;
+    let totalBytes = 0;
+    for (let start = 0; start < files.length; start += 64) {
+        const batch = files.slice(start, start + 64);
+        const sizes = await Promise.all(batch.map(async (filename) => {
+            try {
+                return (await fs.stat(filename)).size;
+            }
+            catch (error) {
+                if (error instanceof Error &&
+                    "code" in error &&
+                    error.code === "ENOENT")
+                    return null;
+                throw error;
+            }
+        }));
+        for (const [index, size] of sizes.entries()) {
+            if (size === null)
+                continue;
+            const filename = batch[index];
+            if (filename === undefined)
+                continue;
+            totalBytes += size;
+            const cursor = state.cursors[filename] ?? 0;
+            scannedBytes += cursor <= size ? cursor : 0;
+            if (!Number.isSafeInteger(totalBytes) ||
+                !Number.isSafeInteger(scannedBytes)) {
+                throw new Error("Codex history size exceeds the supported limit.");
+            }
+        }
+    }
+    return { kind: "scanning", scannedBytes, totalBytes };
+}
 export async function resetHistoryCursors(codexHome, paths) {
     const state = await loadState(paths);
     const sourceDirectory = path.resolve(codexHome);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "iteration-cli",
-  "version": "0.4.1",
+  "version": "0.4.3",
   "type": "module",
   "repository": {
     "type": "git",