iteration-cli 0.4.0

package/dist/history-import.d.ts

@@ -0,0 +1,17 @@
+import { type CollectorConfig, type CollectorPaths } from "./storage.js";
+type HistoryImportSize = Readonly<{
+    files: number;
+    bytes: number;
+}>;
+export type HistoryImportControl = Readonly<{
+    isRunning: () => Promise<boolean>;
+    stop: () => Promise<void>;
+    start: (directory: string) => Promise<unknown>;
+    acquireLock: (paths: CollectorPaths) => Promise<() => Promise<void>>;
+}>;
+export declare function importCodexHistory(config: CollectorConfig, paths: CollectorPaths, confirmed: boolean, control?: HistoryImportControl): Promise<Readonly<{
+    imported: boolean;
+    size: HistoryImportSize;
+    backgroundRunning: boolean | null;
+}>>;
+export {};

package/dist/history-import.js

@@ -0,0 +1,60 @@
+import path from "node:path";
+import { installLaunchAgent, launchAgentStatus, uninstallLaunchAgent, } from "./launch-agent.js";
+import { waitForCollectorLock } from "./service.js";
+import { historyImportSize, loadState, resetHistoryCursors, } from "./storage.js";
+const macControl = {
+    isRunning: launchAgentStatus,
+    stop: uninstallLaunchAgent,
+    start: installLaunchAgent,
+    acquireLock: waitForCollectorLock,
+};
+export async function importCodexHistory(config, paths, confirmed, control = macControl) {
+    const state = await loadState(paths);
+    if (!state.initialized ||
+        state.sourceDirectory !== path.resolve(config.codexHome)) {
+        throw new Error("Connect the Codex source before importing its history.");
+    }
+    const estimate = await historyImportSize(config.codexHome);
+    if (!confirmed)
+        return { imported: false, size: estimate, backgroundRunning: null };
+    const wasRunning = await control.isRunning();
+    let size = null;
+    let stopAttempted = false;
+    let rewound = false;
+    let failure = null;
+    try {
+        if (wasRunning) {
+            stopAttempted = true;
+            await control.stop();
+        }
+        const releaseLock = await control.acquireLock(paths);
+        try {
+            size = await resetHistoryCursors(config.codexHome, paths);
+            rewound = true;
+        }
+        finally {
+            await releaseLock();
+        }
+    }
+    catch (error) {
+        failure = { error };
+    }
+    if (stopAttempted) {
+        try {
+            await control.start(config.dataDirectory);
+        }
+        catch (error) {
+            throw new Error(rewound
+                ? "History checkpoints were rewound, but the collector could not restart. Run iteration start codex to resume uploads."
+                : "The collector could not be restored after a failed history import. Run iteration start codex to resume uploads.", {
+                cause: failure ? new AggregateError([failure.error, error]) : error,
+            });
+        }
+    }
+    if (failure)
+        throw failure.error;
+    if (size === null) {
+        throw new Error("History import did not update checkpoints.");
+    }
+    return { imported: true, size, backgroundRunning: wasRunning };
+}

package/dist/launch-agent.d.ts

@@ -0,0 +1,4 @@
+export declare function launchAgentPlist(cliPath?: string, nodePath?: string, logDirectory?: string, dataDirectory?: string): string;
+export declare function installLaunchAgent(dataDirectory?: string): Promise<string>;
+export declare function uninstallLaunchAgent(): Promise<void>;
+export declare function launchAgentStatus(): Promise<boolean>;

package/dist/launch-agent.js

@@ -0,0 +1,102 @@
+import { promisify } from "node:util";
+import { execFile as execFileCallback } from "node:child_process";
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+const execFile = promisify(execFileCallback);
+const LABEL = "com.iteration.codex-collector";
+function xml(value) {
+    return value
+        .replaceAll("&", "&")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&apos;");
+}
+function plistPath(homeDirectory = os.homedir()) {
+    return path.join(homeDirectory, "Library", "LaunchAgents", `${LABEL}.plist`);
+}
+export function launchAgentPlist(cliPath = fileURLToPath(new URL("./cli.js", import.meta.url)), nodePath = process.execPath, logDirectory = path.join(os.homedir(), "Library", "Logs"), dataDirectory = path.join(os.homedir(), ".config", "iteration", "codex-collector")) {
+    const argumentsList = [
+        nodePath,
+        cliPath,
+        "run",
+        "--data-directory",
+        dataDirectory,
+    ];
+    const argumentsXml = argumentsList
+        .map((argument) => `<string>${xml(argument)}</string>`)
+        .join("");
+    return [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        '<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">',
+        '<plist version="1.0"><dict>',
+        `<key>Label</key><string>${LABEL}</string>`,
+        `<key>ProgramArguments</key><array>${argumentsXml}</array>`,
+        "<key>RunAtLoad</key><true/>",
+        "<key>KeepAlive</key><true/>",
+        "<key>ThrottleInterval</key><integer>10</integer>",
+        `<key>StandardOutPath</key><string>${xml(path.join(logDirectory, "iteration-codex-collector.log"))}</string>`,
+        `<key>StandardErrorPath</key><string>${xml(path.join(logDirectory, "iteration-codex-collector.error.log"))}</string>`,
+        "</dict></plist>",
+    ].join("");
+}
+function launchctlTarget() {
+    const getUserId = process.getuid;
+    if (getUserId === undefined) {
+        throw new Error("Could not determine the current macOS user id.");
+    }
+    const userId = getUserId();
+    if (userId === undefined)
+        throw new Error("Could not determine the current macOS user id.");
+    return `gui/${userId}`;
+}
+async function bootoutIfLoaded(target) {
+    try {
+        await execFile("launchctl", ["print", `${target}/${LABEL}`]);
+    }
+    catch (error) {
+        const details = error;
+        if (String(details.code) === "113" ||
+            details.stderr?.includes("Could not find service"))
+            return;
+        throw error;
+    }
+    await execFile("launchctl", ["bootout", `${target}/${LABEL}`]);
+}
+export async function installLaunchAgent(dataDirectory) {
+    if (process.platform !== "darwin") {
+        throw new Error("Automatic startup is currently supported on macOS only.");
+    }
+    const file = plistPath();
+    const launchAgents = path.dirname(file);
+    const logs = path.join(os.homedir(), "Library", "Logs");
+    await fs.mkdir(launchAgents, { recursive: true, mode: 0o755 });
+    await fs.mkdir(logs, { recursive: true, mode: 0o755 });
+    await fs.writeFile(file, launchAgentPlist(undefined, undefined, undefined, dataDirectory ??
+        path.join(os.homedir(), ".config", "iteration", "codex-collector")), { encoding: "utf8", mode: 0o644 });
+    const target = launchctlTarget();
+    await bootoutIfLoaded(target);
+    await execFile("launchctl", ["bootstrap", target, file]);
+    return file;
+}
+export async function uninstallLaunchAgent() {
+    if (process.platform !== "darwin") {
+        throw new Error("Automatic startup is currently supported on macOS only.");
+    }
+    const file = plistPath();
+    await bootoutIfLoaded(launchctlTarget());
+    await fs.rm(file, { force: true });
+}
+export async function launchAgentStatus() {
+    if (process.platform !== "darwin")
+        return false;
+    try {
+        await execFile("launchctl", ["print", `${launchctlTarget()}/${LABEL}`]);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/rollout.d.ts

@@ -0,0 +1,11 @@
+import { type TraceEvent } from "iteration-trace-contracts";
+export type RolloutParseResult = Readonly<{
+    kind: "event";
+    event: TraceEvent;
+}> | Readonly<{
+    kind: "ignored";
+}> | Readonly<{
+    kind: "gap";
+    reason: string;
+}>;
+export declare function parseRolloutLine(line: string, filename: string, byteOffset: number): RolloutParseResult;

package/dist/rollout.js

@@ -0,0 +1,272 @@
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { parseTracePayload, } from "iteration-trace-contracts";
+import * as z from "zod";
+const recordSchema = z.record(z.string(), z.unknown());
+function parseJsonRecord(value) {
+    const parsed = recordSchema.safeParse(value);
+    return parsed.success ? parsed.data : null;
+}
+function parseJsonString(value) {
+    const parsed = z.string().safeParse(value);
+    return parsed.success ? parsed.data : null;
+}
+function sessionFromPath(filename) {
+    const basename = path.basename(filename, ".jsonl");
+    return basename.startsWith("rollout-")
+        ? basename.slice("rollout-".length)
+        : basename;
+}
+function callIdFrom(record) {
+    return (parseJsonString(record.call_id) ?? parseJsonString(record.tool_call_id));
+}
+function sha256(value) {
+    return createHash("sha256").update(value, "utf8").digest("hex");
+}
+function safePayload(value) {
+    try {
+        const normalized = JSON.parse(JSON.stringify(value));
+        const parsed = parseTracePayload(normalized);
+        return parsed.success ? parsed.value : null;
+    }
+    catch {
+        return null;
+    }
+}
+function mapResponseItem(payload) {
+    const type = parseJsonString(payload.type);
+    if (type === "message") {
+        const role = parseJsonString(payload.role);
+        if (role === "user")
+            return { eventType: "user_message", callId: null };
+        if (role === "assistant")
+            return { eventType: "assistant_message", callId: null };
+        return null;
+    }
+    if (type === "agent_message") {
+        return { eventType: "assistant_message", callId: null };
+    }
+    if (["reasoning", "todo_list"].includes(type ?? ""))
+        return null;
+    if ([
+        "function_call",
+        "custom_tool_call",
+        "mcp_tool_call",
+        "web_search_call",
+        "tool_search_call",
+        "image_generation_call",
+        "local_shell_call",
+    ].includes(type ?? "")) {
+        return { eventType: "tool_call", callId: callIdFrom(payload) };
+    }
+    if ([
+        "function_call_output",
+        "custom_tool_call_output",
+        "mcp_tool_call_output",
+        "web_search_output",
+        "tool_search_output",
+    ].includes(type ?? "")) {
+        return { eventType: "tool_result", callId: callIdFrom(payload) };
+    }
+    return null;
+}
+function mapCompletedItem(item) {
+    const type = parseJsonString(item.type);
+    if ([
+        "command_execution",
+        "mcp_tool_call",
+        "collab_tool_call",
+        "web_search",
+        "file_change",
+    ].includes(type ?? "")) {
+        return { eventType: "tool_result", callId: callIdFrom(item) };
+    }
+    if (type === "agent_message") {
+        return { eventType: "assistant_message", callId: null };
+    }
+    return null;
+}
+function eventTimestamp(record) {
+    const timestamp = parseJsonString(record.timestamp);
+    return timestamp !== null && !Number.isNaN(Date.parse(timestamp))
+        ? timestamp
+        : null;
+}
+export function parseRolloutLine(line, filename, byteOffset) {
+    let decoded;
+    try {
+        decoded = JSON.parse(line);
+    }
+    catch {
+        return { kind: "gap", reason: "malformed_jsonl_record" };
+    }
+    const record = parseJsonRecord(decoded);
+    if (record === null)
+        return { kind: "gap", reason: "record_is_not_an_object" };
+    const recordType = parseJsonString(record.type);
+    const payload = parseJsonRecord(record.payload);
+    if ([
+        "session_meta",
+        "turn_context",
+        "compacted",
+        "token_usage_record",
+        "world_state",
+        "retained_context",
+        "security_risk_score",
+    ].includes(recordType ?? "")) {
+        return { kind: "ignored" };
+    }
+    if (recordType === "response_item" && payload !== null) {
+        const mapped = mapResponseItem(payload);
+        if (mapped === null) {
+            const payloadType = parseJsonString(payload.type) ?? "unknown";
+            if ([
+                "reasoning",
+                "compaction",
+                "context_compaction",
+                "configuration_update",
+                "additional_tools",
+            ].includes(payloadType)) {
+                return { kind: "ignored" };
+            }
+            return {
+                kind: "gap",
+                reason: `unsupported_response_item:${payloadType}`,
+            };
+        }
+        const eventPayload = safePayload(payload);
+        if (eventPayload === null)
+            return { kind: "gap", reason: "invalid_response_item_payload" };
+        const sessionId = sessionFromPath(filename);
+        return {
+            kind: "event",
+            event: {
+                eventId: sha256(`${sessionId}:${byteOffset}`),
+                sessionId,
+                sequence: byteOffset,
+                callId: mapped.callId,
+                projectId: null,
+                occurredAt: eventTimestamp(record),
+                eventType: mapped.eventType,
+                payload: eventPayload,
+            },
+        };
+    }
+    if (recordType === "event_msg" && payload !== null) {
+        const messageType = parseJsonString(payload.type);
+        if (messageType === "user_message") {
+            const eventPayload = safePayload(payload);
+            if (eventPayload === null)
+                return { kind: "gap", reason: "invalid_user_message_payload" };
+            const sessionId = sessionFromPath(filename);
+            return {
+                kind: "event",
+                event: {
+                    eventId: sha256(`${sessionId}:${byteOffset}`),
+                    sessionId,
+                    sequence: byteOffset,
+                    callId: null,
+                    projectId: null,
+                    occurredAt: eventTimestamp(record),
+                    eventType: "user_message",
+                    payload: eventPayload,
+                },
+            };
+        }
+        if (messageType === "agent_message") {
+            const eventPayload = safePayload(payload);
+            if (eventPayload === null)
+                return { kind: "gap", reason: "invalid_agent_message_payload" };
+            const sessionId = sessionFromPath(filename);
+            return {
+                kind: "event",
+                event: {
+                    eventId: sha256(`${sessionId}:${byteOffset}`),
+                    sessionId,
+                    sequence: byteOffset,
+                    callId: null,
+                    projectId: null,
+                    occurredAt: eventTimestamp(record),
+                    eventType: "assistant_message",
+                    payload: eventPayload,
+                },
+            };
+        }
+        if ([
+            "mcp_tool_call_end",
+            "web_search_end",
+            "image_generation_end",
+            "patch_apply_end",
+        ].includes(messageType ?? "")) {
+            const eventPayload = safePayload(payload);
+            if (eventPayload === null)
+                return { kind: "gap", reason: "invalid_tool_result_payload" };
+            const sessionId = sessionFromPath(filename);
+            return {
+                kind: "event",
+                event: {
+                    eventId: sha256(`${sessionId}:${byteOffset}`),
+                    sessionId,
+                    sequence: byteOffset,
+                    callId: callIdFrom(payload),
+                    projectId: null,
+                    occurredAt: eventTimestamp(record),
+                    eventType: "tool_result",
+                    payload: eventPayload,
+                },
+            };
+        }
+        if ([
+            "task_started",
+            "turn_started",
+            "task_complete",
+            "turn_complete",
+            "token_count",
+            "thread_goal_updated",
+            "thread_rolled_back",
+            "turn_aborted",
+            "thread_settings_applied",
+            "agent_reasoning",
+            "agent_reasoning_raw_content",
+        ].includes(messageType ?? "")) {
+            return { kind: "ignored" };
+        }
+        if (messageType !== "item_completed") {
+            return {
+                kind: "gap",
+                reason: `unsupported_event_message:${messageType ?? "unknown"}`,
+            };
+        }
+        const item = parseJsonRecord(payload.item);
+        if (item === null)
+            return { kind: "gap", reason: "item_completed_without_item" };
+        const mapped = mapCompletedItem(item);
+        if (mapped === null) {
+            const itemType = parseJsonString(item.type) ?? "unknown";
+            if (["reasoning", "todo_list"].includes(itemType))
+                return { kind: "ignored" };
+            return { kind: "gap", reason: `unsupported_completed_item:${itemType}` };
+        }
+        const eventPayload = safePayload(item);
+        if (eventPayload === null)
+            return { kind: "gap", reason: "invalid_completed_item_payload" };
+        const sessionId = sessionFromPath(filename);
+        return {
+            kind: "event",
+            event: {
+                eventId: sha256(`${sessionId}:${byteOffset}`),
+                sessionId,
+                sequence: byteOffset,
+                callId: mapped.callId,
+                projectId: null,
+                occurredAt: eventTimestamp(record),
+                eventType: mapped.eventType,
+                payload: eventPayload,
+            },
+        };
+    }
+    return {
+        kind: "gap",
+        reason: `unsupported_rollout_record:${recordType ?? "unknown"}`,
+    };
+}

package/dist/service.d.ts

@@ -0,0 +1,4 @@
+import { type CollectorPaths } from "./storage.js";
+export declare function acquireCollectorLock(paths: CollectorPaths): Promise<() => Promise<void>>;
+export declare function waitForCollectorLock(paths: CollectorPaths): Promise<() => Promise<void>>;
+export declare function runCollector(directory?: string): Promise<void>;

package/dist/service.js

@@ -0,0 +1,119 @@
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { collectorTick } from "./collector.js";
+import { collectorPaths, ensurePrivateDirectory, loadConfig, } from "./storage.js";
+function errorCode(error) {
+    if (error instanceof Error && "code" in error)
+        return String(error.code);
+    return null;
+}
+class CollectorLockBusyError extends Error {
+}
+export async function acquireCollectorLock(paths) {
+    await ensurePrivateDirectory(path.dirname(paths.lock));
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+        try {
+            const handle = await fs.open(paths.lock, "wx", 0o600);
+            await handle.writeFile(`${process.pid}\n`, "utf8");
+            await handle.sync();
+            await handle.close();
+            return async () => {
+                const current = await fs.readFile(paths.lock, "utf8").catch(() => "");
+                if (current.trim() === String(process.pid)) {
+                    await fs.rm(paths.lock, { force: true });
+                }
+            };
+        }
+        catch (error) {
+            if (errorCode(error) !== "EEXIST")
+                throw error;
+            const lockContents = await fs
+                .readFile(paths.lock, "utf8")
+                .catch(() => "");
+            const lockPid = Number(lockContents.trim());
+            if (!Number.isSafeInteger(lockPid) || lockPid < 2) {
+                await fs.rm(paths.lock, { force: true });
+                continue;
+            }
+            try {
+                process.kill(lockPid, 0);
+                throw new CollectorLockBusyError("Codex collector is already running.");
+            }
+            catch (processError) {
+                if (errorCode(processError) !== "ESRCH")
+                    throw processError;
+                await fs.rm(paths.lock, { force: true });
+            }
+        }
+    }
+    throw new Error("Could not obtain the Codex collector lock.");
+}
+export async function waitForCollectorLock(paths) {
+    for (let attempt = 0; attempt < 81; attempt += 1) {
+        try {
+            return await acquireCollectorLock(paths);
+        }
+        catch (error) {
+            if (!(error instanceof CollectorLockBusyError))
+                throw error;
+            if (attempt === 80) {
+                throw new Error("Collector did not stop within 20 seconds; history was not changed.", { cause: error });
+            }
+            await delay(250);
+        }
+    }
+    throw new Error("Could not obtain the collector lock.");
+}
+export async function runCollector(directory) {
+    const paths = collectorPaths(directory);
+    const releaseLock = await acquireCollectorLock(paths);
+    const controller = new AbortController();
+    const stop = () => controller.abort();
+    process.on("SIGTERM", stop);
+    process.on("SIGINT", stop);
+    let retryDelayMs = 5000;
+    try {
+        const config = await loadConfig(paths);
+        while (!controller.signal.aborted) {
+            try {
+                const result = await collectorTick(config, paths, fetch, controller.signal);
+                if (result.discovered > 0 || result.uploaded > 0) {
+                    console.info(JSON.stringify({
+                        level: "info",
+                        discovered: result.discovered,
+                        uploaded: result.uploaded,
+                        outboxBytes: result.outboxBytes,
+                    }));
+                }
+                retryDelayMs = 5000;
+            }
+            catch (error) {
+                if (controller.signal.aborted)
+                    break;
+                console.error(JSON.stringify({
+                    level: "error",
+                    operation: "collector_tick",
+                    errorName: error instanceof Error ? error.name : "UnknownError",
+                    message: error instanceof Error
+                        ? error.message
+                        : "Unknown collector failure.",
+                }));
+                retryDelayMs = Math.min(retryDelayMs * 2, 60_000);
+            }
+            try {
+                await delay(retryDelayMs, undefined, { signal: controller.signal });
+            }
+            catch (error) {
+                if (error instanceof Error && error.name === "AbortError")
+                    break;
+                throw error;
+            }
+        }
+    }
+    finally {
+        process.off("SIGTERM", stop);
+        process.off("SIGINT", stop);
+        await releaseLock();
+    }
+}

package/dist/storage.d.ts

@@ -0,0 +1,44 @@
+import { type TraceEvent } from "iteration-trace-contracts";
+export type CollectorConfig = Readonly<{
+    endpoint: string;
+    token: string;
+    codexHome: string;
+    dataDirectory: string;
+}>;
+export type CollectorState = Readonly<{
+    cursors: Readonly<Record<string, number>>;
+    skipping: readonly string[];
+    sourceDirectory: string | null;
+    initialized: boolean;
+}>;
+export type CollectorPaths = Readonly<{
+    config: string;
+    state: string;
+    outbox: string;
+    lock: string;
+}>;
+export declare function validateCollectorEndpoint(endpoint: string): URL;
+export declare function collectorPaths(directory?: string): CollectorPaths;
+export declare function assertFreshCollectorDirectory(paths: CollectorPaths): Promise<void>;
+export declare function ensurePrivateDirectory(directory: string): Promise<void>;
+export declare function saveConfig(config: CollectorConfig, paths: CollectorPaths): Promise<void>;
+export declare function loadConfig(paths: CollectorPaths): Promise<CollectorConfig>;
+export declare function loadState(paths: CollectorPaths): Promise<CollectorState>;
+export declare function saveState(state: CollectorState, paths: CollectorPaths): Promise<void>;
+export declare function outboxSize(paths: CollectorPaths): Promise<number>;
+export declare function pendingEventIds(paths: CollectorPaths): Promise<Set<string>>;
+export declare function appendEvents(events: readonly TraceEvent[], paths: CollectorPaths): Promise<void>;
+export declare function nextBatch(paths: CollectorPaths, maxBytes: number): Promise<readonly TraceEvent[]>;
+export declare function acknowledgeEvents(acknowledgedIds: ReadonlySet<string>, paths: CollectorPaths): Promise<void>;
+export declare function initializeState(codexHome: string, paths: CollectorPaths, includeHistory: boolean): Promise<void>;
+export declare function prepareInitialState(codexHome: string, includeHistory: boolean): Promise<CollectorState>;
+export declare function ensureStartableState(config: CollectorConfig, paths: CollectorPaths): Promise<boolean>;
+export declare function resetHistoryCursors(codexHome: string, paths: CollectorPaths): Promise<Readonly<{
+    files: number;
+    bytes: number;
+}>>;
+export declare function historyImportSize(codexHome: string): Promise<Readonly<{
+    files: number;
+    bytes: number;
+}>>;
+export declare function rolloutFiles(root: string): Promise<string[]>;