iteration-cli 0.4.3 → 0.5.0

package/dist/storage.js

@@ -1,19 +1,33 @@
+import { once } from "node:events";
 import { createWriteStream, promises as fs } from "node:fs";
 import os from "node:os";
 import path from "node:path";
-import { once } from "node:events";
 import { createInterface } from "node:readline";
-import * as z from "zod";
 import { MAX_TRACE_EVENTS_PER_BATCH, parseTraceEventBatch, } from "iteration-trace-contracts";
+import * as z from "zod";
 const MAX_OUTBOX_BYTES = 64 * 1024 * 1024;
-const configSchema = z
-    .object({
+export function collectorSource(config) {
+    return "claudeHome" in config ? "claude" : "codex";
+}
+export function collectorSourceDirectory(config) {
+    return "claudeHome" in config ? config.claudeHome : config.codexHome;
+}
+const connectionSchema = {
     endpoint: z.string(),
     token: z.string(),
-    codexHome: z.string(),
     dataDirectory: z.string(),
-})
-    .strict();
+    organizationName: z.string().min(1).max(160).optional(),
+};
+const configSchema = z.union([
+    z.object({ ...connectionSchema, codexHome: z.string() }).strict(),
+    z
+        .object({
+        ...connectionSchema,
+        source: z.literal("claude"),
+        claudeHome: z.string(),
+    })
+        .strict(),
+]);
 const stateSchema = z
     .object({
     cursors: z.record(z.string(), z.number().int().min(0).max(Number.MAX_SAFE_INTEGER)),
@@ -22,6 +36,20 @@ const stateSchema = z
     initialized: z.boolean(),
 })
     .strict();
+const healthSchema = z
+    .object({
+    lastUploadAtMs: z.number().int().min(0).nullable(),
+    historyCaughtUpAtMs: z.number().int().min(0).nullable(),
+    lastFailure: z
+        .object({
+        atMs: z.number().int().min(0),
+        kind: z.enum(["http", "network", "response", "collector"]),
+        status: z.number().int().min(100).max(599).nullable(),
+    })
+        .strict()
+        .nullable(),
+})
+    .strict();
 function parseOutboxEvent(line) {
     let decoded;
     try {
@@ -40,8 +68,8 @@ function parseOutboxEvent(line) {
         throw new Error("Trace outbox event is missing.");
     return event;
 }
-function defaultDirectory() {
-    return path.join(os.homedir(), ".config", "iteration", "codex-collector");
+export function defaultCollectorDirectory(source) {
+    return path.join(os.homedir(), ".config", "iteration", `${source}-collector`);
 }
 export function validateCollectorEndpoint(endpoint) {
     let parsed;
@@ -67,16 +95,17 @@ export function validateCollectorEndpoint(endpoint) {
     }
     return parsed;
 }
-export function collectorPaths(directory = defaultDirectory()) {
+export function collectorPaths(directory = defaultCollectorDirectory("codex")) {
     return {
         config: path.join(directory, "config.json"),
         state: path.join(directory, "state.json"),
         outbox: path.join(directory, "outbox.jsonl"),
         lock: path.join(directory, "collector.lock"),
         historyBackfill: path.join(directory, "history-backfill.json"),
+        health: path.join(directory, "health.json"),
     };
 }
-export async function assertFreshCollectorDirectory(paths) {
+export async function assertFreshCollectorDirectory(paths, source = "codex") {
     const state = await loadState(paths);
     const configExists = await fs
         .access(paths.config)
@@ -87,7 +116,7 @@ export async function assertFreshCollectorDirectory(paths) {
         throw error;
     });
     if (configExists) {
-        throw new Error("This collector data directory is already configured. Run iteration status codex or iteration start codex to repair startup. Use a different --data-directory for another workspace, or iteration import-history codex for earlier sessions.");
+        throw new Error(`This collector data directory is already configured. Run iteration status ${source} or iteration start ${source} to repair startup. Use a different --data-directory for another workspace, or iteration import-history ${source} for earlier sessions.`);
     }
     if (state.initialized || (await outboxSize(paths)) > 0) {
         throw new Error("Collector state or pending events exist without a config. Use a different --data-directory; this data cannot be safely assigned to a new workspace.");
@@ -113,6 +142,57 @@ export async function saveConfig(config, paths) {
     await ensurePrivateDirectory(path.dirname(paths.config));
     await writeJsonAtomic(paths.config, config);
 }
+export async function loadHealth(paths) {
+    let contents;
+    try {
+        contents = await fs.readFile(paths.health, "utf8");
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT") {
+            return {
+                lastUploadAtMs: null,
+                historyCaughtUpAtMs: null,
+                lastFailure: null,
+            };
+        }
+        throw new Error(`Could not read collector health at ${paths.health}.`, {
+            cause: error,
+        });
+    }
+    let decoded;
+    try {
+        decoded = JSON.parse(contents);
+    }
+    catch (error) {
+        throw new Error("Collector health is not valid JSON.", { cause: error });
+    }
+    const parsed = healthSchema.safeParse(decoded);
+    if (!parsed.success) {
+        throw new Error("Collector health has an invalid shape.", {
+            cause: parsed.error,
+        });
+    }
+    return parsed.data;
+}
+export async function healthSnapshot(paths) {
+    try {
+        return { kind: "available", health: await loadHealth(paths) };
+    }
+    catch {
+        return {
+            kind: "unavailable",
+            health: {
+                lastUploadAtMs: null,
+                historyCaughtUpAtMs: null,
+                lastFailure: null,
+            },
+        };
+    }
+}
+export async function saveHealth(health, paths) {
+    await ensurePrivateDirectory(path.dirname(paths.health));
+    await writeJsonAtomic(paths.health, health);
+}
 export async function loadConfig(paths) {
     let contents;
     try {
@@ -135,12 +215,12 @@ export async function loadConfig(paths) {
         throw new Error("Collector config has an invalid shape.", {
             cause: parsed.error,
         });
-    const { endpoint, token, codexHome, dataDirectory } = parsed.data;
-    validateCollectorEndpoint(endpoint);
+    validateCollectorEndpoint(parsed.data.endpoint);
+    const { token } = parsed.data;
     if (token.length !== 43 || !/^[A-Za-z0-9_-]+$/.test(token)) {
         throw new Error("Collector token has an invalid format.");
     }
-    return { endpoint, token, codexHome, dataDirectory };
+    return parsed.data;
 }
 export async function loadState(paths) {
     let contents;
@@ -305,24 +385,23 @@ export async function acknowledgeEvents(acknowledgedIds, paths) {
         throw error;
     }
 }
-export async function initializeState(codexHome, paths, includeHistory) {
+export async function initializeState(source, home, paths, includeHistory) {
     const state = await loadState(paths);
-    const sourceDirectory = path.resolve(codexHome);
+    const sourceDirectory = path.resolve(home);
     if (state.initialized && state.sourceDirectory !== sourceDirectory) {
-        throw new Error("Collector state belongs to another Codex directory; use a separate --data-directory.");
+        throw new Error(`Collector state belongs to another ${source} directory; use a separate --data-directory.`);
     }
     if (state.initialized && state.sourceDirectory === sourceDirectory) {
         if (includeHistory) {
-            throw new Error("Codex is already connected. Run iteration import-history codex to import existing sessions.");
+            throw new Error(`${source} is already connected. Run iteration import-history ${source} to import existing sessions.`);
         }
         return;
     }
-    await saveState(await prepareInitialState(codexHome, includeHistory), paths);
+    await saveState(await prepareSourceInitialState(source, home, includeHistory), paths);
 }
-export async function prepareInitialState(codexHome, includeHistory) {
-    const sourceDirectory = path.resolve(codexHome);
-    const sessionsRoot = path.join(sourceDirectory, "sessions");
-    const files = await rolloutFiles(sessionsRoot);
+export async function prepareSourceInitialState(source, home, includeHistory) {
+    const sourceDirectory = path.resolve(home);
+    const files = await sourceFiles(source, sourceDirectory);
     const cursors = {};
     if (!includeHistory) {
         for (const filename of files) {
@@ -337,20 +416,26 @@ export async function prepareInitialState(codexHome, includeHistory) {
     };
 }
 export async function ensureStartableState(config, paths) {
+    const source = collectorSource(config);
+    const directory = collectorSourceDirectory(config);
     const state = await loadState(paths);
     if (state.initialized) {
-        if (state.sourceDirectory !== path.resolve(config.codexHome)) {
-            throw new Error("Collector state belongs to another Codex directory; use a separate --data-directory.");
+        if (state.sourceDirectory !== path.resolve(directory)) {
+            throw new Error(`Collector state belongs to another ${source} directory; use a separate --data-directory.`);
         }
         return false;
     }
     if ((await outboxSize(paths)) > 0) {
         throw new Error("Pending events exist without an initialized checkpoint; repair the collector state before starting uploads.");
     }
-    await initializeState(config.codexHome, paths, true);
+    await saveState(await prepareSourceInitialState(source, directory, true), paths);
     return true;
 }
-async function hasHistoryBackfillMarker(paths) {
+const HISTORY_BACKFILL_VERSION = 4;
+const historyBackfillMarkerSchema = z
+    .object({ version: z.number().int().min(1).max(HISTORY_BACKFILL_VERSION) })
+    .strict();
+async function readHistoryBackfillVersion(paths) {
     let marker;
     try {
         marker = await fs.readFile(paths.historyBackfill, "utf8");
@@ -358,18 +443,28 @@ async function hasHistoryBackfillMarker(paths) {
     catch (error) {
         if (!(error instanceof Error && "code" in error && error.code === "ENOENT"))
             throw error;
-        marker = null;
+        return null;
     }
-    if (marker !== null) {
-        if (marker !== '{"version":1}\n') {
-            throw new Error("Collector history backfill marker has an invalid shape.");
-        }
-        return true;
+    let decoded;
+    try {
+        decoded = JSON.parse(marker);
+    }
+    catch (error) {
+        throw new Error("Collector history backfill marker is not valid JSON.", {
+            cause: error,
+        });
+    }
+    const parsed = historyBackfillMarkerSchema.safeParse(decoded);
+    if (!parsed.success) {
+        throw new Error("Collector history backfill marker has an invalid shape.", {
+            cause: parsed.error,
+        });
     }
-    return false;
+    return parsed.data.version;
 }
 export async function ensureHistoryBackfill(codexHome, paths) {
-    if (await hasHistoryBackfillMarker(paths))
+    const version = await readHistoryBackfillVersion(paths);
+    if (version !== null && version >= HISTORY_BACKFILL_VERSION)
         return false;
     const state = await loadState(paths);
     if (!state.initialized || state.sourceDirectory !== path.resolve(codexHome)) {
@@ -377,17 +472,33 @@ export async function ensureHistoryBackfill(codexHome, paths) {
     }
     await rolloutFiles(path.join(codexHome, "sessions"));
     await saveState({ ...state, cursors: {}, skipping: [] }, paths);
+    await writeJsonAtomic(paths.historyBackfill, {
+        version: HISTORY_BACKFILL_VERSION,
+    });
+    return true;
+}
+export async function ensureClaudeHistoryBackfill(claudeHome, paths) {
+    const version = await readHistoryBackfillVersion(paths);
+    if (version !== null && version >= 1)
+        return false;
+    const state = await loadState(paths);
+    if (!state.initialized ||
+        state.sourceDirectory !== path.resolve(claudeHome)) {
+        throw new Error("Collector state does not match the Claude directory; reconnect Claude before uploading.");
+    }
+    await claudeFiles(path.join(claudeHome, "projects"));
+    await saveState({ ...state, cursors: {}, skipping: [] }, paths);
     await writeJsonAtomic(paths.historyBackfill, { version: 1 });
     return true;
 }
-export async function historyScanProgress(codexHome, paths) {
-    if (!(await hasHistoryBackfillMarker(paths)))
+export async function historyScanProgress(home, paths, source = "codex") {
+    if ((await readHistoryBackfillVersion(paths)) === null)
         return { kind: "waiting" };
     const state = await loadState(paths);
-    if (!state.initialized || state.sourceDirectory !== path.resolve(codexHome)) {
-        throw new Error("Collector state does not match CODEX_HOME.");
+    if (!state.initialized || state.sourceDirectory !== path.resolve(home)) {
+        throw new Error(`Collector state does not match the ${source} directory.`);
     }
-    const files = await rolloutFiles(path.join(codexHome, "sessions"));
+    const files = await sourceFiles(source, home);
     let scannedBytes = 0;
     let totalBytes = 0;
     for (let start = 0; start < files.length; start += 64) {
@@ -415,28 +526,33 @@ export async function historyScanProgress(codexHome, paths) {
             scannedBytes += cursor <= size ? cursor : 0;
             if (!Number.isSafeInteger(totalBytes) ||
                 !Number.isSafeInteger(scannedBytes)) {
-                throw new Error("Codex history size exceeds the supported limit.");
+                throw new Error(`${source} history size exceeds the supported limit.`);
             }
         }
     }
     return { kind: "scanning", scannedBytes, totalBytes };
 }
-export async function resetHistoryCursors(codexHome, paths) {
+export async function resetHistoryCursors(home, paths, source = "codex") {
     const state = await loadState(paths);
-    const sourceDirectory = path.resolve(codexHome);
+    const sourceDirectory = path.resolve(home);
     if (!state.initialized || state.sourceDirectory !== sourceDirectory) {
-        throw new Error("Connect the Codex source before importing its history.");
+        throw new Error(`Connect the ${source} source before importing its history.`);
     }
-    const files = await rolloutFiles(path.join(sourceDirectory, "sessions"));
+    const files = await sourceFiles(source, sourceDirectory);
     const cursors = {};
     let bytes = 0;
     for (const filename of files) {
         cursors[filename] = 0;
         bytes += (await fs.stat(filename)).size;
         if (!Number.isSafeInteger(bytes)) {
-            throw new Error("Codex history size exceeds the supported limit.");
+            throw new Error(`${source} history size exceeds the supported limit.`);
         }
     }
+    const localHealth = await healthSnapshot(paths);
+    if (localHealth.kind === "unavailable" ||
+        localHealth.health.historyCaughtUpAtMs !== null) {
+        await saveHealth({ ...localHealth.health, historyCaughtUpAtMs: null }, paths);
+    }
     await saveState({
         cursors,
         skipping: [],
@@ -445,13 +561,13 @@ export async function resetHistoryCursors(codexHome, paths) {
     }, paths);
     return { files: files.length, bytes };
 }
-export async function historyImportSize(codexHome) {
-    const files = await rolloutFiles(path.join(path.resolve(codexHome), "sessions"));
+export async function historyImportSize(home, source = "codex") {
+    const files = await sourceFiles(source, home);
     let bytes = 0;
     for (const filename of files) {
         bytes += (await fs.stat(filename)).size;
         if (!Number.isSafeInteger(bytes)) {
-            throw new Error("Codex history size exceeds the supported limit.");
+            throw new Error(`${source} history size exceeds the supported limit.`);
         }
     }
     return { files: files.length, bytes };
@@ -485,7 +601,7 @@ async function lastCompleteRecordOffset(filename, size) {
         await handle.close();
     }
 }
-export async function rolloutFiles(root) {
+async function rolloutFiles(root) {
     const files = [];
     const pending = [root];
     while (pending.length > 0) {
@@ -515,3 +631,51 @@ export async function rolloutFiles(root) {
     }
     return files.sort();
 }
+async function directoryEntries(directory) {
+    try {
+        return await fs.readdir(directory, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return [];
+        throw new Error(`Could not scan Claude session directory ${directory}.`, {
+            cause: error,
+        });
+    }
+}
+export async function claudeFiles(root) {
+    const files = [];
+    for (const project of await directoryEntries(root)) {
+        if (!project.isDirectory())
+            continue;
+        const projectDirectory = path.join(root, project.name);
+        for (const entry of await directoryEntries(projectDirectory)) {
+            const filename = path.join(projectDirectory, entry.name);
+            if (entry.isFile() &&
+                /^[^/]+\.jsonl$/.test(entry.name) &&
+                !entry.name.includes(".orphaned-") &&
+                !entry.name.includes(".superseded-")) {
+                files.push(filename);
+            }
+            else if (entry.isDirectory()) {
+                const subagents = path.join(filename, "subagents");
+                for (const agent of await directoryEntries(subagents)) {
+                    if (agent.isFile() && /^agent-.+\.jsonl$/.test(agent.name))
+                        files.push(path.join(subagents, agent.name));
+                }
+            }
+            if (files.length > 10_000)
+                throw new Error("Claude session file count exceeds the scan limit.");
+        }
+    }
+    return files.sort();
+}
+export function sourceFiles(source, home) {
+    const directory = path.resolve(home);
+    if (source === "codex")
+        return rolloutFiles(path.join(directory, "sessions"));
+    if (source === "claude")
+        return claudeFiles(path.join(directory, "projects"));
+    const unhandled = source;
+    throw new Error(`Unsupported collector source: ${unhandled}`);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "iteration-cli",
-  "version": "0.4.3",
+  "version": "0.5.0",
   "type": "module",
   "repository": {
     "type": "git",
@@ -18,7 +18,7 @@
   },
   "dependencies": {
     "zod": "4.6.4",
-    "iteration-trace-contracts": "0.3.0"
+    "iteration-trace-contracts": "0.5.0"
   },
   "scripts": {
     "build": "pnpm --filter iteration-trace-contracts build && tsc -p tsconfig.json"