iteration-cli 0.4.0

package/dist/storage.js

@@ -0,0 +1,445 @@
+import { createWriteStream, promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { once } from "node:events";
+import { createInterface } from "node:readline";
+import * as z from "zod";
+import { MAX_TRACE_EVENTS_PER_BATCH, parseTraceEventBatch, } from "iteration-trace-contracts";
+const MAX_OUTBOX_BYTES = 64 * 1024 * 1024;
+const configSchema = z
+    .object({
+    endpoint: z.string(),
+    token: z.string(),
+    codexHome: z.string(),
+    dataDirectory: z.string(),
+})
+    .strict();
+const stateSchema = z
+    .object({
+    cursors: z.record(z.string(), z.number().int().min(0).max(Number.MAX_SAFE_INTEGER)),
+    skipping: z.array(z.string()),
+    sourceDirectory: z.string().nullable(),
+    initialized: z.boolean(),
+})
+    .strict();
+function parseOutboxEvent(line) {
+    let decoded;
+    try {
+        decoded = JSON.parse(line);
+    }
+    catch (error) {
+        throw new Error("Trace outbox contains a malformed record.", {
+            cause: error,
+        });
+    }
+    const parsed = parseTraceEventBatch({ events: [decoded] });
+    if (!parsed.success)
+        throw new Error(`Trace outbox event is invalid: ${parsed.error}`);
+    const [event] = parsed.value.events;
+    if (event === undefined)
+        throw new Error("Trace outbox event is missing.");
+    return event;
+}
+function defaultDirectory() {
+    return path.join(os.homedir(), ".config", "iteration", "codex-collector");
+}
+export function validateCollectorEndpoint(endpoint) {
+    let parsed;
+    try {
+        parsed = new URL(endpoint);
+    }
+    catch (error) {
+        throw new Error("Collector endpoint must be a valid URL.", {
+            cause: error,
+        });
+    }
+    const localHost = ["localhost", "127.0.0.1", "[::1]"].includes(parsed.hostname);
+    if (parsed.username !== "" ||
+        parsed.password !== "" ||
+        (parsed.protocol !== "https:" &&
+            !(parsed.protocol === "http:" && localHost))) {
+        throw new Error("Collector endpoint must use HTTPS, except for localhost.");
+    }
+    if (parsed.pathname !== "/api/traces/ingest" ||
+        parsed.search !== "" ||
+        parsed.hash !== "") {
+        throw new Error("Collector endpoint must be the trace ingest route without query parameters.");
+    }
+    return parsed;
+}
+export function collectorPaths(directory = defaultDirectory()) {
+    return {
+        config: path.join(directory, "config.json"),
+        state: path.join(directory, "state.json"),
+        outbox: path.join(directory, "outbox.jsonl"),
+        lock: path.join(directory, "collector.lock"),
+    };
+}
+export async function assertFreshCollectorDirectory(paths) {
+    const state = await loadState(paths);
+    const configExists = await fs
+        .access(paths.config)
+        .then(() => true)
+        .catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return false;
+        throw error;
+    });
+    if (configExists) {
+        throw new Error("This collector data directory is already configured. Run iteration status codex or iteration start codex to repair startup. Use a different --data-directory for another workspace, or iteration import-history codex for earlier sessions.");
+    }
+    if (state.initialized || (await outboxSize(paths)) > 0) {
+        throw new Error("Collector state or pending events exist without a config. Use a different --data-directory; this data cannot be safely assigned to a new workspace.");
+    }
+}
+export async function ensurePrivateDirectory(directory) {
+    await fs.mkdir(directory, { recursive: true, mode: 0o700 });
+    await fs.chmod(directory, 0o700);
+}
+async function writeJsonAtomic(filename, value) {
+    const temporary = `${filename}.${process.pid}.tmp`;
+    const handle = await fs.open(temporary, "w", 0o600);
+    try {
+        await handle.writeFile(`${JSON.stringify(value)}\n`, "utf8");
+        await handle.sync();
+    }
+    finally {
+        await handle.close();
+    }
+    await fs.rename(temporary, filename);
+}
+export async function saveConfig(config, paths) {
+    await ensurePrivateDirectory(path.dirname(paths.config));
+    await writeJsonAtomic(paths.config, config);
+}
+export async function loadConfig(paths) {
+    let contents;
+    try {
+        contents = await fs.readFile(paths.config, "utf8");
+    }
+    catch (error) {
+        throw new Error(`Could not read collector config at ${paths.config}.`, {
+            cause: error,
+        });
+    }
+    let decoded;
+    try {
+        decoded = JSON.parse(contents);
+    }
+    catch (error) {
+        throw new Error("Collector config is not valid JSON.", { cause: error });
+    }
+    const parsed = configSchema.safeParse(decoded);
+    if (!parsed.success)
+        throw new Error("Collector config has an invalid shape.", {
+            cause: parsed.error,
+        });
+    const { endpoint, token, codexHome, dataDirectory } = parsed.data;
+    validateCollectorEndpoint(endpoint);
+    if (token.length !== 43 || !/^[A-Za-z0-9_-]+$/.test(token)) {
+        throw new Error("Collector token has an invalid format.");
+    }
+    return { endpoint, token, codexHome, dataDirectory };
+}
+export async function loadState(paths) {
+    let contents;
+    try {
+        contents = await fs.readFile(paths.state, "utf8");
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT") {
+            return {
+                cursors: {},
+                skipping: [],
+                sourceDirectory: null,
+                initialized: false,
+            };
+        }
+        throw new Error(`Could not read collector checkpoint at ${paths.state}.`, {
+            cause: error,
+        });
+    }
+    let decoded;
+    try {
+        decoded = JSON.parse(contents);
+    }
+    catch (error) {
+        throw new Error("Collector checkpoint is not valid JSON.", {
+            cause: error,
+        });
+    }
+    const parsed = stateSchema.safeParse(decoded);
+    if (!parsed.success)
+        throw new Error("Collector checkpoint has an invalid shape.", {
+            cause: parsed.error,
+        });
+    return parsed.data;
+}
+export async function saveState(state, paths) {
+    await ensurePrivateDirectory(path.dirname(paths.state));
+    await writeJsonAtomic(paths.state, state);
+}
+export async function outboxSize(paths) {
+    try {
+        return (await fs.stat(paths.outbox)).size;
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return 0;
+        throw new Error("Could not inspect the local trace outbox.", {
+            cause: error,
+        });
+    }
+}
+export async function pendingEventIds(paths) {
+    const ids = new Set();
+    const size = await outboxSize(paths);
+    if (size > MAX_OUTBOX_BYTES)
+        throw new Error("Trace outbox exceeds its configured capacity.");
+    try {
+        const input = await fs.open(paths.outbox, "r");
+        const stream = input.createReadStream({
+            autoClose: true,
+            encoding: "utf8",
+        });
+        const lines = createInterface({ input: stream, crlfDelay: Infinity });
+        for await (const line of lines) {
+            ids.add(parseOutboxEvent(line).eventId);
+        }
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return ids;
+        throw error;
+    }
+    return ids;
+}
+export async function appendEvents(events, paths) {
+    if (events.length === 0)
+        return;
+    const contents = `${events.map((event) => JSON.stringify(event)).join("\n")}\n`;
+    const newSize = Buffer.byteLength(contents, "utf8");
+    const currentSize = await outboxSize(paths);
+    if (currentSize + newSize > MAX_OUTBOX_BYTES) {
+        throw new Error("Trace outbox is full; upload pending data before scanning more sessions.");
+    }
+    await ensurePrivateDirectory(path.dirname(paths.outbox));
+    const handle = await fs.open(paths.outbox, "a", 0o600);
+    try {
+        await handle.writeFile(contents, "utf8");
+        await handle.sync();
+    }
+    finally {
+        await handle.close();
+    }
+}
+export async function nextBatch(paths, maxBytes) {
+    if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) {
+        throw new Error("Maximum upload batch size must be a positive integer.");
+    }
+    const size = await outboxSize(paths);
+    if (size === 0)
+        return [];
+    if (size > MAX_OUTBOX_BYTES)
+        throw new Error("Trace outbox exceeds its configured capacity.");
+    const input = await fs.open(paths.outbox, "r");
+    const stream = input.createReadStream({ autoClose: true, encoding: "utf8" });
+    const lines = createInterface({ input: stream, crlfDelay: Infinity });
+    const events = [];
+    let bytes = 0;
+    for await (const line of lines) {
+        const lineBytes = Buffer.byteLength(line, "utf8") + 1;
+        if (events.length >= MAX_TRACE_EVENTS_PER_BATCH ||
+            (events.length > 0 && bytes + lineBytes > maxBytes)) {
+            lines.close();
+            stream.destroy();
+            break;
+        }
+        if (lineBytes > maxBytes) {
+            lines.close();
+            stream.destroy();
+            throw new Error("A queued event exceeds the maximum upload batch size.");
+        }
+        events.push(parseOutboxEvent(line));
+        bytes += lineBytes;
+    }
+    return events;
+}
+export async function acknowledgeEvents(acknowledgedIds, paths) {
+    const temporary = `${paths.outbox}.${process.pid}.tmp`;
+    const input = await fs.open(paths.outbox, "r").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return null;
+        throw error;
+    });
+    if (input === null)
+        return;
+    const output = createWriteStream(temporary, { flags: "w", mode: 0o600 });
+    try {
+        const lines = createInterface({
+            input: input.createReadStream({ autoClose: true, encoding: "utf8" }),
+            crlfDelay: Infinity,
+        });
+        for await (const line of lines) {
+            const event = parseOutboxEvent(line);
+            if (!acknowledgedIds.has(event.eventId)) {
+                if (!output.write(`${line}\n`))
+                    await once(output, "drain");
+            }
+        }
+        output.end();
+        await once(output, "close");
+        const durable = await fs.open(temporary, "r+");
+        try {
+            await durable.sync();
+        }
+        finally {
+            await durable.close();
+        }
+        await fs.rename(temporary, paths.outbox);
+    }
+    catch (error) {
+        output.destroy();
+        await fs.rm(temporary, { force: true });
+        throw error;
+    }
+}
+export async function initializeState(codexHome, paths, includeHistory) {
+    const state = await loadState(paths);
+    const sourceDirectory = path.resolve(codexHome);
+    if (state.initialized && state.sourceDirectory !== sourceDirectory) {
+        throw new Error("Collector state belongs to another Codex directory; use a separate --data-directory.");
+    }
+    if (state.initialized && state.sourceDirectory === sourceDirectory) {
+        if (includeHistory) {
+            throw new Error("Codex is already connected. Run iteration import-history codex to import existing sessions.");
+        }
+        return;
+    }
+    await saveState(await prepareInitialState(codexHome, includeHistory), paths);
+}
+export async function prepareInitialState(codexHome, includeHistory) {
+    const sourceDirectory = path.resolve(codexHome);
+    const sessionsRoot = path.join(sourceDirectory, "sessions");
+    const files = await rolloutFiles(sessionsRoot);
+    const cursors = {};
+    for (const filename of files) {
+        cursors[filename] = includeHistory
+            ? 0
+            : await lastCompleteRecordOffset(filename, (await fs.stat(filename)).size);
+    }
+    return {
+        cursors,
+        skipping: [],
+        sourceDirectory,
+        initialized: true,
+    };
+}
+export async function ensureStartableState(config, paths) {
+    const state = await loadState(paths);
+    if (state.initialized) {
+        if (state.sourceDirectory !== path.resolve(config.codexHome)) {
+            throw new Error("Collector state belongs to another Codex directory; use a separate --data-directory.");
+        }
+        return false;
+    }
+    if ((await outboxSize(paths)) > 0) {
+        throw new Error("Pending events exist without an initialized checkpoint; repair the collector state before starting uploads.");
+    }
+    await initializeState(config.codexHome, paths, false);
+    return true;
+}
+export async function resetHistoryCursors(codexHome, paths) {
+    const state = await loadState(paths);
+    const sourceDirectory = path.resolve(codexHome);
+    if (!state.initialized || state.sourceDirectory !== sourceDirectory) {
+        throw new Error("Connect the Codex source before importing its history.");
+    }
+    const files = await rolloutFiles(path.join(sourceDirectory, "sessions"));
+    const cursors = {};
+    let bytes = 0;
+    for (const filename of files) {
+        cursors[filename] = 0;
+        bytes += (await fs.stat(filename)).size;
+        if (!Number.isSafeInteger(bytes)) {
+            throw new Error("Codex history size exceeds the supported limit.");
+        }
+    }
+    await saveState({
+        cursors,
+        skipping: [],
+        sourceDirectory,
+        initialized: true,
+    }, paths);
+    return { files: files.length, bytes };
+}
+export async function historyImportSize(codexHome) {
+    const files = await rolloutFiles(path.join(path.resolve(codexHome), "sessions"));
+    let bytes = 0;
+    for (const filename of files) {
+        bytes += (await fs.stat(filename)).size;
+        if (!Number.isSafeInteger(bytes)) {
+            throw new Error("Codex history size exceeds the supported limit.");
+        }
+    }
+    return { files: files.length, bytes };
+}
+async function lastCompleteRecordOffset(filename, size) {
+    if (size === 0)
+        return 0;
+    const handle = await fs.open(filename, "r");
+    try {
+        const finalByte = Buffer.alloc(1);
+        await handle.read(finalByte, 0, 1, size - 1);
+        if (finalByte[0] === 10)
+            return size;
+        let end = size;
+        const minimum = Math.max(0, size - 32 * 1024 * 1024);
+        while (end > minimum) {
+            const start = Math.max(minimum, end - 64 * 1024);
+            const buffer = Buffer.alloc(end - start);
+            const result = await handle.read(buffer, 0, buffer.length, start);
+            const newline = buffer.subarray(0, result.bytesRead).lastIndexOf(10);
+            if (newline >= 0)
+                return start + newline + 1;
+            end = start;
+        }
+        if (size > 32 * 1024 * 1024) {
+            throw new Error(`Could not find a safe rollout boundary in ${filename}.`);
+        }
+        return 0;
+    }
+    finally {
+        await handle.close();
+    }
+}
+export async function rolloutFiles(root) {
+    const files = [];
+    const pending = [root];
+    while (pending.length > 0) {
+        const directory = pending.pop();
+        if (directory === undefined)
+            continue;
+        let entries;
+        try {
+            entries = await fs.readdir(directory, { withFileTypes: true });
+        }
+        catch (error) {
+            if (error instanceof Error && "code" in error && error.code === "ENOENT")
+                continue;
+            throw new Error(`Could not scan Codex session directory ${directory}.`, {
+                cause: error,
+            });
+        }
+        for (const entry of entries) {
+            const filename = path.join(directory, entry.name);
+            if (entry.isDirectory())
+                pending.push(filename);
+            else if (entry.isFile() && /^rollout-.+\.jsonl$/.test(entry.name))
+                files.push(filename);
+            if (files.length > 10_000)
+                throw new Error("Codex session file count exceeds the scan limit.");
+        }
+    }
+    return files.sort();
+}

package/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "iteration-cli",
+  "version": "0.4.0",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Thytu/Iteration.git",
+    "directory": "packages/cli"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "files": [
+    "dist"
+  ],
+  "bin": {
+    "iteration": "./dist/cli.js"
+  },
+  "dependencies": {
+    "zod": "4.6.4",
+    "iteration-trace-contracts": "0.3.0"
+  },
+  "scripts": {
+    "build": "pnpm --filter iteration-trace-contracts build && tsc -p tsconfig.json"
+  }
+}