@neta-art/cohub-cli 6.11.2 → 7.0.0

package/dist/runtime/archive-store.js

@@ -0,0 +1,353 @@
+import { createHash, randomUUID } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { mkdir, open, readFile, readdir, rename, rm, stat, link } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { RUNTIME_ARCHIVE_SEGMENT_BYTES, harnessArchiveIndexSchema, validateArchiveBoundary, } from "@neta-art/cohub";
+const missing = (error) => error?.code === "ENOENT";
+const hash = (bytes, algorithm = "sha256") => createHash(algorithm).update(bytes).digest("hex");
+export async function checksumNativeFile(path) {
+    const digest = createHash("sha256");
+    for await (const bytes of createReadStream(path))
+        digest.update(bytes);
+    return digest.digest("hex");
+}
+export async function atomicRuntimeJson(path, value) {
+    await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+    const temporary = `${path}.${randomUUID()}.tmp`;
+    try {
+        const file = await open(temporary, "wx", 0o600);
+        try {
+            await file.writeFile(JSON.stringify(value));
+            await file.sync();
+        }
+        finally {
+            await file.close();
+        }
+        await rename(temporary, path);
+        if (process.platform !== "win32") {
+            const directory = await open(dirname(path), "r");
+            try {
+                await directory.sync();
+            }
+            finally {
+                await directory.close();
+            }
+        }
+    }
+    finally {
+        await rm(temporary, { force: true });
+    }
+}
+/** Immutable local segments are the outbox. Model acknowledgements never remove them. */
+export class RuntimeArchiveStore {
+    root;
+    transport;
+    flushing = null;
+    capturing = new Map();
+    constructor(root, transport) {
+        this.root = root;
+        this.transport = transport;
+    }
+    async pendingCount() {
+        const pending = new Set();
+        for (const directory of ["pending", "captures"]) {
+            const names = await readdir(join(this.root, directory)).catch((error) => { if (missing(error))
+                return []; throw error; });
+            for (const name of names)
+                if (name.endsWith(".json"))
+                    pending.add(name);
+        }
+        return pending.size;
+    }
+    async failedCaptureCount() {
+        const names = await readdir(join(this.root, "failed", "captures")).catch((error) => { if (missing(error))
+            return []; throw error; });
+        return names.filter((name) => name.endsWith(".json")).length;
+    }
+    async hasCapture(turnId) { return Boolean(await this.readIndex(this.version(turnId))); }
+    version(turnId) { return join(this.root, "versions", `${turnId}.json`); }
+    blob(sha) { return join(this.root, "objects", sha); }
+    async saveBlob(sha, bytes) {
+        await mkdir(join(this.root, "objects"), { recursive: true, mode: 0o700 });
+        const target = this.blob(sha), temporary = `${target}.${randomUUID()}.tmp`;
+        try {
+            const output = await open(temporary, "wx", 0o600);
+            try {
+                await output.writeFile(bytes);
+                await output.sync();
+            }
+            finally {
+                await output.close();
+            }
+            await link(temporary, target).catch((error) => { if (error.code !== "EEXIST")
+                throw error; });
+        }
+        finally {
+            await rm(temporary, { force: true });
+        }
+    }
+    async readIndex(path) {
+        try {
+            return harnessArchiveIndexSchema.parse(JSON.parse(await readFile(path, "utf8")));
+        }
+        catch (error) {
+            if (missing(error))
+                return null;
+            throw error;
+        }
+    }
+    stage(state, turnId) {
+        const existing = this.capturing.get(turnId);
+        if (existing)
+            return existing;
+        const task = this.capture(state, turnId).finally(() => this.capturing.delete(turnId));
+        this.capturing.set(turnId, task);
+        return task;
+    }
+    async capture(state, turnId) {
+        const identity = { sessionId: state.sessionId, turnId, harness: state.harness };
+        const headPath = join(this.root, "heads", `${state.sessionId}.${state.harness}.json`);
+        const saved = await this.readIndex(this.version(turnId));
+        if (saved) {
+            if (saved.sessionId !== state.sessionId || saved.harness !== state.harness)
+                throw new Error("Archive identity mismatch / 归档身份不匹配");
+            const committed = await stat(join(this.root, "ready", `${turnId}.json`)).catch((error) => { if (missing(error))
+                return null; throw error; });
+            if (!committed)
+                await atomicRuntimeJson(join(this.root, "pending", `${turnId}.json`), saved);
+            if (!await this.readIndex(headPath))
+                await atomicRuntimeJson(headPath, saved);
+            return identity;
+        }
+        const previous = await this.readIndex(headPath);
+        const file = await open(state.path, "r");
+        let index;
+        try {
+            const before = await file.stat();
+            if (!before.isFile() || !before.size)
+                throw new Error("Native archive is empty / 原生归档为空");
+            const buffer = Buffer.alloc(RUNTIME_ARCHIVE_SEGMENT_BYTES);
+            let offset = 0;
+            let digest = createHash("sha256");
+            let parent = null;
+            // Hash the old prefix, not just its size: equal-size and growing rewrites are valid.
+            if (previous && previous.nativeSessionId === state.nativeSessionId && previous.sizeBytes <= before.size) {
+                while (offset < previous.sizeBytes) {
+                    const { bytesRead } = await file.read(buffer, 0, Math.min(buffer.length, previous.sizeBytes - offset), offset);
+                    if (!bytesRead)
+                        throw new Error("Native file changed during capture / 归档时文件发生变化");
+                    digest.update(buffer.subarray(0, bytesRead));
+                    offset += bytesRead;
+                }
+                if (digest.copy().digest("hex") === previous.sha256)
+                    parent = previous;
+            }
+            if (!parent) {
+                offset = 0;
+                digest = createHash("sha256");
+            }
+            const segments = [];
+            await mkdir(join(this.root, "objects"), { recursive: true, mode: 0o700 });
+            while (offset < before.size) {
+                const { bytesRead } = await file.read(buffer, 0, Math.min(buffer.length, before.size - offset), offset);
+                if (!bytesRead)
+                    throw new Error("Native file changed during capture / 归档时文件发生变化");
+                const bytes = buffer.subarray(0, bytesRead);
+                digest.update(bytes);
+                const segment = { offset, sizeBytes: bytesRead, sha256: hash(bytes), md5: hash(bytes, "md5") };
+                await this.saveBlob(segment.sha256, bytes);
+                segments.push(segment);
+                offset += bytesRead;
+            }
+            const after = await stat(state.path);
+            if (before.ino !== after.ino || before.size !== after.size || before.mtimeMs !== after.mtimeMs)
+                throw new Error("Native file changed during capture / 归档时文件发生变化");
+            if (process.platform !== "win32") {
+                const directory = await open(join(this.root, "objects"), "r");
+                try {
+                    await directory.sync();
+                }
+                finally {
+                    await directory.close();
+                }
+            }
+            index = harnessArchiveIndexSchema.parse({ ...identity, version: 1, nativeSessionId: state.nativeSessionId,
+                nativeFormat: state.harness === "pi" ? "pi.jsonl" : "codex.rollout", parentTurnId: parent?.turnId ?? null,
+                sizeBytes: before.size, sha256: digest.digest("hex"), segments });
+            validateArchiveBoundary(index, parent);
+        }
+        finally {
+            await file.close();
+        }
+        // Publish the outbox before advancing the local head. Neither points at mutable files.
+        await atomicRuntimeJson(join(this.root, "pending", `${turnId}.json`), index);
+        await atomicRuntimeJson(this.version(turnId), index);
+        await atomicRuntimeJson(headPath, index);
+        return identity;
+    }
+    async flush(signal) {
+        if (!this.transport)
+            return;
+        if (this.flushing)
+            return this.flushing;
+        this.flushing = this.drain(signal).finally(() => { this.flushing = null; });
+        return this.flushing;
+    }
+    async drain(signal) {
+        const transport = this.transport;
+        if (!transport)
+            return;
+        const names = await readdir(join(this.root, "pending")).catch((error) => { if (missing(error))
+            return []; throw error; });
+        const pending = new Map();
+        for (const name of names) {
+            if (!name.endsWith(".json"))
+                continue;
+            const index = await this.readIndex(join(this.root, "pending", name));
+            if (index)
+                pending.set(index.turnId, index);
+        }
+        const children = new Map();
+        const queue = [];
+        for (const index of pending.values()) {
+            if (index.parentTurnId && pending.has(index.parentTurnId)) {
+                const siblings = children.get(index.parentTurnId) ?? [];
+                siblings.push(index);
+                children.set(index.parentTurnId, siblings);
+            }
+            else
+                queue.push(index);
+        }
+        if (pending.size && !queue.length)
+            throw new Error("Cyclic archive outbox / 归档队列引用循环");
+        for (let cursor = 0; cursor < queue.length; cursor++) {
+            signal.throwIfAborted();
+            const index = queue[cursor];
+            if (!index)
+                continue;
+            try {
+                const { uploads } = await transport.prepareRuntimeArchive(index, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]) });
+                if (uploads.length > index.segments.length)
+                    throw new Error("Upload plan mismatch / 上传计划不匹配");
+                const expected = new Set(index.segments.map((segment) => JSON.stringify(segment)));
+                for (const { segment, uploadUrl, headers } of uploads) {
+                    if (!expected.has(JSON.stringify(segment)))
+                        throw new Error("Upload segment mismatch / 上传分段不匹配");
+                    signal.throwIfAborted();
+                    const bytes = await readFile(this.blob(segment.sha256));
+                    if (bytes.length !== segment.sizeBytes || hash(bytes) !== segment.sha256)
+                        throw new Error("Local archive segment is corrupt / 本地归档分段已损坏");
+                    const response = await (transport.fetchObject ?? fetch)(uploadUrl, { method: "PUT", headers, body: bytes, redirect: "error", signal: AbortSignal.any([signal, AbortSignal.timeout(60_000)]) });
+                    // An immutable segment may already exist after a lost acknowledgement. Commit verifies it.
+                    if (!response.ok && ![409, 412].includes(response.status))
+                        throw new Error(`Archive upload failed / 归档上传失败: ${response.status}`);
+                }
+                await transport.commitRuntimeArchive(index, { signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]) });
+                await atomicRuntimeJson(join(this.root, "ready", `${index.turnId}.json`), { turnId: index.turnId, sha256: index.sha256 });
+                await rm(join(this.root, "pending", `${index.turnId}.json`), { force: true });
+                queue.push(...children.get(index.turnId) ?? []);
+            }
+            catch (error) {
+                if (!signal.aborted)
+                    console.error("Archive pending; native segments retained / 归档待重试，原始分段已保留:", error);
+            }
+        }
+    }
+    async restore(reference, target, signal) {
+        if (!this.transport)
+            throw new Error("Archive transport unavailable / 归档传输不可用");
+        const timeout = (ms) => signal ? AbortSignal.any([signal, AbortSignal.timeout(ms)]) : AbortSignal.timeout(ms);
+        const pages = [];
+        const visited = new Set();
+        let turnId = reference.turnId;
+        while (turnId) {
+            signal?.throwIfAborted();
+            if (visited.has(turnId))
+                throw new Error("Cyclic archive / 归档引用循环");
+            visited.add(turnId);
+            const page = await this.transport.getRuntimeArchive(reference.sessionId, turnId, { signal: timeout(30_000) });
+            const index = harnessArchiveIndexSchema.parse(page.index);
+            if (index.turnId !== turnId || index.sessionId !== reference.sessionId || index.harness !== reference.harness)
+                throw new Error("Archive identity mismatch / 归档身份不匹配");
+            pages.push({ ...page, index });
+            turnId = index.parentTurnId;
+        }
+        const head = pages[0]?.index;
+        if (!head)
+            throw new Error("Archive missing / 归档不存在");
+        await mkdir(dirname(target), { recursive: true, mode: 0o700 });
+        const temporary = `${target}.${randomUUID()}.restoring`;
+        const file = await open(temporary, "wx", 0o600);
+        try {
+            let parent = null;
+            const digest = createHash("sha256");
+            for (const page of pages.reverse()) {
+                validateArchiveBoundary(page.index, parent);
+                if (page.segments.length !== page.index.segments.length)
+                    throw new Error("Missing archive segments / 归档分段缺失");
+                for (const [ordinal, expected] of page.index.segments.entries()) {
+                    signal?.throwIfAborted();
+                    const cached = await readFile(this.blob(expected.sha256)).catch((error) => { if (missing(error))
+                        return null; throw error; });
+                    if (cached) {
+                        if (cached.length !== expected.sizeBytes || hash(cached) !== expected.sha256)
+                            throw new Error("Cached archive segment is corrupt / 缓存归档分段已损坏");
+                        digest.update(cached);
+                        await file.writeFile(cached);
+                        continue;
+                    }
+                    let link = page.segments[ordinal];
+                    if (!link || JSON.stringify(link.segment) !== JSON.stringify(expected))
+                        throw new Error("Archive segment identity mismatch / 归档分段标识不匹配");
+                    let response = await (this.transport.fetchObject ?? fetch)(link.downloadUrl, { redirect: "error", signal: timeout(60_000) });
+                    if ([401, 403].includes(response.status)) {
+                        const refreshed = await this.transport.getRuntimeArchive(reference.sessionId, page.index.turnId, { signal: timeout(30_000) });
+                        link = refreshed.segments[ordinal];
+                        if (!link || JSON.stringify(link.segment) !== JSON.stringify(expected))
+                            throw new Error("Archive segment missing / 归档分段缺失");
+                        response = await (this.transport.fetchObject ?? fetch)(link.downloadUrl, { redirect: "error", signal: timeout(60_000) });
+                    }
+                    if (!response.ok || !response.body)
+                        throw new Error(`Archive download failed / 归档下载失败: ${response.status}`);
+                    const segmentHash = createHash("sha256");
+                    let size = 0;
+                    const chunks = [];
+                    for await (const chunk of response.body) {
+                        size += chunk.length;
+                        if (size > expected.sizeBytes)
+                            throw new Error("Archive size mismatch / 归档大小不匹配");
+                        chunks.push(chunk);
+                        segmentHash.update(chunk);
+                        digest.update(chunk);
+                        await file.writeFile(chunk);
+                    }
+                    if (size !== expected.sizeBytes || segmentHash.digest("hex") !== expected.sha256)
+                        throw new Error("Archive checksum mismatch / 归档校验失败");
+                    await this.saveBlob(expected.sha256, Buffer.concat(chunks));
+                }
+                if (digest.copy().digest("hex") !== page.index.sha256)
+                    throw new Error("Archive version checksum mismatch / 归档版本校验失败");
+                parent = page.index;
+            }
+            if ((await file.stat()).size !== head.sizeBytes)
+                throw new Error("Archive length mismatch / 归档长度不匹配");
+            await file.sync();
+            await file.close();
+            await link(temporary, target);
+            if (process.platform !== "win32") {
+                const directory = await open(dirname(target), "r");
+                try {
+                    await directory.sync();
+                }
+                finally {
+                    await directory.close();
+                }
+            }
+            return head;
+        }
+        finally {
+            await file.close();
+            await rm(temporary, { force: true });
+        }
+    }
+}

package/dist/runtime/codex-usage.d.ts

@@ -0,0 +1,9 @@
+import type { RuntimeMessage } from "@neta-art/cohub";
+declare const fields: readonly ["inputTokens", "outputTokens", "cachedInputTokens", "cacheWriteInputTokens", "totalTokens"];
+export type CodexTokenTotals = Record<typeof fields[number], number>;
+export declare const codexTokenTotals: (value: unknown) => CodexTokenTotals;
+export declare const subtractCodexTokens: (total: CodexTokenTotals, base: CodexTokenTotals) => CodexTokenTotals;
+export declare function codexUsage(total: CodexTokenTotals): NonNullable<RuntimeMessage["usage"]>;
+/** Seed portable imports from the original native counters, not a prior turn's `last`. */
+export declare function codexArchiveTotals(records: Record<string, unknown>[]): CodexTokenTotals | undefined;
+export {};

package/dist/runtime/codex-usage.js

@@ -0,0 +1,23 @@
+import { record } from "./json-rpc.js";
+const fields = ["inputTokens", "outputTokens", "cachedInputTokens", "cacheWriteInputTokens", "totalTokens"];
+const tokens = (value) => typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : 0;
+export const codexTokenTotals = (value) => {
+    const input = record(value);
+    return Object.fromEntries(fields.map((key) => [key, tokens(input[key])]));
+};
+export const subtractCodexTokens = (total, base) => Object.fromEntries(fields.map((key) => [key, Math.max(0, total[key] - base[key])]));
+export function codexUsage(total) {
+    return { input: Math.max(0, total.inputTokens - total.cachedInputTokens - total.cacheWriteInputTokens), output: total.outputTokens, cacheRead: total.cachedInputTokens, cacheWrite: total.cacheWriteInputTokens, totalTokens: total.totalTokens };
+}
+/** Seed portable imports from the original native counters, not a prior turn's `last`. */
+export function codexArchiveTotals(records) {
+    for (let i = records.length - 1; i >= 0; i--) {
+        const entry = records[i];
+        if (entry?.type !== "token_usage_record")
+            continue;
+        const value = record(record(entry.payload).thread_token_usage);
+        if (typeof value.total_tokens !== "number")
+            continue;
+        return codexTokenTotals({ inputTokens: value.input_tokens, outputTokens: value.output_tokens, cachedInputTokens: value.cached_input_tokens, cacheWriteInputTokens: value.cache_write_input_tokens, totalTokens: value.total_tokens });
+    }
+}

package/dist/runtime/connection.d.ts

@@ -0,0 +1,16 @@
+import { type RuntimeCapabilities } from "@neta-art/cohub";
+import { type HarnessOptions } from "./harness.js";
+import { type RuntimeSessionStore } from "./session-store.js";
+export type RuntimeConnectionOptions = {
+    spaceId: string;
+    cwd: string;
+    url: string;
+    capabilities: RuntimeCapabilities;
+    harnesses: HarnessOptions;
+    token: () => Promise<string>;
+    signal: AbortSignal;
+    store: RuntimeSessionStore;
+    onReady: () => void;
+    leaseConflictTimeoutMs?: number;
+};
+export declare function serveRuntime(options: RuntimeConnectionOptions): Promise<void>;