@neta-art/cohub-cli 8.2.0 → 8.3.0

package/dist/runtime/native/results.js

@@ -0,0 +1,128 @@
+import { readdir, readFile, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { RUNTIME_RECOVERY_BATCH_SIZE, runtimeEventSchema } from "@neta-art/cohub";
+import { atomicRuntimeJson } from "../archive-store.js";
+import { confirmQuiescentProcessGroup } from "../process-group.js";
+import { readTranscript } from "./adapters.js";
+export class ContextRequiredError extends Error {
+}
+const missing = (error) => error?.code === "ENOENT";
+/** One small receipt per Cohub-executed Turn until the server acknowledges its result. */
+export class ExecutionResults {
+    root;
+    constructor(root) {
+        this.root = root;
+    }
+    path(sessionId) { return join(this.root, "executions", `${sessionId}.json`); }
+    async read(sessionId) {
+        try {
+            const receipt = JSON.parse(await readFile(this.path(sessionId), "utf8"));
+            if (receipt.version !== 2 || receipt.sessionId !== sessionId)
+                throw new Error("Invalid execution receipt");
+            return receipt;
+        }
+        catch (error) {
+            if (missing(error))
+                return null;
+            throw error;
+        }
+    }
+    /**
+     * Admit a new Turn. An unacknowledged earlier Turn is released once the server has settled it;
+     * otherwise the server must decide first, so a result is never overwritten unseen.
+     */
+    async admit(input) {
+        const receipt = await this.read(input.sessionId);
+        if (!receipt || receipt.turnId === input.turnId)
+            return null;
+        const { resolvedTurnIds, settledTurnIds, complete } = input.context;
+        const resolved = resolvedTurnIds?.includes(receipt.turnId) === true;
+        if (resolved || settledTurnIds?.includes(receipt.turnId)) {
+            await rm(this.path(input.sessionId), { force: true });
+            return resolved ? receipt.session.path : null;
+        }
+        if (complete === false)
+            throw new ContextRequiredError("Server resolution is required for the pending native execution");
+        throw new Error(`Local turn ${receipt.turnId} has unconfirmed results; reconcile it before continuing`);
+    }
+    async start(session, input, requestId) {
+        await atomicRuntimeJson(this.path(input.sessionId), { version: 2, requestId, sessionId: input.sessionId, turnId: input.turnId, session });
+    }
+    async record(session, input, requestId, events, uncertainCleanup) {
+        await atomicRuntimeJson(this.path(input.sessionId), { version: 2, requestId, sessionId: input.sessionId, turnId: input.turnId, session, events, ...(uncertainCleanup ? { uncertainCleanup } : {}) });
+    }
+    async recover(input, requestId) {
+        const receipt = await this.read(input.sessionId);
+        if (!receipt || receipt.turnId !== input.turnId || receipt.session.harness !== input.harness)
+            return null;
+        if (requestId && receipt.requestId != null && receipt.requestId !== requestId)
+            throw new Error("Runtime result execution identity mismatch");
+        if (receipt.events) {
+            // A quiescent process group upgrades the receipt to confirmed; a live one stays uncertain.
+            if (receipt.uncertainCleanup && !receipt.uncertainCleanup.confirmedAt) {
+                if (!await confirmQuiescentProcessGroup(receipt.uncertainCleanup.processGroupId))
+                    return null;
+                receipt.uncertainCleanup.confirmedAt = new Date().toISOString();
+                await atomicRuntimeJson(this.path(input.sessionId), receipt);
+            }
+            return { session: receipt.session, events: receipt.events.map((event) => runtimeEventSchema.parse(event)) };
+        }
+        const events = await this.fromTranscript(receipt);
+        if (!events)
+            return null;
+        await this.record(receipt.session, receipt, receipt.requestId, events);
+        return { session: receipt.session, events };
+    }
+    async acknowledge(sessionId, turnId) {
+        const receipt = await this.read(sessionId);
+        if (receipt?.turnId !== turnId)
+            throw new Error("Runtime acknowledgement identity mismatch");
+        await rm(this.path(sessionId), { force: true });
+    }
+    async pendingTurnIds(sessionId) {
+        const receipt = await this.read(sessionId);
+        return receipt ? [receipt.turnId] : [];
+    }
+    async *pendingBatches() {
+        const names = await readdir(join(this.root, "executions")).catch((error) => { if (missing(error))
+            return []; throw error; });
+        let batch = [];
+        for (const name of names) {
+            if (!name.endsWith(".json"))
+                continue;
+            const receipt = await this.read(name.slice(0, -5)).catch(() => null);
+            if (!receipt)
+                continue;
+            batch.push({ sessionId: receipt.sessionId, turnId: receipt.turnId, harness: receipt.session.harness });
+            if (batch.length >= RUNTIME_RECOVERY_BATCH_SIZE) {
+                yield batch;
+                batch = [];
+            }
+        }
+        if (batch.length)
+            yield batch;
+    }
+    async fromTranscript(receipt) {
+        const transcript = await readTranscript(receipt.session.path, receipt.session.harness).catch(() => null);
+        const turn = transcript?.turns.find((entry) => entry.cloudTurnId === receipt.turnId);
+        const result = turn?.result;
+        if (!result || result.status === "interrupted" || turn !== transcript?.turns.at(-1))
+            return null;
+        const messages = result.messages.filter((message) => message.content.length);
+        if (!messages.length)
+            return null;
+        const message = (entry, ordinal) => ({
+            ordinal,
+            content: entry.content,
+            provider: entry.provider ?? null,
+            model: entry.model ?? null,
+            usage: (entry.usage ?? null),
+            stopReason: entry.stopReason ?? "stop",
+            errorMessage: entry.errorMessage ?? null,
+        });
+        return [
+            ...messages.slice(0, -1).map((entry, index) => ({ type: "message.commit", message: message(entry, index) })),
+            { type: "turn.end", message: message(messages.at(-1), messages.length - 1), resume: "native", archive: null },
+        ];
+    }
+}

package/dist/runtime/native/sessions.d.ts

@@ -0,0 +1,45 @@
+import { type RuntimeContext } from "@neta-art/cohub";
+import type { SessionTurnProjectionClient } from "../turn-projection.js";
+import type { RuntimeArchiveStore } from "../archive-store.js";
+import type { NativeIngest, TranscriptState } from "./ingest.js";
+/** How the native file for a Turn was obtained; reported with the Turn result. */
+export type ResumeKind = "native" | "restored" | "handoff" | "new";
+export type NativeSession = {
+    harness: "pi" | "codex";
+    nativeSessionId: string;
+    path: string;
+    cwd: string;
+};
+export type PreparedSession = {
+    session: NativeSession;
+    resume: ResumeKind;
+};
+/** Chooses the native file a Turn runs in. 1. */
+export declare class NativeSessions {
+    private readonly options;
+    private readonly projections;
+    constructor(options: {
+        root: string;
+        spaceId: string;
+        projectionSource: SessionTurnProjectionClient;
+        archives: RuntimeArchiveStore;
+        ingest: NativeIngest;
+        onRestoreFailed?: (error: unknown, context: {
+            sessionId: string;
+            harness: "pi" | "codex";
+            unrestorable: boolean;
+        }) => void;
+    });
+    prepare(input: {
+        sessionId: string;
+        turnId: string;
+        harness: "pi" | "codex";
+        cwd: string;
+        provider?: string | null;
+        context: RuntimeContext;
+        resumable: (transcript: TranscriptState) => boolean;
+        signal?: AbortSignal;
+    }): Promise<PreparedSession>;
+    private restore;
+    private project;
+}

package/dist/runtime/native/sessions.js

@@ -0,0 +1,92 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, rename, rm, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import { serializeProjectionRecords } from "@neta-art/cohub";
+import { ArchiveNotRestorableError, importNativeArchive } from "./archive.js";
+import { piSessionDirectoryName, readTranscript } from "./adapters.js";
+import { ProjectionStore, rebindProjectionNativeSession } from "../projection-store.js";
+/** Chooses the native file a Turn runs in. 1. */
+export class NativeSessions {
+    options;
+    projections;
+    constructor(options) {
+        this.options = options;
+        this.projections = new ProjectionStore(options.projectionSource);
+    }
+    async prepare(input) {
+        const existing = this.options.ingest.transcriptFor(input.sessionId, input.harness, input.context.throughTurnId);
+        if (existing && input.resumable(existing)) {
+            return { session: { harness: existing.harness, nativeSessionId: existing.nativeSessionId, path: existing.path, cwd: existing.cwd }, resume: "native" };
+        }
+        const archive = input.context.archive;
+        if (archive && archive.harness === input.harness && archive.sessionId === input.sessionId && archive.turnId === input.context.throughTurnId) {
+            const restored = await this.restore(input, archive, input.signal).catch((error) => {
+                input.signal?.throwIfAborted();
+                // A structurally unrestorable archive falls back to projection; so does a transient
+                // failure, which simply tries the archive again on the next Turn.
+                this.options.onRestoreFailed?.(error, { sessionId: input.sessionId, harness: input.harness, unrestorable: error instanceof ArchiveNotRestorableError });
+                return null;
+            });
+            if (restored) {
+                // A restored copy is history the server already has, even where it carries no markers.
+                const through = (await readTranscript(restored.path, restored.harness).catch(() => null))?.turns.at(-1)?.key ?? null;
+                this.options.ingest.adopt(restored, input.sessionId, { turnKey: through, turnId: input.context.throughTurnId });
+                return { session: restored, resume: "restored" };
+            }
+        }
+        const projected = await this.project(input);
+        // A new Codex thread gets its file when it starts; the executor adopts it then.
+        if (projected.resume !== "new" || input.harness === "pi")
+            this.options.ingest.adopt(projected.session, input.sessionId);
+        return projected;
+    }
+    async restore(input, archive, signal) {
+        const id = randomUUID();
+        const raw = join(this.options.root, "native", "restored", `${id}.jsonl`);
+        const path = nativePath(input.harness, id, input.cwd);
+        try {
+            const restored = await this.options.archives.restore(archive, raw, signal);
+            const imported = await importNativeArchive({ source: raw, target: path, harness: input.harness, nativeSessionId: restored.nativeSessionId, id, cwd: input.cwd, signal });
+            return { harness: input.harness, nativeSessionId: imported.nativeSessionId, path, cwd: input.cwd };
+        }
+        finally {
+            await rm(raw, { force: true }).catch(() => undefined);
+        }
+    }
+    async project(input) {
+        const nativeSessionId = randomUUID();
+        const path = nativePath(input.harness, nativeSessionId, input.cwd);
+        const projection = rebindProjectionNativeSession(await this.projections.project({
+            spaceId: this.options.spaceId,
+            sessionId: input.sessionId,
+            turnId: input.turnId,
+            nativeSessionId,
+            cwd: input.cwd,
+            provider: input.provider,
+            target: input.harness,
+            throughTurnId: input.context.throughTurnId,
+        }, input.signal), nativeSessionId);
+        const session = { harness: input.harness, nativeSessionId, path, cwd: input.cwd };
+        // An empty Codex thread is created by Codex itself when the Turn starts.
+        if (!projection.turns.length && input.harness === "codex")
+            return { session, resume: "new" };
+        await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+        const temporary = `${path}.${randomUUID()}.projection`;
+        await writeFile(temporary, serializeProjectionRecords(projection.projection.records), { mode: 0o600 });
+        await rename(temporary, path);
+        return { session, resume: projection.turns.length ? "handoff" : "new" };
+    }
+}
+/** Where a harness keeps a new transcript for the bound directory. */
+function nativePath(harness, nativeSessionId, cwd) {
+    const stamp = new Date().toISOString();
+    if (harness === "pi") {
+        const agent = process.env.PI_CODING_AGENT_DIR?.trim() || join(homedir(), ".pi", "agent");
+        const directory = process.env.PI_CODING_AGENT_SESSION_DIR?.trim() || join(agent, "sessions", piSessionDirectoryName(cwd));
+        return join(directory, `${stamp.replace(/[:.]/g, "-")}_${nativeSessionId}.jsonl`);
+    }
+    const home = process.env.CODEX_HOME?.trim() || join(homedir(), ".codex");
+    const day = stamp.replace(/\.\d{3}Z$/, "").replaceAll(":", "-");
+    return join(home, "sessions", day.slice(0, 4), day.slice(5, 7), day.slice(8, 10), `rollout-${day}-${nativeSessionId}.jsonl`);
+}

package/dist/runtime/{native-transcript.d.ts → native/transcript.d.ts}

@@ -4,6 +4,11 @@ export type NativeTranscriptTurn = {
     parentKey: string | null;
     cloudTurnId?: string;
     userContent: ContentBlock[];
+    /**
+     * Codex records a prompt in several steps and the final form last; until this is set the user
+     * content may still change.
+     */
+    userFinal?: boolean;
     messages: NativeTurnMessage[];
     startedAt: string;
     startBytes: number;
@@ -12,11 +17,8 @@ export type NativeTranscriptTurn = {
     boundaries: Record<number, string>;
     sha256: string;
     result: NativeTurnComplete | null;
-    /** Codex lineage: the rollout file this Turn was read from. */
     path?: string;
-    /** Codex lineage: the rollout id of the file this Turn was read from. */
     rolloutId?: string;
-    /** Codex lineage: merged order across rollout files. */
     sequence?: number;
 };
 export type NativeTranscript = {
@@ -25,9 +27,7 @@ export type NativeTranscript = {
     cloudSessionId?: string;
     turns: NativeTranscriptTurn[];
     prefixes: ReadonlyMap<number, string>;
-    /** Codex lineage: every rollout file oldest-to-leaf, including the leaf itself. */
     lineagePaths?: string[];
-    /** Codex lineage: rollout ids matching `lineagePaths` order. */
     lineageRolloutIds?: string[];
 };
 export declare const NATIVE_MAX_RECORD_BYTES: number;

package/dist/runtime/{native-transcript.js → native/transcript.js}

@@ -6,9 +6,9 @@ import { homedir } from "node:os";
 import { basename, join } from "node:path";
 import { createZstdDecompress } from "node:zlib";
 import { pipeline } from "node:stream/promises";
-import { piContent } from "./harness.js";
-import { codexTokenTotals, codexUsage } from "./codex-usage.js";
-import { record } from "./json-rpc.js";
+import { piContent } from "../harness.js";
+import { codexTokenTotals, codexUsage } from "../codex-usage.js";
+import { record } from "../json-rpc.js";
 const text = (value) => typeof value === "string" ? value : "";
 const list = (value) => Array.isArray(value) ? value : [];
 const iso = (value) => {
@@ -265,6 +265,7 @@ function parsePiTranscript(lines, options) {
     let cloudSessionId = text(record(header.cohub).sessionId) || text(record(header.affinity).sessionId);
     let current = null;
     let messages = [];
+    let pendingMarker = null;
     let completedAt = iso(header.timestamp);
     const finish = (settled) => {
         if (!current)
@@ -277,6 +278,10 @@ function parsePiTranscript(lines, options) {
     for (const line of branch) {
         const entry = line.value;
         if (entry.type !== "message") {
+            // Cohub writes a marker entry immediately before a user message it sends, so a Turn it
+            // started can be matched to the cloud Turn without guessing at byte offsets.
+            if (entry.type === "custom" && text(entry.customType) === "cohub.turn")
+                pendingMarker = text(record(entry.data).turnId) || null;
             if (current) {
                 current.endBytes = line.endBytes;
                 current.sha256 = line.sha256;
@@ -298,7 +303,9 @@ function parsePiTranscript(lines, options) {
         if (message.role === "user") {
             finish(true);
             completedAt = timestamp;
-            const cloudTurnId = text(record(message.meta).turnId);
+            const metaTurnId = text(record(message.meta).turnId);
+            const cloudTurnId = metaTurnId || pendingMarker || "";
+            pendingMarker = null;
             if (cloudTurnId && !text(record(header.cohub).sessionId) && !text(record(header.affinity).sessionId))
                 cloudSessionId = text(record(message.meta).sourceSessionId) || cloudSessionId;
             current = { key: text(entry.id), parentKey: turns.at(-1)?.key ?? null, ...(cloudTurnId ? { cloudTurnId } : {}), userContent: typeof message.content === "string" ? [{ type: "text", text: message.content }] : piContent(message.content), messages: [], startedAt: completedAt, startBytes: line.startBytes, endBytes: line.endBytes, contentEndBytes: line.endBytes, boundaries: {}, sha256: line.sha256, result: null };
@@ -347,8 +354,8 @@ async function resolveCodexLineage(leafPath, signal) {
         if (segments.length >= CODEX_HISTORY_MAX_DEPTH)
             throw new Error("Codex history chain is too deep; retain the original files");
         const header = await readCodexHeader(path, signal);
-        // Every segment of one conversation must belong to the same project: a cross-project
-        // reference would import another workspace's history into this one.
+        // Every segment of one conversation must belong to the same project: a cross-project reference
+        // would import another workspace's history into this one.
         if (header.cwd && leafCwd != null && header.cwd !== leafCwd)
             throw new Error("Codex lineage crosses projects; retain the original files");
         leafCwd ??= header.cwd;
@@ -468,9 +475,28 @@ function parseCodexLines(lines, segment) {
                 assistant().content.push({ type: "tool_result", tool_use_id: text(payload.call_id), content: typeof payload.output === "string" ? payload.output : JSON.stringify(payload.output ?? null) });
             }
         }
-        if (entry.type === "event_msg" && payload.type === "user_message" && !userFromResponse) {
-            current.userContent = [{ type: "text", text: text(payload.message) }];
+        if (entry.type === "event_msg" && payload.type === "item_completed") {
+            // Cohub tags a Turn it starts with its own cloud Turn id, so the file itself records which
+            // Turns already belong to a cloud Turn (and must not be ingested twice).
+            const item = record(payload.item);
+            if (item.type === "UserMessage") {
+                const clientId = text(item.client_id);
+                if (clientId)
+                    current.cloudTurnId = clientId;
+                // The typed input, without the environment context Codex injects as a user item.
+                const typed = codexContent(item.content);
+                if (typed.length) {
+                    current.userContent = typed;
+                    userFromResponse = true;
+                }
+                current.userFinal = true;
+            }
+        }
+        if (entry.type === "event_msg" && payload.type === "user_message") {
+            if (!userFromResponse)
+                current.userContent = [{ type: "text", text: text(payload.message) }];
             userFromResponse = true;
+            current.userFinal = true;
         }
         if (entry.type === "event_msg" && ["turn_complete", "task_complete", "turn_aborted"].includes(text(payload.type))) {
             if (payload.turn_id && payload.turn_id !== current.key)
@@ -500,8 +526,8 @@ function parseCodexLines(lines, segment) {
     return { turns, ...(cloudSessionId ? { cloudSessionId } : {}), settled };
 }
 const codexSegmentMemo = new Map();
-// Ancestor prefixes are immutable, so the cache is a pure win; parsed objects inflate in
-// memory several-fold over raw bytes, hence a conservative budget.
+// Ancestor prefixes are immutable, so the cache is a pure win; parsed objects inflate in memory
+// several-fold over raw bytes, hence a conservative budget.
 const CODEX_MEMO_MAX_BYTES = 128 * 1024 * 1024;
 /** Ancestors are immutable; only their bounded prefixes are worth caching, bounded by bytes. */
 async function parseCodexSegment(segment, signal) {
@@ -520,7 +546,7 @@ async function parseCodexSegment(segment, signal) {
         throw new Error("Native transcript is empty");
     const parsed = parseCodexLines(read.lines, segment);
     // A paginated ancestor stamps every record with its ordinal; the referenced prefix must end
-    // exactly at the ordinal bound. Records without the stamp cannot validate a paginated bound.
+    // exactly at the ordinal bound.
     if (segment.endBytes != null && segment.endOrdinal != null) {
         const ordinal = read.lines.at(-1)?.value.ordinal;
         if (typeof ordinal !== "number" || ordinal !== segment.endOrdinal - 1)

package/dist/runtime/native/translate.d.ts

@@ -0,0 +1,44 @@
+import type { ContentBlock, NativeTurnProgress, RuntimeExecutionEvent, RuntimeMessage } from "@neta-art/cohub";
+import { type JsonRecord } from "../json-rpc.js";
+/** Harness events mapped onto Cohub's streaming protocol. */
+export type Translator = {
+    push(event: JsonRecord): boolean;
+    last(): RuntimeMessage;
+};
+/** Pi emits the same event shapes to its extensions as over RPC. */
+export declare function piTranslator(emit: (event: RuntimeExecutionEvent) => void): Translator;
+/** Codex app-server notifications for one Turn of one thread. */
+export declare function codexTranslator(emit: (event: RuntimeExecutionEvent) => void, identity: {
+    provider: string | null;
+    model: string | null;
+}): Translator;
+type PreviewMessage = {
+    content: ContentBlock[];
+    provider: string | null;
+    model: string | null;
+};
+/** Where progress goes; `null` while the Turn is not yet recorded, so nothing can be sent yet. */
+export type ProgressSink = (progress: NativeTurnProgress) => Promise<unknown> | null;
+/**
+ * The live preview of a Turn a terminal user runs: harness events folded into messages and sent as
+ * ephemeral progress, at most once per interval and only from the first message that changed.
+ */
+export declare class LiveProgress {
+    private readonly sink;
+    private readonly intervalMs;
+    private messages;
+    private dirtyFrom;
+    private synced;
+    private timer;
+    private inFlight;
+    private disposed;
+    constructor(sink: ProgressSink, intervalMs: number);
+    apply(event: RuntimeExecutionEvent): void;
+    replace(messages: PreviewMessage[]): void;
+    dispose(): void;
+    private touched;
+    private schedule;
+    private flush;
+    private at;
+}
+export {};