@neta-art/cohub-cli 8.1.1 → 8.2.0

package/dist/runtime/native-sync.js

@@ -1,12 +1,13 @@
 import { readFile, readdir, stat } from "node:fs/promises";
 import { homedir } from "node:os";
-import { join } from "node:path";
+import { basename, join } from "node:path";
 import { createClient } from "../client.js";
 import { currentIdentityKey } from "../space.js";
 import { canonicalRuntimeRoot, getRuntimeSpaceBinding } from "./space-binding.js";
-import { readNativeTranscript, readNativeTranscriptHeader } from "./native-transcript.js";
+import { readNativeTranscript, readNativeTranscriptHeader, codexSessionsDirectory, codexArchivedSessionsDirectory, codexRolloutIdOf } from "./native-transcript.js";
 import { findRuntimeNativeSession } from "./session-store.js";
 import { listNativeSyncStores, nativeIdentityHash, NativeSyncStore } from "./native-sync-store.js";
+import { serializeDiagnosticError } from "./diagnostics.js";
 export const nativeRuntimeRoot = (spaceId) => join(homedir(), ".local", "state", "cohub", "runtime", spaceId);
 export const nativeSyncConfigPath = (runtimeRoot, identity) => join(runtimeRoot, "native", nativeIdentityHash(identity), "config.json");
 const piSessionDirectory = (cwd) => {
@@ -17,10 +18,11 @@ const piSessionDirectory = (cwd) => {
     const safePath = `--${cwd.replace(/^[/\\\\]/, "").replace(/[/\\\\:]/g, "-")}--`;
     return join(agentDir, "sessions", safePath);
 };
-const codexSessionDirectory = () => join(process.env.CODEX_HOME?.trim() || join(homedir(), ".codex"), "sessions");
 const LARGE_TRANSCRIPT_BYTES = 64 * 1024 * 1024;
 async function nativeTranscriptPaths(root, harness, signal) {
-    const pending = [harness === "pi" ? piSessionDirectory(root) : codexSessionDirectory()];
+    // Codex keeps rollouts active or archived, plain or `.zst`-compressed; Pi writes plain JSONL.
+    const pending = harness === "pi" ? [piSessionDirectory(root)] : [codexSessionsDirectory(), codexArchivedSessionsDirectory()];
+    const accepts = harness === "pi" ? (name) => name.endsWith(".jsonl") : (name) => name.endsWith(".jsonl") || name.endsWith(".jsonl.zst");
     const paths = [];
     const errors = [];
     while (pending.length) {
@@ -38,7 +40,7 @@ async function nativeTranscriptPaths(root, harness, signal) {
             const path = join(directory, entry.name);
             if (entry.isDirectory())
                 pending.push(path);
-            else if (entry.isFile() && entry.name.endsWith(".jsonl"))
+            else if (entry.isFile() && accepts(entry.name))
                 paths.push(path);
         }
     }
@@ -48,10 +50,68 @@ async function nativeTranscriptPaths(root, harness, signal) {
 export async function discoverNativeImportCandidates(root, harnesses, options = {}) {
     const candidates = [];
     const errors = [];
+    /** Same-thread rollouts left out of import; they may carry Turns no candidate contains. */
+    const superseded = [];
     for (const harness of harnesses) {
         options.signal?.throwIfAborted();
         const scanned = await nativeTranscriptPaths(root, harness, options.signal);
         errors.push(...scanned.errors);
+        // One rollout chain per thread: every rollout referenced by a same-thread file is
+        // superseded by it (rollover / revert); a parent referenced only from a different thread
+        // (fork) still owns its own Turns and stays importable. Headers are read once and reused.
+        const headers = new Map();
+        const absorbedRollouts = new Set();
+        if (harness === "codex") {
+            // Bounded concurrency: one descriptor per open file across the global Codex sessions tree.
+            let next = 0;
+            const readHeaders = async () => {
+                for (;;) {
+                    const path = scanned.paths[next++];
+                    if (!path)
+                        return;
+                    try {
+                        headers.set(path, await readNativeTranscriptHeader(path, harness, options.signal));
+                    }
+                    catch (error) {
+                        if (options.signal?.aborted)
+                            throw error;
+                        errors.push({ harness, path, message: error instanceof Error ? error.message : String(error) });
+                    }
+                }
+            };
+            await Promise.all(Array.from({ length: Math.min(16, scanned.paths.length) }, readHeaders));
+            const threadByRollout = new Map();
+            for (const header of headers.values())
+                if (header.rolloutId)
+                    threadByRollout.set(header.rolloutId, header.nativeSessionId);
+            for (const header of headers.values()) {
+                const base = header.historyBase;
+                if (base && threadByRollout.get(base.rolloutId) === header.nativeSessionId)
+                    absorbedRollouts.add(base.rolloutId);
+            }
+        }
+        // One candidate per thread: several same-thread rollouts can survive absorption (two
+        // siblings reverting from the same parent). The newest by rollout filename timestamp wins;
+        // the losers are reported, never imported twice under one nativeSessionId.
+        const supersededSiblings = new Set();
+        if (harness === "codex") {
+            const byThread = new Map();
+            for (const [path, header] of headers) {
+                if (absorbedRollouts.has(header.rolloutId ?? ""))
+                    continue;
+                const list = byThread.get(header.nativeSessionId) ?? [];
+                list.push({ path, rolloutId: header.rolloutId ?? "", sortKey: basename(path) });
+                byThread.set(header.nativeSessionId, list);
+            }
+            for (const list of byThread.values()) {
+                if (list.length < 2)
+                    continue;
+                const newest = list.reduce((a, b) => (a.sortKey > b.sortKey ? a : b));
+                for (const sibling of list)
+                    if (sibling !== newest)
+                        supersededSiblings.add(sibling.path);
+            }
+        }
         let start = 0;
         while (start < scanned.paths.length) {
             options.signal?.throwIfAborted();
@@ -60,9 +120,16 @@ export async function discoverNativeImportCandidates(root, harnesses, options =
             const concurrency = sizes.some((size) => size > LARGE_TRANSCRIPT_BYTES) ? 1 : 2;
             const batch = await Promise.all(paths.slice(0, concurrency).map(async (path) => {
                 try {
-                    const header = await readNativeTranscriptHeader(path, harness, options.signal);
+                    // Pre-scan failures are already reported; do not read the header again.
+                    const header = headers.get(path) ?? (harness === "pi" ? await readNativeTranscriptHeader(path, harness, options.signal) : null);
+                    if (!header)
+                        return null;
                     if (await canonicalRuntimeRoot(header.cwd) !== root)
                         return null;
+                    if (header.rolloutId && absorbedRollouts.has(header.rolloutId))
+                        return null;
+                    if (supersededSiblings.has(path))
+                        return { superseded: { path, nativeSessionId: header.nativeSessionId, message: "Superseded by a newer rollout of the same conversation; its Turns are not imported" } };
                     const transcript = await readNativeTranscript(path, harness, { settled: true, signal: options.signal });
                     if (transcript.turns.length === 0)
                         return { error: { harness, path, message: "Transcript contains no importable Turns" } };
@@ -77,10 +144,10 @@ export async function discoverNativeImportCandidates(root, harnesses, options =
                 options.signal?.throwIfAborted();
                 if (!result)
                     continue;
-                if ("error" in result) {
-                    if (result.error)
-                        errors.push(result.error);
-                }
+                if ("superseded" in result)
+                    superseded.push(result.superseded);
+                else if ("error" in result)
+                    errors.push(result.error);
                 else
                     candidates.push(result);
                 options.onProgress?.({ harness, scanned: start + batch.length, candidates: candidates.length });
@@ -88,7 +155,7 @@ export async function discoverNativeImportCandidates(root, harnesses, options =
             start += concurrency;
         }
     }
-    return { candidates, errors };
+    return { candidates, errors, superseded };
 }
 export function nativeArchiveTransport(spaceId, identity) {
     const client = createClient().space(spaceId);
@@ -149,21 +216,30 @@ export async function captureNativeSession(input) {
     const transcript = await readNativeTranscript(path, input.harness, input);
     if (await canonicalPathOrNull(transcript.cwd) !== root || input.nativeSessionId && transcript.nativeSessionId !== input.nativeSessionId)
         throw new Error("Native transcript belongs to another project or Session");
+    // The path stays in the cache key: restored Pi working copies share a native ID but not a file.
     const key = JSON.stringify([identity, space.spaceId, input.harness, transcript.nativeSessionId, path]);
     let store = nativeStores.get(key);
     if (!store) {
         const transport = nativeArchiveTransport(space.spaceId, identity);
+        const lineage = input.harness === "codex" ? transcript.lineageRolloutIds ?? [] : [];
         const candidates = (await listNativeSyncStores(runtimeRoot, space.spaceId, identity, transport))
             .filter((candidate) => candidate.options.harness === input.harness && candidate.options.nativeSessionId === transcript.nativeSessionId);
-        for (const candidate of candidates)
-            if ((await candidate.binding()).path === path) {
+        // A Codex rollover moves appends to a fresh rollout that references the old one: a store
+        // bound anywhere in the new leaf's lineage is the same conversation, so keep using it —
+        // the binding migrates inside capture. Exact-path matches win; lineage matches follow.
+        for (const candidate of candidates) {
+            const binding = await candidate.binding();
+            if (binding.path === path) {
                 store = candidate;
                 break;
             }
+            if (!store && lineage.includes(codexRolloutIdOf(basename(binding.path)) ?? ""))
+                store = candidate;
+        }
         if (!store) {
             const managed = await findRuntimeNativeSession(runtimeRoot, input.harness, transcript.nativeSessionId, path);
             const managedPath = managed ? await canonicalPathOrNull(managed.path) : null;
-            if (candidates.length && managedPath !== path)
+            if (candidates.length && managedPath !== path && !(managedPath && lineage.includes(codexRolloutIdOf(basename(managedPath)) ?? "")))
                 throw new Error("Native path changed; original bindings retained");
             // Restored Pi working copies can share a native ID. Existing Cohub sidecars disambiguate them.
             store = new NativeSyncStore({ runtimeRoot, spaceId: space.spaceId, identity, harness: input.harness, nativeSessionId: transcript.nativeSessionId,
@@ -173,7 +249,7 @@ export async function captureNativeSession(input) {
             nativeStores.delete(nativeStores.keys().next().value ?? "");
         nativeStores.set(key, store);
     }
-    await store.capture(path, transcript, { sessionStartedAt: input.sessionStartedAt, origin: input.origin });
+    await store.capture(path, transcript, { sessionStartedAt: input.sessionStartedAt, origin: input.origin, signal: input.signal });
     return store;
 }
 /** The existing Runtime supervisor retries receipts even after the original terminal has exited. */
@@ -187,6 +263,35 @@ export function nativeWebSocketTransport(spaceId, identity, send) {
         heartbeatNativeTurn: async (sessionId, turnId) => await send({ type: "heartbeat", sessionId, turnId }),
     };
 }
+export const NATIVE_SYNC_SESSION_CONCURRENCY = 4;
+export async function runNativeSyncPool(items, concurrency, signal, processItem) {
+    let next = 0;
+    const worker = async () => {
+        while (!signal.aborted) {
+            const index = next++;
+            if (index >= items.length)
+                return;
+            await processItem(items[index]);
+        }
+    };
+    const results = await Promise.allSettled(Array.from({ length: Math.min(items.length, Math.max(1, concurrency)) }, worker));
+    const failure = results.find((result) => result.status === "rejected");
+    if (failure)
+        throw failure.reason;
+}
+export function summarizeNativeSyncErrors(errors) {
+    const groups = new Map();
+    for (const error of errors) {
+        const value = serializeDiagnosticError(error);
+        const key = JSON.stringify([value.name, value.code, value.status, value.causeCode, value.message]);
+        const group = groups.get(key);
+        if (group)
+            group.count += 1;
+        else
+            groups.set(key, { name: value.name ?? "Error", message: value.message, ...(value.code ? { code: value.code } : {}), ...(value.status ? { status: value.status } : {}), count: 1 });
+    }
+    return [...groups.values()];
+}
 export async function flushNativeSessions(spaceId, identity, signal, report, transportOverride) {
     if (currentIdentityKey() !== identity)
         return false;
@@ -199,12 +304,12 @@ export async function flushNativeSessions(spaceId, identity, signal, report, tra
         return false;
     const stores = await listNativeSyncStores(runtimeRoot, spaceId, identity, transport);
     let failed = false;
-    for (const store of stores) {
-        signal.throwIfAborted();
-        const binding = await store.binding();
-        if (!config.harnesses.includes(binding.harness))
-            continue;
+    await runNativeSyncPool(stores, NATIVE_SYNC_SESSION_CONCURRENCY, signal, async (store) => {
         try {
+            signal.throwIfAborted();
+            const binding = await store.binding();
+            if (!config.harnesses.includes(binding.harness))
+                return;
             // Codex hooks trigger captures; re-reading here also closes gaps after reconnects or missed hooks.
             if (binding.harness === "codex")
                 await store.capture(binding.path, await readNativeTranscript(binding.path, "codex"));
@@ -216,6 +321,6 @@ export async function flushNativeSessions(spaceId, identity, signal, report, tra
                 report(error);
             }
         }
-    }
+    });
     return failed;
 }

package/dist/runtime/native-transcript.d.ts

@@ -12,6 +12,12 @@ export type NativeTranscriptTurn = {
     boundaries: Record<number, string>;
     sha256: string;
     result: NativeTurnComplete | null;
+    /** Codex lineage: the rollout file this Turn was read from. */
+    path?: string;
+    /** Codex lineage: the rollout id of the file this Turn was read from. */
+    rolloutId?: string;
+    /** Codex lineage: merged order across rollout files. */
+    sequence?: number;
 };
 export type NativeTranscript = {
     nativeSessionId: string;
@@ -19,14 +25,36 @@ export type NativeTranscript = {
     cloudSessionId?: string;
     turns: NativeTranscriptTurn[];
     prefixes: ReadonlyMap<number, string>;
+    /** Codex lineage: every rollout file oldest-to-leaf, including the leaf itself. */
+    lineagePaths?: string[];
+    /** Codex lineage: rollout ids matching `lineagePaths` order. */
+    lineageRolloutIds?: string[];
 };
 export declare const NATIVE_MAX_RECORD_BYTES: number;
 export declare const NATIVE_MAX_TRANSCRIPT_BYTES: number;
-/** Read only the session header before deciding whether a global Codex file belongs to this project. */
-export declare function readNativeTranscriptHeader(path: string, harness: "pi" | "codex", signal?: AbortSignal): Promise<{
+export declare const codexSessionsDirectory: () => string;
+export declare const codexArchivedSessionsDirectory: () => string;
+/** Ordinary rollout names encode one id; reverted threads append `_rollout-id`. */
+export declare function codexRolloutIdOf(fileName: string): string | null;
+export type CodexHistoryBase = {
+    rolloutId: string;
+    endOrdinal: number;
+    endBytes: number;
+};
+export type NativeTranscriptHeader = {
     nativeSessionId: string;
     cwd: string;
-}>;
+    rolloutId?: string;
+    historyBase?: CodexHistoryBase;
+};
+/** Read only the session header before deciding whether a global Codex file belongs to this project. */
+export declare function readNativeTranscriptHeader(path: string, harness: "pi" | "codex", signal?: AbortSignal): Promise<NativeTranscriptHeader>;
+/** Raw JSONL bytes of a rollout, transparently decompressing `.zst` and honoring a lineage cutoff. */
+export declare function codexRolloutBytes(path: string, endBytes: number | null, signal?: AbortSignal): AsyncGenerator<Buffer>;
+/** Cache location for a rollout's decompressed copy; shared by capture and archival. */
+export declare function codexRolloutCachePath(source: string, cacheDirectory: string): string;
+/** Archives need plain JSONL bytes; decompress a cold `.zst` rollout into a cache once. */
+export declare function ensurePlainCodexRollout(path: string, cacheDirectory: string, signal?: AbortSignal, maxBytes?: number): Promise<string>;
 /** Partial trailing records are retried, never parsed or acknowledged as complete. */
 export declare function readNativeTranscript(path: string, harness: "pi" | "codex", options?: {
     settled?: boolean;