@neta-art/cohub-cli 10.7.0 → 10.8.1

package/dist/runtime/native/ingest.js

@@ -2,8 +2,9 @@ import { appendFile, mkdir, open, readFile, readdir, rename, stat } from "node:f
 import { join } from "node:path";
 import { NATIVE_INGEST_MAX_TURNS, SETTLED_TURN_STATUSES } from "@neta-art/cohub";
 import { acceptsDirectory, acceptsTranscript, readSessionHeader, readTranscript, transcriptRoots } from "./adapters.js";
+import { SizeLimitError } from "./transcript.js";
 import { nativeTurnId } from "./identity.js";
-import { transcriptFormat } from "../harnesses/index.js";
+import { harnessAdapter, transcriptFormat } from "../harnesses/index.js";
 import { containsNativePath } from "./paths.js";
 const missing = (error) => error?.code === "ENOENT";
 /** Transcripts read in parallel per lane. The live lane never waits behind a backfill. */
@@ -27,6 +28,7 @@ const RETRY_MAX_MS = 5 * 60_000;
  */
 const INGEST_BATCH_TURNS = Math.min(50, NATIVE_INGEST_MAX_TURNS);
 const IMPORT_FAILURES_KEPT = 100;
+const fileVersion = (info) => `${info.ino}:${info.size}:${info.mtimeMs}`;
 /** Headers are read this many at a time; each costs one small read. */
 const HEADER_CONCURRENCY = 16;
 /** Whether a session in `cwd` belongs to the directory bound at `root`. */
@@ -105,8 +107,12 @@ export class NativeIngest {
     reads = new Set();
     timers = new Map();
     failures = new Map();
+    /** File versions that hit a size limit; they are not read again until the file changes. */
+    oversized = new Map();
     aliveAt = new Map();
     runningPaths = new Set();
+    /** Files a live Cohub execution is writing in a format that cannot mark its Turn. */
+    executing = new Set();
     writing = Promise.resolve();
     foreign = new Set();
     importQueue = [];
@@ -194,18 +200,49 @@ export class NativeIngest {
     }
     /**
      * Record that Cohub created or chose this transcript for a Session, before any Turn of it runs.
+     * `through` says the server has the file's Turns up to `turnKey` (none when null) and `turnId` heads them.
      */
     adopt(session, sessionId, through) {
         const state = this.states.get(session.path) ?? this.blank(session.path, session.nativeSessionId, session.cwd, session.harness, "user", { size: -1, mtimeMs: 0, ino: 0 });
         Object.assign(state, { sessionId, nativeSessionId: session.nativeSessionId });
-        if (through?.turnKey)
+        if (through)
             Object.assign(state, { throughTurnKey: through.turnKey, throughTurnId: through.turnId, headTurnId: through.turnId });
         if (!this.states.has(session.path))
             state.origin = "cohub";
+        // The harness wrote elsewhere than expected; the execution follows the file.
+        for (const other of this.states.values()) {
+            if (other === state || other.harness !== session.harness || other.nativeSessionId !== session.nativeSessionId || !other.executingTurnId)
+                continue;
+            state.executingTurnId = other.executingTurnId;
+            other.executingTurnId = null;
+            if (this.executing.delete(other.path))
+                this.executing.add(session.path);
+            this.markDirty(other.path);
+        }
         this.states.set(session.path, state);
         this.markDirty(session.path);
     }
+    /** A Cohub Turn starts writing this file; a format that cannot mark it is not read until it ends. */
+    begin(session, sessionId, turnId) {
+        if (transcriptFormat(session.harness).marksCohubTurns)
+            return;
+        this.adopt(session, sessionId);
+        const state = this.states.get(session.path);
+        state.executingTurnId = turnId;
+        this.executing.add(session.path);
+        this.markDirty(session.path);
+    }
+    /** A Cohub Turn ended, whether or not it produced a result. */
+    end(turnId) {
+        for (const state of this.states.values()) {
+            if (state.executingTurnId !== turnId)
+                continue;
+            this.executing.delete(state.path);
+            this.touch(state.path);
+        }
+    }
     advance(path, turnId) {
+        this.end(turnId);
         const state = this.states.get(path);
         if (!state)
             return;
@@ -287,21 +324,23 @@ export class NativeIngest {
         }
         this.drainLive();
         if (this.job.state === "running")
-            await this.startImport(this.job.filter, this.job.concurrency);
+            await this.startImport(this.job.filter, this.job.concurrency, { resume: true });
     }
-    async plan(filter = {}, signal) {
+    /** `resume` is automatic resumption: it skips file versions that already hit a size limit, which only an explicit import retries. */
+    async plan(filter = {}, signal, options = {}) {
         const harnesses = this.harnesses.filter((harness) => !filter.harnesses?.length || filter.harnesses.includes(harness));
         const candidates = await discoverTranscripts(this.options.cwd, harnesses, { headers: true, nested: this.nested, signal });
         // A transcript whose first read failed is still untracked (size -1), so a later import retries it.
         return candidates.filter((candidate) => (this.states.get(candidate.path)?.size ?? -1) < 0 && !this.live.has(candidate.path)
+            && !(options.resume && this.oversized.get(candidate.path) === fileVersion(candidate))
             && (!filter.nativeSessionId || candidate.nativeSessionId === filter.nativeSessionId));
     }
     /**
      * Start or resume a backfill. What was imported is tracked, so resuming simply plans again; the
      * server recognizes any Turn it already has, so an interrupted batch is never duplicated.
      */
-    async startImport(filter = {}, concurrency = DEFAULT_IMPORT_CONCURRENCY) {
-        const queue = await this.plan(filter);
+    async startImport(filter = {}, concurrency = DEFAULT_IMPORT_CONCURRENCY, options = {}) {
+        const queue = await this.plan(filter, undefined, options);
         const resuming = this.job.state === "running" || this.job.state === "paused";
         const same = resuming && JSON.stringify(this.job.filter) === JSON.stringify(filter);
         const base = same ? this.job : { ...this.job, done: 0, turns: 0, bytes: 0, failed: [], startedAt: new Date().toISOString() };
@@ -417,7 +456,7 @@ export class NativeIngest {
     }
     retryLater(path, error) {
         this.report(path, error);
-        if (this.stopped || !this.transport)
+        if (this.stopped || !this.transport || error instanceof SizeLimitError)
             return;
         const attempts = (this.failures.get(path) ?? 0) + 1;
         if (attempts > RETRY_ATTEMPTS) {
@@ -469,12 +508,26 @@ export class NativeIngest {
         }
     }
     async read(path, lane, force) {
-        if (this.foreign.has(path))
+        if (this.foreign.has(path) || this.executing.has(path))
             return;
         const info = await stat(path).catch((error) => { if (missing(error))
             return null; throw error; });
         if (!info?.isFile())
             return;
+        const version = fileVersion(info);
+        if (lane === "live" && this.oversized.get(path) === version)
+            return;
+        this.oversized.delete(path);
+        try {
+            await this.readVersion(path, lane, force, info);
+        }
+        catch (error) {
+            if (error instanceof SizeLimitError)
+                this.oversized.set(path, version);
+            throw error;
+        }
+    }
+    async readVersion(path, lane, force, info) {
         const previous = this.states.get(path);
         // Unchanged bytes can still hold a Turn whose end the control plane has only now reported.
         if (previous && this.unchanged(previous, info) && !(force && previous.runningTurnId))
@@ -565,6 +618,8 @@ export class NativeIngest {
         const certain = this.options.hooks?.settled?.(state, mtimeMs);
         if (certain != null)
             return certain;
+        if (tail.ended)
+            return true;
         if (!format.replyEnded?.(tail))
             return false;
         const idleMs = Date.now() - mtimeMs;
@@ -598,9 +653,20 @@ export class NativeIngest {
         const known = pending.length ? await this.known(transport, pending.flatMap((turn) => turn.cloudTurnId ? [turn.cloudTurnId, derived(turn)] : [derived(turn)])) : new Map();
         const cohub = (turn) => turn.cloudTurnId && known.has(turn.cloudTurnId) ? turn.cloudTurnId : null;
         const tail = turns.at(-1);
+        // A Cohub file whose latest Cohub Turn the file cannot name (its format marks none, or it predates
+        // the marker) holds only Turns the server already has, up to its current end.
+        const cohubHead = state.executingTurnId ?? state.headTurnId;
+        const unmarked = !transcriptFormat(state.harness).marksCohubTurns || state.size < 0;
+        if (state.origin === "cohub" && cohubHead && unmarked && state.throughTurnId !== cohubHead && !turns.some((turn) => turn.cloudTurnId === cohubHead)) {
+            const through = tail ? cohub(tail) ?? cohubHead : cohubHead;
+            Object.assign(state, { throughTurnKey: tail?.key ?? state.throughTurnKey, throughTurnId: through, headTurnId: through, executingTurnId: null });
+            return;
+        }
+        state.executingTurnId = null;
         const head = !tail ? null : pending.length ? cohub(tail) ?? derived(tail) : state.throughTurnId;
         const candidates = [];
-        let parentTurnId = from > 0 ? state.throughTurnId : null;
+        // A head adopted without a transcript key (Cohub's Turn left no record) still parents what follows.
+        let parentTurnId = from > 0 || !state.throughTurnKey ? state.throughTurnId : null;
         for (const [offset, turn] of pending.entries()) {
             const marked = cohub(turn);
             if (marked) {
@@ -726,7 +792,7 @@ export class NativeIngest {
         return Date.parse(turn.startedAt) < Date.parse(this.enabledAt);
     }
     async archive(state, turn, turnId) {
-        if (!state.sessionId)
+        if (!state.sessionId || !harnessAdapter(state.harness).session.archive)
             return;
         try {
             const source = turn.path ?? state.path;

package/dist/runtime/native/sessions.js

@@ -31,7 +31,10 @@ export class NativeSessions {
             if (restored) {
                 // A restored copy is history the server already has, even where it carries no markers.
                 const through = (await readTranscript(restored.path, restored.harness).catch(() => null))?.turns.at(-1)?.key ?? null;
-                this.options.ingest.adopt(restored, input.sessionId, { turnKey: through, turnId: input.context.throughTurnId });
+                if (through)
+                    this.options.ingest.adopt(restored, input.sessionId, { turnKey: through, turnId: input.context.throughTurnId });
+                else
+                    this.options.ingest.adopt(restored, input.sessionId);
                 return { session: restored, resume: "restored" };
             }
         }
@@ -45,6 +48,8 @@ export class NativeSessions {
         const id = randomUUID();
         const raw = join(this.options.root, "native", "restored", `${id}.jsonl`);
         const format = harnessAdapter(input.harness).session.archive;
+        if (!format)
+            throw new ArchiveNotRestorableError(`${input.harness} archives are not restorable`);
         try {
             const restored = await this.options.archives.restore(archive, raw, signal);
             const path = format.target({ id, nativeSessionId: restored.nativeSessionId, cwd: input.cwd });

package/dist/runtime/native/transcript.d.ts

@@ -18,6 +18,8 @@ export type NativeTranscriptTurn = {
     boundaries: Record<number, string>;
     sha256: string;
     result: NativeTurnComplete | null;
+    /** The file itself recorded this Turn's end, so it settles without waiting for a quiet period. */
+    ended?: boolean;
     path?: string;
     rolloutId?: string;
     sequence?: number;
@@ -31,6 +33,18 @@ export type NativeTranscript = {
     lineagePaths?: string[];
     lineageRolloutIds?: string[];
 };
+export type TranscriptLine = {
+    value: JsonRecord;
+    startBytes: number;
+    endBytes: number;
+    sha256: string;
+};
+type Line = TranscriptLine;
+export declare const iso: (value: unknown) => string;
+/** A size limit that the same bytes will hit again; retrying before the file changes is wasted work. */
+export declare class SizeLimitError extends Error {
+    name: string;
+}
 export declare const NATIVE_MAX_RECORD_BYTES: number;
 export declare const NATIVE_MAX_TRANSCRIPT_BYTES: number;
 export declare const codexSessionsDirectory: () => string;
@@ -50,6 +64,8 @@ export type NativeTranscriptHeader = {
 };
 /** Read exactly the first JSONL record of a stream without re-concatenating per chunk. */
 export declare function readFirstRecord(stream: AsyncIterable<Buffer>): Promise<JsonRecord>;
+/** The first record a format without a header line identifies its session by; `null` when none is found early. */
+export declare function readFirstMatchingRecord(stream: AsyncIterable<Buffer>, match: (value: JsonRecord) => boolean): Promise<JsonRecord | null>;
 /** Read only the session header before deciding whether a global Codex file belongs to this project. */
 export declare function readNativeTranscriptHeader(path: string, harness: "pi" | "codex", signal?: AbortSignal): Promise<NativeTranscriptHeader>;
 /** Raw JSONL bytes of a rollout, transparently decompressing `.zst` and honoring a lineage cutoff. */
@@ -58,9 +74,16 @@ export declare function codexRolloutBytes(path: string, endBytes: number | null,
 export declare function codexRolloutCachePath(source: string, cacheDirectory: string): string;
 /** Archives need plain JSONL bytes; decompress a cold `.zst` rollout into a cache once. */
 export declare function ensurePlainCodexRollout(path: string, cacheDirectory: string, signal?: AbortSignal, maxBytes?: number): Promise<string>;
+type TranscriptLines = {
+    lines: Line[];
+    prefixes: Map<number, string>;
+};
+/** Newline framing with exact byte offsets and rolling checksums; a partial tail stays unparsed. */
+export declare function readTranscriptLines(stream: AsyncIterable<Buffer>): Promise<TranscriptLines>;
 /** Partial trailing records are retried, never parsed or acknowledged as complete. */
 export declare function readNativeTranscript(path: string, harness: "pi" | "codex", options?: {
     settled?: boolean;
     leafId?: string | null;
     signal?: AbortSignal;
 }): Promise<NativeTranscript>;
+export {};

package/dist/runtime/native/transcript.js

@@ -12,16 +12,22 @@ import { codexTokenTotals, codexUsage } from "../codex-usage.js";
 import { record } from "../json-rpc.js";
 const text = (value) => typeof value === "string" ? value : "";
 const list = (value) => Array.isArray(value) ? value : [];
-const iso = (value) => {
+export const iso = (value) => {
     const date = new Date(typeof value === "number" || typeof value === "string" ? value : 0);
     if (!Number.isFinite(date.getTime()))
         throw new Error("Invalid native timestamp");
     return date.toISOString();
 };
 const missing = (error) => error?.code === "ENOENT";
+/** A size limit that the same bytes will hit again; retrying before the file changes is wasted work. */
+export class SizeLimitError extends Error {
+    name = "SizeLimitError";
+}
 export const NATIVE_MAX_RECORD_BYTES = 256 * 1024 * 1024;
 export const NATIVE_MAX_TRANSCRIPT_BYTES = 1024 * 1024 * 1024;
 const NATIVE_HEADER_MAX_BYTES = 1024 * 1024;
+/** Formats without a header line are searched this far for the record naming the session's project. */
+const NATIVE_HEADER_SCAN_MAX_BYTES = 16 * 1024 * 1024;
 const CODEX_HISTORY_MAX_DEPTH = 64;
 const CODEX_ROLLOUT_INDEX_TTL_MS = 2_000;
 const CODEX_ROLLOUT_NAME = /^rollout-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-([0-9a-f-]{36})(?:_([0-9a-f-]{36}))?\.(?:jsonl|jsonl\.zst)$/;
@@ -64,6 +70,35 @@ export async function readFirstRecord(stream) {
     }
     throw new Error("Native transcript is empty");
 }
+/** The first record a format without a header line identifies its session by; `null` when none is found early. */
+export async function readFirstMatchingRecord(stream, match) {
+    let fragments = [], scanned = 0;
+    for await (const chunk of stream) {
+        const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        let start = 0;
+        for (let end = bytes.indexOf(10, start); end >= 0; end = bytes.indexOf(10, start)) {
+            const line = Buffer.concat([...fragments, bytes.subarray(start, end)]);
+            fragments = [];
+            start = end + 1;
+            if (!line.length)
+                continue;
+            let value;
+            try {
+                value = record(JSON.parse(line.toString("utf8")));
+            }
+            catch {
+                return null;
+            }
+            if (match(value))
+                return value;
+        }
+        fragments.push(bytes.subarray(start));
+        scanned += bytes.length;
+        if (scanned > NATIVE_HEADER_SCAN_MAX_BYTES)
+            return null;
+    }
+    return null;
+}
 async function readCodexHeader(path, signal) {
     const value = await readFirstRecord(codexRolloutBytes(path, null, signal));
     const payload = record(value.payload);
@@ -172,7 +207,7 @@ export async function ensurePlainCodexRollout(path, cacheDirectory, signal, maxB
             transform(chunk, _encoding, push) {
                 written += chunk.length;
                 if (written > maxBytes) {
-                    push(new Error("Decompressed rollout exceeds the local limit"));
+                    push(new SizeLimitError("Decompressed rollout exceeds the local limit"));
                     return;
                 }
                 push(null, chunk);
@@ -188,7 +223,7 @@ export async function ensurePlainCodexRollout(path, cacheDirectory, signal, maxB
     }
 }
 /** Newline framing with exact byte offsets and rolling checksums; a partial tail stays unparsed. */
-async function readTranscriptLines(stream) {
+export async function readTranscriptLines(stream) {
     const lines = [];
     const prefixes = new Map();
     let fragments = [], pendingBytes = 0, offset = 0;
@@ -200,9 +235,9 @@ async function readTranscriptLines(stream) {
         pendingBytes += bytes.length;
         checksum.update(bytes);
         if (pendingBytes > NATIVE_MAX_RECORD_BYTES)
-            throw new Error("Native record exceeds the 256 MiB local parse limit; original retained");
+            throw new SizeLimitError("Native record exceeds the 256 MiB local parse limit; original retained");
         if (offset + pendingBytes > NATIVE_MAX_TRANSCRIPT_BYTES)
-            throw new Error("Native transcript exceeds the 1 GiB local parse limit; original retained");
+            throw new SizeLimitError("Native transcript exceeds the 1 GiB local parse limit; original retained");
     };
     for await (const chunk of stream) {
         let start = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@neta-art/cohub-cli",
-  "version": "10.7.0",
+  "version": "10.8.1",
   "description": "CLI for Cohub — spaces, sessions, and agent collaboration.",
   "type": "module",
   "engines": {
@@ -18,8 +18,8 @@
     "NOTICE"
   ],
   "dependencies": {
-    "@neta-art/cohub": "11.5.1",
-    "@neta-art/generation": "^0.1.39",
+    "@neta-art/cohub": "12.1.0",
+    "@neta-art/generation": "^0.1.40",
     "commander": "^15.0.0",
     "sharp": "^0.35.4",
     "ws": "^8.21.3"