@spexcode/transcript 0.7.0-next.12 → 0.7.0-next.14

package/dist/parsers.d.ts

@@ -45,6 +45,7 @@ export declare class IntervalCollector {
     private readonly evicted;
     private readonly synthesized;
     sawTimestamp: boolean;
+    sawRecord: boolean;
     omittedTurns: number;
     omittedBytes: number;
     outOfOrderEvents: number;

package/dist/parsers.js

@@ -133,7 +133,14 @@ export function claudeEvent(value) {
             return { at: eventAt, turn: { id: idOf(entry) ?? idOf(message), at: eventAt, role: 'user', text, tools: [] } };
     }
     if (entry.type === 'assistant' && message.role === 'assistant') {
-        const turn = { id: idOf(entry) ?? idOf(message), at: eventAt, role: 'assistant', tools: [] };
+        // ONE API MESSAGE IS ONE TURN, however many lines Claude wrote it as. It writes one CONTENT BLOCK per
+        // line — prose on one, each tool call on its own — and every line of the same message repeats that
+        // message's `id` while carrying its own `uuid`. Keying on the line made one message up to six turns:
+        // measured over twelve recent threads, 5,077 assistant lines carry only 2,396 distinct message ids, so a
+        // turn count read off the lines is more than twice the truth. Keying on the message id lets the collector
+        // fold the fragments back together with the rule it already has for a re-emitted turn — text kept, calls
+        // merged by their own ids — so nothing here has to accumulate.
+        const turn = { id: idOf(message) ?? idOf(entry), at: eventAt, role: 'assistant', tools: [] };
         for (const blockValue of items(message.content)) {
             const block = object(blockValue);
             if (block?.type === 'text')
@@ -202,6 +209,17 @@ export function codexEvent(value) {
     }
     return null;
 }
+// A file edit names the paths it touched, and its result is the diff the app-server already computed. Both
+// come off `FileUpdateChange[]`, which is the only shape on the union whose payload is a list of records
+// rather than one field.
+const filePaths = (changes) => {
+    const paths = items(changes).map((change) => string(object(change)?.path)).filter(Boolean);
+    return paths.length ? paths.join('\n') : undefined;
+};
+const fileDiffs = (changes) => {
+    const diffs = items(changes).map((change) => string(object(change)?.diff)).filter(Boolean);
+    return diffs.length ? diffs.join('\n') : undefined;
+};
 // Codex app-server notifications are a different native stream from rollout lines. Keep this mapping stateless:
 // a caller that needs streamed prose uses codexAppServerStream below, while file and in-memory sources still
 // share the same one-record parser contract.
@@ -244,27 +262,35 @@ export function codexAppServerEvent(value) {
         const text = string(item.text);
         return { at: eventAt, turn: { id, at: eventAt, role: 'assistant', text: text ?? undefined, tools: [] } };
     }
-    const toolTypes = new Set(['commandExecution', 'functionCall', 'customToolCall', 'mcpToolCall', 'dynamicToolCall']);
+    // The tool-bearing variants of the app-server's own `ThreadItem` union. `functionCall` and `customToolCall`
+    // used to be listed here and are NOT members of it — those are ROLLOUT record types, and no such string
+    // exists anywhere in the app-server binary. `fileChange` is a real one that was missing, so a codex file
+    // edit appeared as no call at all.
+    const toolTypes = new Set(['commandExecution', 'fileChange', 'mcpToolCall', 'dynamicToolCall']);
     if (!toolTypes.has(type))
         return null;
     if (method === 'item/started') {
         const name = type === 'commandExecution' ? 'command'
-            : string(item.name) ?? string(item.tool) ?? (type === 'mcpToolCall' ? 'mcp' : 'tool');
+            : type === 'fileChange' ? 'edit'
+                : string(item.name) ?? string(item.tool) ?? (type === 'mcpToolCall' ? 'mcp' : 'tool');
         const input = item.arguments !== undefined ? item.arguments
             : item.input !== undefined ? item.input
                 : item.command !== undefined ? item.command
-                    : undefined;
+                    : type === 'fileChange' ? filePaths(item.changes)
+                        : undefined;
         return { at: eventAt, turn: { id, at: eventAt, role: 'assistant', tools: [{ id, name, input: input === undefined ? undefined : compact(input), outputLines: 0, outputBytes: 0 }] } };
     }
     let output = undefined;
     if (type === 'commandExecution')
         output = item.aggregatedOutput;
-    else if (type === 'functionCall' || type === 'customToolCall')
-        output = item.output ?? item.result;
+    // an MCP result is `{content, structuredContent, _meta}`: the text is in `content`, and handing the wrapper
+    // to `resultText` printed the JSON envelope instead of what the tool said
     else if (type === 'mcpToolCall')
-        output = item.result ?? item.error;
+        output = object(item.result)?.content ?? item.result ?? object(item.error)?.message ?? item.error;
     else if (type === 'dynamicToolCall')
         output = item.contentItems ?? item.output ?? item.error;
+    else if (type === 'fileChange')
+        output = fileDiffs(item.changes);
     // the item status is the app-server's own verdict: `failed`, or `declined` when the person refused the call —
     // a declined call has no output, so the empty result is what ends its "running"
     const status = string(item.status);
@@ -498,6 +524,7 @@ export class IntervalCollector {
     evicted = new Set();
     synthesized = new Map(); // `<role>@<at>` → how many turns already wore it
     sawTimestamp = false;
+    sawRecord = false; // a record this harness's parser RECOGNIZED arrived, with or without a clock
     omittedTurns = 0;
     omittedBytes = 0;
     outOfOrderEvents = 0;
@@ -509,6 +536,7 @@ export class IntervalCollector {
         this.range.to = to; }
     // returns true once the source has moved past `to` (the caller may then bound its lookahead)
     add(event) {
+        this.sawRecord = true;
         const eventAt = event.at;
         if (eventAt === null)
             return this.pastRange;
@@ -581,7 +609,13 @@ export class IntervalCollector {
         return this.pastRange;
     }
     finish(revision, harness) {
-        if (!this.sawTimestamp)
+        // THE CLOCK GATE IS ABOUT THE HARNESS, NOT THE MOMENT. It catches a source whose conversational records
+        // carry no usable time, which makes interval reads impossible. It must NOT catch a thread that has simply
+        // not spoken yet: every Claude transcript opens with clockless bookkeeping (`mode`, `permission-mode`,
+        // `file-history-snapshot`) before its first message — 40 of 40 real threads on this box — and those lines
+        // are not records this parser recognizes at all, so failing on them put an error frame on the page for the
+        // first moments of EVERY new session, which is exactly when someone is watching.
+        if (this.sawRecord && !this.sawTimestamp)
             throw new TranscriptReadError('invalid', `${harness} transcript has no reliable timestamps; interval reads are unavailable`);
         return {
             revision,

package/dist/readers.d.ts

@@ -4,13 +4,18 @@ export declare function codexRolloutPath(threadId: string, root?: string, archiv
 export declare function piSessionPath(threadId: string, root?: string): string | null;
 export declare function geminiTranscriptPath(threadId: string, root?: string): string | null;
 export declare function openclawTranscriptPath(threadId: string, root?: string): string | null;
+export declare const claudeTranscriptReader: (root?: string) => TranscriptReader;
+export declare const codexTranscriptReader: (root?: string) => TranscriptReader;
+export declare const piTranscriptReader: (root?: string) => TranscriptReader;
+export declare const geminiTranscriptReader: (root?: string) => TranscriptReader;
+export declare const openclawTranscriptReader: (root?: string) => TranscriptReader;
 export declare const claudeTranscript: TranscriptReader;
 export declare const codexTranscript: TranscriptReader;
 export declare const piTranscript: TranscriptReader;
 export declare const geminiTranscript: TranscriptReader;
 export declare const openclawTranscript: TranscriptReader;
 export declare function opencodeTranscriptReader(root?: string, load?: (threadId: string) => string): TranscriptReader;
-export declare const opencodeTranscript: TranscriptReader;
 export declare function hermesTranscriptReader(root?: string, load?: (threadId: string) => string): TranscriptReader;
+export declare const opencodeTranscript: TranscriptReader;
 export declare const hermesTranscript: TranscriptReader;
 export declare function unsupportedTranscript(harness: string): TranscriptReader;

package/dist/readers.js

@@ -108,29 +108,9 @@ const openclawRoot = () => process.env.OPENCLAW_STATE_DIR || join(homedir(), '.o
 export function openclawTranscriptPath(threadId, root = openclawRoot()) { return findJsonl(root, threadId); }
 const opencodeStoreRoot = () => process.env.SPEXCODE_OPENCODE_DATA_DIR
     || join(process.env.XDG_DATA_HOME || join(homedir(), '.local', 'share'), 'opencode');
-function opencodeStoreRevision(root) {
-    try {
-        const database = statSync(join(root, 'opencode.db'));
-        let writeAheadLog = '0:0';
-        try {
-            const stat = statSync(join(root, 'opencode.db-wal'));
-            writeAheadLog = `${stat.size}:${Math.floor(stat.mtimeMs)}`;
-        }
-        catch { /* a checkpointed database has no separate write-ahead log */ }
-        return `${database.size}:${Math.floor(database.mtimeMs)}:${writeAheadLog}`;
-    }
-    catch {
-        return null;
-    }
-}
 // The export is read RAW: `--sanitize` replaces every prose and tool-output part with a `[redacted:…]` token,
 // which made the whole conversation unreadable; the reader hands over the same local bytes the other harnesses'
 // files hold, and nothing here leaves the machine that ran the thread.
-function opencodeExport(threadId) {
-    return execFileSync(process.env.SPEXCODE_OPENCODE_CMD || 'opencode', ['export', threadId], {
-        encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'],
-    });
-}
 const fileRevision = (path) => {
     try {
         const stat = statSync(path);
@@ -147,7 +127,14 @@ const intervalOffsets = new Map();
 // One pass over the bytes from `scan.position` to the end of the file. Every complete line is parsed as JSON
 // and handed to `onLine` with its byte offset; `onLine` returning true stops the scan early (a bounded
 // lookahead), which abandons the rest — only a one-shot read does that.
-function scanLines(harness, fd, scan, onLine) {
+// ONE UNREADABLE LINE IS OMITTED PAYLOAD, NOT AN UNREADABLE TRANSCRIPT. A native log is written by another
+// process and can carry a line that is not JSON — a truncated record from a crash, a line someone appended by
+// hand. Throwing on it makes the whole thread unreadable forever, which is the loudest possible failure and
+// the least useful one: the person loses a conversation over one bad line. The reader already has an honest
+// word for this — the line's bytes are counted as omitted and the read reports `truncated`, exactly as it does
+// for a result past the cap. A file that is not this format at all still fails loudly, because nothing in it
+// parses and `finish()` refuses a read that never saw a timestamp.
+function scanLines(fd, scan, onLine, onUnparsable) {
     const chunk = Buffer.allocUnsafe(64 * 1024);
     let { position, carry } = scan;
     let lineStart = position - carry.length;
@@ -171,8 +158,9 @@ function scanLines(harness, fd, scan, onLine) {
             try {
                 value = JSON.parse(line);
             }
-            catch (error) {
-                throw new TranscriptReadError('invalid', `${harness} transcript cannot be parsed: ${error instanceof Error ? error.message : String(error)}`);
+            catch {
+                onUnparsable(Buffer.byteLength(line));
+                continue;
             }
             if (onLine(value, lineOffset))
                 return { position, carry: Buffer.alloc(0) };
@@ -217,8 +205,13 @@ class LineFileCursor {
         catch (error) {
             throw new TranscriptReadError('unreadable', `${this.harness} transcript is unreadable: ${error instanceof Error ? error.message : String(error)}`);
         }
+        // AN EMPTY FILE IS A THREAD THAT HAS NOT SPOKEN YET, not a broken one. The harness creates the transcript
+        // before it writes the first record, so the moments right after a session starts — exactly when a person is
+        // watching — read as zero bytes. Failing there put an error on the page for a conversation that simply had
+        // not begun. Zero bytes is unambiguous in a way a garbled file is not: there is nothing to misread, so this
+        // is the one place the no-timestamp gate is skipped rather than tripped.
         if (size <= 0)
-            throw new TranscriptReadError('unreadable', `${this.harness} transcript is unreadable: file is empty`);
+            return { revision: fileRevision(this.path) ?? '0', from: this.from, to, turns: [], truncated: false, omittedTurns: 0, omittedBytes: 0, outOfOrderEvents: 0 };
         // a source that shrank was rewritten underneath the cursor: forget the position and read the interval afresh
         if (!this.started || size < this.scan.position) {
             this.restart(size);
@@ -228,8 +221,9 @@ class LineFileCursor {
         let fd = null;
         try {
             fd = openSync(this.path, 'r');
-            let postRangeLines = 0;
-            this.scan = scanLines(this.harness, fd, this.scan, (value, offset) => {
+            let postRangeLines = 0, parsedLines = 0, unparsableLines = 0;
+            this.scan = scanLines(fd, this.scan, (value, offset) => {
+                parsedLines++;
                 const event = this.parse(value);
                 if (!event)
                     return false;
@@ -238,7 +232,11 @@ class LineFileCursor {
                     intervalOffsets.set(this.seekKey, offset);
                 const pastRange = this.collector.add(event);
                 return pastRange && ++postRangeLines >= lookahead;
-            });
+            }, (bytes) => { unparsableLines++; this.collector.omittedBytes += bytes; });
+            // a file in which NOTHING is JSON is not this thread's transcript at all — that is the loud case, and it
+            // stays loud; a file whose lines parse but say nothing conversational yet is simply not started
+            if (!parsedLines && unparsableLines > 0)
+                throw new TranscriptReadError('invalid', `${this.harness} transcript cannot be parsed: no line is JSON`);
         }
         catch (error) {
             if (error instanceof TranscriptReadError)
@@ -274,100 +272,100 @@ function lineFileReader(harness, locate, parse) {
         },
     };
 }
-export const claudeTranscript = lineFileReader('claude', (threadId) => claudeTranscriptPath(threadId), claudeEvent);
-export const codexTranscript = lineFileReader('codex', (threadId) => codexRolloutPath(threadId), codexEvent);
-export const piTranscript = lineFileReader('pi', (threadId) => piSessionPath(threadId), piEvent);
-export const geminiTranscript = lineFileReader('gemini', (threadId) => geminiTranscriptPath(threadId), geminiEvent);
-export const openclawTranscript = lineFileReader('openclaw', (threadId) => openclawTranscriptPath(threadId), openclawEvent);
-// OpenCode has no per-thread file: the store's revision is the change token, and one export per
-// revision is parsed and kept, so repeated interval reads of a quiet thread cost nothing new.
-const opencodeExports = new Map();
-export function opencodeTranscriptReader(root = opencodeStoreRoot(), load = opencodeExport) {
-    const reader = {
-        revision: () => opencodeStoreRevision(root),
-        read: async (threadId, range) => {
-            const revision = opencodeStoreRevision(root);
-            if (!revision)
-                throw new TranscriptReadError('missing', `opencode transcript for ${threadId} is unavailable: store was not found`);
-            const key = `${root}:${threadId}`;
-            let cached = opencodeExports.get(key);
-            if (!cached || cached.revision !== revision) {
-                let exported;
-                try {
-                    exported = load(threadId);
-                }
-                catch (error) {
-                    throw new TranscriptReadError('unreadable', `opencode transcript could not be exported: ${error instanceof Error ? error.message : String(error)}`);
-                }
-                let value;
-                try {
-                    value = JSON.parse(exported);
-                }
-                catch (error) {
-                    throw new TranscriptReadError('invalid', `opencode transcript cannot be parsed: ${error instanceof Error ? error.message : String(error)}`);
-                }
-                cached = { revision, events: opencodeEvents(value) };
-                opencodeExports.set(key, cached);
-            }
-            const collector = new IntervalCollector(range);
-            for (const event of cached.events)
-                collector.add(event);
-            return collector.finish(revision, 'opencode');
-        },
-    };
-    return {
-        ...reader,
-        // no file grows here: an open interval is re-collected from the cached export, which is one export per revision
-        tail: (threadId, from) => ({ advance: (to) => reader.read(threadId, { from, to }), close: () => { } }),
-    };
-}
-export const opencodeTranscript = opencodeTranscriptReader();
-const hermesRoot = () => process.env.HERMES_HOME || join(homedir(), '.hermes', 'profiles', 'default');
-function hermesRevision(root) {
-    try {
-        const stat = statSync(join(root, 'state.db'));
-        return `${stat.size}:${Math.floor(stat.mtimeMs)}`;
-    }
-    catch {
-        return null;
+// A ROOT IS A PARAMETER OF EVERY READER, not of some of them. Each locator already takes one; only the store
+// readers exposed it, so anything wanting a second pi root — a producer under an isolated agent dir, a test —
+// had no way to ask. Passing nothing keeps the old behaviour exactly: the locator's own default is evaluated
+// per call, so a late `CLAUDE_CONFIG_DIR` is still picked up.
+export const claudeTranscriptReader = (root) => lineFileReader('claude', (threadId) => claudeTranscriptPath(threadId, root ?? projectTranscriptRoot()), claudeEvent);
+export const codexTranscriptReader = (root) => lineFileReader('codex', (threadId) => codexRolloutPath(threadId, root ?? codexSessionsDir()), codexEvent);
+export const piTranscriptReader = (root) => lineFileReader('pi', (threadId) => piSessionPath(threadId, root ?? piSessionsRoot()), piEvent);
+export const geminiTranscriptReader = (root) => lineFileReader('gemini', (threadId) => geminiTranscriptPath(threadId, root ?? geminiRoot()), geminiEvent);
+export const openclawTranscriptReader = (root) => lineFileReader('openclaw', (threadId) => openclawTranscriptPath(threadId, root ?? openclawRoot()), openclawEvent);
+export const claudeTranscript = claudeTranscriptReader();
+export const codexTranscript = codexTranscriptReader();
+export const piTranscript = piTranscriptReader();
+export const geminiTranscript = geminiTranscriptReader();
+export const openclawTranscript = openclawTranscriptReader();
+function storeRevision(root, files) {
+    const legs = [];
+    for (const [index, name] of files.entries()) {
+        try {
+            const stat = statSync(join(root, name));
+            legs.push(`${stat.size}:${Math.floor(stat.mtimeMs)}`);
+        }
+        catch {
+            // the database itself must exist; a checkpointed store simply has no separate write-ahead log
+            if (index === 0)
+                return null;
+            legs.push('0:0');
+        }
     }
+    return legs.join(':');
 }
-function hermesExport(threadId) {
-    return execFileSync(process.env.SPEXCODE_HERMES_CMD || 'hermes', ['sessions', 'export', '--format', 'jsonl', '--session-id', threadId, '--yes'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
-}
-const hermesExports = new Map();
-export function hermesTranscriptReader(root = hermesRoot(), load = hermesExport) {
+const storeExports = new Map();
+function storeReader(source, root, load) {
     const read = async (threadId, range) => {
-        const revision = hermesRevision(root);
+        const revision = storeRevision(root, source.files);
         if (!revision)
-            throw new TranscriptReadError('missing', `hermes transcript for ${threadId} is unavailable: state.db was not found`);
-        const key = `${root}:${threadId}`;
-        let cached = hermesExports.get(key);
+            throw new TranscriptReadError('missing', `${source.harness} transcript for ${threadId} is unavailable: ${source.missing}`);
+        const key = `${source.harness}:${root}:${threadId}`;
+        let cached = storeExports.get(key);
         if (!cached || cached.revision !== revision) {
             let exported;
             try {
                 exported = load(threadId);
             }
             catch (error) {
-                throw new TranscriptReadError('unreadable', `hermes transcript could not be exported: ${error instanceof Error ? error.message : String(error)}`);
+                throw new TranscriptReadError('unreadable', `${source.harness} transcript could not be exported: ${error instanceof Error ? error.message : String(error)}`);
             }
             let value;
             try {
                 value = JSON.parse(exported);
             }
             catch (error) {
-                throw new TranscriptReadError('invalid', `hermes transcript cannot be parsed: ${error instanceof Error ? error.message : String(error)}`);
+                throw new TranscriptReadError('invalid', `${source.harness} transcript cannot be parsed: ${error instanceof Error ? error.message : String(error)}`);
             }
-            cached = { revision, events: hermesEvents(value) };
-            hermesExports.set(key, cached);
+            cached = { revision, events: source.parse(value) };
+            storeExports.set(key, cached);
         }
         const collector = new IntervalCollector(range);
         for (const event of cached.events)
             collector.add(event);
-        return collector.finish(revision, 'hermes');
+        return collector.finish(revision, source.harness);
+    };
+    return {
+        revision: () => storeRevision(root, source.files),
+        read,
+        // no file grows here: an open interval is re-collected from the cached export, one export per revision
+        tail: (threadId, from) => ({ advance: (to) => read(threadId, { from, to }), close: () => { } }),
     };
-    return { revision: () => hermesRevision(root), read, tail: (threadId, from) => ({ advance: (to) => read(threadId, { from, to }), close: () => { } }) };
 }
+// The export is read RAW: OpenCode's `--sanitize` replaces every prose and tool-output part with a
+// `[redacted:…]` token, which made the whole conversation unreadable; the reader hands over the same local
+// bytes the other harnesses' files hold, and nothing here leaves the machine that ran the thread.
+const OPENCODE_STORE = {
+    harness: 'opencode',
+    defaultRoot: opencodeStoreRoot,
+    files: ['opencode.db', 'opencode.db-wal'],
+    missing: 'store was not found',
+    load: (threadId) => execFileSync(process.env.SPEXCODE_OPENCODE_CMD || 'opencode', ['export', threadId], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }),
+    parse: opencodeEvents,
+};
+const HERMES_STORE = {
+    harness: 'hermes',
+    defaultRoot: () => process.env.HERMES_HOME || join(homedir(), '.hermes', 'profiles', 'default'),
+    files: ['state.db', 'state.db-wal'],
+    missing: 'state.db was not found',
+    load: (threadId) => execFileSync(process.env.SPEXCODE_HERMES_CMD || 'hermes', ['sessions', 'export', '--format', 'jsonl', '--session-id', threadId, '--yes'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }),
+    parse: hermesEvents,
+};
+export function opencodeTranscriptReader(root = OPENCODE_STORE.defaultRoot(), load = OPENCODE_STORE.load) {
+    return storeReader(OPENCODE_STORE, root, load);
+}
+export function hermesTranscriptReader(root = HERMES_STORE.defaultRoot(), load = HERMES_STORE.load) {
+    return storeReader(HERMES_STORE, root, load);
+}
+export const opencodeTranscript = opencodeTranscriptReader();
 export const hermesTranscript = hermesTranscriptReader();
 export function unsupportedTranscript(harness) {
     const refuse = async () => { throw new TranscriptReadError('unsupported', `${harness} does not support transcript access`); };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spexcode/transcript",
-  "version": "0.7.0-next.12",
+  "version": "0.7.0-next.14",
   "type": "module",
   "description": "Normalized agent transcripts: one parser per harness, a bounded interval reader over a native thread file or an in-memory event stream, and the full/delta frame protocol every transport and renderer share.",
   "files": [