@phnx-labs/agents-cli 1.22.120 → 1.22.121

package/dist/lib/session/timeline-pass.js

@@ -1,149 +1,173 @@
-/**
- * The daemon's incremental timeline pass (PHNX-3939).
- *
- * Runs inside the existing `SessionStateService` tick, which is already
- * reader-gated — no watcher, no work — and already visits every live session.
- * This adds the one thing the tick did not do: fold each live transcript's
- * NEW BYTES into its cached {@link TimelineState} and store the bounded
- * projection a session row merges.
- *
- * Three properties keep it off the request path and off the CPU:
- *   - **Reader-gated and budgeted.** Nothing runs unless a `sessions watch`
- *     consumer is attached, and at most `budget` sessions are folded per tick.
- *   - **Appended bytes only.** For the resumable harnesses (Claude and Codex,
- *     line-delimited JSON) a fold reads `size - offset` bytes, not the file.
- *     The same resume rule the tool index already uses (`tool-index.ts`).
- *   - **Complete records only.** Only newline-terminated lines are folded and
- *     the offset advances past those alone, so the 973,963-byte record in one
- *     live transcript on this fleet cannot be folded half-written; the next tick
- *     picks it up whole.
- */
 import * as fs from 'fs';
 import { readSessionTimelineEntry, writeSessionTimeline } from './db.js';
-import { parseClaudeContent, parseCodexItemsContent, parseSession } from './parse.js';
+import { parseClaudeContent, parseCodexItemsContent, parseSession } from '@phnx-labs/sessions-cli/reader';
 import { toolEvidenceSourcePath } from './tool-store.js';
-import { compactTimelineState, emptyTimelineState, foldTimeline, projectSessionFiles, projectTimeline, unavailableTimeline, TIMELINE_EXTRACTOR_VERSION, } from './timeline.js';
-/** Sessions folded per tick. The tick's own deadline is 30 s; this stays well inside it. */
+import { compactTimelineState, emptyTimelineState, foldTimeline, projectGlance, projectSessionFiles, projectTimeline, unavailableTimeline, TIMELINE_EXTRACTOR_VERSION, } from '@phnx-labs/sessions-cli/reader';
+import { appendInlineImage, dropOpenInlineImage, finishInlineImage, hasOpenInlineImage, materializeInlineImages, readSessionSubagents, takePendingImagePaths } from './glance-files.js';
 const TIMELINE_PASS_MAX_PER_TICK = 8;
-/** Bytes one session may consume in one tick. A bigger backlog catches up over ticks. */
 export const TIMELINE_PASS_MAX_BYTES_PER_SESSION = 4 * 1024 * 1024;
-/**
- * Bytes the whole pass may consume in one tick, across every session.
- *
- * The per-session cap alone is not a bound on the TICK: on a cold cache eight
- * live sessions each fold from offset 0, and eight multi-megabyte transcripts
- * parsed back to back is what blew the 30 s `session-state` deadline and parked
- * the service on this fleet (observed 2026-09-06). A tick budget makes a cold
- * start catch up over a few ticks instead of stalling one.
- */
 const TIMELINE_PASS_MAX_BYTES_PER_TICK = 8 * 1024 * 1024;
-/**
- * Ceiling on a whole-file re-parse for a harness with no resumable reader.
- * Beyond it the timeline is reported `partial` rather than wedging the tick —
- * the same honesty rule the tool index applies at its own in-memory limit.
- */
 export const TIMELINE_PASS_MAX_WHOLE_FILE_BYTES = 16 * 1024 * 1024;
-/** Events folded from one non-resumable whole-file parse before reporting `partial`. */
 const TIMELINE_PASS_MAX_EVENTS = 20_000;
-/**
- * How long a non-resumable harness's timeline may go stale before the pass
- * re-parses its transcript whole.
- *
- * Kimi and Grok expose no byte offset to resume from, so their only option is a
- * full parse — and a full parse of a large, actively-appended transcript wedges
- * the Node event loop for seconds (the PHNX-3411 incident, which is why the tool
- * index deferred those two harnesses to their own budgeted pass). Re-parsing
- * every 15 s would reproduce it. One re-parse per minute keeps such a session's
- * timeline current enough for a card that moves about once a minute, at a
- * quarter of the cost.
- */
 export const TIMELINE_PASS_NON_RESUMABLE_MIN_INTERVAL_MS = 60_000;
-/**
- * Harnesses whose transcript is append-only line-delimited JSON that the fold
- * can resume from a byte offset. Everything else is re-parsed whole (bounded)
- * because its parser exposes no offset — the same split as
- * `isResumableToolSource` in `tool-index.ts`.
- */
 function isResumableTimelineSource(agent) {
     return agent === 'claude' || agent === 'codex';
 }
-/** Harnesses that write no parseable transcript at all — stated, never faked as empty. */
 const NO_TRANSCRIPT_REASON = {
     openclaw: 'OpenClaw writes no parseable transcript, so there are no steps to fold',
 };
-/** Read `[start, end)` of a file, keeping only complete newline-terminated lines. */
-function readCompleteLines(filePath, start, end, maxBytes) {
+const MAX_PARTIAL_TEXT = 256 * 1024;
+function utf8Cut(buffer, length) {
+    let i = length - 1;
+    while (i >= 0 && i >= length - 4 && (buffer[i] & 0xc0) === 0x80)
+        i--;
+    if (i < 0 || i < length - 4)
+        return length;
+    const byte = buffer[i];
+    const need = byte < 0x80 ? 1 : byte < 0xe0 ? 2 : byte < 0xf0 ? 3 : 4;
+    return length - i < need ? i : length;
+}
+function readCompleteLines(filePath, start, end, maxBytes, allowPartial) {
     const stop = Math.min(end, start + maxBytes);
     if (stop <= start)
-        return { text: '', offset: start };
+        return { text: '', offset: start, partial: false };
     const fd = fs.openSync(filePath, 'r');
     try {
         const buffer = Buffer.alloc(stop - start);
         const read = fs.readSync(fd, buffer, 0, buffer.length, start);
         const lastNewline = buffer.lastIndexOf(0x0a, read - 1);
-        // Nothing complete in this window: leave the offset where it was so the
-        // record is folded whole once its closing newline lands.
-        if (lastNewline < 0)
-            return { text: '', offset: start };
+        if (lastNewline >= 0) {
+            return {
+                text: buffer.toString('utf8', 0, lastNewline + 1),
+                offset: start + lastNewline + 1,
+                partial: false,
+            };
+        }
+        const atEof = start + read >= end;
+        if (!allowPartial || atEof || read < stop - start)
+            return { text: '', offset: start, partial: false };
+        const cut = utf8Cut(buffer, read);
+        if (cut <= 0)
+            return { text: '', offset: start, partial: false };
         return {
-            text: buffer.toString('utf8', 0, lastNewline + 1),
-            offset: start + lastNewline + 1,
+            text: buffer.toString('utf8', 0, cut),
+            offset: start + cut,
+            partial: true,
         };
     }
     finally {
         fs.closeSync(fd);
     }
 }
-/**
- * Normalize one transcript chunk into the events the fold consumes.
- *
- * Interrupts are user steps in the timeline and the harness file ledger is the
- * Files card, so both are opted in — the published stream every OTHER consumer
- * reads stays unchanged. Codex reads its typed `item_completed` stream, NOT the
- * `response_item` records `parseCodexContent` reads: the two describe the same
- * turn, so folding both would double-count every call (see
- * `parseCodexItemsContent`).
- */
+function resumePartialLine(prior, chunk, sessionId, captureImages) {
+    let text = prior?.text ?? '';
+    let skipping = prior?.skippingData ?? false;
+    let discarded = prior?.discarded === true;
+    const lines = [];
+    let i = 0;
+    const lostImage = skipping && captureImages && !!sessionId && !hasOpenInlineImage(sessionId);
+    if (skipping) {
+        const quote = chunk.indexOf('"');
+        if (quote < 0) {
+            if (captureImages && sessionId && !lostImage)
+                appendInlineImage(sessionId, chunk, 'image/png');
+            return { lines, partial: { text, skippingData: true, ...(discarded ? { discarded: true } : {}) } };
+        }
+        if (captureImages && sessionId && !lostImage) {
+            appendInlineImage(sessionId, chunk.slice(0, quote), mediaTypeOf(text));
+            finishInlineImage(sessionId);
+        }
+        text += '"';
+        skipping = false;
+        i = quote + 1;
+    }
+    while (i < chunk.length) {
+        if (discarded) {
+            const nl = chunk.indexOf('\n', i);
+            if (sessionId)
+                dropOpenInlineImage(sessionId);
+            if (nl < 0)
+                return { lines, partial: { text: '', skippingData: false, discarded: true } };
+            discarded = false;
+            text = '';
+            i = nl + 1;
+            continue;
+        }
+        const nl = chunk.indexOf('\n', i);
+        const regionEnd = nl < 0 ? chunk.length : nl;
+        const region = chunk.slice(i, regionEnd);
+        const tail = text.slice(-32);
+        const hay = tail + region;
+        const marker = hay.match(/"data"\s*:\s*"/);
+        const markerAt = marker?.index ?? -1;
+        const markerEnd = marker ? markerAt + marker[0].length : -1;
+        if (marker && markerEnd >= tail.length && isImageData(hay.slice(0, markerAt))) {
+            const dataStart = markerEnd - tail.length;
+            const rest = dataStart < 0 ? region : region.slice(dataStart);
+            if (dataStart > 0)
+                text += region.slice(0, dataStart);
+            const quote = rest.indexOf('"');
+            if (quote < 0) {
+                if (captureImages && sessionId)
+                    appendInlineImage(sessionId, rest, mediaTypeOf(text));
+                const next = capped(text, true);
+                if (next.discarded && sessionId)
+                    dropOpenInlineImage(sessionId);
+                return { lines, partial: next };
+            }
+            if (captureImages && sessionId) {
+                appendInlineImage(sessionId, rest.slice(0, quote), mediaTypeOf(text));
+                finishInlineImage(sessionId);
+            }
+            text += '"';
+            i += (dataStart < 0 ? 0 : dataStart) + quote + 1;
+            continue;
+        }
+        if (nl < 0) {
+            text += region;
+            if (text.length > MAX_PARTIAL_TEXT) {
+                if (sessionId)
+                    dropOpenInlineImage(sessionId);
+                return { lines, partial: { text: '', skippingData: false, discarded: true } };
+            }
+            return { lines, partial: text ? { text, skippingData: false } : undefined };
+        }
+        text += region;
+        if (text.trim())
+            lines.push(text);
+        text = '';
+        i = nl + 1;
+    }
+    return { lines, partial: undefined };
+}
+function capped(text, skippingData) {
+    if (text.length <= MAX_PARTIAL_TEXT)
+        return { text, skippingData };
+    return { text: '', skippingData: false, discarded: true };
+}
+function mediaTypeOf(text) {
+    return text.match(/"media_type"\s*:\s*"(image\/(?:png|jpeg|gif|webp))"/)?.[1] ?? 'image/png';
+}
+function isImageData(before) {
+    return before.slice(-180).includes('base64');
+}
 function eventsForChunk(agent, text) {
     if (agent === 'claude') {
-        return parseClaudeContent(text, { includeInterrupts: true, includeFileHistory: true });
+        return parseClaudeContent(text, { includeInterrupts: true, includeFileHistory: true, includeInlineImages: true });
     }
     return parseCodexItemsContent(text);
 }
-/**
- * Every event of one transcript, read the way the timeline fold needs them.
- *
- * The single source of truth for "which reader does this harness's timeline
- * use", shared by the daemon pass's non-resumable branch and
- * `agents sessions trace --steps` — so the CLI door and the cached row can never
- * fold two different event streams for the same session.
- */
 export function parseTimelineEvents(filePath, agent) {
     if (isResumableTimelineSource(agent)) {
         return eventsForChunk(agent, fs.readFileSync(filePath, 'utf8'));
     }
-    return parseSession(filePath, agent, { includeInterrupts: true, includeFileHistory: true });
+    return parseSession(filePath, agent, { includeInterrupts: true, includeFileHistory: true, includeInlineImages: true });
 }
-/** MiB, for the human-readable reasons a row carries. */
 function mib(bytes) {
     return Math.round((bytes / (1024 * 1024)) * 10) / 10;
 }
-/**
- * Fold one session's new transcript bytes and return the entry to cache.
- * Exported for tests: this is where the resume decision and the whole-file
- * fallback live, and both are worth pinning without a daemon.
- *
- * `byteBudget` is the TICK's remaining allowance. Each branch derives its own
- * ceiling from it — the resumable chunk read is additionally capped per session,
- * while a whole-file re-parse is gated on a budget that can actually reach a
- * real transcript. A session that genuinely does not fit gets a `partial` row
- * stating why, never silence.
- */
 function foldSessionTimeline(session, filePath, fileSize, prior, nowMs = Date.now(), byteBudget = TIMELINE_PASS_MAX_BYTES_PER_TICK) {
     const agent = (session.kind ?? 'claude');
-    // A settled state stamped at the file's current size, so an unavailable or
-    // over-limit session is recognized as up to date next tick instead of being
-    // re-decided every 15 s.
     const settledAt = () => ({ ...emptyTimelineState(), offset: fileSize });
     const unavailable = NO_TRANSCRIPT_REASON[agent];
     if (unavailable) {
@@ -153,29 +177,38 @@ function foldSessionTimeline(session, filePath, fileSize, prior, nowMs = Date.no
     let events;
     let offset;
     let bytesRead;
+    let partial;
     if (isResumableTimelineSource(agent)) {
         const resumable = prior?.state
             && prior.state.version === TIMELINE_EXTRACTOR_VERSION
             && prior.state.offset <= fileSize;
         state = resumable ? prior.state : emptyTimelineState();
+        if (!resumable && session.sessionId)
+            dropOpenInlineImage(session.sessionId);
         const start = resumable ? state.offset : 0;
         const maxBytes = Math.min(byteBudget, TIMELINE_PASS_MAX_BYTES_PER_SESSION);
-        const chunk = readCompleteLines(filePath, start, fileSize, maxBytes);
-        // Nothing COMPLETE in the window this tick could read — a record still being
-        // written, or a byte budget too small to reach the next newline. Leave the
-        // cached row and its offset alone; writing here would stamp an empty fold
-        // over a real one and move the resume point past bytes never folded.
+        const chunk = readCompleteLines(filePath, start, fileSize, maxBytes, maxBytes === TIMELINE_PASS_MAX_BYTES_PER_SESSION);
         if (!chunk.text && chunk.offset === start)
             return undefined;
-        events = eventsForChunk(agent, chunk.text);
+        const resumed = resumePartialLine(resumable ? state.partialLine : undefined, chunk.text, session.sessionId, agent === 'claude');
+        partial = chunk.partial ? resumed.partial : undefined;
+        if (!chunk.partial && session.sessionId && resumed.partial?.discarded)
+            dropOpenInlineImage(session.sessionId);
+        events = resumed.lines.length ? eventsForChunk(agent, resumed.lines.map(line => line + '\n').join('')) : [];
         offset = chunk.offset;
         bytesRead = chunk.offset - start;
+        if (agent === 'claude' && session.sessionId && !partial) {
+            const paths = takePendingImagePaths(session.sessionId);
+            for (const event of events) {
+                if (event.type !== 'attachment' || event.path || !paths.length)
+                    continue;
+                const cached = paths.shift();
+                event.path = cached.path;
+                event.sizeBytes = cached.size;
+            }
+        }
     }
     else {
-        // No resumable reader for this harness: re-parse whole, bounded, from a
-        // FRESH state — a partial re-fold onto a prior state would double-count.
-        // Rate-limited so an actively-appended large transcript is not re-parsed on
-        // every 15 s tick (see TIMELINE_PASS_NON_RESUMABLE_MIN_INTERVAL_MS).
         if (prior && nowMs - prior.computedAt < TIMELINE_PASS_NON_RESUMABLE_MIN_INTERVAL_MS)
             return undefined;
         if (fileSize > TIMELINE_PASS_MAX_WHOLE_FILE_BYTES) {
@@ -187,17 +220,8 @@ function foldSessionTimeline(session, filePath, fileSize, prior, nowMs = Date.no
                 bytesRead: 0,
             };
         }
-        // Eligibility is the smaller of what the tick has left and what a whole-file
-        // fold is allowed to cost — NOT the per-session allowance, which is capped
-        // at 4 MiB and so could never reach a transcript in (4 MiB, 16 MiB] however
-        // idle the tick was. That gap left every such session on kimi/grok/gemini/
-        // cursor/opencode/droid/warp/forge/copilot/goose/antigravity with no row at
-        // all, silently counted `reused`.
         const wholeFileAllowance = Math.min(byteBudget, TIMELINE_PASS_MAX_WHOLE_FILE_BYTES);
         if (fileSize > wholeFileAllowance) {
-            // It does not fit THIS tick. Say so on the row rather than write nothing:
-            // the state is left at offset 0, so a tick with more budget upgrades it to
-            // a real fold instead of the session staying invisible forever.
             return {
                 entry: {
                     timeline: {
@@ -212,17 +236,30 @@ function foldSessionTimeline(session, filePath, fileSize, prior, nowMs = Date.no
         state = emptyTimelineState();
         events = parseTimelineEvents(filePath, agent);
         offset = fileSize;
-        // The whole file was read, not the delta — charge the tick for all of it.
         bytesRead = fileSize;
     }
+    if (agent === 'claude' && session.sessionId)
+        materializeInlineImages(events, session.sessionId);
     const folded = compactTimelineState(foldTimeline(events, state, {
         attachments: session.attachments,
         maxEvents: isResumableTimelineSource(agent) ? undefined : TIMELINE_PASS_MAX_EVENTS,
         offset,
     }));
+    if (partial)
+        folded.partialLine = partial;
+    else
+        delete folded.partialLine;
     const files = projectSessionFiles(folded);
+    const glance = projectGlance(folded.glance);
+    if (offset < fileSize)
+        delete glance.activityHistogram;
+    const subagents = agent === 'claude'
+        ? readSessionSubagents(filePath, session.pidAlive === true, folded.glance.failedAgentCalls, nowMs)
+        : undefined;
     return {
         entry: {
+            ...glance,
+            ...(subagents?.length ? { subagents } : {}),
             timeline: projectTimeline(folded, session.activity),
             ...(folded.request ? { request: folded.request } : {}),
             ...(files ? { files } : {}),
@@ -231,15 +268,6 @@ function foldSessionTimeline(session, filePath, fileSize, prior, nowMs = Date.no
         bytesRead,
     };
 }
-/**
- * Fold the timelines of the live local sessions, bounded, once per tick.
- * Returns what it did so the daemon tick can log it like every other pass.
- *
- * Async only because resolving the session list needs the session-cache module,
- * which is loaded lazily to keep this off the CLI's startup graph. When the
- * caller already HAS the rows (the daemon tick does — it just gathered them),
- * {@link runTimelinePassSync} is the same work with no dynamic import.
- */
 export async function runTimelinePass(opts = {}) {
     const { readActiveSessionsCache, isActiveSessionsJournalReaderRecent } = await import('./session-cache.js');
     const now = opts.nowMs ?? Date.now();
@@ -252,7 +280,6 @@ export async function runTimelinePass(opts = {}) {
     }
     return runTimelinePassSync({ ...opts, sessions });
 }
-/** {@link runTimelinePass} over an explicit row list. Pure of the session cache. */
 export function runTimelinePassSync(opts) {
     const result = { computed: 0, reused: 0, skipped: 0 };
     const sessions = opts.sessions;
@@ -272,11 +299,6 @@ export function runTimelinePassSync(opts) {
         const file = session.sessionFile;
         if (!id || !file)
             continue;
-        // Stamp on the file the PARSER reads, not the row's `sessionFile`: Kimi's row
-        // points at `state.json` while its content is in a sibling `wire.jsonl`, and
-        // Grok's at `summary.json` beside `chat_history.jsonl`. Stamping the wrong
-        // file makes an actively-growing session look unchanged forever. Same helper
-        // the tool index uses, so the two agree on what a session's bytes are.
         const source = toolEvidenceSourcePath(file, (session.kind ?? 'claude'));
         let stamp;
         try {
@@ -285,11 +307,10 @@ export function runTimelinePassSync(opts) {
         }
         catch {
             result.skipped++;
-            continue; // transcript unreadable this tick — try again next time
+            continue;
         }
         const prior = readSessionTimelineEntry(id);
-        // Cached against these exact bytes: nothing was appended, nothing to fold.
-        if (prior && prior.state.offset === stamp.fileSize) {
+        if (prior && prior.state.version === TIMELINE_EXTRACTOR_VERSION && prior.state.offset === stamp.fileSize) {
             result.reused++;
             continue;
         }
@@ -299,22 +320,14 @@ export function runTimelinePassSync(opts) {
             fold = foldSessionTimeline(session, file, stamp.fileSize, prior, now, byteBudget);
         }
         catch (err) {
-            // A novel record shape, or a transcript rotated between the stat and the
-            // read. That is a real failure, not "nothing new" — say so and count it a
-            // skip, or it retries forever behind a healthy-looking log line.
             console.log(`timeline pass: fold failed for ${id} (${session.kind ?? 'claude'}): ${err.message}`);
             result.skipped++;
             continue;
         }
         if (!fold) {
-            // Nothing complete to fold yet, or a non-resumable harness inside its
-            // re-parse interval — the cached row stays current, so this is not a skip.
             result.reused++;
             continue;
         }
-        // Debited by the bytes actually read, so the tick's bound is real: a
-        // whole-file re-parse charges the whole file, and a session that read
-        // nothing (over budget, unavailable harness) charges nothing.
         byteBudget -= fold.bytesRead;
         writeSessionTimeline({ id, ...stamp, timeline: fold.entry, computedAtMs: now });
         result.computed++;