@threadbase-sh/streamer 1.27.2 → 1.28.0

package/dist/index.d.cts

@@ -6,8 +6,9 @@ import { Logger as Logger$1 } from 'pino';
 import { HttpBindings } from '@hono/node-server';
 import { ServerResponse, IncomingMessage } from 'http';
 import { WebSocket } from 'ws';
-import { FileStatEntry, ConversationMeta } from '@threadbase-sh/scanner';
+import { ConversationMessage, FileStatEntry, ConversationMeta } from '@threadbase-sh/scanner';
 import Database from 'better-sqlite3';
+import { Stats } from 'fs';
 import { z } from 'zod';
 import { Pool } from 'pg';
 
@@ -190,6 +191,7 @@ type WSMessage = {
     type: "conversation_events";
     sessionId: string;
     lines: string[];
+    seqs?: (number | null)[];
 } | {
     type: "question";
     sessionId: string;
@@ -367,6 +369,15 @@ interface SessionRunner {
     dispose(): void;
 }
 
+interface LineSpan {
+    /** Absolute byte offset of the line's first byte in the file. */
+    byteOffset: number;
+    /** Byte length of the line content, excluding the trailing "\n". */
+    byteLength: number;
+    /** The decoded line text (no trailing "\n"). */
+    text: string;
+}
+
 interface ConversationCacheOptions {
     filterAgentConversations?: boolean;
     agentEntrypoints?: ReadonlySet<string>;
@@ -401,6 +412,25 @@ interface CachedTail {
     messages: CachedTailMessage[];
     tailSize: number;
 }
+/** A row of `conversation_file_state` — per-file offset-index resume state. */
+interface FileStateRow {
+    path: string;
+    identity: string;
+    size: number;
+    mtime_ms: number;
+    byte_offset: number;
+    last_message_index: number;
+}
+/** A row of `conversation_message_index` — one indexed message's byte span. */
+interface MessageIndexRow {
+    conversation_id: string;
+    message_index: number;
+    byte_offset: number;
+    byte_length: number;
+    uuid: string | null;
+    role: string | null;
+    ts: number | null;
+}
 interface ScannerMeta {
     id: string;
     sessionId?: string;
@@ -423,6 +453,8 @@ declare class ConversationCache {
     private tailSize;
     private fileIndex;
     private fileIndexLoaded;
+    private indexParseState;
+    private backfillInFlight;
     private tailSeq;
     private nameSeq;
     private stmts;
@@ -436,6 +468,78 @@ declare class ConversationCache {
      * share the same connection. Internal API; not part of the public surface.
      */
     getDatabase(): Database.Database;
+    getFileState(path: string): FileStateRow | null;
+    upsertFileState(row: FileStateRow): void;
+    /** Drop a file's index rows + file_state (truncation / identity change). */
+    deleteFileIndex(path: string, conversationId: string): void;
+    /** Append/replace index rows in one transaction. */
+    appendMessageIndexRows(rows: MessageIndexRow[]): void;
+    /** Rows for message_index in [fromIndex, toIndex), ordered ascending. */
+    getMessageIndexWindow(conversationId: string, fromIndex: number, toIndex: number): MessageIndexRow[];
+    getIndexedMessageCount(conversationId: string): number;
+    /**
+     * Conversation id for a JSONL path — the filename stem (matches the pseudo-id
+     * updateFromLine derives and the uuid the detail read path resolves). The
+     * offset index keys on this so the window select and the cursor agree.
+     */
+    static conversationIdForFile(filePath: string): string;
+    /**
+     * Incremental offset-index writer: extend the index for a burst of appended
+     * lines (one watcher read) using their byte spans. Each line is classified
+     * with the scanner's parseJsonlLine (a running per-file reducer state), so the
+     * message ordering can never drift from the scanner's. Message lines get an
+     * index row at the next message_index; non-message lines (summary/sidecar)
+     * get no row but still advance byte_offset. file_state is updated to the end
+     * of the last consumed span.
+     *
+     * Requires an up-to-date `stat` (identity/size/mtime) for the file so the read
+     * path can detect truncation/replacement.
+     *
+     * `readFrom` is the absolute byte offset the watcher read started at, and
+     * `endOffset` is where it ended (readFrom + consumed, i.e. the watcher's new
+     * entry.offset). CONTIGUITY GUARD: the read must begin exactly where the index
+     * left off (`readFrom === existing.byte_offset`, or 0 with no state). If it
+     * doesn't — the watcher attached at EOF after the server was down, or an
+     * append raced an in-flight backfill — extending would assign wrong
+     * message_index values over a hole. In that case this writes nothing and
+     * returns null so the caller drops the index and backfills.
+     *
+     * On success returns the message_index assigned to each input span (null for a
+     * non-message line) so the caller can stamp WS `seq`. Empty array when spans
+     * is empty. `endOffset` is stored verbatim as byte_offset so the watcher's
+     * offset and file_state.byte_offset are the same number by construction.
+     */
+    extendMessageIndex(filePath: string, spans: LineSpan[], stat: Stats, readFrom: number, endOffset: number): (number | null)[] | null;
+    clearIndexParseState(filePath: string): void;
+    /**
+     * On-demand full backfill of the offset index for a file with no/stale
+     * file_state (cold conversation, or after a truncation/replacement). Rebuilds
+     * from byte 0: drops any existing rows, walks the whole file in chunks with a
+     * running parse state, yields to the event loop every ~1000 lines so a large
+     * file never blocks, and writes index rows + file_state.
+     *
+     * Single-flighted per path: concurrent callers await the same walk. The
+     * triggering detail request is served by the scanner fallback while this runs.
+     */
+    backfillIndex(filePath: string): Promise<void>;
+    private runBackfill;
+    /**
+     * Windowed detail read straight from the offset index — the hot path.
+     * Returns the parsed messages for message_index in [fromIndex, toIndex) plus
+     * the total indexed count, or null when the index can't serve this file (no
+     * file_state, identity/size mismatch = truncation/replacement, or cold index)
+     * so the caller falls back to the scanner and enqueues a backfill.
+     *
+     * On a match it SQL-selects the window's byte ranges and preads exactly those
+     * ranges from the JSONL (never the whole file), parsing only the sliced lines.
+     * Returns messages in the same ConversationMessage shape parseJsonlLine
+     * produces during a scan, so the payload is identical to the scanner path.
+     */
+    readMessageWindow(filePath: string, fromIndex: number, toIndex: number): {
+        messages: ConversationMessage[];
+        total: number;
+        fromIndex: number;
+    } | null;
     private agentEntrypointsKey;
     private classifyAgentFile;
     isAgentFileCached(filePath: string): boolean;
@@ -874,6 +978,7 @@ declare class StreamerServer {
     private wsHub;
     private fileWatcher;
     private sessionFileMap;
+    private pendingLineSeqs;
     private pendingQuestions;
     private pendingQuestionKey;
     private pendingPermission;
@@ -882,6 +987,7 @@ declare class StreamerServer {
     private allScanners;
     private scannerReady;
     private scannerStale;
+    private refreshInFlight;
     private binding;
     private cacheReady;
     private inFlightCacheWrites;
@@ -945,6 +1051,7 @@ declare class StreamerServer {
     private bindWithRetry;
     private bindWithRetryLoop;
     private trackCacheWrite;
+    private refreshFileGuarded;
     close(): Promise<void>;
     private handleRequest;
     private handlePairStart;
@@ -1008,6 +1115,15 @@ interface ConversationWatcherEvents {
      * downstream cache write + WebSocket broadcast.
      */
     onNewLines?: (filePath: string, lines: string[]) => void;
+    /**
+     * Like onNewLines but also carries each line's absolute byte span in the
+     * file (for the offset index). Fires ALONGSIDE onNewLines/onNewLine (it does
+     * not replace them) so the cache tail write and the index extend can both
+     * consume the same read. `readFrom` is the absolute byte offset the read
+     * started at; `spans` are complete lines only (a torn trailing line is held
+     * for the next read).
+     */
+    onNewLineSpans?: (filePath: string, spans: LineSpan[], readFrom: number, endOffset: number) => void;
     /** Fires when chokidar reports an add/change/unlink at the directory level. */
     onConversationChanged?: (filePath: string) => void | Promise<void>;
     /** Fires when a tailed file is deleted (per-file watcher unlink event). */
@@ -1031,6 +1147,7 @@ declare class ConversationWatcher {
     private directories;
     private onNewLine;
     private onNewLines;
+    private onNewLineSpans;
     private onConversationChanged;
     private onFileDeleted;
     private onError;

package/dist/index.d.ts

@@ -6,8 +6,9 @@ import { Logger as Logger$1 } from 'pino';
 import { HttpBindings } from '@hono/node-server';
 import { ServerResponse, IncomingMessage } from 'http';
 import { WebSocket } from 'ws';
-import { FileStatEntry, ConversationMeta } from '@threadbase-sh/scanner';
+import { ConversationMessage, FileStatEntry, ConversationMeta } from '@threadbase-sh/scanner';
 import Database from 'better-sqlite3';
+import { Stats } from 'fs';
 import { z } from 'zod';
 import { Pool } from 'pg';
 
@@ -190,6 +191,7 @@ type WSMessage = {
     type: "conversation_events";
     sessionId: string;
     lines: string[];
+    seqs?: (number | null)[];
 } | {
     type: "question";
     sessionId: string;
@@ -367,6 +369,15 @@ interface SessionRunner {
     dispose(): void;
 }
 
+interface LineSpan {
+    /** Absolute byte offset of the line's first byte in the file. */
+    byteOffset: number;
+    /** Byte length of the line content, excluding the trailing "\n". */
+    byteLength: number;
+    /** The decoded line text (no trailing "\n"). */
+    text: string;
+}
+
 interface ConversationCacheOptions {
     filterAgentConversations?: boolean;
     agentEntrypoints?: ReadonlySet<string>;
@@ -401,6 +412,25 @@ interface CachedTail {
     messages: CachedTailMessage[];
     tailSize: number;
 }
+/** A row of `conversation_file_state` — per-file offset-index resume state. */
+interface FileStateRow {
+    path: string;
+    identity: string;
+    size: number;
+    mtime_ms: number;
+    byte_offset: number;
+    last_message_index: number;
+}
+/** A row of `conversation_message_index` — one indexed message's byte span. */
+interface MessageIndexRow {
+    conversation_id: string;
+    message_index: number;
+    byte_offset: number;
+    byte_length: number;
+    uuid: string | null;
+    role: string | null;
+    ts: number | null;
+}
 interface ScannerMeta {
     id: string;
     sessionId?: string;
@@ -423,6 +453,8 @@ declare class ConversationCache {
     private tailSize;
     private fileIndex;
     private fileIndexLoaded;
+    private indexParseState;
+    private backfillInFlight;
     private tailSeq;
     private nameSeq;
     private stmts;
@@ -436,6 +468,78 @@ declare class ConversationCache {
      * share the same connection. Internal API; not part of the public surface.
      */
     getDatabase(): Database.Database;
+    getFileState(path: string): FileStateRow | null;
+    upsertFileState(row: FileStateRow): void;
+    /** Drop a file's index rows + file_state (truncation / identity change). */
+    deleteFileIndex(path: string, conversationId: string): void;
+    /** Append/replace index rows in one transaction. */
+    appendMessageIndexRows(rows: MessageIndexRow[]): void;
+    /** Rows for message_index in [fromIndex, toIndex), ordered ascending. */
+    getMessageIndexWindow(conversationId: string, fromIndex: number, toIndex: number): MessageIndexRow[];
+    getIndexedMessageCount(conversationId: string): number;
+    /**
+     * Conversation id for a JSONL path — the filename stem (matches the pseudo-id
+     * updateFromLine derives and the uuid the detail read path resolves). The
+     * offset index keys on this so the window select and the cursor agree.
+     */
+    static conversationIdForFile(filePath: string): string;
+    /**
+     * Incremental offset-index writer: extend the index for a burst of appended
+     * lines (one watcher read) using their byte spans. Each line is classified
+     * with the scanner's parseJsonlLine (a running per-file reducer state), so the
+     * message ordering can never drift from the scanner's. Message lines get an
+     * index row at the next message_index; non-message lines (summary/sidecar)
+     * get no row but still advance byte_offset. file_state is updated to the end
+     * of the last consumed span.
+     *
+     * Requires an up-to-date `stat` (identity/size/mtime) for the file so the read
+     * path can detect truncation/replacement.
+     *
+     * `readFrom` is the absolute byte offset the watcher read started at, and
+     * `endOffset` is where it ended (readFrom + consumed, i.e. the watcher's new
+     * entry.offset). CONTIGUITY GUARD: the read must begin exactly where the index
+     * left off (`readFrom === existing.byte_offset`, or 0 with no state). If it
+     * doesn't — the watcher attached at EOF after the server was down, or an
+     * append raced an in-flight backfill — extending would assign wrong
+     * message_index values over a hole. In that case this writes nothing and
+     * returns null so the caller drops the index and backfills.
+     *
+     * On success returns the message_index assigned to each input span (null for a
+     * non-message line) so the caller can stamp WS `seq`. Empty array when spans
+     * is empty. `endOffset` is stored verbatim as byte_offset so the watcher's
+     * offset and file_state.byte_offset are the same number by construction.
+     */
+    extendMessageIndex(filePath: string, spans: LineSpan[], stat: Stats, readFrom: number, endOffset: number): (number | null)[] | null;
+    clearIndexParseState(filePath: string): void;
+    /**
+     * On-demand full backfill of the offset index for a file with no/stale
+     * file_state (cold conversation, or after a truncation/replacement). Rebuilds
+     * from byte 0: drops any existing rows, walks the whole file in chunks with a
+     * running parse state, yields to the event loop every ~1000 lines so a large
+     * file never blocks, and writes index rows + file_state.
+     *
+     * Single-flighted per path: concurrent callers await the same walk. The
+     * triggering detail request is served by the scanner fallback while this runs.
+     */
+    backfillIndex(filePath: string): Promise<void>;
+    private runBackfill;
+    /**
+     * Windowed detail read straight from the offset index — the hot path.
+     * Returns the parsed messages for message_index in [fromIndex, toIndex) plus
+     * the total indexed count, or null when the index can't serve this file (no
+     * file_state, identity/size mismatch = truncation/replacement, or cold index)
+     * so the caller falls back to the scanner and enqueues a backfill.
+     *
+     * On a match it SQL-selects the window's byte ranges and preads exactly those
+     * ranges from the JSONL (never the whole file), parsing only the sliced lines.
+     * Returns messages in the same ConversationMessage shape parseJsonlLine
+     * produces during a scan, so the payload is identical to the scanner path.
+     */
+    readMessageWindow(filePath: string, fromIndex: number, toIndex: number): {
+        messages: ConversationMessage[];
+        total: number;
+        fromIndex: number;
+    } | null;
     private agentEntrypointsKey;
     private classifyAgentFile;
     isAgentFileCached(filePath: string): boolean;
@@ -874,6 +978,7 @@ declare class StreamerServer {
     private wsHub;
     private fileWatcher;
     private sessionFileMap;
+    private pendingLineSeqs;
     private pendingQuestions;
     private pendingQuestionKey;
     private pendingPermission;
@@ -882,6 +987,7 @@ declare class StreamerServer {
     private allScanners;
     private scannerReady;
     private scannerStale;
+    private refreshInFlight;
     private binding;
     private cacheReady;
     private inFlightCacheWrites;
@@ -945,6 +1051,7 @@ declare class StreamerServer {
     private bindWithRetry;
     private bindWithRetryLoop;
     private trackCacheWrite;
+    private refreshFileGuarded;
     close(): Promise<void>;
     private handleRequest;
     private handlePairStart;
@@ -1008,6 +1115,15 @@ interface ConversationWatcherEvents {
      * downstream cache write + WebSocket broadcast.
      */
     onNewLines?: (filePath: string, lines: string[]) => void;
+    /**
+     * Like onNewLines but also carries each line's absolute byte span in the
+     * file (for the offset index). Fires ALONGSIDE onNewLines/onNewLine (it does
+     * not replace them) so the cache tail write and the index extend can both
+     * consume the same read. `readFrom` is the absolute byte offset the read
+     * started at; `spans` are complete lines only (a torn trailing line is held
+     * for the next read).
+     */
+    onNewLineSpans?: (filePath: string, spans: LineSpan[], readFrom: number, endOffset: number) => void;
     /** Fires when chokidar reports an add/change/unlink at the directory level. */
     onConversationChanged?: (filePath: string) => void | Promise<void>;
     /** Fires when a tailed file is deleted (per-file watcher unlink event). */
@@ -1031,6 +1147,7 @@ declare class ConversationWatcher {
     private directories;
     private onNewLine;
     private onNewLines;
+    private onNewLineSpans;
     private onConversationChanged;
     private onFileDeleted;
     private onError;