@maudecode/talaria-web 1.16.1 → 1.18.0

package/dist/sessions/state-db.js

@@ -7,6 +7,7 @@
  */
 import { existsSync, openSync, readSync, closeSync, statSync } from 'node:fs';
 import { DatabaseSync } from 'node:sqlite';
+import { Worker } from 'node:worker_threads';
 import { str } from '../util.js';
 import { MESSAGING_SOURCES } from './source-kind.js';
 import { looksLikeDefaultCliTitle, normalizeSourceName } from './titles.js';
@@ -255,6 +256,9 @@ function tableColumns(db, table) {
     }
 }
 const warnedNoSourceColumn = new Set();
+let readDelayMs = 0;
+/** Test hook: each listing read (and each listing worker request) blocks its thread this long first, standing in for a slow `state.db` (TAL-920). */
+export function setStateDbReadDelayForTests(ms) { readDelayMs = ms; }
 /**
  * Python `read_importable_agent_session_rows`. The recency slice is bounded to
  * `limit * 8` candidates before the messages aggregate runs; subagent parents
@@ -269,6 +273,8 @@ export function readImportableAgentSessionRows(dbPath, opts = {}) {
     const limit = opts.limit === undefined ? 200 : opts.limit;
     const exclude = opts.excludeSources === undefined ? ['cron', 'webui'] : opts.excludeSources;
     const include = opts.includeSources ?? null;
+    if (readDelayMs > 0)
+        Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, readDelayMs);
     const db = openStateDbReadonly(dbPath);
     try {
         const sessionCols = tableColumns(db, 'sessions');
@@ -579,6 +585,114 @@ export function cheapChangeFingerprint(dbPath, onError) {
         return null;
     }
 }
+let listingWorker = null;
+let nextCallId = 0;
+/** Set while `stopStateDbWorker` waits for the worker to exit; calls then reject instead of starting another worker. */
+let stoppingWorker = false;
+/** Reject every call still waiting on `current` and forget it, so the next call starts a fresh worker. */
+function retireListingWorker(current, error) {
+    if (listingWorker === current)
+        listingWorker = null;
+    for (const call of current.pending.values())
+        call.reject(error);
+    current.pending.clear();
+}
+/** The lazily started, long-lived listing worker; requests run one at a time in arrival order. */
+function startedListingWorker() {
+    if (listingWorker)
+        return listingWorker;
+    // Source runs (vitest) load the TypeScript worker; the build runs the compiled one beside this module.
+    const ext = import.meta.url.endsWith('.ts') ? 'ts' : 'js';
+    const current = { worker: new Worker(new URL(`./state-db-worker.${ext}`, import.meta.url)), pending: new Map() };
+    current.worker.unref();
+    current.worker.on('message', (reply) => {
+        const call = current.pending.get(reply.id);
+        if (!call)
+            return;
+        current.pending.delete(reply.id);
+        if (!current.pending.size)
+            current.worker.unref();
+        call.resolve(reply);
+    });
+    current.worker.on('error', (error) => { retireListingWorker(current, error); });
+    current.worker.on('exit', (code) => { retireListingWorker(current, new Error(`state.db worker exited with code ${String(code)}`)); });
+    listingWorker = current;
+    return current;
+}
+function callListingWorker(call) {
+    // A multi-read caller the stop interrupted goes on to its next read; that read must not restart the worker.
+    if (stoppingWorker)
+        return Promise.reject(new Error('state.db worker stopped'));
+    const current = startedListingWorker();
+    const id = ++nextCallId;
+    return new Promise((resolve, reject) => {
+        current.pending.set(id, { resolve, reject });
+        // A call in flight keeps the process alive until it settles; an idle worker never does.
+        current.worker.ref();
+        current.worker.postMessage({ ...call, id, delayMs: readDelayMs });
+    });
+}
+/** `readImportableAgentSessionRows` on the listing worker, so a slow `state.db` never blocks the event loop. Rejects when the worker fails. */
+export async function readImportableAgentSessionRowsAsync(dbPath, opts = {}) {
+    const { log, ...query } = opts;
+    const reply = await callListingWorker({ op: 'rows', dbPath, opts: query });
+    for (const line of reply.logs)
+        log?.(line);
+    if (reply.error !== undefined)
+        throw new Error(reply.error);
+    return reply.value;
+}
+/** `cheapChangeFingerprint` on the listing worker; a worker failure reports as a failed change check. */
+export async function cheapChangeFingerprintAsync(dbPath, onError) {
+    let reply;
+    try {
+        reply = await callListingWorker({ op: 'fingerprint', dbPath });
+    }
+    catch (error) {
+        onError?.('change check failed', error);
+        return null;
+    }
+    for (const [reason, message] of reply.failures)
+        onError?.(reason, new Error(message));
+    return reply.value;
+}
+/** `stateDbSessionSources` on the listing worker; a worker failure is an unreadable state.db (null), so owners stay locked. */
+export async function stateDbSessionSourcesAsync(dbPath, sessionIds) {
+    try {
+        return (await callListingWorker({ op: 'sources', dbPath, ids: [...sessionIds] })).value;
+    }
+    catch {
+        return null;
+    }
+}
+/** `agentSessionRowsExisting` on the listing worker; a worker failure assumes every row present, so nothing is pruned. */
+export async function agentSessionRowsExistingAsync(dbPath, sessionIds) {
+    const ids = [...sessionIds];
+    try {
+        return (await callListingWorker({ op: 'existing', dbPath, ids })).value;
+    }
+    catch {
+        return new Set(ids.map((s) => s.trim()).filter(Boolean));
+    }
+}
+/** Test hook: the listing worker exits abruptly while this call is in flight. */
+export function crashStateDbWorkerForTests() {
+    return callListingWorker({ op: 'crash' });
+}
+/** Stop the listing worker (server shutdown). Calls in flight reject; a later call starts a new worker. */
+export async function stopStateDbWorker() {
+    const current = listingWorker;
+    if (!current)
+        return;
+    stoppingWorker = true;
+    try {
+        retireListingWorker(current, new Error('state.db worker stopped'));
+        await current.worker.terminate();
+    }
+    finally {
+        stoppingWorker = false;
+    }
+}
 // ── message reader ────────────────────────────────────────────────────────
 const STATE_DB_CONTENT_JSON_PREFIX = '\0json:';
 // `display_kind` / `display_metadata`: the Agent's own tag for rows it writes itself (steers, delivered notifications; TAL-371).
@@ -618,7 +732,8 @@ function projectStateDbMessage(row, hasId) {
             continue;
         msg[col] = JSON_MESSAGE_COLUMNS.has(col) ? jsonLoadsIfString(value) : value;
     }
-    if (hasId && row.id !== null && row.id !== undefined && typeof msg.api_content === 'string' && msg.api_content)
+    // TAL-493: the merge tells rows committed after the session's last read by this id.
+    if (hasId && row.id !== null && row.id !== undefined)
         msg._state_db_row_id = row.id;
     if (msg.role === 'tool' && msg.tool_name && !msg.name)
         msg.name = msg.tool_name;
@@ -628,25 +743,44 @@ function projectStateDbMessage(row, hasId) {
  * Python `get_state_db_session_messages(sid, stitch_continuations=True)`: the session's active rows in durable order,
  * walking compatible compression/close parents so a continued CLI conversation reads as one transcript.
  */
+/** The state.db file's identity (inode and creation time); a recreated or replaced database gets a new one. */
+function stateDbGeneration(dbPath) {
+    try {
+        const st = statSync(dbPath);
+        return `${String(st.ino)}:${String(st.birthtimeMs)}`;
+    }
+    catch {
+        return null;
+    }
+}
 export function stateDbSessionMessages(dbPath, sid, opts = {}) {
+    return stateDbSessionRead(dbPath, sid, opts).rows;
+}
+/**
+ * `stateDbSessionMessages`, whether the read succeeded (`ok`), and whether its `messages` table has an `id` column
+ * (TAL-493: an empty id-capable read is a baseline of 0; a missing database or a failed read says nothing).
+ */
+export function stateDbSessionRead(dbPath, sid, opts = {}) {
+    const none = { rows: [], idCapable: false, ok: false };
     const id = sid.trim();
     if (!id || !existsSync(dbPath))
-        return [];
+        return none;
     let db;
     try {
         db = openStateDbReadonly(dbPath);
     }
     catch {
-        return [];
+        return none;
     }
     try {
         const available = tableColumns(db, 'messages');
         if (!['role', 'content', 'timestamp'].every((c) => available.has(c)))
-            return [];
+            return { ...none, ok: available.size > 0 };
         const hasId = available.has('id');
         const selected = [...(hasId ? ['id'] : []), 'role', 'content', 'timestamp', ...OPTIONAL_MESSAGE_COLUMNS.filter((c) => available.has(c))];
-        const chain = [id];
-        if (opts.stitch ?? true) {
+        // TAL-529: an explicit lineage (the session and its compression continuations) replaces the parent walk.
+        const chain = opts.lineage?.length ? [...opts.lineage] : [id];
+        if (!opts.lineage?.length && (opts.stitch ?? true)) {
             const sessionCols = tableColumns(db, 'sessions');
             if (['parent_session_id', 'end_reason', 'started_at', 'source'].every((c) => sessionCols.has(c))) {
                 const select = db.prepare('SELECT id, source, started_at, parent_session_id, ended_at, end_reason FROM sessions WHERE id = ?');
@@ -668,10 +802,161 @@ export function stateDbSessionMessages(dbPath, sid, opts = {}) {
         const activeClause = available.has('active') ? ' AND (active IS NULL OR active != 0)' : '';
         const order = hasId ? 'id' : 'timestamp';
         const rows = db.prepare(`SELECT ${selected.join(', ')}, session_id FROM messages WHERE session_id IN (${chain.map(() => '?').join(', ')})${activeClause} ORDER BY ${order} ASC`).all(...chain);
-        return rows.map((row) => projectStateDbMessage(row, hasId));
+        const projected = rows.map((row) => projectStateDbMessage(row, hasId));
+        // TAL-709: an in-place compaction archives the live rows and re-inserts each as a copy that keeps its role and timestamp
+        // (content may be pruned). A copy takes the id of the oldest archived row it copies, so the merge sees a row it has
+        // already read instead of a new one. Only a compaction archive (`compacted = 1`) vouches for a changed copy: a rewound
+        // row (`compacted = 0`, also the compaction's carried tail) counts only for the same tool call or the same content, so
+        // an edit that kept its timestamp stays new.
+        if (hasId && available.has('active')) {
+            // One indexed lookup per live row ((session_id, active, timestamp)): the archive is never loaded, however many
+            // compactions it holds.
+            const sameCall = available.has('tool_call_id') ? " AND COALESCE(a.tool_call_id, '') = COALESCE(m.tool_call_id, '')" : '';
+            const vouched = [available.has('compacted') ? 'a.compacted = 1' : '', available.has('tool_call_id') ? 'm.tool_call_id IS NOT NULL' : '', 'a.content = m.content'].filter(Boolean).join(' OR ');
+            const twins = db.prepare(`SELECT m.id AS id, MIN(a.id) AS twin FROM messages m JOIN messages a ON a.session_id = m.session_id AND a.active = 0 AND a.role = m.role AND a.timestamp = m.timestamp AND a.id < m.id${sameCall} WHERE m.session_id IN (${chain.map(() => '?').join(', ')}) AND (m.active IS NULL OR m.active != 0) AND (${vouched}) GROUP BY m.id`).all(...chain);
+            const twinOf = new Map(twins.map((row) => [row.id, row.twin]));
+            for (const [i, row] of rows.entries()) {
+                const twin = twinOf.get(row.id);
+                if (typeof twin === 'number')
+                    projected[i]._state_db_row_id = twin;
+            }
+        }
+        // TAL-493: the database file's identity, so a marker taken in a replaced state.db is never applied to its successor.
+        if (hasId) {
+            const generation = stateDbGeneration(dbPath);
+            for (const m of projected)
+                m._state_db_generation = generation;
+        }
+        return { rows: projected, idCapable: hasId, ok: true };
     }
     catch {
-        return [];
+        return none;
+    }
+    finally {
+        db.close();
+    }
+}
+/**
+ * TAL-529: `sid` and each compression continuation the Agent rotated it to, down to the live tip (Python
+ * `resolve_live_compression_tip`). The walk stops at the last unambiguous row: no continuation, several, a cycle, or 20
+ * hops. A missing or unreadable database is `[sid]`.
+ */
+export function stateDbCompressionLineage(dbPath, sid) {
+    const lineage = [sid.trim()];
+    if (!lineage[0] || !existsSync(dbPath))
+        return lineage;
+    let db;
+    try {
+        db = openStateDbReadonly(dbPath);
+    }
+    catch {
+        return lineage;
+    }
+    try {
+        const sessionCols = tableColumns(db, 'sessions');
+        if (!['id', 'parent_session_id', 'end_reason', 'ended_at', 'started_at', 'source'].every((c) => sessionCols.has(c)))
+            return lineage;
+        const sessionSource = sessionCols.has('session_source') ? 'session_source' : 'NULL AS session_source';
+        const cols = `id, source, ${sessionSource}, started_at, parent_session_id, ended_at, end_reason`;
+        let current = db.prepare(`SELECT ${cols} FROM sessions WHERE id = ?`).get(lineage[0]);
+        const children = db.prepare(`SELECT ${cols} FROM sessions WHERE parent_session_id = ?`);
+        for (let hop = 0; hop < 20 && current; hop += 1) {
+            const parent = current;
+            const next = children.all(str(parent.id)).filter((child) => isContinuationSession(parent, child));
+            if (next.length !== 1 || lineage.includes(str(next[0].id)))
+                break;
+            current = next[0];
+            lineage.push(str(current.id));
+        }
+        return lineage;
+    }
+    catch {
+        return lineage.slice(0, 1);
+    }
+    finally {
+        db.close();
+    }
+}
+function lineageReportRow(row, role) {
+    const ended = row.ended_at ?? null;
+    return {
+        session_id: row.id, role, title: row.title ?? null, source: row.source ?? null, started_at: row.started_at ?? null, updated_at: ended ?? row.started_at ?? null,
+        end_reason: row.end_reason ?? null, active: ended === null, archived: false,
+    };
+}
+const emptyLineageReport = (sid) => ({
+    mutation: false, found: false, session_id: sid, lineage_key: sid, tip_session_id: sid, total_segments: 0, materialized_segments: 0, segments: [], children: [], manual_review: false,
+});
+/**
+ * Python `read_session_lineage_report`: a read-only report of `sid`'s compression lineage. The walk follows up to
+ * `maxHops` continuation parents (the tip first, the root last), then lists each segment's non-continuation children,
+ * newest first. A cycle, a hop limit hit, or a continuation off the walked path marks `manual_review`. A missing,
+ * unreadable, or too-old database, or an unknown id, answers `found: false`.
+ */
+export function stateDbLineageReport(dbPath, rawSid, maxHops = 20) {
+    const sid = rawSid.trim();
+    if (!sid || !existsSync(dbPath))
+        return emptyLineageReport(sid);
+    let db;
+    try {
+        db = openStateDbReadonly(dbPath);
+    }
+    catch {
+        return emptyLineageReport(sid);
+    }
+    try {
+        const cols = tableColumns(db, 'sessions');
+        if (!['id', 'parent_session_id', 'end_reason'].every((c) => cols.has(c)))
+            return emptyLineageReport(sid);
+        const col = (name, fallback = 'NULL') => (cols.has(name) ? `s.${name}` : `${fallback} AS ${name}`);
+        const select = `SELECT s.id, ${col('source')}, ${col('session_source')}, ${col('title')}, ${col('started_at', '0')}, s.parent_session_id, ${col('ended_at')}, s.end_reason FROM sessions s`;
+        const byId = db.prepare(`${select} WHERE s.id = ?`);
+        const fetch = (id) => (id ? byId.get(str(id)) ?? null : null);
+        const target = fetch(sid);
+        if (!target)
+            return emptyLineageReport(sid);
+        const segments = [target];
+        const seen = new Set([sid]);
+        let manualReview = true;
+        for (let hop = 0; hop < maxHops; hop += 1) {
+            const current = segments[segments.length - 1];
+            const parentId = str(current.parent_session_id);
+            const parent = fetch(parentId);
+            if (!parent || seen.has(parentId)) {
+                manualReview = Boolean(parentId && seen.has(parentId));
+                break;
+            }
+            if (!isContinuationSession(parent, current)) {
+                manualReview = false;
+                break;
+            }
+            segments.push(parent);
+            seen.add(parentId);
+        }
+        const segmentIds = segments.map((row) => str(row.id));
+        const children = db.prepare(`${select} WHERE s.parent_session_id IN (${segmentIds.map(() => '?').join(', ')})`).all(...segmentIds);
+        const childRows = [];
+        for (const parent of segments) {
+            const own = children.filter((child) => str(child.parent_session_id) === str(parent.id) && !seen.has(str(child.id)));
+            own.sort((a, b) => (Number(b.started_at) || 0) - (Number(a.started_at) || 0));
+            for (const child of own) {
+                // A continuation off the walked path means a branched lineage or an older selected segment: flag it, list nothing.
+                if (isContinuationSession(parent, child))
+                    manualReview = true;
+                else
+                    childRows.push(child);
+            }
+        }
+        return {
+            mutation: false, found: true, session_id: sid, lineage_key: segments[segments.length - 1].id, tip_session_id: segments[0].id,
+            total_segments: segments.length, materialized_segments: segments.length,
+            segments: segments.map((row, idx) => lineageReportRow(row, idx === 0 ? 'tip' : 'hidden_segment')),
+            children: childRows.map((row) => lineageReportRow(row, 'child_session')),
+            manual_review: manualReview,
+        };
+    }
+    catch {
+        return emptyLineageReport(sid);
     }
     finally {
         db.close();
@@ -703,4 +988,128 @@ export function stateDbSessionRow(dbPath, sid) {
         db.close();
     }
 }
+/**
+ * TAL-550 (Python `_handle_insights`): usage of the non-Web sessions active since `cutoff`, shaped like session index
+ * entries. Activity is the latest of start, end, and last message, so a long-lived gateway session still counts.
+ * `webui` rows mirror index sessions and are skipped; a missing column reads as empty and a missing or unreadable
+ * state.db answers no rows.
+ */
+export function insightsSessionRows(dbPath, cutoff) {
+    if (!existsSync(dbPath))
+        return [];
+    let db;
+    try {
+        db = openStateDbReadonly(dbPath);
+    }
+    catch {
+        return [];
+    }
+    try {
+        const cols = tableColumns(db, 'sessions');
+        if (!cols.has('id') || !cols.has('started_at'))
+            return [];
+        const col = (name) => (cols.has(name) ? `s.${name}` : 'NULL');
+        const messageCols = tableColumns(db, 'messages');
+        const hasMessages = messageCols.has('session_id') && messageCols.has('timestamp');
+        let indexed = false;
+        try {
+            indexed = hasMessages && db.prepare('PRAGMA index_list(messages)').all().some((r) => r.name === 'idx_messages_session');
+        }
+        catch {
+            indexed = false;
+        }
+        // Without the index a per-session lookup scans messages once per session; one grouped scan replaces it.
+        const lastMessage = !hasMessages ? 'NULL' : indexed ? '(SELECT MAX(m.timestamp) FROM messages m WHERE m.session_id = s.id)' : 'lm.last_message_at';
+        const join = hasMessages && !indexed ? 'LEFT JOIN (SELECT session_id, MAX(timestamp) AS last_message_at FROM messages GROUP BY session_id) lm ON lm.session_id = s.id' : '';
+        const source = cols.has('source') ? "LOWER(COALESCE(s.source, '')) != 'webui'" : '1';
+        const rows = db.prepare(`SELECT * FROM (SELECT s.id AS id, ${col('model')} AS model, ${col('message_count')} AS message_count, ${col('input_tokens')} AS input_tokens,
+      ${col('output_tokens')} AS output_tokens, ${col('cache_read_tokens')} AS cache_read_tokens, ${col('estimated_cost_usd')} AS estimated_cost_usd, s.started_at AS started_at,
+      MAX(COALESCE(s.started_at, 0), COALESCE(${col('ended_at')}, 0), COALESCE(${lastMessage}, 0)) AS last_activity
+      FROM sessions s ${join} WHERE ${source}) WHERE last_activity >= ?`).all(cutoff);
+        return rows.map((r) => ({
+            session_id: str(r.id), model: r.model, message_count: r.message_count, input_tokens: r.input_tokens, output_tokens: r.output_tokens,
+            cache_read_tokens: r.cache_read_tokens, estimated_cost: r.estimated_cost_usd, created_at: r.started_at, updated_at: r.last_activity,
+        }));
+    }
+    catch {
+        return [];
+    }
+    finally {
+        db.close();
+    }
+}
+/** A state.db `timestamp` in unix seconds: a number, or an ISO-8601 string (Python `fromisoformat`); null when unreadable. */
+export function stateDbTimestampSeconds(value) {
+    if (typeof value === 'number')
+        return Number.isFinite(value) ? value : null;
+    if (typeof value !== 'string' || !value.trim())
+        return null;
+    const ms = Date.parse(value.trim());
+    return Number.isFinite(ms) ? ms / 1000 : null;
+}
+/** Python `CONVERSATION_ROUND_THRESHOLD`: the handoff dock is offered from this many rounds. */
+export const CONVERSATION_ROUND_THRESHOLD = 10;
+/**
+ * Python `count_conversation_rounds` (TAL-258): a round is a user message answered by the assistant; consecutive user
+ * messages merge into one. With `since`, only rows stamped after it count (an unreadable stamp still counts). A missing
+ * or unreadable database is 0.
+ */
+export function countConversationRounds(dbPath, sid, since = null) {
+    if (!existsSync(dbPath))
+        return 0;
+    let rows;
+    try {
+        const db = openStateDbReadonly(dbPath);
+        try {
+            rows = db.prepare('SELECT role, timestamp FROM messages WHERE session_id = ? ORDER BY timestamp ASC').all(sid);
+        }
+        finally {
+            db.close();
+        }
+    }
+    catch {
+        return 0;
+    }
+    let rounds = 0;
+    let seenUser = false;
+    let answered = false;
+    for (const row of rows) {
+        if (since !== null) {
+            const ts = stateDbTimestampSeconds(row.timestamp);
+            if (ts !== null && ts <= since)
+                continue;
+        }
+        const role = lower(row.role);
+        if (role === 'user') {
+            if (seenUser && answered) {
+                rounds += 1;
+                answered = false;
+            }
+            seenUser = true;
+        }
+        else if (role === 'assistant' && seenUser) {
+            answered = true;
+        }
+    }
+    return seenUser && answered ? rounds + 1 : rounds;
+}
+/** The content of the session's newest row when that row is a `tool` row (the handoff marker's tail dedupe, TAL-258); else null. */
+export function stateDbTailToolContent(dbPath, sid) {
+    if (!existsSync(dbPath))
+        return null;
+    try {
+        const db = openStateDbReadonly(dbPath);
+        try {
+            const row = db.prepare('SELECT role, content FROM messages WHERE session_id = ? ORDER BY rowid DESC LIMIT 1').get(sid);
+            const content = row?.role === 'tool' ? decodeStateDbContent(row.content) : null;
+            return typeof content === 'string' ? content : null;
+        }
+        finally {
+            db.close();
+        }
+    }
+    catch {
+        return null;
+    }
+}
 //# sourceMappingURL=state-db.js.map