@pasko70/pibo 3.4.4 → 3.5.0

package/dist/data/async-chat-reads.js

@@ -0,0 +1,38 @@
+import { BoundedWorkerClient } from "./bounded-worker-client.js";
+export const CHAT_READ_METHODS = {
+    timeline: ["listEvents", "listSessionEvents", "listAllSessionEvents", "listMessageTurnTimings", "scanMessageTurnTimings", "listTraceEvents", "isPayloadAttachedToTraceNode", "countEventsByType", "getLatestEventSequence", "getLatestStreamId"],
+    history: ["listProductHistoryEntries", "getProductHistoryCoverage"],
+    navigation: ["unreadCountsPage", "sessionIndexPage"],
+};
+/** Read traffic owns a separate connection and queue from admission and output commits. */
+export class AsyncChatReadQueries {
+    path;
+    payloadRootDir;
+    client;
+    closed = false;
+    restarts = 0;
+    restartAfter = 0;
+    timeline;
+    navigation;
+    history;
+    constructor(path, payloadRootDir) {
+        this.path = path;
+        this.payloadRootDir = payloadRootDir;
+        this.client = this.startWorker();
+        const group = (name) => Object.fromEntries(CHAT_READ_METHODS[name].map(method => [method, (...args) => this.getClient().request({ group: name, method, args }, { priority: "background", fairnessKey: typeof args[0] === "string" ? args[0] : args[0]?.piboSessionId })]));
+        this.timeline = group("timeline");
+        this.history = group("history");
+        this.navigation = group("navigation");
+    }
+    startWorker() {
+        this.restartAfter = Date.now() + 1000;
+        return new BoundedWorkerClient(new URL("./chat-read-worker.js", import.meta.url), { maxPending: 64, reservedControlRequests: 2, reservedControlBytes: 16384, maxPendingBytes: 1024 * 1024, maxMessageBytes: 4 * 1024 * 1024, maxAgeMs: 2000, workerOptions: { workerData: { path: this.path, payloadRootDir: this.payloadRootDir } } });
+    }
+    getClient() { if (!this.closed && this.client.status().closed && this.client.status().exited && Date.now() >= this.restartAfter) {
+        this.restarts++;
+        this.client = this.startWorker();
+    } return this.client; }
+    maintenance(action) { return this.getClient().request({ group: "maintenance", method: action, args: [] }, { priority: "control" }); }
+    status() { return { ...this.client.status(), restarts: this.restarts }; }
+    async close() { this.closed = true; await this.client.close(); }
+}

package/dist/data/async-chat-storage.js

@@ -0,0 +1,91 @@
+import { BoundedWorkerClient, StorageUnavailableError } from "./bounded-worker-client.js";
+/** Admission/output writer; bulk reads must use a separate worker connection. */
+export class AsyncChatStorage {
+    writerClient;
+    reader;
+    workerUrl;
+    workerOptions;
+    closed = false;
+    writerRestarts = 0;
+    readerRestarts = 0;
+    writerRestartAfter = 0;
+    readerRestartAfter = 0;
+    constructor(path, payloadRootDir, options = {}) {
+        if (path === ":memory:")
+            throw new Error("Worker storage requires a file-backed database.");
+        // Admission carries the bounded message plus its event projection; account for both in IPC.
+        this.workerOptions = { maxMessageBytes: 4 * 1024 * 1024, reservedControlRequests: 8, reservedControlBytes: 256 * 1024, admissionWindowMs: 1, ...options, workerOptions: { ...options.workerOptions, workerData: { path, payloadRootDir } } };
+        this.workerUrl = new URL("./chat-storage-worker.js", import.meta.url);
+        this.writerClient = this.startWriter();
+    }
+    /**
+     * A crashed worker, or one that breached its execution deadline, must not disable durable
+     * admission permanently. Replacement waits for the failed worker to exit and is bounded to one
+     * per second, so two writers never own the same SQLite file. Recovery is safe without replay
+     * protection because callers reconcile the same client transaction ID and admission is
+     * idempotent by request key.
+     */
+    get writer() {
+        const status = this.writerClient.status();
+        if (!this.closed && status.closed && status.exited && Date.now() >= this.writerRestartAfter) {
+            this.writerRestarts++;
+            this.writerClient = this.startWriter();
+        }
+        return this.writerClient;
+    }
+    get readerClient() {
+        const status = this.reader?.status();
+        if (!this.reader || (!this.closed && status.closed && status.exited && Date.now() >= this.readerRestartAfter)) {
+            if (this.reader)
+                this.readerRestarts++;
+            this.reader = this.startReader();
+        }
+        return this.reader;
+    }
+    startWriter() {
+        this.writerRestartAfter = Date.now() + 1000;
+        return new BoundedWorkerClient(this.workerUrl, this.workerOptions);
+    }
+    startReader() {
+        this.readerRestartAfter = Date.now() + 1000;
+        return new BoundedWorkerClient(this.workerUrl, this.workerOptions);
+    }
+    resolveRoom(roomId, required = false) {
+        return this.writer.request({ type: "resolveRoom", roomId, required });
+    }
+    admit(input, session, text, durableCommand) {
+        if (durableCommand && Buffer.byteLength(text) > 1024 * 1024)
+            return Promise.reject(Object.assign(new Error("Message exceeds the durable command byte limit."), { code: "command_too_large" }));
+        return this.writer.request({ type: "admit", input, session, text, durableCommand }, { fairnessKey: input.roomId, priority: durableCommand?.delivery === "steer" ? "control" : "admission" });
+    }
+    append(input) {
+        return this.writer.request({ type: "append", input });
+    }
+    find(roomId, actorId, clientTxnId) {
+        if (this.closed)
+            return Promise.reject(new StorageUnavailableError("storage_closed", "Storage worker is closed."));
+        return this.readerClient.request({ type: "find", roomId, actorId, clientTxnId }, { priority: "admission" });
+    }
+    ingestUser(input) {
+        return this.writer.request({ type: "ingestUser", input });
+    }
+    ingestOutput(input) {
+        return this.writer.request({ type: "ingestOutput", input }, { priority: "output", fairnessKey: input.roomId });
+    }
+    cancelPendingCommands(sessionId) { return this.writer.request({ type: "cancelPendingCommands", sessionId }, { priority: "control" }); }
+    commandReceiptPage(sessionId) { return this.writer.request({ type: "commandReceiptPage", sessionId }, { priority: "control" }); }
+    commandReceipts(sessionId) { return this.writer.request({ type: "commandReceipts", sessionId }, { priority: "control" }); }
+    commandReceipt(id) { return this.writer.request({ type: "commandReceipt", id }, { priority: "control" }); }
+    claimCommand(owner, leaseMs) { return this.writer.request({ type: "claimCommand", owner, leaseMs }, { priority: "background" }); }
+    transitionCommand(id, owner, token, state, error) { return this.writer.request({ type: "transitionCommand", id, owner, token, state, error }, { priority: "control" }); }
+    heartbeatCommand(id, owner, token, leaseMs) { return this.writer.request({ type: "heartbeatCommand", id, owner, token, leaseMs }, { priority: "control" }); }
+    status() {
+        const writer = this.writer.status();
+        const reader = this.reader?.status();
+        return { ready: writer.ready && (!reader || reader.ready), closed: writer.closed && (!reader || reader.closed), restarts: { writer: this.writerRestarts, reader: this.readerRestarts }, writer, reader };
+    }
+    async close() {
+        this.closed = true;
+        await Promise.all([this.writerClient.close(), this.reader?.close()]);
+    }
+}

package/dist/data/async-telemetry-maintenance.js

@@ -0,0 +1,9 @@
+import { BoundedWorkerClient } from "./bounded-worker-client.js";
+export class AsyncTelemetryMaintenance {
+    client;
+    constructor(path) { this.client = new BoundedWorkerClient(new URL("./telemetry-maintenance-worker.js", import.meta.url), { maxPending: 8, maxPendingBytes: 64 * 1024, maxMessageBytes: 16 * 1024, maxAgeMs: 2000, workerOptions: { workerData: { path } } }); }
+    command(action, cutoff) { return this.client.request({ action, cutoff }, { priority: "control" }); }
+    preview(cutoff) { return this.client.request({ action: "preview", cutoff }, { priority: "background" }); }
+    status() { return this.client.status(); }
+    close() { return this.client.close(); }
+}

package/dist/data/bounded-worker-client.js

@@ -0,0 +1,256 @@
+import { Worker } from "node:worker_threads";
+export class StorageUnavailableError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = "StorageUnavailableError";
+    }
+}
+/** Count bounded structured data without creating a second serialized payload. */
+export function boundedMessageBytes(value, maximum) {
+    let bytes = 0;
+    let nodes = 0;
+    const stack = [{ value }];
+    const seen = new Set();
+    while (stack.length) {
+        const frame = stack.pop();
+        const item = frame.value;
+        if (frame.leave) {
+            seen.delete(item);
+            continue;
+        }
+        if (++nodes > 100_000)
+            throw new StorageUnavailableError("storage_payload_limit", "Storage message has too many fields.");
+        if (typeof item === "string")
+            bytes += Buffer.byteLength(item, "utf8") + 8;
+        else if (item === null || item === undefined || typeof item === "boolean" || typeof item === "number")
+            bytes += 8;
+        else if (ArrayBuffer.isView(item))
+            bytes += item.byteLength + 16;
+        else if (item instanceof ArrayBuffer)
+            bytes += item.byteLength + 16;
+        else if (typeof item === "object") {
+            if (seen.has(item))
+                throw new StorageUnavailableError("storage_payload_limit", "Storage messages must be acyclic.");
+            seen.add(item);
+            stack.push({ value: item, leave: true });
+            bytes += 16;
+            for (const key in item) {
+                if (!Object.hasOwn(item, key))
+                    continue;
+                const child = item[key];
+                bytes += Buffer.byteLength(key, "utf8") + 8;
+                stack.push({ value: child });
+                if (stack.length > 100_000 || bytes > maximum)
+                    throw new StorageUnavailableError("storage_payload_limit", "Storage message exceeds its budget.");
+            }
+        }
+        else
+            throw new StorageUnavailableError("storage_payload_limit", "Unsupported storage message value.");
+        if (bytes > maximum)
+            throw new StorageUnavailableError("storage_payload_limit", "Storage message exceeds its byte budget; use payload references.");
+    }
+    return bytes;
+}
+/** One in-flight RPC keeps the worker's hidden message queue bounded. */
+export class BoundedWorkerClient {
+    worker;
+    queue = [];
+    inFlight;
+    pendingBytes = 0;
+    nextId = 0;
+    dispatchSequence = 0;
+    dispatchTimer;
+    lastServed = new Map();
+    ready = false;
+    closed = false;
+    exited = false;
+    termination;
+    startupTimer;
+    completed = 0;
+    rejected = 0;
+    lastResponseAt;
+    workerIdentity;
+    maximum;
+    constructor(url, options = {}) {
+        this.maximum = {
+            maxPending: options.maxPending ?? 128,
+            admissionWindowMs: options.admissionWindowMs ?? 0,
+            reservedControlRequests: Math.min(options.reservedControlRequests ?? 0, (options.maxPending ?? 128) - 1),
+            reservedControlBytes: Math.min(options.reservedControlBytes ?? 0, (options.maxPendingBytes ?? 8 * 1024 * 1024) - 1),
+            maxPendingBytes: options.maxPendingBytes ?? 8 * 1024 * 1024,
+            maxMessageBytes: options.maxMessageBytes ?? 1024 * 1024,
+            maxAgeMs: options.maxAgeMs ?? 500,
+            agingMs: options.agingMs ?? 100,
+            startupTimeoutMs: options.startupTimeoutMs ?? 10_000,
+        };
+        for (const [key, limit] of Object.entries(this.maximum))
+            if (!Number.isFinite(limit) || (key.startsWith("reservedControl") || key === "admissionWindowMs" ? limit < 0 : limit <= 0))
+                throw new Error("Storage budgets must be positive finite numbers.");
+        this.worker = new Worker(url, { resourceLimits: { maxOldGenerationSizeMb: 256 }, ...options.workerOptions });
+        this.worker.unref();
+        this.startupTimer = setTimeout(() => this.fail(new StorageUnavailableError("storage_worker_failed", "Storage worker startup timed out.")), this.maximum.startupTimeoutMs);
+        this.worker.on("message", (message) => {
+            this.lastResponseAt = performance.now();
+            if (message.worker)
+                this.workerIdentity = message.worker;
+            if (message.ready) {
+                clearTimeout(this.startupTimer);
+                this.ready = true;
+                this.workerIdentity = message.worker;
+                this.pump();
+                return;
+            }
+            const pending = this.inFlight;
+            if (!pending || message.id !== pending.id)
+                return;
+            this.inFlight = undefined;
+            this.pendingBytes -= pending.bytes;
+            clearTimeout(pending.timer);
+            if (!pending.settled) {
+                pending.settled = true;
+                if (message.error) {
+                    this.rejected++;
+                    pending.reject(Object.assign(new Error(message.error.message ?? "Storage operation failed."), { code: message.error.code ?? "storage_operation_failed" }));
+                }
+                else {
+                    try {
+                        boundedMessageBytes(message.value, this.maximum.maxMessageBytes);
+                        this.completed++;
+                        pending.resolve(message.value);
+                    }
+                    catch (error) {
+                        this.rejected++;
+                        pending.reject(error);
+                    }
+                }
+            }
+            this.pump();
+        });
+        this.worker.on("error", () => this.fail(new StorageUnavailableError("storage_worker_failed", "Storage worker failed; in-flight commit state must be reconciled.")));
+        this.worker.on("exit", () => { this.exited = true; if (!this.closed)
+            this.fail(new StorageUnavailableError("storage_worker_failed", "Storage worker exited.")); });
+    }
+    request(command, options = {}) {
+        if (this.closed)
+            return Promise.reject(new StorageUnavailableError("storage_closed", "Storage worker is closed."));
+        let bytes;
+        try {
+            bytes = boundedMessageBytes(command, this.maximum.maxMessageBytes);
+        }
+        catch (error) {
+            return Promise.reject(error);
+        }
+        if (this.queue.length + Number(Boolean(this.inFlight)) >= this.maximum.maxPending - (options.priority === "control" ? 0 : this.maximum.reservedControlRequests) || this.pendingBytes + bytes > this.maximum.maxPendingBytes - (options.priority === "control" ? 0 : this.maximum.reservedControlBytes)) {
+            this.rejected++;
+            return Promise.reject(new StorageUnavailableError("storage_overloaded", "Storage capacity exhausted; retry with the same transaction ID."));
+        }
+        const age = Math.min(options.timeoutMs ?? this.maximum.maxAgeMs, this.maximum.maxAgeMs);
+        if (!Number.isFinite(age) || age <= 0)
+            return Promise.reject(new StorageUnavailableError("storage_deadline", "Storage deadline elapsed."));
+        return new Promise((resolve, reject) => {
+            const now = performance.now();
+            const pending = {
+                id: ++this.nextId, command, bytes, queuedAt: now, deadline: now + age,
+                priority: options.priority === "control" ? -1 : options.priority === "background" ? 2 : options.priority === "output" ? 1 : 0,
+                fairnessKey: `${options.priority ?? "admission"}:${options.fairnessKey?.slice(0, 256) ?? "default"}`,
+                resolve: value => resolve(value), reject, settled: false,
+                timer: setTimeout(() => {
+                    if (pending.settled)
+                        return;
+                    pending.settled = true;
+                    this.rejected++;
+                    if (this.inFlight === pending) {
+                        reject(new StorageUnavailableError("storage_unknown", "Storage response deadline elapsed; reconcile the same transaction ID before retrying."));
+                        this.fail(new StorageUnavailableError("storage_worker_failed", "Storage worker exceeded its execution deadline."));
+                    }
+                    else {
+                        const index = this.queue.indexOf(pending);
+                        if (index >= 0) {
+                            this.queue.splice(index, 1);
+                            this.pendingBytes -= pending.bytes;
+                        }
+                        reject(new StorageUnavailableError("storage_deadline", "Storage queue deadline elapsed before execution."));
+                    }
+                }, age),
+            };
+            this.pendingBytes += bytes;
+            this.queue.push(pending);
+            this.schedulePump(options.priority === "control");
+        });
+    }
+    status() {
+        return { ready: this.ready, closed: this.closed, exited: this.exited, queued: this.queue.length, inFlight: Boolean(this.inFlight), pendingBytes: this.pendingBytes, oldestAgeMs: Math.max(0, ...this.queue.map(entry => performance.now() - entry.queuedAt), this.inFlight ? performance.now() - this.inFlight.queuedAt : 0), lastResponseAgeMs: this.lastResponseAt === undefined ? undefined : performance.now() - this.lastResponseAt, completed: this.completed, rejected: this.rejected, worker: this.workerIdentity, limits: { ...this.maximum } };
+    }
+    async close() {
+        this.fail(new StorageUnavailableError("storage_closed", "Storage worker closed; reconcile any in-flight transaction."), false);
+        await (this.termination ??= this.worker.terminate());
+    }
+    fail(error, terminateWorker = true) {
+        if (this.closed)
+            return;
+        this.closed = true;
+        clearTimeout(this.startupTimer);
+        if (this.dispatchTimer)
+            clearTimeout(this.dispatchTimer);
+        this.dispatchTimer = undefined;
+        for (const pending of [...this.queue, ...(this.inFlight ? [this.inFlight] : [])]) {
+            clearTimeout(pending.timer);
+            if (!pending.settled) {
+                pending.settled = true;
+                pending.reject(pending === this.inFlight ? new StorageUnavailableError("storage_unknown", "Storage worker stopped during execution; reconcile the transaction ID.") : error);
+                this.rejected++;
+            }
+        }
+        this.queue.length = 0;
+        this.lastServed.clear();
+        this.inFlight = undefined;
+        this.pendingBytes = 0;
+        if (terminateWorker)
+            this.termination ??= this.worker.terminate();
+    }
+    schedulePump(control) {
+        if (control || !this.maximum.admissionWindowMs) {
+            this.pump();
+            return;
+        }
+        if (this.dispatchTimer || this.inFlight || !this.ready || this.closed)
+            return;
+        // Collect a bounded burst before choosing its Room; short control RPCs bypass this window.
+        this.dispatchTimer = setTimeout(() => { this.dispatchTimer = undefined; this.pump(); }, this.maximum.admissionWindowMs);
+        this.dispatchTimer.unref?.();
+    }
+    pump() {
+        if (!this.ready || this.closed || this.inFlight || !this.queue.length)
+            return;
+        if (this.dispatchTimer)
+            clearTimeout(this.dispatchTimer);
+        this.dispatchTimer = undefined;
+        const now = performance.now();
+        let index = 0;
+        const score = (entry) => entry.priority - Math.floor((now - entry.queuedAt) / this.maximum.agingMs);
+        for (let i = 1; i < this.queue.length; i++) {
+            const candidate = this.queue[i], current = this.queue[index];
+            if (score(candidate) < score(current) || (score(candidate) === score(current) && (this.lastServed.get(candidate.fairnessKey) ?? 0) < (this.lastServed.get(current.fairnessKey) ?? 0)))
+                index = i;
+        }
+        const pending = this.queue.splice(index, 1)[0];
+        this.inFlight = pending;
+        this.lastServed.delete(pending.fairnessKey);
+        this.lastServed.set(pending.fairnessKey, ++this.dispatchSequence);
+        const liveKeys = new Set([...this.queue.map(item => item.fairnessKey), pending.fairnessKey]);
+        for (const key of this.lastServed.keys()) {
+            if (this.lastServed.size <= this.maximum.maxPending)
+                break;
+            if (!liveKeys.has(key))
+                this.lastServed.delete(key);
+        }
+        try {
+            this.worker.postMessage({ id: pending.id, command: pending.command, deadline: pending.deadline, maxResultBytes: this.maximum.maxMessageBytes });
+        }
+        catch {
+            this.fail(new StorageUnavailableError("storage_worker_failed", "Storage IPC failed."));
+        }
+    }
+}

package/dist/data/chat-read-projections.js

@@ -0,0 +1,159 @@
+export const CHAT_READ_PROJECTION_SCHEMA = `
+CREATE INDEX IF NOT EXISTS idx_event_log_timing_sequence ON event_log(session_id,session_sequence,stream_id) WHERE type IN ('message_queued','message_steered','message_started','message_finished','session_error','thinking_finished','assistant_message');
+CREATE INDEX IF NOT EXISTS idx_event_log_semantic_sequence ON event_log(session_id,session_sequence DESC,stream_id DESC) WHERE type NOT IN ('assistant_delta','thinking_delta','tool_execution_updated');
+CREATE TABLE IF NOT EXISTS chat_trace_revisions(session_id TEXT PRIMARY KEY,revision INTEGER NOT NULL);
+CREATE TRIGGER IF NOT EXISTS chat_trace_insert AFTER INSERT ON event_log WHEN NEW.session_id IS NOT NULL BEGIN
+ INSERT INTO chat_trace_revisions VALUES(NEW.session_id,1) ON CONFLICT(session_id) DO UPDATE SET revision=revision+1;
+END;
+CREATE TRIGGER IF NOT EXISTS chat_trace_update AFTER UPDATE ON event_log WHEN NEW.session_id IS NOT NULL BEGIN
+ INSERT INTO chat_trace_revisions VALUES(NEW.session_id,1) ON CONFLICT(session_id) DO UPDATE SET revision=revision+1;
+ UPDATE chat_trace_revisions SET revision=revision+1 WHERE session_id=OLD.session_id AND OLD.session_id IS NOT NEW.session_id;
+END;
+CREATE TRIGGER IF NOT EXISTS chat_trace_delete AFTER DELETE ON event_log WHEN OLD.session_id IS NOT NULL BEGIN
+ INSERT INTO chat_trace_revisions VALUES(OLD.session_id,1) ON CONFLICT(session_id) DO UPDATE SET revision=revision+1;
+END;
+CREATE INDEX IF NOT EXISTS idx_event_log_unread_stream ON event_log(stream_id) WHERE (retention_class='chat_message' AND type IN ('user.message.accepted','assistant_message')) OR type='session_error';
+CREATE TABLE IF NOT EXISTS chat_unread_index(stream_id INTEGER PRIMARY KEY,session_id TEXT NOT NULL);
+CREATE INDEX IF NOT EXISTS idx_chat_unread_session_stream ON chat_unread_index(session_id,stream_id);
+CREATE TABLE IF NOT EXISTS chat_unread_counts(session_id TEXT PRIMARY KEY,unread_count INTEGER NOT NULL DEFAULT 0);
+CREATE TRIGGER IF NOT EXISTS chat_unread_count_insert AFTER INSERT ON chat_unread_index BEGIN
+ INSERT INTO chat_unread_counts(session_id,unread_count) VALUES(NEW.session_id,CASE WHEN NEW.stream_id>COALESCE((SELECT last_read_stream_id FROM app_session_read_state WHERE session_id=NEW.session_id),0) THEN 1 ELSE 0 END)
+ ON CONFLICT(session_id) DO UPDATE SET unread_count=unread_count+excluded.unread_count;
+END;
+CREATE TRIGGER IF NOT EXISTS chat_unread_count_delete AFTER DELETE ON chat_unread_index BEGIN
+ UPDATE chat_unread_counts SET unread_count=MAX(0,unread_count-CASE WHEN OLD.stream_id>COALESCE((SELECT last_read_stream_id FROM app_session_read_state WHERE session_id=OLD.session_id),0) THEN 1 ELSE 0 END) WHERE session_id=OLD.session_id;
+END;
+CREATE TRIGGER IF NOT EXISTS chat_unread_event_insert AFTER INSERT ON event_log WHEN NEW.session_id IS NOT NULL AND ((NEW.retention_class='chat_message' AND NEW.type IN ('user.message.accepted','assistant_message')) OR NEW.type='session_error') BEGIN
+ INSERT OR IGNORE INTO chat_unread_index VALUES(NEW.stream_id,NEW.session_id);
+END;
+CREATE TRIGGER IF NOT EXISTS chat_unread_event_delete AFTER DELETE ON event_log BEGIN DELETE FROM chat_unread_index WHERE stream_id=OLD.stream_id; END;
+CREATE TRIGGER IF NOT EXISTS chat_unread_event_update AFTER UPDATE ON event_log BEGIN
+ DELETE FROM chat_unread_index WHERE stream_id=OLD.stream_id;
+ INSERT OR IGNORE INTO chat_unread_index SELECT NEW.stream_id,NEW.session_id WHERE NEW.session_id IS NOT NULL AND ((NEW.retention_class='chat_message' AND NEW.type IN ('user.message.accepted','assistant_message')) OR NEW.type='session_error');
+END;
+CREATE TRIGGER IF NOT EXISTS chat_unread_mark_insert AFTER INSERT ON app_session_read_state BEGIN
+ INSERT INTO chat_unread_counts(session_id,unread_count) SELECT NEW.session_id,COUNT(*) FROM chat_unread_index WHERE session_id=NEW.session_id AND stream_id>NEW.last_read_stream_id
+ ON CONFLICT(session_id) DO UPDATE SET unread_count=excluded.unread_count;
+END;
+CREATE TRIGGER IF NOT EXISTS chat_unread_mark_update AFTER UPDATE OF last_read_stream_id ON app_session_read_state BEGIN
+ INSERT INTO chat_unread_counts(session_id,unread_count) SELECT NEW.session_id,COUNT(*) FROM chat_unread_index WHERE session_id=NEW.session_id AND stream_id>NEW.last_read_stream_id
+ ON CONFLICT(session_id) DO UPDATE SET unread_count=excluded.unread_count;
+END;
+CREATE TRIGGER IF NOT EXISTS chat_unread_mark_delete AFTER DELETE ON app_session_read_state BEGIN
+ INSERT INTO chat_unread_counts(session_id,unread_count) SELECT OLD.session_id,COUNT(*) FROM chat_unread_index WHERE session_id=OLD.session_id
+ ON CONFLICT(session_id) DO UPDATE SET unread_count=excluded.unread_count;
+END;
+CREATE TABLE IF NOT EXISTS chat_history_index (
+ message_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, event_sequence INTEGER NOT NULL,
+ source_stream_id INTEGER, role TEXT NOT NULL, created_at TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_chat_history_index_session_event ON chat_history_index(session_id,event_sequence DESC,message_id DESC);
+CREATE INDEX IF NOT EXISTS idx_chat_history_index_session_created ON chat_history_index(session_id,created_at,message_id);
+CREATE INDEX IF NOT EXISTS idx_chat_history_index_source ON chat_history_index(source_stream_id);
+CREATE TABLE IF NOT EXISTS chat_history_counts (session_id TEXT PRIMARY KEY,message_count INTEGER NOT NULL DEFAULT 0,revision INTEGER NOT NULL DEFAULT 0);
+CREATE TABLE IF NOT EXISTS chat_read_backfill (id INTEGER PRIMARY KEY CHECK(id=1),cursor INTEGER NOT NULL DEFAULT 0,target INTEGER NOT NULL,event_cursor INTEGER NOT NULL DEFAULT 0,event_target INTEGER NOT NULL,paused INTEGER NOT NULL DEFAULT 0);
+INSERT OR IGNORE INTO chat_read_backfill(id,target,event_target) SELECT 1,COALESCE(MAX(rowid),0),(SELECT COALESCE(MAX(stream_id),0) FROM event_log) FROM chat_messages;
+CREATE TRIGGER IF NOT EXISTS chat_history_count_insert AFTER INSERT ON chat_history_index BEGIN
+ INSERT INTO chat_history_counts(session_id,message_count,revision) VALUES(NEW.session_id,1,1)
+ ON CONFLICT(session_id) DO UPDATE SET message_count=message_count+1,revision=revision+1;
+END;
+CREATE TRIGGER IF NOT EXISTS chat_history_count_delete AFTER DELETE ON chat_history_index BEGIN
+ UPDATE chat_history_counts SET message_count=MAX(0,message_count-1),revision=revision+1 WHERE session_id=OLD.session_id;
+END;
+CREATE TRIGGER IF NOT EXISTS chat_history_message_insert AFTER INSERT ON chat_messages BEGIN
+ INSERT OR IGNORE INTO chat_history_index(message_id,session_id,event_sequence,source_stream_id,role,created_at)
+ VALUES(NEW.id,NEW.session_id,COALESCE((SELECT session_sequence FROM event_log WHERE stream_id=NEW.source_stream_id),NEW.sequence),NEW.source_stream_id,NEW.role,NEW.created_at);
+END;
+CREATE TRIGGER IF NOT EXISTS chat_history_message_update AFTER UPDATE ON chat_messages BEGIN
+ DELETE FROM chat_history_index WHERE message_id=OLD.id;
+ INSERT INTO chat_history_index(message_id,session_id,event_sequence,source_stream_id,role,created_at)
+ VALUES(NEW.id,NEW.session_id,COALESCE((SELECT session_sequence FROM event_log WHERE stream_id=NEW.source_stream_id),NEW.sequence),NEW.source_stream_id,NEW.role,NEW.created_at);
+END;
+CREATE TRIGGER IF NOT EXISTS chat_history_message_delete AFTER DELETE ON chat_messages BEGIN
+ DELETE FROM chat_history_index WHERE message_id=OLD.id;
+END;
+CREATE TRIGGER IF NOT EXISTS chat_history_event_sequence AFTER UPDATE OF session_sequence ON event_log
+WHEN OLD.session_sequence IS NOT NEW.session_sequence BEGIN
+ UPDATE chat_history_index SET event_sequence=NEW.session_sequence WHERE source_stream_id=NEW.stream_id;
+ UPDATE chat_history_counts SET revision=revision+1 WHERE session_id=NEW.session_id;
+END;
+`;
+/** Resumable bounded backfill; concurrent product writes maintain their rows through triggers. */
+export class ChatReadProjectionStore {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    status() {
+        const r = this.db.prepare("SELECT cursor,target,event_cursor,event_target,paused FROM chat_read_backfill WHERE id=1").get();
+        return { ...r, paused: Boolean(r.paused), historyComplete: r.cursor >= r.target, unreadComplete: r.event_cursor >= r.event_target, complete: r.cursor >= r.target && r.event_cursor >= r.event_target };
+    }
+    setPaused(paused) { this.db.prepare("UPDATE chat_read_backfill SET paused=? WHERE id=1").run(paused ? 1 : 0); }
+    step(limit = 128, maxMs = 4) {
+        if (!Number.isSafeInteger(limit) || limit < 1 || limit > 512)
+            throw Error("Backfill row budget must be 1..512");
+        const state = this.status();
+        if (state.paused || state.complete)
+            return { ...state, processed: 0 };
+        this.db.exec("BEGIN IMMEDIATE");
+        try {
+            const current = this.status();
+            if (current.paused || current.complete) {
+                this.db.exec("COMMIT");
+                return { ...current, processed: 0 };
+            }
+            if (current.historyComplete) {
+                const rows = this.db.prepare("SELECT stream_id,session_id FROM event_log INDEXED BY idx_event_log_unread_stream WHERE stream_id>? AND stream_id<=? AND ((retention_class='chat_message' AND type IN ('user.message.accepted','assistant_message')) OR type='session_error') ORDER BY stream_id LIMIT ?").all(current.event_cursor, current.event_target, limit);
+                const insert = this.db.prepare("INSERT OR IGNORE INTO chat_unread_index VALUES(?,?)");
+                let processed = 0;
+                const started = performance.now();
+                for (const row of rows) {
+                    if (row.session_id)
+                        insert.run(row.stream_id, row.session_id);
+                    processed++;
+                    if (performance.now() - started >= maxMs)
+                        break;
+                }
+                const cursor = processed === rows.length && rows.length < limit ? current.event_target : rows[processed - 1].stream_id;
+                this.db.prepare("UPDATE chat_read_backfill SET event_cursor=? WHERE id=1").run(cursor);
+                this.db.exec("COMMIT");
+                return { ...this.status(), processed };
+            }
+            const rows = this.db.prepare("SELECT rowid AS cursor,id FROM chat_messages WHERE rowid>? AND rowid<=? ORDER BY rowid LIMIT ?").all(current.cursor, current.target, limit);
+            const insert = this.db.prepare(`INSERT OR IGNORE INTO chat_history_index(message_id,session_id,event_sequence,source_stream_id,role,created_at)
+    SELECT m.id,m.session_id,COALESCE(e.session_sequence,m.sequence),m.source_stream_id,m.role,m.created_at FROM chat_messages m LEFT JOIN event_log e ON e.stream_id=m.source_stream_id WHERE m.id=?`);
+            let processed = 0;
+            const started = performance.now();
+            for (const row of rows) {
+                insert.run(row.id);
+                processed++;
+                if (performance.now() - started >= maxMs)
+                    break;
+            }
+            const cursor = processed === rows.length && rows.length < limit ? current.target : rows[processed - 1].cursor;
+            this.db.prepare("UPDATE chat_read_backfill SET cursor=? WHERE id=1").run(cursor);
+            this.db.exec("COMMIT");
+            return { ...this.status(), processed };
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+}
+export const CHAT_NAVIGATION_REVISION_SCHEMA = `
+CREATE INDEX IF NOT EXISTS idx_sessions_navigation_cursor ON sessions(room_id,id) WHERE deleted_at IS NULL;
+CREATE INDEX IF NOT EXISTS idx_sessions_navigation_read ON sessions(room_id,last_activity_at DESC,created_at DESC) WHERE deleted_at IS NULL;
+CREATE TABLE IF NOT EXISTS chat_navigation_clock(id INTEGER PRIMARY KEY CHECK(id=1),revision INTEGER NOT NULL);
+INSERT OR IGNORE INTO chat_navigation_clock VALUES(1,0);
+CREATE TRIGGER IF NOT EXISTS chat_navigation_session_insert AFTER INSERT ON sessions BEGIN UPDATE chat_navigation_clock SET revision=revision+1 WHERE id=1; END;
+CREATE TRIGGER IF NOT EXISTS chat_navigation_session_delete AFTER DELETE ON sessions BEGIN UPDATE chat_navigation_clock SET revision=revision+1 WHERE id=1; END;
+CREATE TRIGGER IF NOT EXISTS chat_navigation_session_update AFTER UPDATE ON sessions
+WHEN OLD.title IS NOT NEW.title OR OLD.profile IS NOT NEW.profile OR OLD.parent_id IS NOT NEW.parent_id OR OLD.origin_id IS NOT NEW.origin_id OR OLD.metadata_json IS NOT NEW.metadata_json OR OLD.deleted_at IS NOT NEW.deleted_at OR OLD.active_model_json IS NOT NEW.active_model_json OR OLD.room_id IS NOT NEW.room_id
+BEGIN UPDATE chat_navigation_clock SET revision=revision+1 WHERE id=1; END;
+CREATE TRIGGER IF NOT EXISTS chat_navigation_binding_insert AFTER INSERT ON session_runtime_bindings BEGIN UPDATE chat_navigation_clock SET revision=revision+1 WHERE id=1; END;
+CREATE TRIGGER IF NOT EXISTS chat_navigation_binding_update AFTER UPDATE ON session_runtime_bindings
+WHEN OLD.runtime_instance_id IS NOT NEW.runtime_instance_id OR OLD.native_session_id IS NOT NEW.native_session_id OR OLD.revision IS NOT NEW.revision
+BEGIN UPDATE chat_navigation_clock SET revision=revision+1 WHERE id=1; END;
+CREATE TRIGGER IF NOT EXISTS chat_navigation_binding_delete AFTER DELETE ON session_runtime_bindings BEGIN UPDATE chat_navigation_clock SET revision=revision+1 WHERE id=1; END;
+`;

package/dist/data/chat-read-worker.js

@@ -0,0 +1,73 @@
+import { ChatReadStateService } from "../apps/chat/data/read-state-service.js";
+import { DatabaseSync } from "node:sqlite";
+import { ChatReadProjectionStore } from "./chat-read-projections.js";
+import { parentPort, workerData, threadId } from "node:worker_threads";
+import { PiboDataStore } from "./pibo-store.js";
+import { boundedMessageBytes } from "./bounded-worker-client.js";
+import { CHAT_READ_METHODS } from "./async-chat-reads.js";
+import { ChatTimelineQueryService } from "../apps/chat/data/timeline-query-service.js";
+import { ChatHistoryQueryService } from "../apps/chat/data/history-query-service.js";
+if (!parentPort)
+    throw Error("Chat reads require a worker");
+const store = new PiboDataStore(workerData.path, { payloadRootDir: workerData.payloadRootDir, readOnly: true });
+store.db.exec("PRAGMA busy_timeout=10");
+// Timeline hydration must match the in-process default. A smaller worker budget silently replaces
+// message bodies with their preview and StoredChatEvent carries no payload reference to recover them,
+// so the same query would answer with less content than the fallback it replaced. The IPC bound stays
+// owned by maxMessageBytes plus the bounded page-halving recovery in the event stream replay loop.
+const services = { navigation: new ChatReadStateService(store), timeline: new ChatTimelineQueryService(store), history: new ChatHistoryQueryService(store, 2 * 1024 * 1024) };
+let operations = 0;
+const maintenanceDb = new DatabaseSync(workerData.path);
+maintenanceDb.exec("PRAGMA busy_timeout=10; PRAGMA foreign_keys=ON; PRAGMA synchronous=FULL");
+const maintenance = new ChatReadProjectionStore(maintenanceDb);
+let maintenanceFailures = 0;
+let maintenanceDelay = 25;
+let maintenanceTimer;
+function scheduleMaintenance() {
+    if (maintenanceTimer)
+        return;
+    const state = maintenance.status();
+    if (state.complete || state.paused)
+        return;
+    maintenanceTimer = setTimeout(() => { maintenanceTimer = undefined; try {
+        maintenance.step();
+        maintenanceDelay = 25;
+    }
+    catch {
+        maintenanceFailures++;
+        maintenanceDelay = Math.min(1000, maintenanceDelay * 2);
+    } scheduleMaintenance(); }, maintenanceDelay);
+    maintenanceTimer.unref();
+}
+scheduleMaintenance();
+parentPort.on("message", (request) => {
+    try {
+        if (performance.now() >= request.deadline)
+            throw Object.assign(Error("Read deadline elapsed"), { code: "storage_deadline" });
+        const { group, method, args } = request.command;
+        if (String(group) === "maintenance") {
+            if (method === "pause")
+                maintenance.setPaused(true);
+            else if (method === "resume") {
+                maintenance.setPaused(false);
+                scheduleMaintenance();
+            }
+            else if (method !== "status")
+                throw Error("Unsupported maintenance operation");
+            parentPort.postMessage({ id: request.id, value: { ...maintenance.status(), failures: maintenanceFailures } });
+            return;
+        }
+        if (!Object.hasOwn(services, group) || !CHAT_READ_METHODS[group].includes(method))
+            throw Error("Unsupported read operation");
+        const service = services[group];
+        const value = service[method].apply(service, args);
+        boundedMessageBytes(value, request.maxResultBytes);
+        operations++;
+        parentPort.postMessage({ id: request.id, value, worker: { pid: process.pid, threadId, operations, readOnly: true, maintenance: { ...maintenance.status(), failures: maintenanceFailures } } });
+    }
+    catch (error) {
+        const code = error && typeof error === "object" && "code" in error && typeof error.code === "string" && error.code.startsWith("storage_") ? error.code : "storage_read_failed";
+        parentPort.postMessage({ id: request.id, error: { code, message: "Bounded chat read failed; retry the page." } });
+    }
+});
+parentPort.postMessage({ ready: true, worker: { pid: process.pid, threadId, operations, readOnly: true, maintenance: { ...maintenance.status(), failures: maintenanceFailures } } });