@pasko70/pibo 3.4.4 → 3.5.0

package/dist/data/chat-storage-worker.js

@@ -0,0 +1,146 @@
+import { MessageCommandStore } from "./message-command-store.js";
+import { parentPort, workerData } from "node:worker_threads";
+import { ChatRoomService } from "../apps/chat/data/room-service.js";
+import { ChatSessionQueryService } from "../apps/chat/data/session-query-service.js";
+import { isPiboRoomArchived } from "../apps/chat/types/rooms.js";
+import { PiboDataStore } from "./pibo-store.js";
+import { ChatEventCommandService, chatClientTransactionKey } from "../apps/chat/data/event-command-service.js";
+import { ChatDataIngestService } from "./ingest-service.js";
+import { boundedMessageBytes } from "./bounded-worker-client.js";
+const port = parentPort;
+if (!port)
+    throw new Error("Chat storage requires a dedicated worker.");
+const config = workerData;
+const store = new PiboDataStore(config.path, { payloadRootDir: config.payloadRootDir });
+// Lock waiting is bounded independently of SQL execution; retries yield this worker.
+store.db.exec("PRAGMA busy_timeout=10");
+const commands = new ChatEventCommandService(store);
+const ingest = new ChatDataIngestService(store);
+const rooms = new ChatRoomService(store);
+const sessions = new ChatSessionQueryService(store);
+const messageCommands = new MessageCommandStore(store);
+let active = false;
+let operations = 0;
+let busyRetries = 0;
+let lastOperationMs = 0;
+function execute(command) {
+    switch (command.type) {
+        case "cancelPendingCommands": return messageCommands.cancelPending(command.sessionId);
+        case "commandReceiptPage": return { receipts: messageCommands.list(command.sessionId), queue: messageCommands.queueStatus(command.sessionId) };
+        case "commandReceipts": return messageCommands.list(command.sessionId);
+        case "commandReceipt": return messageCommands.get(command.id);
+        case "claimCommand": return messageCommands.claim(command.owner, command.leaseMs);
+        case "transitionCommand": return messageCommands.transition(command.id, command.owner, command.token, command.state, command.error);
+        case "heartbeatCommand": return messageCommands.heartbeat(command.id, command.owner, command.token, command.leaseMs);
+        case "resolveRoom": {
+            const room = command.roomId ? rooms.getRoom(command.roomId) : undefined;
+            if (room)
+                return room;
+            if (command.required)
+                throw Object.assign(new Error("Room not found"), { code: "room_not_found" });
+            return store.transaction(() => rooms.ensureDefaultRoom());
+        }
+        case "admit": {
+            const room = command.input.roomId ? rooms.getRoom(command.input.roomId) : undefined;
+            if (!room)
+                throw Object.assign(new Error("Room not found"), { code: "room_not_found" });
+            if (isPiboRoomArchived(room))
+                throw Object.assign(new Error("Archived rooms are read-only"), { code: "room_read_only" });
+            const key = command.input.clientTxnId ? chatClientTransactionKey(room.id, command.input.actorId, command.input.clientTxnId) : undefined;
+            const commandInput = command.durableCommand ? { sessionId: command.session.id, roomId: room.id, text: command.text, delivery: command.durableCommand.delivery } : undefined;
+            const receipt = key && commandInput ? messageCommands.find(key, messageCommands.fingerprint(commandInput)) : undefined;
+            const existing = key ? store.eventLog.findByIdempotencyKey(key) : undefined;
+            if (existing && commandInput && !receipt)
+                throw Object.assign(new Error("Transaction belongs to the legacy admission contract."), { code: "command_conflict" });
+            if (existing)
+                return { event: commands.findByClientTxn(room.id, command.input.actorId, command.input.clientTxnId), created: false, receipt };
+            const preparedCommand = commandInput ? messageCommands.prepare(commandInput) : undefined;
+            const createdAt = command.input.createdAt ?? new Date().toISOString();
+            const preparedPayload = ingest.prepareUserMessagePayload(command.text, createdAt);
+            return store.transaction(() => {
+                const concurrent = key ? store.eventLog.findByIdempotencyKey(key) : undefined;
+                if (concurrent) {
+                    const receipt = preparedCommand ? messageCommands.find(key, preparedCommand.fingerprint) : undefined;
+                    if (preparedCommand && !receipt)
+                        throw Object.assign(new Error("Transaction belongs to the legacy admission contract."), { code: "command_conflict" });
+                    return { event: commands.findByClientTxn(room.id, command.input.actorId, command.input.clientTxnId), created: false, receipt };
+                }
+                const event = commands.appendEvent({ ...command.input, createdAt });
+                sessions.upsertSession(command.session, command.durableCommand ? sessions.getSession(command.session.id)?.status ?? "idle" : "idle", command.session.updatedAt, { preserveRuntimeBinding: true });
+                ingest.ingestUserMessageAccepted({ session: command.session, roomId: room.id, actorId: command.input.actorId ?? "", text: command.text, clientTxnId: command.input.clientTxnId, eventId: command.durableCommand?.eventId, legacyEvent: event, preparedPayload });
+                const receipt = preparedCommand && command.durableCommand ? messageCommands.insert({ key: key ?? `chat:command:${command.durableCommand.eventId}`, ...preparedCommand, sessionId: command.session.id, roomId: room.id, eventId: command.durableCommand.eventId, streamId: event.streamId, delivery: command.durableCommand.delivery }) : undefined;
+                return { event, created: true, receipt };
+            });
+        }
+        case "append": return store.transaction(() => {
+            const key = command.input.clientTxnId
+                ? chatClientTransactionKey(command.input.roomId, command.input.actorId, command.input.clientTxnId)
+                : command.input.eventId ? `chat:event:${command.input.eventId}` : undefined;
+            const existing = key ? store.eventLog.findByIdempotencyKey(key) : undefined;
+            const event = commands.appendEvent(command.input);
+            return { event, created: !existing };
+        });
+        case "find": return commands.findByClientTxn(command.roomId, command.actorId, command.clientTxnId);
+        case "ingestUser": return ingest.ingestUserMessageAccepted(command.input);
+        case "ingestOutput": {
+            const result = ingest.ingestOutputEvent(command.input);
+            messageCommands.recordOutput(command.input.session.id, "eventId" in command.input.event ? command.input.event.eventId : undefined, command.input.event.type);
+            const row = store.db.prepare("SELECT created_at, event_id FROM event_log WHERE stream_id = ?").get(result.streamId);
+            if (!row)
+                throw new Error(`Missing output event ${result.streamId} after ingest.`);
+            return { ...result, stored: { createdAt: row.created_at, eventId: row.event_id ?? String(result.streamId) } };
+        }
+        case "status": return { operations, busyRetries, lastOperationMs, pid: process.pid, synchronous: store.db.prepare("PRAGMA synchronous").get(), journalMode: store.db.prepare("PRAGMA journal_mode").get() };
+    }
+}
+function respond(request, response) {
+    port.postMessage({ id: request.id, ...response, worker: workerStatus() });
+    active = false;
+}
+function workerStatus() {
+    return { pid: process.pid, operations, busyRetries, lastOperationMs, busyTimeoutMs: 10 };
+}
+function attempt(request) {
+    if (performance.now() >= request.deadline) {
+        respond(request, { error: { code: "storage_deadline", message: "Storage execution deadline elapsed before commit." } });
+        return;
+    }
+    const start = performance.now();
+    try {
+        const value = execute(request.command);
+        lastOperationMs = performance.now() - start;
+        operations++;
+        boundedMessageBytes(value, request.maxResultBytes);
+        respond(request, { value });
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : "";
+        if (/database is (?:locked|busy)/i.test(message) && performance.now() + 15 < request.deadline) {
+            busyRetries++;
+            setTimeout(() => attempt(request), 5 + Math.floor(Math.random() * 10));
+            return;
+        }
+        const domainCode = error && typeof error === "object" && "code" in error ? String(error.code) : "";
+        if (domainCode === "room_not_found" || domainCode === "room_read_only" || (domainCode.startsWith("storage_") || domainCode.startsWith("command_")) || domainCode === "pibo_output_identity_collision") {
+            respond(request, { error: { code: domainCode, message } });
+            return;
+        }
+        respond(request, { error: { code: /database is (?:locked|busy)/i.test(message) ? "storage_busy" : "storage_operation_failed", message: "Storage operation failed; reconcile the transaction ID before retrying." } });
+    }
+}
+port.on("message", (request) => {
+    if (active) {
+        port.postMessage({ id: request.id, error: { code: "storage_overloaded", message: "Storage worker already owns a request." } });
+        return;
+    }
+    active = true;
+    attempt(request);
+});
+port.postMessage({
+    ready: true,
+    worker: {
+        ...workerStatus(),
+        journalMode: store.db.prepare("PRAGMA journal_mode").get(),
+        synchronous: store.db.prepare("PRAGMA synchronous").get(),
+    },
+});

package/dist/data/ingest-service.js

@@ -17,6 +17,8 @@ export class PiboOutputIdentityCollisionError extends Error {
 export function outputPersistenceErrorIsRetryable(error) {
     if (error instanceof PiboOutputIdentityCollisionError)
         return false;
+    if (error && typeof error === "object" && "code" in error && error.code === "pibo_output_identity_collision")
+        return false;
     if (error instanceof AggregateError) {
         return error.errors.length === 0 || error.errors.some(outputPersistenceErrorIsRetryable);
     }
@@ -43,10 +45,11 @@ export class ChatDataIngestService {
                 duplicate: true,
             };
         }
+        const now = input.legacyEvent?.createdAt ?? new Date().toISOString();
+        const preparedPayload = input.preparedPayload ?? this.prepareUserMessagePayload(input.text, now);
         return this.store.transaction(() => {
-            const now = input.legacyEvent?.createdAt ?? new Date().toISOString();
-            this.store.sessions.upsertSession({ session: input.session, roomId: input.roomId, firstMessagePreview: input.text, lastActivityAt: now });
-            const payloadRef = this.writeTextPayloadIfLarge(input.text, now, "chat_message");
+            this.store.sessions.upsertSession({ session: input.session, roomId: input.roomId, firstMessagePreview: input.text, lastActivityAt: now, preserveRuntimeBinding: true });
+            const payloadRef = preparedPayload ? this.store.payloads.commitPreparedPayload(preparedPayload).id : undefined;
             const event = this.store.eventLog.appendEvent({
                 sessionId: input.session.id,
                 sessionSequence: this.nextEventSequence(input.session.id),
@@ -79,6 +82,7 @@ export class ChatDataIngestService {
                 roomId: input.roomId,
                 sequence: this.nextMessageSequence(input.session.id),
                 role: "user",
+                turnId: input.eventId,
                 actorId: input.actorId,
                 status: "complete",
                 createdAt: now,
@@ -91,10 +95,13 @@ export class ChatDataIngestService {
                     inlineText: payloadRef ? undefined : input.text,
                 }),
             });
-            this.upsertNavigation(input.session, input.roomId, previewText(input.text), now, "running");
+            this.upsertNavigation(input.session, input.roomId, previewText(input.text), now, input.eventId ? undefined : "running");
             return { streamId: event.streamId, messageId, duplicate: false };
         });
     }
+    prepareUserMessagePayload(text, createdAt) {
+        return this.prepareTextPayloadIfLarge(text, createdAt, "chat_message");
+    }
     ingestOutputEvent(input) {
         const event = input.event;
         const idempotencyKey = outputIdempotencyKey(event);
@@ -140,13 +147,17 @@ export class ChatDataIngestService {
                 };
             }
         }
+        const now = input.createdAt ?? new Date().toISOString();
+        const payload = payloadForOutputEvent(event);
+        const preparedPayload = payload ? this.preparePayloadIfLarge(payload.value, payload.contentType, now, retentionClassForOutputEvent(event)) : undefined;
         return this.store.transaction(() => {
-            const now = input.createdAt ?? new Date().toISOString();
+            if (event.type === "compaction_end" && !event.aborted && !event.errorMessage && !event.compactionStats) {
+                event.compactionStats = this.compactionStats(input.session.id, event.result);
+            }
             if (input.roomId) {
-                this.store.sessions.upsertSession({ session: input.session, roomId: input.roomId, lastActivityAt: now, status: outputSessionStatus(event) });
+                this.store.sessions.upsertSession({ session: input.session, roomId: input.roomId, lastActivityAt: now, status: outputSessionStatus(event), preserveRuntimeBinding: true });
             }
-            const payload = payloadForOutputEvent(event);
-            const payloadRef = payload ? this.writePayloadIfLarge(payload.value, payload.contentType, now, retentionClassForOutputEvent(event)) : undefined;
+            const payloadRef = preparedPayload ? this.store.payloads.commitPreparedPayload(preparedPayload).id : undefined;
             const storedEvent = this.store.eventLog.appendEvent({
                 sessionId: input.session.id,
                 sessionSequence: this.nextEventSequence(input.session.id),
@@ -233,6 +244,34 @@ export class ChatDataIngestService {
             return { streamId: storedEvent.streamId, duplicate: false, messageId, observationId };
         });
     }
+    compactionStats(sessionId, result) {
+        const boundary = this.store.db.prepare(`
+			SELECT COALESCE(MAX(session_sequence), 0) AS sequence
+			FROM event_log
+			WHERE session_id = ?
+				AND type = 'compaction_end'
+				AND COALESCE(json_extract(attributes_json, '$.aborted'), 0) = 0
+				AND json_extract(attributes_json, '$.errorMessage') IS NULL
+		`).get(sessionId);
+        const segmentWhere = `session_id = ? AND type = 'tool_execution_finished' AND session_sequence > ?`;
+        const count = this.store.db.prepare(`SELECT COUNT(*) AS count FROM event_log WHERE ${segmentWhere}`)
+            .get(sessionId, boundary.sequence);
+        const maxRow = this.store.db.prepare(`
+			SELECT attributes_json
+			FROM event_log
+			WHERE ${segmentWhere}
+				AND json_type(attributes_json, '$.toolMetrics.outputTokens') IN ('integer', 'real')
+			ORDER BY json_extract(attributes_json, '$.toolMetrics.outputTokens') DESC
+			LIMIT 1
+		`).get(sessionId, boundary.sequence);
+        const maxToolOutput = maxRow ? toolMetricsFromAttributes(maxRow.attributes_json) : undefined;
+        return compactObject({
+            toolCallCount: count.count,
+            maxToolOutputTokens: maxToolOutput?.outputTokens,
+            maxToolOutputTokenBasis: maxToolOutput?.tokenBasis,
+            compactionTokens: compactionTokenCount(result),
+        });
+    }
     nextEventSequence(sessionId) {
         const row = this.store.db.prepare("SELECT COALESCE(MAX(session_sequence), 0) + 1 AS next_sequence FROM event_log WHERE session_id = ?").get(sessionId);
         return row.next_sequence;
@@ -241,22 +280,22 @@ export class ChatDataIngestService {
         const row = this.store.db.prepare("SELECT COALESCE(MAX(sequence), 0) + 1 AS next_sequence FROM chat_messages WHERE session_id = ?").get(sessionId);
         return row.next_sequence;
     }
-    writeTextPayloadIfLarge(text, createdAt, retentionClass) {
+    prepareTextPayloadIfLarge(text, createdAt, retentionClass) {
         if (Buffer.byteLength(text, "utf8") <= INLINE_MESSAGE_PAYLOAD_THRESHOLD_BYTES)
             return undefined;
-        return this.store.payloads.writePayload({
+        return this.store.payloads.preparePayload({
             value: text,
             contentType: "text/plain; charset=utf-8",
             retentionClass,
             createdAt,
-        }).id;
+        });
     }
-    writePayloadIfLarge(value, contentType, createdAt, retentionClass) {
+    preparePayloadIfLarge(value, contentType, createdAt, retentionClass) {
         const bytes = Buffer.byteLength(typeof value === "string" ? value : JSON.stringify(value), "utf8");
         const threshold = typeof value === "string" ? INLINE_MESSAGE_PAYLOAD_THRESHOLD_BYTES : INLINE_JSON_PAYLOAD_THRESHOLD_BYTES;
         if (bytes <= threshold)
             return undefined;
-        return this.store.payloads.writePayload({ value, contentType, retentionClass, createdAt }).id;
+        return this.store.payloads.preparePayload({ value, contentType, retentionClass, createdAt });
     }
     upsertNavigation(session, roomId, lastMessagePreview, now, status) {
         this.store.navigation.upsertSession({
@@ -275,6 +314,32 @@ export class ChatDataIngestService {
         });
     }
 }
+function toolMetricsFromAttributes(attributesJson) {
+    try {
+        const attributes = JSON.parse(attributesJson);
+        if (!isRecord(attributes) || !isRecord(attributes.toolMetrics))
+            return undefined;
+        const outputTokens = nonNegativeFiniteNumber(attributes.toolMetrics.outputTokens);
+        const tokenBasis = typeof attributes.toolMetrics.tokenBasis === "string"
+            ? attributes.toolMetrics.tokenBasis
+            : undefined;
+        return { outputTokens, tokenBasis };
+    }
+    catch {
+        return undefined;
+    }
+}
+function compactionTokenCount(result) {
+    if (!isRecord(result))
+        return undefined;
+    return nonNegativeFiniteNumber(result.tokensBefore);
+}
+function nonNegativeFiniteNumber(value) {
+    return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
+}
+function isRecord(value) {
+    return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
 function deterministicId(prefix, value) {
     return `${prefix}_${createHash("sha256").update(value).digest("hex").slice(0, 32)}`;
 }
@@ -435,7 +500,7 @@ function specificAttributesForOutputEvent(event) {
     if (event.type === "session_error")
         return { error: event.error, ...(event.errorDetails ? { errorDetails: event.errorDetails } : {}) };
     if (event.type === "compaction_start" || event.type === "compaction_end")
-        return { compactionIndex: event.compactionIndex, reason: event.reason, aborted: "aborted" in event ? event.aborted : undefined, errorMessage: "errorMessage" in event ? event.errorMessage : undefined };
+        return { compactionIndex: event.compactionIndex, reason: event.reason, aborted: "aborted" in event ? event.aborted : undefined, errorMessage: "errorMessage" in event ? event.errorMessage : undefined, compactionStats: "compactionStats" in event ? event.compactionStats : undefined };
     return {};
 }
 function observationKindForOutputEvent(event) {

package/dist/data/message-command-store.js

@@ -0,0 +1,148 @@
+import { createHash, randomUUID } from "node:crypto";
+export const MESSAGE_COMMAND_SCHEMA = `
+CREATE TABLE IF NOT EXISTS message_commands (
+ id TEXT PRIMARY KEY, request_key TEXT NOT NULL UNIQUE, fingerprint TEXT NOT NULL,
+ session_id TEXT NOT NULL, room_id TEXT NOT NULL, event_id TEXT NOT NULL,
+ stream_id INTEGER NOT NULL, payload_ref TEXT NOT NULL REFERENCES payloads(id), payload_bytes INTEGER NOT NULL,
+ delivery TEXT NOT NULL CHECK(delivery IN ('queue','steer')),
+ state TEXT NOT NULL CHECK(state IN ('accepted','waiting_slot','initializing','session_queue','running','completed','failed','interrupted')),
+ owner TEXT, token INTEGER NOT NULL DEFAULT 0, lease_until INTEGER NOT NULL DEFAULT 0,
+ created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, error TEXT,
+ UNIQUE(session_id,event_id)
+);
+CREATE INDEX IF NOT EXISTS message_commands_dispatch_order ON message_commands(session_id,state,stream_id,delivery);
+CREATE INDEX IF NOT EXISTS message_commands_recent ON message_commands(session_id,stream_id DESC);
+CREATE INDEX IF NOT EXISTS message_commands_pending ON message_commands(state,created_at,id);
+CREATE INDEX IF NOT EXISTS message_commands_session ON message_commands(session_id,state,created_at,id);
+CREATE INDEX IF NOT EXISTS message_commands_lease ON message_commands(lease_until) WHERE owner IS NOT NULL;
+CREATE TABLE IF NOT EXISTS message_dispatch_clock (id INTEGER PRIMARY KEY CHECK(id=1), sequence INTEGER NOT NULL);
+INSERT OR IGNORE INTO message_dispatch_clock VALUES (1,0);
+CREATE TABLE IF NOT EXISTS message_dispatch_rooms (room_id TEXT PRIMARY KEY, sequence INTEGER NOT NULL);
+
+`;
+export const MESSAGE_COMMAND_LIMITS = Object.freeze({
+    queue: { count: 1000, roomCount: 256, sessionCount: 64, bytes: 64 * 1024 * 1024, roomBytes: 16 * 1024 * 1024, sessionBytes: 4 * 1024 * 1024, ageMs: 60 * 60 * 1000, roomAgeMs: 15 * 60 * 1000, sessionAgeMs: 10 * 60 * 1000 },
+    steer: { count: 64, roomCount: 16, sessionCount: 4, bytes: 4 * 1024 * 1024, roomBytes: 1024 * 1024, sessionBytes: 1024 * 1024, ageMs: 60 * 1000, roomAgeMs: 60 * 1000, sessionAgeMs: 60 * 1000 },
+    dispatch: { queue: 10, roomQueue: 5, steer: 2, roomSteer: 1 },
+});
+const active = "'accepted','waiting_slot','initializing','session_queue','running'";
+/** Durable receipts are independent of optional trace/telemetry retention. Only the storage worker owns this store. */
+export class MessageCommandStore {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    fingerprint(input) {
+        if (Buffer.byteLength(input.text) > 1024 * 1024)
+            throw domainError("command_too_large", "Message exceeds the durable command byte limit.");
+        return createHash("sha256").update(JSON.stringify(input)).digest("hex");
+    }
+    prepare(input) {
+        return {
+            fingerprint: this.fingerprint(input),
+            payload: this.store.payloads.preparePayload({ value: input.text, contentType: "text/plain", retentionClass: "message_command" }),
+        };
+    }
+    find(key, fingerprint) {
+        const row = this.store.db.prepare("SELECT * FROM message_commands WHERE request_key = ?").get(key);
+        if (row && fingerprint && row.fingerprint !== fingerprint)
+            throw domainError("command_conflict", "Client transaction ID is already bound to different message content, delivery, or session.");
+        return row && receipt(row);
+    }
+    list(sessionId) {
+        const pending = this.store.db.prepare(`SELECT * FROM message_commands WHERE session_id=? AND state IN (${active},'interrupted') ORDER BY stream_id DESC LIMIT 70`).all(sessionId);
+        const terminal = this.store.db.prepare("SELECT * FROM message_commands WHERE session_id=? AND state IN ('completed','failed') ORDER BY stream_id DESC LIMIT 64").all(sessionId);
+        return [...pending, ...terminal].sort((a, b) => b.stream_id - a.stream_id).map(receipt);
+    }
+    queueStatus(sessionId) {
+        const rows = this.store.db.prepare(`SELECT state,delivery,payload_bytes,created_at FROM message_commands WHERE session_id=? AND state IN (${active}) LIMIT 69`).all(sessionId);
+        const summarize = (delivery) => {
+            const scoped = rows.filter(row => row.delivery === delivery), limits = MESSAGE_COMMAND_LIMITS[delivery];
+            return { count: scoped.length, bytes: scoped.reduce((sum, row) => sum + row.payload_bytes, 0), oldestWaitMs: Math.max(0, ...scoped.filter(row => row.state === "accepted" || row.state === "waiting_slot").map(row => Date.now() - row.created_at)), limits: { count: limits.sessionCount, bytes: limits.sessionBytes, waitMs: limits.sessionAgeMs } };
+        };
+        return { queue: summarize("queue"), steer: summarize("steer") };
+    }
+    get(id) {
+        const row = this.store.db.prepare("SELECT * FROM message_commands WHERE id = ?").get(id);
+        return row && receipt(row);
+    }
+    insert(input) {
+        const prior = this.find(input.key, input.fingerprint);
+        if (prior)
+            return prior;
+        const limit = MESSAGE_COMMAND_LIMITS[input.delivery];
+        const rows = this.store.db.prepare(`SELECT session_id, room_id, payload_bytes, created_at, state FROM message_commands WHERE state IN (${active}) AND delivery=? LIMIT ?`).all(input.delivery, limit.count + 1);
+        const now = Date.now();
+        const exceeds = (scope, count, bytes, ageMs) => scope.length >= count
+            || scope.reduce((n, r) => n + r.payload_bytes, input.payload.byteSize) > bytes
+            || scope.some(r => (r.state === "accepted" || r.state === "waiting_slot") && now - r.created_at >= ageMs);
+        if (exceeds(rows, limit.count, limit.bytes, limit.ageMs)
+            || exceeds(rows.filter(r => r.room_id === input.roomId), limit.roomCount, limit.roomBytes, limit.roomAgeMs)
+            || exceeds(rows.filter(r => r.session_id === input.sessionId), limit.sessionCount, limit.sessionBytes, limit.sessionAgeMs)) {
+            throw domainError("command_overloaded", "Durable message queue count, byte or wait-age capacity reached; retry the same transaction later.");
+        }
+        const id = `cmd_${randomUUID()}`;
+        const payload = this.store.payloads.commitPreparedPayload(input.payload);
+        this.store.db.prepare(`INSERT INTO message_commands (id,request_key,fingerprint,session_id,room_id,event_id,stream_id,payload_ref,payload_bytes,delivery,state,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,'accepted',?,?)`).run(id, input.key, input.fingerprint, input.sessionId, input.roomId, input.eventId, input.streamId, payload.id, payload.byteSize, input.delivery, now, now);
+        return this.get(id);
+    }
+    claim(owner, leaseMs) {
+        const limits = MESSAGE_COMMAND_LIMITS.dispatch;
+        const candidateSql = `WITH occupied AS (
+		 SELECT room_id,delivery FROM message_commands WHERE owner IS NOT NULL AND state IN (${active})
+		) SELECT c.* FROM message_commands c LEFT JOIN message_dispatch_rooms r ON r.room_id=c.room_id
+		 WHERE c.state='accepted'
+		 AND (SELECT count(*) FROM occupied o WHERE o.delivery=c.delivery) < CASE c.delivery WHEN 'steer' THEN ${limits.steer} ELSE ${limits.queue} END
+		 AND (SELECT count(*) FROM occupied o WHERE o.delivery=c.delivery AND o.room_id=c.room_id) < CASE c.delivery WHEN 'steer' THEN ${limits.roomSteer} ELSE ${limits.roomQueue} END
+		 AND NOT EXISTS (SELECT 1 FROM message_commands p WHERE p.session_id=c.session_id AND p.state IN (${active},'interrupted') AND p.stream_id<c.stream_id AND (c.delivery='queue' OR p.delivery='steer'))
+		 ORDER BY CASE c.delivery WHEN 'steer' THEN 0 ELSE 1 END,COALESCE(r.sequence,0),c.stream_id LIMIT 1`;
+        // Idle polling must not take the SQLite writer lock. The claim transaction rechecks.
+        if (!this.store.db.prepare(candidateSql).get() && !this.store.db.prepare("SELECT 1 FROM message_commands WHERE owner IS NOT NULL AND lease_until <= ? LIMIT 1").get(Date.now()))
+            return undefined;
+        const row = this.store.transaction(() => {
+            const now = Date.now();
+            // Never replay an expired command that may already have reached a provider or tool.
+            this.store.db.prepare(`UPDATE message_commands SET state=CASE WHEN state='waiting_slot' THEN 'accepted' ELSE 'interrupted' END, error=CASE WHEN state='waiting_slot' THEN NULL ELSE 'Runtime ownership expired; execution requires reconciliation.' END, owner = NULL,lease_until=0,updated_at=? WHERE id IN (SELECT id FROM message_commands WHERE owner IS NOT NULL AND lease_until <= ? ORDER BY lease_until LIMIT 100)`).run(now, now);
+            const candidate = this.store.db.prepare(candidateSql).get();
+            if (!candidate)
+                return undefined;
+            this.store.db.prepare("UPDATE message_dispatch_clock SET sequence=sequence+1 WHERE id=1").run();
+            this.store.db.prepare("INSERT INTO message_dispatch_rooms(room_id,sequence) SELECT ?,sequence FROM message_dispatch_clock WHERE id=1 ON CONFLICT(room_id) DO UPDATE SET sequence=excluded.sequence").run(candidate.room_id);
+            this.store.db.prepare("UPDATE message_commands SET state='waiting_slot',owner = ?,token=token+1,lease_until=?,updated_at=? WHERE id=?").run(owner, now + leaseMs, now, candidate.id);
+            return this.store.db.prepare("SELECT * FROM message_commands WHERE id=?").get(candidate.id);
+        });
+        if (!row)
+            return undefined;
+        try {
+            const text = Buffer.from(this.store.payloads.readPayloadBytesBounded(row.payload_ref, 1024 * 1024)).toString("utf8");
+            return { ...receipt(row), token: row.token, text, delivery: row.delivery };
+        }
+        catch {
+            this.transition(row.id, owner, row.token, "failed", "Durable message payload is unavailable or corrupt.");
+            return undefined;
+        }
+    }
+    cancelPending(sessionId) {
+        return Number(this.store.db.prepare("UPDATE message_commands SET state='failed',error='Cancelled before runtime dispatch.',owner = NULL,lease_until=0,updated_at=? WHERE session_id=? AND state IN ('accepted','waiting_slot')").run(Date.now(), sessionId).changes);
+    }
+    transition(id, owner, token, state, error) {
+        return Number(this.store.db.prepare(`UPDATE message_commands SET state=?,error=?,updated_at=?,owner = CASE WHEN ? IN ('completed','failed','interrupted') THEN NULL ELSE owner END WHERE id=? AND owner = ? AND token=? AND lease_until>? AND state IN (${active})`).run(state, error?.slice(0, 500) ?? null, Date.now(), state, id, owner, token, Date.now()).changes) === 1;
+    }
+    heartbeat(id, owner, token, leaseMs) {
+        return Number(this.store.db.prepare(`UPDATE message_commands SET lease_until=? WHERE id=? AND owner = ? AND token=? AND lease_until>? AND state IN (${active})`).run(Date.now() + leaseMs, id, owner, token, Date.now()).changes) === 1;
+    }
+    recordOutput(sessionId, eventId, type) {
+        if (!eventId)
+            return;
+        const states = { message_queued: "session_queue", message_started: "running", message_finished: "completed", session_error: "failed", message_steered: "completed" };
+        const state = states[type];
+        if (!state)
+            return;
+        const eligible = state === "session_queue" ? "'initializing','waiting_slot'" : state === "running" ? "'initializing','waiting_slot','session_queue'" : `${active},'interrupted'`;
+        this.store.db.prepare(`UPDATE message_commands SET state=?,error=NULL,updated_at=?,owner = CASE WHEN ? IN ('completed','failed') THEN NULL ELSE owner END WHERE session_id=? AND event_id=? AND state IN (${eligible})`).run(state, Date.now(), state, sessionId, eventId);
+    }
+}
+function receipt(row) {
+    return { id: row.id, sessionId: row.session_id, roomId: row.room_id, eventId: row.event_id, streamId: row.stream_id, state: row.state, createdAt: row.created_at, updatedAt: row.updated_at, ...(row.error ? { error: row.error } : {}) };
+}
+function domainError(code, message) { return Object.assign(new Error(message), { code }); }

package/dist/data/payload-store.js

@@ -1,6 +1,7 @@
+import { pipeline } from "node:stream";
 import { createHash, randomUUID } from "node:crypto";
-import { gunzipSync, gzipSync } from "node:zlib";
-import { closeSync, existsSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { createGunzip, gunzipSync, gzipSync } from "node:zlib";
+import { createReadStream, closeSync, existsSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { dirname, extname, join, resolve } from "node:path";
 import { piboHomePath } from "../core/pibo-home.js";
 const MAX_SYNC_PAYLOAD_GZIP_BYTES = 512 * 1024;
@@ -23,30 +24,55 @@ export class PiboPayloadMetadataConflictError extends Error {
 export class PayloadStore {
     db;
     rootDir;
-    constructor(db, rootDir = piboHomePath("payloads")) {
+    constructor(db, rootDir = piboHomePath("payloads"), readOnly = false) {
         this.db = db;
         this.rootDir = rootDir === ":memory:" ? rootDir : resolve(rootDir);
-        if (this.rootDir !== ":memory:")
+        if (!readOnly && this.rootDir !== ":memory:")
             mkdirSync(this.rootDir, { recursive: true });
     }
     writePayload(input) {
+        return this.commitPreparedPayload(this.preparePayload(input));
+    }
+    /** Performs hashing, compression and atomic file publication before a DB transaction begins. */
+    preparePayload(input) {
         const contentType = input.contentType ?? defaultContentType(input.value);
         const createdAt = input.createdAt ?? new Date().toISOString();
         const bytes = payloadToBytes(input.value, contentType);
         const sha256 = createHash("sha256").update(bytes).digest("hex");
         const existing = this.findByIdentity(sha256, contentType, input.retentionClass);
         if (existing) {
-            this.db.prepare("UPDATE payloads SET ref_count = ref_count + 1 WHERE id = ?").run(existing.id);
-            return this.getPayload(existing.id) ?? existing;
+            return { ...existing, refCount: 1, status: "staged" };
         }
         const shouldCompress = bytes.byteLength <= MAX_SYNC_PAYLOAD_GZIP_BYTES;
         const encoding = shouldCompress ? "gzip" : "identity";
         const bytesToStore = shouldCompress ? gzipSync(bytes) : bytes;
-        const compressedByteSize = shouldCompress ? bytesToStore.byteLength : null;
+        const compressedByteSize = shouldCompress ? bytesToStore.byteLength : undefined;
         const relativePath = buildRelativePayloadPath(sha256, contentType, input.retentionClass, encoding);
         const absolutePath = this.rootDir === ":memory:" ? relativePath : join(this.rootDir, relativePath);
         writePayloadFile(absolutePath, bytesToStore);
-        const id = input.id ?? `payload_${randomUUID()}`;
+        return {
+            id: input.id ?? `payload_${randomUUID()}`,
+            sha256,
+            storageKind: "file",
+            storagePath: relativePath,
+            contentType,
+            encoding,
+            byteSize: bytes.byteLength,
+            compressedByteSize,
+            previewText: previewTextFromValue(input.value),
+            retentionClass: input.retentionClass,
+            refCount: 1,
+            status: "staged",
+            createdAt,
+        };
+    }
+    /** Links a previously published payload file with only bounded metadata SQL. */
+    commitPreparedPayload(prepared) {
+        const existing = this.findByIdentity(prepared.sha256, prepared.contentType, prepared.retentionClass);
+        if (existing) {
+            this.db.prepare("UPDATE payloads SET ref_count = ref_count + 1 WHERE id = ?").run(existing.id);
+            return this.getPayload(existing.id) ?? existing;
+        }
         this.db.prepare(`
 			INSERT INTO payloads (
 				id,
@@ -64,10 +90,10 @@ export class PayloadStore {
 				created_at,
 				last_verified_at
 			) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-		`).run(id, sha256, "file", relativePath, contentType, encoding, bytes.byteLength, compressedByteSize, previewTextFromValue(input.value) ?? null, input.retentionClass, 1, "committed", createdAt, createdAt);
-        const stored = this.getPayload(id);
+		`).run(prepared.id, prepared.sha256, "file", prepared.storagePath ?? null, prepared.contentType, prepared.encoding, prepared.byteSize, prepared.compressedByteSize ?? null, prepared.previewText ?? null, prepared.retentionClass, 1, "committed", prepared.createdAt, prepared.createdAt);
+        const stored = this.getPayload(prepared.id);
         if (!stored)
-            throw new Error(`Failed to persist payload \"${id}\"`);
+            throw new Error(`Failed to persist payload \"${prepared.id}\"`);
         return stored;
     }
     getPayload(id) {
@@ -88,6 +114,61 @@ export class PayloadStore {
             return bytes;
         throw new Error(`Unsupported payload encoding \"${payload.encoding}\"`);
     }
+    openPayloadStream(id) {
+        const payload = this.getPayload(id);
+        if (!payload?.storagePath)
+            throw new Error("Payload not found");
+        if (payload.encoding !== "identity" && payload.encoding !== "gzip")
+            throw new Error("Unsupported payload encoding");
+        const source = createReadStream(this.rootDir === ":memory:" ? payload.storagePath : join(this.rootDir, payload.storagePath));
+        if (payload.encoding === "identity")
+            return source;
+        const decoded = createGunzip();
+        pipeline(source, decoded, () => { }); // Pipeline closes both ends on cancellation or read failure.
+        return decoded;
+    }
+    /** Read a bounded uncompressed range without materializing the entire payload. */
+    async readPayloadRange(id, offset, limit) {
+        if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1)
+            throw new RangeError("Invalid payload range");
+        const payload = this.getPayload(id);
+        if (!payload?.storagePath)
+            throw new Error("Payload not found");
+        if (offset >= payload.byteSize)
+            return Buffer.alloc(0);
+        const path = this.rootDir === ":memory:" ? payload.storagePath : join(this.rootDir, payload.storagePath);
+        const end = Math.min(payload.byteSize, offset + limit);
+        const source = createReadStream(path, payload.encoding === "identity" ? { start: offset, end: end - 1 } : {});
+        if (payload.encoding !== "identity" && payload.encoding !== "gzip") {
+            source.destroy();
+            throw new Error("Unsupported payload encoding");
+        }
+        const stream = payload.encoding === "gzip" ? source.pipe(createGunzip()) : source;
+        const forwardError = (error) => stream.destroy(error);
+        if (stream !== source)
+            source.on("error", forwardError);
+        let position = payload.encoding === "identity" ? offset : 0;
+        const parts = [];
+        try {
+            for await (const value of stream) {
+                const bytes = Buffer.from(value);
+                const from = Math.max(0, offset - position), to = Math.min(bytes.length, end - position);
+                if (to > from)
+                    parts.push(Buffer.from(bytes.subarray(from, to)));
+                position += bytes.length;
+                if (position >= end)
+                    break;
+            }
+        }
+        finally {
+            stream.destroy();
+            source.destroy();
+        }
+        const result = Buffer.concat(parts);
+        if (result.length !== end - offset)
+            throw new Error("Incomplete payload range");
+        return result;
+    }
     readPayloadText(id) {
         return Buffer.from(this.readPayloadBytes(id)).toString("utf8");
     }