@pasko70/pibo 3.4.4 → 3.5.0

package/dist/agent-runtime/routed-session.js

@@ -1,3 +1,4 @@
+import { previouslyClearedMessages } from "../core/events.js";
 import { PiboSteeringUnavailableError, } from "../core/events.js";
 import { PIBO_PROVIDER_RECOVERY_PROMPT, isPiboProviderFallbackError } from "../core/provider-recovery.js";
 import { normalizeSessionErrorDetails, runtimeSessionErrorDetails } from "../core/session-errors.js";
@@ -166,8 +167,14 @@ export class RuntimeRoutedSession {
         if (this.sessionIdentityOperationInFlight && !this.forkCandidatesRequest) {
             throw new Error("Pibo session cannot accept messages while a session identity operation is in progress.");
         }
+        const messages = this.queue.filter(item => item.kind === "message");
+        const bytes = Buffer.byteLength(event.text);
+        if (bytes > 1024 * 1024 || messages.length >= 64 || messages.reduce((total, item) => total + Buffer.byteLength(item.event.text), bytes) > 4 * 1024 * 1024
+            || messages.some(item => Date.now() - item.acceptedAt >= 10 * 60 * 1000)) {
+            throw Object.assign(new Error("Session runtime queue count, byte or wait-age capacity reached."), { code: "runtime_capacity_unavailable" });
+        }
         onAccepted();
-        this.queue.push({ kind: "message", event });
+        this.queue.push({ kind: "message", event, acceptedAt: Date.now() });
         const output = {
             type: "message_queued",
             piboSessionId: this.piboSessionId,
@@ -593,6 +600,7 @@ export class RuntimeRoutedSession {
         if (inFlight?.event.id === eventId) {
             if (!inFlight.cancellation) {
                 inFlight.cancelled = true;
+                inFlight.capacityAbort.abort(new Error("Message cancelled while waiting for capacity."));
                 this.notifyMessagesInterrupted([inFlight.event], "message cancelled");
                 const active = this.activeMessage?.id === eventId;
                 inFlight.cancellation = (async () => {
@@ -710,6 +718,7 @@ export class RuntimeRoutedSession {
                     result: event.result,
                     aborted: event.aborted,
                     errorMessage: event.errorMessage,
+                    compactionStats: event.compactionStats,
                 }));
                 return;
             case "approval_requested":
@@ -815,11 +824,29 @@ export class RuntimeRoutedSession {
             this.notifyState();
         }
     }
+    async promptWithinCapacity(event, inFlight, input) {
+        const provider = this.runtimeSession.getStatus().activeModel?.provider ?? this.runtimeSession.runtimeInstanceId;
+        const lease = await this.options.acquireProviderCapacity?.(provider, inFlight.capacityAbort.signal);
+        try {
+            if (this.disposed || inFlight.cancelled)
+                return;
+            if (!inFlight.started) {
+                inFlight.started = true;
+                this.emit({ type: "message_started", piboSessionId: this.piboSessionId, eventId: event.id, text: event.text, source: event.source, provenance: event.provenance });
+            }
+            if (this.disposed || inFlight.cancelled)
+                return;
+            await this.runtimeSession.prompt(input);
+        }
+        finally {
+            lease?.release();
+        }
+    }
     async promptWithModelFallbacks(event, inFlight) {
         const canSetModel = Boolean(this.runtimeSession.controls?.setModel);
         const fallbackModels = this.modelFallbacks.filter((model) => !sameModel(model, this.primaryModel));
         if (!canSetModel || fallbackModels.length === 0) {
-            await this.runtimeSession.prompt({
+            await this.promptWithinCapacity(event, inFlight, {
                 text: event.text,
                 source: promptSource(event.source),
                 capabilityScope: event.capabilityScope,
@@ -845,7 +872,7 @@ export class RuntimeRoutedSession {
                 this.pendingProviderFailure = undefined;
                 let promptError;
                 try {
-                    await this.runtimeSession.prompt({
+                    await this.promptWithinCapacity(event, inFlight, {
                         text: attempt === 0 ? event.text : PIBO_PROVIDER_RECOVERY_PROMPT,
                         source: attempt === 0 ? promptSource(event.source) : "rpc",
                         capabilityScope: event.capabilityScope,
@@ -907,14 +934,6 @@ export class RuntimeRoutedSession {
             this.activeMessageFailed = false;
             this.beginRunReminderTurnGuard(event);
             this.resetContentIndices();
-            this.emit({
-                type: "message_started",
-                piboSessionId: this.piboSessionId,
-                eventId: event.id,
-                text: event.text,
-                source: event.source,
-                provenance: event.provenance,
-            });
             if (inFlight.cancelled)
                 return;
             await this.promptWithModelFallbacks(event, inFlight);
@@ -963,6 +982,7 @@ export class RuntimeRoutedSession {
         const inFlight = {
             event,
             cancelled: false,
+            capacityAbort: new AbortController(),
             settled: new Promise((resolve) => { resolveSettled = resolve; }),
             resolveSettled: () => { resolveSettled?.(); },
         };
@@ -1129,8 +1149,12 @@ export class RuntimeRoutedSession {
                 ? await this.options.logoutRuntimeAuth(input)
                 : await this.pluginRegistry.logoutAgentRuntimeAuth(this.runtimeSession.runtimeInstanceId, input),
             getProviderUsage: () => this.getActionProviderUsage(),
-            clearQueue: () => this.clearQueue(),
+            clearQueue: () => this.clearQueue() + previouslyClearedMessages(event),
             abort: async () => {
+                if (this.inFlightMessage && !this.inFlightMessage.started) {
+                    await this.cancelMessage(this.inFlightMessage.event.id);
+                    return;
+                }
                 if (this.activeMessage)
                     this.notifyMessagesInterrupted([this.activeMessage], "abort requested");
                 await this.runtimeSession.abort();
@@ -1187,6 +1211,7 @@ export class RuntimeRoutedSession {
         if (this.disposed)
             return false;
         const activeMessage = this.activeMessage;
+        this.inFlightMessage?.capacityAbort.abort(new Error(reason));
         this.notifyMessagesInterrupted(this.activeAndQueuedMessages(), reason);
         if (activeMessage) {
             const error = `Session disposed while a message was active: ${reason}`;

package/dist/agent-runtimes/pi/adapter.js

@@ -253,6 +253,7 @@ export function semanticEventFromPibo(event) {
                 result: event.result,
                 aborted: event.aborted,
                 errorMessage: event.errorMessage,
+                compactionStats: event.compactionStats,
             };
         case "session_error":
             return { type: "error", message: event.error, details: event.errorDetails };

package/dist/agent-runtimes/pi/routed-session.js

@@ -863,8 +863,6 @@ export class RoutedSession {
             if (assistantMessageEnded && !assistantMessageFailed)
                 this.flushPendingProviderWebSearchFinishes();
             const usageEvent = assistantMessageEnded ? normalizeAssistantUsageEvent(this.piboSessionId, candidate?.message) : undefined;
-            if (usageEvent)
-                this.emit(this.withActiveMessage(usageEvent));
             // Pi gets the first chance to recover through its short retry/compaction loop.
             // Keep the final error pending so the routed turn can continue durable recovery.
             if (assistantMessageEnded && normalized?.type === "session_error") {
@@ -889,6 +887,8 @@ export class RoutedSession {
                     this.emit(this.withActiveMessage(normalized));
                 }
             }
+            if (usageEvent)
+                this.emit(this.withActiveMessage(usageEvent));
             if (this.forwardPiEvents) {
                 this.emit({ type: "pi_event", piboSessionId: this.piboSessionId, event });
             }

package/dist/apps/chat/bounded-event-stream.js

@@ -0,0 +1,98 @@
+/** Owns the only application queue; HWM zero prevents a second hidden stream queue. */
+export class BoundedEventStream {
+    onClose;
+    maximumBytes;
+    maximumAgeMs;
+    maximumFrames;
+    stream;
+    writer;
+    controller;
+    queue = [];
+    bytes = 0;
+    stopped = false;
+    finishing = false;
+    pendingPull;
+    capacityWaiter;
+    timer;
+    constructor(onClose, maximumBytes = 2 * 1024 * 1024, maximumAgeMs = 5000, maximumFrames = 4096) {
+        this.onClose = onClose;
+        this.maximumBytes = maximumBytes;
+        this.maximumAgeMs = maximumAgeMs;
+        this.maximumFrames = maximumFrames;
+        this.stream = new ReadableStream({
+            start: controller => { this.controller = controller; },
+            pull: () => {
+                const frame = this.queue.shift();
+                if (frame) {
+                    this.bytes -= frame.value.byteLength;
+                    this.controller.enqueue(frame.value);
+                    this.releaseCapacity();
+                    if (this.finishing && !this.queue.length)
+                        this.stop("complete");
+                    return;
+                }
+                if (this.finishing || this.stopped) {
+                    this.stop("complete");
+                    return;
+                }
+                return new Promise(resolve => { this.pendingPull = resolve; });
+            },
+            cancel: () => this.stop("cancel"),
+        }, { highWaterMark: 0 });
+        const self = this;
+        this.writer = { enqueue: value => { this.enqueue(value); }, close: () => this.finish(), error: () => this.stop("error"), get desiredSize() { return self.stopped ? null : self.maximumBytes - self.bytes; } };
+        this.timer = setInterval(() => { if (this.queue.length && performance.now() - this.queue[0].at > this.maximumAgeMs)
+            this.stop("slow_age"); }, Math.min(1000, Math.max(5, maximumAgeMs / 2)));
+        this.timer.unref();
+    }
+    get closed() { return this.stopped || this.finishing; }
+    enqueue(value) {
+        if (this.closed)
+            return;
+        if (value.byteLength > this.maximumBytes || this.bytes + value.byteLength > this.maximumBytes || this.queue.length >= this.maximumFrames) {
+            this.stop("slow_bytes");
+            return;
+        }
+        if (this.pendingPull) {
+            const resolve = this.pendingPull;
+            this.pendingPull = undefined;
+            this.controller.enqueue(value);
+            resolve();
+            return;
+        }
+        this.queue.push({ value, at: performance.now() });
+        this.bytes += value.byteLength;
+    }
+    async waitForCapacity() { if (this.closed)
+        return false; if (this.bytes < this.maximumBytes / 2)
+        return true; return new Promise(resolve => { this.capacityWaiter = resolve; }); }
+    releaseCapacity() { if (this.bytes < this.maximumBytes / 2) {
+        this.capacityWaiter?.(!this.closed);
+        this.capacityWaiter = undefined;
+    } }
+    finish() { if (this.stopped)
+        return; this.finishing = true; if (!this.queue.length)
+        this.stop("complete"); }
+    fail() { this.stop("error"); }
+    stop(reason) {
+        if (this.stopped)
+            return;
+        this.stopped = true;
+        clearInterval(this.timer);
+        this.queue.length = 0;
+        this.bytes = 0;
+        try {
+            if (reason === "complete")
+                this.controller.close();
+            else if (reason !== "cancel")
+                this.controller.error(Error("Event stream budget exhausted; reconnect using the last received cursor."));
+        }
+        catch { }
+        this.pendingPull?.();
+        this.pendingPull = undefined;
+        this.capacityWaiter?.(false);
+        this.capacityWaiter = undefined;
+        this.onClose(reason);
+    }
+    status() { return { frames: this.queue.length, bytes: this.bytes, oldestAgeMs: this.queue.length ? performance.now() - this.queue[0].at : 0, closed: this.closed }; }
+}

package/dist/apps/chat/chat-settings-routes.js

@@ -8,7 +8,7 @@ import { parseToolMetricTokenCalculation } from "../../shared/tool-call-token-se
 import { PiboWebHttpError, readJsonBody, responseJson } from "../../web/http.js";
 import { CHAT_WEB_API_PREFIX } from "./chat-api-routes.js";
 import { normalizeBasePromptMarkdown, normalizeBasePromptMode, normalizeCompactionPromptMarkdown, normalizeCompactionPromptMode, updateChatModelDefaults, } from "./chat-request-normalizers.js";
-import { pruneTelemetryOlderThan } from "./telemetry-retention-service.js";
+import { pruneTelemetryOlderThanAsync } from "./telemetry-retention-service.js";
 export function chatSettingsRoute(pathname, method) {
     if (pathname === `${CHAT_WEB_API_PREFIX}/model-defaults` && method === "PATCH")
         return { kind: "model-defaults" };
@@ -70,8 +70,8 @@ export async function handleChatSettingsRoute(input) {
         const days = sanitizeTelemetryRetentionDays(body.days);
         if (!days)
             throw new PiboWebHttpError("Invalid telemetry retention days", 400);
-        const result = pruneTelemetryOlderThan({ dataStore: input.dataStore, days, apply: body.dryRun !== true });
-        if (result.applied)
+        const result = await pruneTelemetryOlderThanAsync({ dataStore: input.dataStore, days, apply: body.dryRun !== true, signal: request.signal });
+        if (result.applied && result.completed !== false)
             updateTelemetryRetentionLastPrunedAt(new Date().toISOString());
         return responseJson({ telemetryRetention: result });
     }

package/dist/apps/chat/data/chat-data-mappers.js

@@ -3,13 +3,31 @@ import { roomWorkspaceFromMetadata } from "../types/rooms.js";
 import { tracePayloadRefForStoredPayload } from "../trace-v2.js";
 import { qualifiedToolNodeId } from "../../../shared/trace-tool-identity.js";
 const MAX_TRACE_EVENT_HYDRATION_BYTES = 1024 * 1024;
-export function storedPiboEventFromV2Row(row, payloadStore) {
+export function storedPiboEventFromV2Row(row, payloadStore, hydrationBytes = MAX_TRACE_EVENT_HYDRATION_BYTES) {
+    if (typeof hydrationBytes !== "number" || !Number.isFinite(hydrationBytes))
+        hydrationBytes = MAX_TRACE_EVENT_HYDRATION_BYTES;
     const attributes = parseJsonObject(row.attributes_json);
-    const payload = outputPayloadFromV2Row(row, attributes, readPersistedPayload(row, payloadStore));
+    if (hydrationBytes < MAX_TRACE_EVENT_HYDRATION_BYTES && typeof attributes.inlineText === "string" && Buffer.byteLength(attributes.inlineText) > hydrationBytes)
+        attributes.inlineText = row.preview_text ?? "";
+    const payload = outputPayloadFromV2Row(row, attributes, readPersistedPayload(row, payloadStore, hydrationBytes));
     if (!payload)
         return undefined;
+    const storedPayloadRef = storedPayloadReference(row, attributes, payloadStore);
+    const renderSequence = numberAttribute(attributes, "renderSequence") ?? payload.renderSequence;
+    return { id: String(row.stream_id), piboSessionId: row.session_id ?? undefined, eventSequence: row.session_sequence ?? undefined, renderSequence, eventId: row.event_id ?? undefined, streamId: row.stream_id, storedPayloadRef, type: row.type, createdAt: row.created_at, payload };
+}
+export function storedChatEventFromV2Row(row, payloadStore, hydrationBytes = MAX_TRACE_EVENT_HYDRATION_BYTES) {
+    if (typeof hydrationBytes !== "number" || !Number.isFinite(hydrationBytes))
+        hydrationBytes = MAX_TRACE_EVENT_HYDRATION_BYTES;
+    const attributes = parseJsonObject(row.attributes_json);
+    if (hydrationBytes < MAX_TRACE_EVENT_HYDRATION_BYTES && typeof attributes.inlineText === "string" && Buffer.byteLength(attributes.inlineText) > hydrationBytes)
+        attributes.inlineText = row.preview_text ?? "";
+    const storedPayloadRef = storedPayloadReference(row, attributes, payloadStore);
+    return { storedPayloadRef, streamId: row.stream_id, roomId: row.room_id ?? undefined, piboSessionId: row.session_id ?? undefined, eventId: row.event_id ?? `evt_${row.stream_id}`, eventType: row.type, actorType: actorTypeValue(row.actor_type), actorId: row.actor_id ?? undefined, clientTxnId: typeof attributes.clientTxnId === "string" ? attributes.clientTxnId : undefined, createdAt: row.created_at, retentionClass: retentionClassValue(row.retention_class), payload: (outputPayloadFromV2Row(row, attributes, readPersistedPayload(row, payloadStore, hydrationBytes)) ?? null) };
+}
+function storedPayloadReference(row, attributes, payloadStore) {
     const payloadIdentity = tracePayloadIdentityForEvent(row, attributes);
-    const storedPayloadRef = row.payload_ref && row.session_id && payloadStore && payloadIdentity
+    return row.payload_ref && row.session_id && payloadStore && payloadIdentity
         ? tracePayloadRefForStoredPayload({
             payloadStore,
             piboSessionId: row.session_id,
@@ -17,12 +35,6 @@ export function storedPiboEventFromV2Row(row, payloadStore) {
             ...payloadIdentity,
         })
         : undefined;
-    const renderSequence = numberAttribute(attributes, "renderSequence") ?? payload.renderSequence;
-    return { id: String(row.stream_id), piboSessionId: row.session_id ?? undefined, eventSequence: row.session_sequence ?? undefined, renderSequence, eventId: row.event_id ?? undefined, streamId: row.stream_id, storedPayloadRef, type: row.type, createdAt: row.created_at, payload };
-}
-export function storedChatEventFromV2Row(row, payloadStore) {
-    const attributes = parseJsonObject(row.attributes_json);
-    return { streamId: row.stream_id, roomId: row.room_id ?? undefined, piboSessionId: row.session_id ?? undefined, eventId: row.event_id ?? `evt_${row.stream_id}`, eventType: row.type, actorType: actorTypeValue(row.actor_type), actorId: row.actor_id ?? undefined, clientTxnId: typeof attributes.clientTxnId === "string" ? attributes.clientTxnId : undefined, createdAt: row.created_at, retentionClass: retentionClassValue(row.retention_class), payload: (outputPayloadFromV2Row(row, attributes, readPersistedPayload(row, payloadStore)) ?? null) };
 }
 function outputPayloadFromV2Row(row, attributes, persistedPayload) {
     const inlinePayload = attributes.inlinePayload ?? persistedPayload;
@@ -112,7 +124,7 @@ function outputPayloadFromV2Row(row, attributes, persistedPayload) {
     if (row.type === "compaction_start")
         return { ...base, type: "compaction_start", compactionIndex: numberAttribute(attributes, "compactionIndex"), reason: stringAttribute(attributes, "reason") ?? row.preview_text ?? "unknown" };
     if (row.type === "compaction_end")
-        return { ...base, type: "compaction_end", compactionIndex: numberAttribute(attributes, "compactionIndex"), reason: stringAttribute(attributes, "reason") ?? row.preview_text ?? "unknown", result: inlinePayload, aborted: booleanAttribute(attributes, "aborted") ?? false, errorMessage: stringAttribute(attributes, "errorMessage") };
+        return { ...base, type: "compaction_end", compactionIndex: numberAttribute(attributes, "compactionIndex"), reason: stringAttribute(attributes, "reason") ?? row.preview_text ?? "unknown", result: inlinePayload, aborted: booleanAttribute(attributes, "aborted") ?? false, errorMessage: stringAttribute(attributes, "errorMessage"), compactionStats: isRecord(attributes.compactionStats) ? attributes.compactionStats : undefined };
     if (row.type === "session_error") {
         const error = stringAttribute(attributes, "error") ?? row.preview_text ?? "Error";
         return { ...base, type: "session_error", error, errorDetails: normalizeSessionErrorDetails(error, isRecord(attributes.errorDetails) ? attributes.errorDetails : undefined) };
@@ -121,14 +133,14 @@ function outputPayloadFromV2Row(row, attributes, persistedPayload) {
         return { type: "user.message.accepted", piboSessionId, roomId: row.room_id ?? undefined, text: stringAttribute(attributes, "inlineText") ?? row.preview_text ?? "", clientTxnId: stringAttribute(attributes, "clientTxnId") };
     return { ...base, type: row.type };
 }
-function readPersistedPayload(row, payloadStore) {
+function readPersistedPayload(row, payloadStore, hydrationBytes = MAX_TRACE_EVENT_HYDRATION_BYTES) {
     if (!row.payload_ref || !payloadStore)
         return undefined;
     try {
         const metadata = payloadStore.getPayload(row.payload_ref);
-        if (!metadata || metadata.byteSize > MAX_TRACE_EVENT_HYDRATION_BYTES)
+        if (!metadata || metadata.byteSize > hydrationBytes)
             return undefined;
-        const text = Buffer.from(payloadStore.readPayloadBytesBounded(row.payload_ref, MAX_TRACE_EVENT_HYDRATION_BYTES)).toString("utf8");
+        const text = Buffer.from(payloadStore.readPayloadBytesBounded(row.payload_ref, hydrationBytes)).toString("utf8");
         return metadata.contentType.includes("json") ? JSON.parse(text) : text;
     }
     catch {
@@ -136,6 +148,13 @@ function readPersistedPayload(row, payloadStore) {
     }
 }
 function tracePayloadIdentityForEvent(row, attributes) {
+    const semanticId = stringAttribute(attributes, "semanticEventId") ?? row.event_id;
+    if (semanticId && (row.type === "assistant_message" || row.type === "thinking_finished")) {
+        const thinking = row.type === "thinking_finished";
+        const index = numberAttribute(attributes, thinking ? "thinkingIndex" : "assistantIndex") ?? numberAttribute(attributes, "contentIndex");
+        const identity = index === undefined ? semanticId : `${semanticId}:${thinking ? "thinking" : "assistant"}:${index}`;
+        return { nodeId: `${thinking ? "reasoning" : "assistant"}:${identity}`, payloadKind: thinking ? "reasoning" : "output" };
+    }
     const toolCallId = stringAttribute(attributes, "toolCallId");
     if (!toolCallId)
         return undefined;

package/dist/apps/chat/data/event-command-service.js

@@ -1,6 +1,9 @@
 import { randomUUID } from "node:crypto";
 import { isLiveOnlyOutputEvent } from "../output-event-policy.js";
 import { compactObject, nextSessionSequence, previewForPayload, storedChatEventFromV2Row } from "./chat-data-mappers.js";
+export function chatClientTransactionKey(roomId, actorId, clientTxnId) {
+    return `chat:user.accepted:${roomId ?? ""}:${actorId ?? ""}:${clientTxnId}`;
+}
 export class ChatEventCommandService {
     store;
     constructor(store) {
@@ -9,10 +12,7 @@ export class ChatEventCommandService {
     appendEvent(input) {
         const createdAt = input.createdAt ?? new Date().toISOString();
         const eventId = input.eventId ?? `ce_${randomUUID()}`;
-        const idempotencyKey = input.clientTxnId ? `chat:user.accepted:${input.roomId ?? ""}:${input.actorId ?? ""}:${input.clientTxnId}` : `chat:event:${eventId}`;
-        const existing = input.clientTxnId ? this.findByClientTxn(input.roomId, input.actorId, input.clientTxnId) : undefined;
-        if (existing)
-            return existing;
+        const idempotencyKey = input.clientTxnId ? chatClientTransactionKey(input.roomId, input.actorId, input.clientTxnId) : `chat:event:${eventId}`;
         const stored = this.store.eventLog.appendEvent({
             sessionId: input.piboSessionId,
             sessionSequence: input.piboSessionId ? nextSessionSequence(this.store, input.piboSessionId) : undefined,
@@ -30,21 +30,7 @@ export class ChatEventCommandService {
             createdAt,
             indexedAt: createdAt,
         });
-        return storedChatEventFromV2Row({
-            stream_id: stored.streamId,
-            session_id: stored.sessionId ?? null,
-            session_sequence: stored.sessionSequence ?? null,
-            room_id: stored.roomId ?? null,
-            type: stored.type,
-            actor_type: stored.actorType ?? null,
-            actor_id: stored.actorId ?? null,
-            event_id: stored.eventId ?? null,
-            idempotency_key: stored.idempotencyKey ?? null,
-            retention_class: stored.retentionClass,
-            preview_text: stored.previewText ?? null,
-            attributes_json: JSON.stringify(stored.attributes),
-            created_at: stored.createdAt,
-        });
+        return chatEventFromStoredRow(stored);
     }
     appendOutputEvent(event, _input = {}) {
         // Runtime output ingestion is owned by ChatDataIngestService. Avoid a second event_log insert here.
@@ -55,8 +41,14 @@ export class ChatEventCommandService {
     findByClientTxn(roomId, actorId, clientTxnId) {
         if (!roomId || !actorId)
             return undefined;
-        const row = this.store.db.prepare("SELECT * FROM event_log WHERE room_id = ? AND actor_id = ? AND json_extract(attributes_json, '$.clientTxnId') = ? ORDER BY stream_id ASC LIMIT 1").get(roomId, actorId, clientTxnId);
-        return row ? storedChatEventFromV2Row(row) : undefined;
+        // Empty IDs are not valid HTTP transaction keys. Retain the legacy direct-call
+        // lookup contract without adding a scan fallback to ordinary indexed misses.
+        if (clientTxnId === "") {
+            const row = this.store.db.prepare("SELECT * FROM event_log WHERE room_id = ? AND actor_id = ? AND json_extract(attributes_json, '$.clientTxnId') = ? ORDER BY stream_id ASC LIMIT 1").get(roomId, actorId, clientTxnId);
+            return row ? storedChatEventFromV2Row(row) : undefined;
+        }
+        const row = this.store.eventLog.findByIdempotencyKey(chatClientTransactionKey(roomId, actorId, clientTxnId));
+        return row ? chatEventFromStoredRow(row) : undefined;
     }
     deleteSessions(piboSessionIds) {
         if (!piboSessionIds.length)
@@ -73,3 +65,20 @@ export class ChatEventCommandService {
         return Number(result.changes ?? 0);
     }
 }
+function chatEventFromStoredRow(stored) {
+    return storedChatEventFromV2Row({
+        stream_id: stored.streamId,
+        session_id: stored.sessionId ?? null,
+        session_sequence: stored.sessionSequence ?? null,
+        room_id: stored.roomId ?? null,
+        type: stored.type,
+        actor_type: stored.actorType ?? null,
+        actor_id: stored.actorId ?? null,
+        event_id: stored.eventId ?? null,
+        idempotency_key: stored.idempotencyKey ?? null,
+        retention_class: stored.retentionClass,
+        preview_text: stored.previewText ?? null,
+        attributes_json: JSON.stringify(stored.attributes),
+        created_at: stored.createdAt,
+    });
+}

package/dist/apps/chat/data/history-query-service.js

@@ -1,7 +1,11 @@
+import { tracePayloadRefForStoredPayload } from "../trace-v2.js";
+import { ChatReadProjectionStore } from "../../../data/chat-read-projections.js";
 export class ChatHistoryQueryService {
     store;
-    constructor(store) {
+    pageByteBudget;
+    constructor(store, pageByteBudget = Number.POSITIVE_INFINITY) {
         this.store = store;
+        this.pageByteBudget = pageByteBudget;
     }
     listProductHistoryEntries(input) {
         const limit = Math.max(1, Math.min(input.limit ?? 200, 1000));
@@ -11,7 +15,12 @@ export class ChatHistoryQueryService {
             clauses.push("COALESCE(e.session_sequence, m.sequence) < ?");
             values.push(input.beforeSequence);
         }
-        const rows = this.store.db.prepare(`
+        const complete = new ChatReadProjectionStore(this.store.db).status().historyComplete;
+        const rows = complete ? this.store.db.prepare(`
+          SELECT m.*,h.event_sequence FROM chat_history_index h JOIN chat_messages m ON m.id=h.message_id
+          WHERE h.session_id=? AND h.role IN ('user','assistant','system') ${input.beforeSequence !== undefined ? "AND h.event_sequence < ?" : ""}
+          ORDER BY h.event_sequence DESC,h.message_id DESC LIMIT ?
+        `).all(input.piboSessionId, ...(input.beforeSequence !== undefined ? [input.beforeSequence] : []), limit) : this.store.db.prepare(`
 			SELECT m.*, e.session_sequence AS event_sequence
 			FROM chat_messages m
 			LEFT JOIN event_log e ON e.stream_id = m.source_stream_id
@@ -24,7 +33,9 @@ export class ChatHistoryQueryService {
             if (!role)
                 return [];
             const attributes = parseObject(row.attributes_json);
-            const text = this.readMessageText(row, attributes);
+            const text = this.readMessageText(row, attributes, Math.min(64 * 1024, Math.floor(this.pageByteBudget / limit)));
+            const contentBytes = row.content_payload_ref ? this.store.payloads.getPayload(row.content_payload_ref)?.byteSize : typeof attributes.inlineText === "string" ? Buffer.byteLength(attributes.inlineText) : Buffer.byteLength(row.content_preview ?? "");
+            const contentTruncated = contentBytes !== undefined && Buffer.byteLength(text) < contentBytes;
             return [{
                     id: `product:${row.id}`,
                     type: "message",
@@ -42,42 +53,40 @@ export class ChatHistoryQueryService {
                         sourceStreamId: row.source_stream_id ?? undefined,
                         completedAt: row.completed_at ?? undefined,
                         payloadRef: row.content_payload_ref ?? undefined,
+                        tracePayloadRef: contentTruncated && row.content_payload_ref ? tracePayloadRefForStoredPayload({ payloadStore: this.store.payloads, piboSessionId: row.session_id, payloadId: row.content_payload_ref, nodeId: `product:${row.id}`, payloadKind: "output" }) : undefined,
+                        contentTruncated: contentTruncated || undefined,
+                        contentBytes: contentTruncated ? contentBytes : undefined,
                     }),
                 }];
         });
     }
     getProductHistoryCoverage(piboSessionId) {
-        const row = this.store.db.prepare(`
-			SELECT
-				COUNT(*) AS message_count,
-				MIN(e.session_sequence) AS first_event_sequence,
-				MAX(e.session_sequence) AS last_event_sequence,
-				MIN(m.created_at) AS first_created_at,
-				MAX(m.created_at) AS last_created_at
-			FROM chat_messages m
-			LEFT JOIN event_log e ON e.stream_id = m.source_stream_id
-			WHERE m.session_id = ?
-		`).get(piboSessionId);
-        return {
-            messageCount: Number(row.message_count ?? 0),
-            firstEventSequence: row.first_event_sequence ?? undefined,
-            lastEventSequence: row.last_event_sequence ?? undefined,
-            firstCreatedAt: row.first_created_at ?? undefined,
-            lastCreatedAt: row.last_created_at ?? undefined,
-        };
+        const complete = new ChatReadProjectionStore(this.store.db).status().historyComplete;
+        const count = this.store.db.prepare("SELECT message_count,revision FROM chat_history_counts WHERE session_id=?").get(piboSessionId);
+        const edge = (column, order) => this.store.db.prepare(`SELECT ${column} AS value FROM chat_history_index WHERE session_id=? ORDER BY ${column} ${order},message_id ${order} LIMIT 1`).get(piboSessionId);
+        return { messageCount: count?.message_count ?? 0, revision: count?.revision ?? 0, complete,
+            firstEventSequence: edge("event_sequence", "ASC")?.value,
+            lastEventSequence: edge("event_sequence", "DESC")?.value,
+            firstCreatedAt: edge("created_at", "ASC")?.value,
+            lastCreatedAt: edge("created_at", "DESC")?.value };
     }
-    readMessageText(row, attributes) {
+    readMessageText(row, attributes, maximum) {
         if (row.content_payload_ref) {
             try {
-                return this.store.payloads.readPayloadText(row.content_payload_ref);
+                if (!Number.isFinite(this.pageByteBudget))
+                    return this.store.payloads.readPayloadText(row.content_payload_ref);
+                const payload = this.store.payloads.getPayload(row.content_payload_ref);
+                if (!payload || payload.byteSize > maximum)
+                    return (row.content_preview ?? "").slice(0, Math.min(2048, maximum / 4));
+                return Buffer.from(this.store.payloads.readPayloadBytesBounded(row.content_payload_ref, maximum)).toString("utf8");
             }
             catch {
                 // Retain the durable preview if the external payload was removed or corrupted.
             }
         }
         if (typeof attributes.inlineText === "string")
-            return attributes.inlineText;
-        return row.content_preview ?? "";
+            return Number.isFinite(this.pageByteBudget) ? attributes.inlineText.slice(0, Math.floor(maximum / 4)) : attributes.inlineText;
+        return Number.isFinite(this.pageByteBudget) ? (row.content_preview ?? "").slice(0, Math.floor(maximum / 4)) : (row.content_preview ?? "");
     }
 }
 function historyRole(role) {

package/dist/apps/chat/data/read-state-service.js

@@ -1,3 +1,5 @@
+import { ChatSessionQueryService } from "./session-query-service.js";
+import { ChatReadProjectionStore } from "../../../data/chat-read-projections.js";
 export class ChatReadStateService {
     store;
     constructor(store) {
@@ -14,11 +16,22 @@ export class ChatReadStateService {
 				updated_at = excluded.updated_at
 		`).run(piboSessionId, lastReadStreamId, now, now);
     }
+    sessionIndexPage(input) { return new ChatSessionQueryService(this.store).listSessionIndexPage(input); }
+    unreadCountsPage(input) { return [...this.countUnreadMessagesBySession(input)]; }
     countUnreadMessagesBySession(input) {
         const counts = new Map();
         const uniqueIds = [...new Set(input.piboSessionIds)];
         if (!uniqueIds.length)
             return counts;
+        if (new ChatReadProjectionStore(this.store.db).status().unreadComplete) {
+            for (let offset = 0; offset < uniqueIds.length; offset += 400) {
+                const ids = uniqueIds.slice(offset, offset + 400);
+                const rows = this.store.db.prepare(`SELECT session_id,unread_count FROM chat_unread_counts WHERE session_id IN (${ids.map(() => "?").join(",")}) AND unread_count>0`).all(...ids);
+                for (const row of rows)
+                    counts.set(row.session_id, row.unread_count);
+            }
+            return counts;
+        }
         for (let offset = 0; offset < uniqueIds.length; offset += 400) {
             const ids = uniqueIds.slice(offset, offset + 400);
             const requested = ids.map(() => "(?)").join(", ");

package/dist/apps/chat/data/session-query-service.js

@@ -6,20 +6,21 @@ export class ChatSessionQueryService {
     constructor(store) {
         this.store = store;
     }
-    upsertSession(session, status = "idle", lastActivityAt = session.updatedAt) {
+    upsertSession(session, status = "idle", lastActivityAt = session.updatedAt, options = {}) {
         const roomId = chatRoomIdFromMetadata(session.metadata) ?? "room_default";
-        this.store.sessions.upsertSession({ session, roomId, status, lastActivityAt });
+        this.store.sessions.upsertSession({ session, roomId, status, lastActivityAt, preserveRuntimeBinding: options.preserveRuntimeBinding });
         this.upsertNavigation(session, roomId, status, lastActivityAt);
     }
     upsertSessionsIfChanged(sessions) {
         let written = 0;
         let skipped = 0;
         for (const session of sessions) {
-            if (this.sessionIndexMatches(session)) {
+            const status = this.getSession(session.id)?.status ?? "idle";
+            if (this.sessionIndexMatches(session, status)) {
                 skipped++;
                 continue;
             }
-            this.upsertSession(session);
+            this.upsertSession(session, status);
             written++;
         }
         return { checked: sessions.length, written, skipped };
@@ -28,7 +29,7 @@ export class ChatSessionQueryService {
         if (session) {
             const status = statusFromOutputEvent(event);
             if (status)
-                this.upsertSession(session, status, createdAt);
+                this.upsertSession(session, status, createdAt, { preserveRuntimeBinding: true });
             else
                 this.touchSession(session, createdAt);
         }
@@ -36,7 +37,7 @@ export class ChatSessionQueryService {
     }
     touchSession(session, lastActivityAt) {
         const roomId = chatRoomIdFromMetadata(session.metadata) ?? "room_default";
-        this.store.sessions.upsertSession({ session, roomId, lastActivityAt });
+        this.store.sessions.upsertSession({ session, roomId, lastActivityAt, preserveRuntimeBinding: true });
         this.store.navigation.upsertSession({
             roomId,
             sessionId: session.id,
@@ -50,19 +51,26 @@ export class ChatSessionQueryService {
             updatedAt: lastActivityAt,
         });
     }
-    listSessions() {
+    listSessions(roomId) {
         const rows = this.store.db.prepare(`
-			SELECT s.*, b.runtime_instance_id, b.runtime_adapter_id, b.native_session_id, b.binding_state
+			SELECT s.id,s.pi_session_id,s.parent_id,s.channel,s.kind,s.profile,s.created_at,s.updated_at,s.last_activity_at,s.status,b.runtime_instance_id,b.runtime_adapter_id,b.native_session_id,b.binding_state
 			FROM sessions s
 			LEFT JOIN session_runtime_bindings b ON b.pibo_session_id = s.id
-			WHERE s.deleted_at IS NULL
+			WHERE s.deleted_at IS NULL ${roomId ? "AND s.room_id=?" : ""}
 			ORDER BY s.last_activity_at DESC, s.created_at DESC
-		`).all();
+		`).all(...(roomId ? [roomId] : []));
+        return rows.map(sessionFromRow);
+    }
+    listSessionIndexPage(input) {
+        const limit = Math.max(1, Math.min(input.limit ?? 500, 500));
+        const rows = this.store.db.prepare(`SELECT s.id,s.pi_session_id,s.parent_id,s.channel,s.kind,s.profile,s.created_at,s.updated_at,s.last_activity_at,s.status,b.runtime_instance_id,b.runtime_adapter_id,b.native_session_id,b.binding_state
+		FROM sessions s LEFT JOIN session_runtime_bindings b ON b.pibo_session_id=s.id
+		WHERE s.deleted_at IS NULL AND s.room_id=? AND s.id>? ORDER BY s.id LIMIT ?`).all(input.roomId, input.afterId ?? "", limit);
         return rows.map(sessionFromRow);
     }
     getSession(piboSessionId) {
         const row = this.store.db.prepare(`
-			SELECT s.*, b.runtime_instance_id, b.runtime_adapter_id, b.native_session_id, b.binding_state
+			SELECT s.id,s.pi_session_id,s.parent_id,s.channel,s.kind,s.profile,s.created_at,s.updated_at,s.last_activity_at,s.status,b.runtime_instance_id,b.runtime_adapter_id,b.native_session_id,b.binding_state
 			FROM sessions s
 			LEFT JOIN session_runtime_bindings b ON b.pibo_session_id = s.id
 			WHERE s.id = ? AND s.deleted_at IS NULL