@pasko70/pibo 3.1.0 → 3.1.2

package/dist/core/output-render-sequence.js

@@ -3,6 +3,7 @@ const RENDER_SEQUENCE_SLOTS_PER_MILLISECOND = 1_000;
 const MAX_TRACKED_SESSIONS = 1_024;
 const MAX_RECENT_SEGMENTS_PER_SESSION = 1_024;
 const MAX_RECENT_TOOL_INVOCATIONS_PER_SESSION = 1_024;
+const MAX_RECENT_OUTPUT_PARTS_PER_SESSION = 1_024;
 export function outputRenderHighWaterStore(value) {
     if (!value || typeof value !== "object")
         return undefined;
@@ -32,7 +33,8 @@ export class OutputRenderSequencer {
     position(event) {
         const state = this.sessionState(event.piboSessionId);
         this.trackActiveTurn(state, event);
-        const eventWithToolIdentity = this.positionToolInvocation(state, event);
+        const eventWithPartIdentity = this.positionOutputPart(state, event);
+        const eventWithToolIdentity = this.positionToolInvocation(state, eventWithPartIdentity);
         const key = outputRenderSegmentKey(eventWithToolIdentity, state.activeEventId);
         const supplied = validRenderSequence(eventWithToolIdentity.renderSequence)
             ? eventWithToolIdentity.renderSequence
@@ -90,11 +92,61 @@ export class OutputRenderSequencer {
             activeSegments: new Set(),
             completedSegments: new Set(),
             toolInvocations: new Map(),
+            outputParts: new Map(),
             lastSequence: 0,
         };
         this.sessions.set(piboSessionId, state);
         return state;
     }
+    positionOutputPart(state, event) {
+        const transition = outputPartTransition(event, state.activeEventId);
+        if (!transition)
+            return event;
+        const key = outputPartCounterKey(transition.eventId, transition.kind);
+        const parts = state.outputParts.get(key) ?? [];
+        if (!state.outputParts.has(key))
+            state.outputParts.set(key, parts);
+        const exact = parts.find((part) => part.fingerprints.has(transition.fingerprint));
+        const latestOpen = [...parts].reverse().find((part) => !part.closed);
+        const localMaximum = parts.reduce((maximum, part) => Math.max(maximum, part.index), -1);
+        const localIndex = exact?.index
+            ?? latestOpen?.index
+            ?? (transition.proposedIndex > localMaximum ? transition.proposedIndex : localMaximum + 1);
+        const index = validRenderSequence(event.renderSequence) || exact || latestOpen
+            ? localIndex
+            : this.highWaterStore?.claimOrAttachOutputPart?.({ ...transition, proposedIndex: localIndex }) ?? localIndex;
+        let part = parts.find((candidate) => candidate.index === index);
+        if (!part) {
+            part = { index, fingerprints: new Set(), closed: false };
+            parts.push(part);
+        }
+        part.fingerprints.add(transition.fingerprint);
+        if (transition.terminal)
+            part.closed = true;
+        this.trimOutputParts(state);
+        return withOutputPartIndex(event, transition.kind, index);
+    }
+    trimOutputParts(state) {
+        let total = 0;
+        for (const parts of state.outputParts.values())
+            total += parts.length;
+        while (total > MAX_RECENT_OUTPUT_PARTS_PER_SESSION) {
+            let removed = false;
+            for (const [key, parts] of state.outputParts) {
+                const closedIndex = parts.findIndex((part) => part.closed);
+                if (closedIndex === -1)
+                    continue;
+                parts.splice(closedIndex, 1);
+                total -= 1;
+                if (!parts.length)
+                    state.outputParts.delete(key);
+                removed = true;
+                break;
+            }
+            if (!removed)
+                break;
+        }
+    }
     positionToolInvocation(state, event) {
         if (!isToolIdentityEvent(event) || !event.toolCallId)
             return event;
@@ -130,7 +182,7 @@ export class OutputRenderSequencer {
             if (!latest) {
                 invocation = this.createToolInvocation(invocations, event.piboSessionId, eventId, event.toolCallId, transition);
             }
-            else if (event.type === "tool_call" && (latest.seen.has("tool_call") || (latest.closed && latest.persistedIdentity))) {
+            else if (event.type === "tool_call" && toolCallStartsNewInvocation(latest, transition?.callFingerprint)) {
                 invocation = this.createToolInvocation(invocations, event.piboSessionId, eventId, event.toolCallId, transition);
             }
             else if (event.type === "tool_execution_started" && latest.closed && latest.seen.has("tool_execution_started")) {
@@ -143,6 +195,8 @@ export class OutputRenderSequencer {
             }
         }
         invocation.seen.add(event.type);
+        if (event.type === "tool_call" && transition?.callFingerprint)
+            invocation.callFingerprint = transition.callFingerprint;
         if (event.type === "tool_execution_finished")
             invocation.closed = true;
         this.trimToolInvocations(state);
@@ -243,6 +297,110 @@ function createInvocation(invocations, ordinal) {
     invocations.push(invocation);
     return invocation;
 }
+function toolCallStartsNewInvocation(invocation, callFingerprint) {
+    if (!invocation.seen.has("tool_call"))
+        return invocation.closed && invocation.persistedIdentity;
+    if (invocation.closed)
+        return true;
+    if (invocation.callFingerprint === undefined)
+        return false;
+    return callFingerprint === undefined || invocation.callFingerprint !== callFingerprint;
+}
+function outputPartTransition(event, activeEventId) {
+    const eventId = ("eventId" in event ? event.eventId : undefined) ?? activeEventId;
+    if (!eventId)
+        return undefined;
+    if (event.type === "assistant_delta" || event.type === "assistant_message") {
+        return {
+            piboSessionId: event.piboSessionId,
+            eventId,
+            kind: "assistant",
+            proposedIndex: event.assistantIndex ?? event.contentIndex ?? 0,
+            fingerprint: outputPartFingerprint(event),
+            identityFingerprint: outputIdentityFingerprint(event),
+            terminal: event.type === "assistant_message",
+        };
+    }
+    if (event.type === "thinking_started" || event.type === "thinking_delta" || event.type === "thinking_finished") {
+        return {
+            piboSessionId: event.piboSessionId,
+            eventId,
+            kind: "thinking",
+            proposedIndex: event.thinkingIndex ?? event.contentIndex ?? 0,
+            fingerprint: outputPartFingerprint(event),
+            identityFingerprint: outputIdentityFingerprint(event),
+            terminal: event.type === "thinking_finished",
+        };
+    }
+    if (event.type === "assistant_usage") {
+        return {
+            piboSessionId: event.piboSessionId,
+            eventId,
+            kind: "usage",
+            proposedIndex: event.usageIndex ?? 0,
+            fingerprint: outputPartFingerprint(event),
+            identityFingerprint: outputIdentityFingerprint(event),
+            terminal: true,
+        };
+    }
+    if (event.type === "compaction_start" || event.type === "compaction_end") {
+        return {
+            piboSessionId: event.piboSessionId,
+            eventId,
+            kind: "compaction",
+            proposedIndex: event.compactionIndex ?? 0,
+            fingerprint: outputPartFingerprint(event),
+            identityFingerprint: outputIdentityFingerprint(event),
+            terminal: event.type === "compaction_end",
+        };
+    }
+    return undefined;
+}
+export function outputIdentityFingerprint(event) {
+    const payload = { ...event };
+    delete payload.renderSequence;
+    return createHash("sha256").update(stableJson(payload)).digest("hex");
+}
+export function outputPartFingerprint(event) {
+    const payload = { ...event };
+    delete payload.renderSequence;
+    delete payload.assistantIndex;
+    delete payload.thinkingIndex;
+    delete payload.usageIndex;
+    delete payload.compactionIndex;
+    delete payload.contentIndex;
+    return createHash("sha256").update(stableJson(payload)).digest("hex");
+}
+function stableJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(stableJson).join(",")}]`;
+    if (value && typeof value === "object") {
+        return `{${Object.entries(value)
+            .filter(([, entry]) => entry !== undefined)
+            .sort(([left], [right]) => left.localeCompare(right))
+            .map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`)
+            .join(",")}}`;
+    }
+    return JSON.stringify(value) ?? "null";
+}
+function outputPartCounterKey(eventId, kind) {
+    return JSON.stringify([eventId, kind]);
+}
+function withOutputPartIndex(event, kind, index) {
+    if (kind === "assistant" && (event.type === "assistant_delta" || event.type === "assistant_message")) {
+        return event.assistantIndex === index ? event : { ...event, assistantIndex: index };
+    }
+    if (kind === "thinking" && (event.type === "thinking_started" || event.type === "thinking_delta" || event.type === "thinking_finished")) {
+        return event.thinkingIndex === index ? event : { ...event, thinkingIndex: index };
+    }
+    if (kind === "usage" && event.type === "assistant_usage") {
+        return event.usageIndex === index ? event : { ...event, usageIndex: index };
+    }
+    if (kind === "compaction" && (event.type === "compaction_start" || event.type === "compaction_end")) {
+        return event.compactionIndex === index ? event : { ...event, compactionIndex: index };
+    }
+    return event;
+}
 function segmentRemainsActive(event) {
     return event.type === "message_started"
         || event.type === "assistant_delta"
@@ -278,6 +436,8 @@ function toolInvocationCounterKey(eventId, toolCallId) {
     return JSON.stringify([eventId ?? "unscoped", toolCallId]);
 }
 function toolCallFingerprint(event) {
+    if (!event.argsComplete)
+        return undefined;
     return createHash("sha256")
         .update(JSON.stringify([event.toolName, event.argsComplete, event.args]))
         .digest("hex");

package/dist/data/ingest-service.js

@@ -1,4 +1,5 @@
 import { createHash, randomUUID } from "node:crypto";
+import { outputIdentityFingerprint, outputPartFingerprint } from "../core/output-render-sequence.js";
 import { rootSessionId } from "./session-store.js";
 export class PiboOutputIdentityCollisionError extends Error {
     idempotencyKey;
@@ -89,7 +90,8 @@ export class ChatDataIngestService {
     ingestOutputEvent(input) {
         const event = input.event;
         const idempotencyKey = outputIdempotencyKey(event);
-        const identityFingerprint = outputEventFingerprint(event);
+        const identityFingerprint = outputIdentityFingerprint(event);
+        const partFingerprint = isOutputPartEvent(event) ? outputPartFingerprint(event) : undefined;
         if (idempotencyKey) {
             const existing = this.store.eventLog.findByIdempotencyKey(idempotencyKey);
             if (existing) {
@@ -156,6 +158,7 @@ export class ChatDataIngestService {
                 previewText: previewTextForOutputEvent(event),
                 attributes: compactObject({
                     identityFingerprint,
+                    outputPartFingerprint: partFingerprint,
                     eventIdentityScoped: eventIdForOutputEvent(event) !== undefined,
                     semanticEventId: eventIdForOutputEvent(event),
                     legacyStreamId: input.legacyStreamId,
@@ -270,11 +273,6 @@ function deterministicId(prefix, value) {
 function hashJson(value) {
     return createHash("sha256").update(canonicalJson(value)).digest("hex").slice(0, 16);
 }
-function outputEventFingerprint(event) {
-    const identity = { ...event };
-    delete identity.renderSequence;
-    return createHash("sha256").update(canonicalJson(identity)).digest("hex");
-}
 function canonicalJson(value) {
     if (value === null || typeof value !== "object")
         return JSON.stringify(value) ?? "null";
@@ -470,6 +468,16 @@ function observationStatusForOutputEvent(event) {
         return "error";
     return "completed";
 }
+function isOutputPartEvent(event) {
+    return event.type === "assistant_delta"
+        || event.type === "assistant_message"
+        || event.type === "thinking_started"
+        || event.type === "thinking_delta"
+        || event.type === "thinking_finished"
+        || event.type === "assistant_usage"
+        || event.type === "compaction_start"
+        || event.type === "compaction_end";
+}
 function isTerminalOutputEvent(event) {
     return !event.type.endsWith("_started") && event.type !== "tool_call" && event.type !== "assistant_delta" && event.type !== "thinking_delta" && event.type !== "tool_execution_updated";
 }

package/dist/data/schema.js

@@ -199,6 +199,16 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
 			FOREIGN KEY (pibo_session_id) REFERENCES sessions(id) ON DELETE CASCADE
 		);
 
+		CREATE TABLE IF NOT EXISTS session_output_part_counters (
+			pibo_session_id TEXT NOT NULL,
+			event_id TEXT NOT NULL,
+			part_kind TEXT NOT NULL CHECK(part_kind IN ('assistant', 'thinking', 'usage', 'compaction')),
+			next_index INTEGER NOT NULL CHECK(next_index >= 0),
+			updated_at TEXT NOT NULL,
+			PRIMARY KEY (pibo_session_id, event_id, part_kind),
+			FOREIGN KEY (pibo_session_id) REFERENCES sessions(id) ON DELETE CASCADE
+		);
+
 		CREATE TABLE IF NOT EXISTS session_tool_invocation_counters (
 			pibo_session_id TEXT NOT NULL,
 			event_id TEXT NOT NULL,

package/dist/sessions/pibo-data-store.js

@@ -147,6 +147,65 @@ export class PiboDataSessionStore {
 			`).run(sequence, new Date().toISOString(), piboSessionId);
         });
     }
+    claimOrAttachOutputPart(input) {
+        return this.dataStore.transaction(() => {
+            const indexAttribute = outputPartIndexAttribute(input.kind);
+            const eventTypes = outputPartEventTypes(input.kind);
+            const placeholders = eventTypes.map(() => "?").join(", ");
+            const rows = this.db.prepare(`
+				SELECT type, attributes_json
+				FROM event_log
+				WHERE session_id = ? AND event_id = ? AND type IN (${placeholders})
+				ORDER BY stream_id ASC
+			`).all(input.piboSessionId, input.eventId, ...eventTypes);
+            let maximum = -1;
+            const latestTypeByIndex = new Map();
+            for (const row of rows) {
+                const attributes = parseJsonObject(row.attributes_json);
+                const index = attributes[indexAttribute];
+                if (typeof index !== "number" || !Number.isSafeInteger(index) || index < 0)
+                    continue;
+                maximum = Math.max(maximum, index);
+                if (attributes.outputPartFingerprint === input.fingerprint
+                    || attributes.identityFingerprint === input.identityFingerprint) {
+                    this.observeOutputPartIndex(input, index);
+                    return index;
+                }
+                latestTypeByIndex.set(index, row.type);
+            }
+            const latestOpen = [...latestTypeByIndex.entries()]
+                .filter(([, eventType]) => !outputPartEventIsTerminal(eventType))
+                .sort(([left], [right]) => right - left)[0]?.[0];
+            if (latestOpen !== undefined) {
+                this.observeOutputPartIndex(input, latestOpen);
+                return latestOpen;
+            }
+            const minimum = Math.max(input.proposedIndex, maximum + 1);
+            const row = this.db.prepare(`
+				INSERT INTO session_output_part_counters (
+					pibo_session_id, event_id, part_kind, next_index, updated_at
+				) VALUES (?, ?, ?, ? + 1, ?)
+				ON CONFLICT(pibo_session_id, event_id, part_kind) DO UPDATE SET
+					next_index = MAX(session_output_part_counters.next_index, ?) + 1,
+					updated_at = excluded.updated_at
+				RETURNING next_index - 1 AS part_index
+			`).get(input.piboSessionId, input.eventId, input.kind, minimum, new Date().toISOString(), minimum);
+            return row.part_index;
+        });
+    }
+    observeOutputPartIndex(input, index) {
+        this.db.prepare(`
+			INSERT INTO session_output_part_counters (
+				pibo_session_id, event_id, part_kind, next_index, updated_at
+			) VALUES (?, ?, ?, ?, ?)
+			ON CONFLICT(pibo_session_id, event_id, part_kind) DO UPDATE SET
+				next_index = MAX(session_output_part_counters.next_index, excluded.next_index),
+				updated_at = CASE
+					WHEN excluded.next_index > session_output_part_counters.next_index THEN excluded.updated_at
+					ELSE session_output_part_counters.updated_at
+				END
+		`).run(input.piboSessionId, input.eventId, input.kind, index + 1, new Date().toISOString());
+    }
     claimOutputToolInvocationOrdinal(piboSessionId, eventId, toolCallId) {
         return this.dataStore.transaction(() => {
             const durable = this.db.prepare(`
@@ -196,7 +255,9 @@ export class PiboDataSessionStore {
             let ordinal;
             if (input.eventType === "tool_call"
                 && latestOpen
-                && (latestOpen.seen_call === 0 || latestOpen.call_fingerprint === input.callFingerprint)) {
+                && (latestOpen.seen_call === 0
+                    || latestOpen.call_fingerprint === null
+                    || latestOpen.call_fingerprint === input.callFingerprint)) {
                 ordinal = latestOpen.invocation_ordinal;
             }
             else if (input.eventType !== "tool_call" && latestOpen) {
@@ -562,6 +623,28 @@ function runtimeBindingFromRow(row) {
 function isRuntimeBindingState(value) {
     return value === "unbound" || value === "bound" || value === "missing" || value === "error";
 }
+function outputPartIndexAttribute(kind) {
+    switch (kind) {
+        case "assistant": return "assistantIndex";
+        case "thinking": return "thinkingIndex";
+        case "usage": return "usageIndex";
+        case "compaction": return "compactionIndex";
+    }
+}
+function outputPartEventTypes(kind) {
+    switch (kind) {
+        case "assistant": return ["assistant_delta", "assistant_message"];
+        case "thinking": return ["thinking_started", "thinking_delta", "thinking_finished"];
+        case "usage": return ["assistant_usage"];
+        case "compaction": return ["compaction_start", "compaction_end"];
+    }
+}
+function outputPartEventIsTerminal(eventType) {
+    return eventType === "assistant_message"
+        || eventType === "thinking_finished"
+        || eventType === "assistant_usage"
+        || eventType === "compaction_end";
+}
 function parseJsonObject(json) {
     if (!json)
         return {};

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@pasko70/pibo",
-  "version": "3.1.0",
+  "version": "3.1.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@pasko70/pibo",
-      "version": "3.1.0",
+      "version": "3.1.2",
       "workspaces": [
         "packages/workflows"
       ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pasko70/pibo",
-  "version": "3.1.0",
+  "version": "3.1.2",
   "type": "module",
   "workspaces": [
     "packages/workflows"