@pasko70/pibo 3.5.0 → 3.6.0

package/dist/debug/output-repair.js

@@ -43,6 +43,7 @@ export function repairOutputTurn(input) {
                 actorId: "pibo-debug-repair",
                 event: current.terminalEvent,
                 createdAt,
+                persistenceProvenance: { producer: "debug-repair", projection: "product-history", phase: "operator-repair" },
             });
             const audit = data.eventLog.appendEvent({
                 sessionId: input.piboSessionId,

package/dist/debug/session.js

@@ -48,6 +48,7 @@ export function inspectDebugSession(input, stores, options = {}) {
             `pibo debug messages ${parsed.piboSessionId} list`,
             `pibo debug final ${parsed.piboSessionId}`,
             `pibo debug trace ${parsed.piboSessionId} --check`,
+            `pibo debug cache ${parsed.piboSessionId}`,
             `pibo debug failures ${parsed.piboSessionId}`,
             `pibo debug events ${parsed.piboSessionId} --limit 20`,
         ];
@@ -114,6 +115,7 @@ export function inspectDebugSessionRuntime(input, stores) {
             nextCommands: [
                 `pibo debug trace ${parsed.piboSessionId} --check`,
                 `pibo debug trace ${parsed.piboSessionId} --native-history --check`,
+                `pibo debug cache ${parsed.piboSessionId}`,
                 `pibo debug events ${parsed.piboSessionId} --limit 20`,
             ],
         };

package/dist/debug/storage-backup.js

@@ -1,5 +1,6 @@
 import { parseArgs } from "node:util";
 import { createStorageBackup, readStorageBackupManifest, verifyStorageBackup, restoreStorageBackup } from "../data/storage-backup.js";
+import { recordStorageMaintenance } from "../data/storage-maintenance.js";
 export async function runStorageBackupCli(args) {
     const action = args[0];
     if (!action || args.includes("--help") || args.includes("-h")) {
@@ -19,12 +20,19 @@ No source deletion, vacuum, or overwrite of an existing restore destination.`);
     const required = (name) => { const value = values[name]; if (typeof value !== "string" || !value)
         throw Error(`Missing --${name}`); return value; };
     let result;
-    if (action === "create")
-        result = await createStorageBackup({ source: required("source"), payloadRoot: required("payload-root"), destination: required("destination"), resume: values.resume, maxBytes: values["max-bytes"] ? Number(values["max-bytes"]) : undefined, maxPayloads: values["max-payloads"] ? Number(values["max-payloads"]) : undefined, maxMilliseconds: values["max-ms"] ? Number(values["max-ms"]) : undefined, maxWalGrowthBytes: values["max-wal-growth"] ? Number(values["max-wal-growth"]) : undefined });
+    if (action === "create") {
+        const source = required("source");
+        result = await createStorageBackup({ source, payloadRoot: required("payload-root"), destination: required("destination"), resume: values.resume, maxBytes: values["max-bytes"] ? Number(values["max-bytes"]) : undefined, maxPayloads: values["max-payloads"] ? Number(values["max-payloads"]) : undefined, maxMilliseconds: values["max-ms"] ? Number(values["max-ms"]) : undefined, maxWalGrowthBytes: values["max-wal-growth"] ? Number(values["max-wal-growth"]) : undefined });
+        recordStorageMaintenance(source, { operation: "backup", at: new Date().toISOString(), status: result.status, destination: required("destination") });
+    }
     else if (action === "inspect")
         result = await readStorageBackupManifest(required("archive"));
-    else if (action === "verify")
-        result = await verifyStorageBackup(required("archive"), AbortSignal.timeout(60000));
+    else if (action === "verify") {
+        const archive = required("archive");
+        result = await verifyStorageBackup(archive, AbortSignal.timeout(60000));
+        const manifest = await readStorageBackupManifest(archive);
+        recordStorageMaintenance(manifest.source, { operation: "backup", at: new Date().toISOString(), status: "verified", archive });
+    }
     else if (action === "restore")
         result = await restoreStorageBackup(required("archive"), required("destination"), AbortSignal.timeout(60000));
     else

package/dist/debug/storage-maintenance.js

@@ -0,0 +1,78 @@
+import { parseArgs } from "node:util";
+import { piboHomePath } from "../core/pibo-home.js";
+import { checkpointStorage, inspectStorageStatus, maintainStorageRetention, verifyStorage } from "../data/storage-maintenance.js";
+import { resolveDebugStore } from "./stores.js";
+export async function runStorageMaintenanceCli(args) {
+    const action = args[0];
+    if (!action || args.includes("--help") || args.includes("-h")) {
+        printHelp();
+        return;
+    }
+    const { values } = parseArgs({ args: args.slice(1), strict: true, options: {
+            store: { type: "string" }, path: { type: "string" }, json: { type: "boolean" },
+            "timeout-ms": { type: "string" }, full: { type: "boolean" }, quick: { type: "boolean" },
+            apply: { type: "boolean" }, "dry-run": { type: "boolean" }, mode: { type: "string" },
+            before: { type: "string" }, limit: { type: "string" }, "payload-root": { type: "string" },
+            "database-warn-bytes": { type: "string" }, "wal-warn-bytes": { type: "string" }, "payload-warn-bytes": { type: "string" },
+        } });
+    if (values.apply && values["dry-run"])
+        throw new Error("Choose either --dry-run or --apply");
+    const path = typeof values.path === "string" ? values.path : resolveStorePath(values.store);
+    let result;
+    if (action === "status" || action === "doctor") {
+        result = inspectStorageStatus({ path, databaseWarnBytes: numberValue(values["database-warn-bytes"]), walWarnBytes: numberValue(values["wal-warn-bytes"]), payloadWarnBytes: numberValue(values["payload-warn-bytes"]) });
+    }
+    else if (action === "verify") {
+        if (values.full && values.quick)
+            throw new Error("Choose either --quick or --full");
+        result = await verifyStorage({ path, mode: values.full ? "full" : "quick", timeoutMs: numberValue(values["timeout-ms"]) });
+    }
+    else if (action === "checkpoint") {
+        const mode = values.mode ?? "passive";
+        if (!new Set(["passive", "restart", "truncate"]).has(mode))
+            throw new Error("Checkpoint --mode must be passive, restart, or truncate");
+        result = checkpointStorage({ path, mode: mode, apply: values.apply });
+    }
+    else if (action === "retention") {
+        if (typeof values.before !== "string")
+            throw new Error("Storage retention requires --before <iso-date>");
+        result = await maintainStorageRetention({ path, before: values.before, limit: numberValue(values.limit), apply: values.apply, payloadRoot: typeof values["payload-root"] === "string" ? values["payload-root"] : piboHomePath("payloads") });
+    }
+    else
+        throw new Error(`Unknown storage action "${action}"; run pibo debug storage --help`);
+    if (values.json)
+        console.log(JSON.stringify(result, null, 2));
+    else
+        console.log(formatText(result));
+}
+function resolveStorePath(value) {
+    if (!value || value === "pibo-data")
+        return resolveDebugStore("pibo-data").path;
+    if (value === "reliability")
+        return resolveDebugStore("reliability").path;
+    throw new Error("--store must be pibo-data or reliability; use --path for another SQLite store");
+}
+function numberValue(value) { if (value === undefined)
+    return undefined; const parsed = Number(value); if (!Number.isSafeInteger(parsed) || parsed < 0)
+    throw new Error(`Invalid numeric value "${value}"`); return parsed; }
+function formatText(value) { const object = value; const lines = [`pibo debug storage ${String(object.resultType ?? "result").split(".").at(-1)}`]; for (const [key, item] of Object.entries(object)) {
+    if (key === "resultType" || typeof item === "object")
+        continue;
+    lines.push(`${key}\t${String(item)}`);
+} if (Array.isArray(object.warnings))
+    lines.push(...object.warnings.map((warning) => `warning\t${String(warning)}`)); lines.push("", "Use --json for bounded row, size, projection, and progress detail."); return lines.join("\n"); }
+function printHelp() {
+    console.log(`pibo debug storage - bounded SQLite health and maintenance
+
+Commands:
+  status [--store pibo-data|reliability | --path <sqlite>] [--json]
+  doctor [--store pibo-data|reliability | --path <sqlite>] [--json]
+  verify [--quick|--full] [--timeout-ms <n>] [--store <name>|--path <sqlite>] [--json]
+  checkpoint [--mode passive|restart|truncate] [--dry-run|--apply] [--store <name>|--path <sqlite>] [--json]
+  retention --before <iso-date> [--limit <1..10000>] [--dry-run|--apply] [--payload-root <dir>] [--store <name>|--path <sqlite>] [--json]
+
+Status is read-only and reports DB/WAL/SHM/payload size, pages/freelist, bounded row counts, payload-reference integrity, maintenance metadata, and degraded thresholds.
+Verification runs in a cancellable worker. Timeout is partial and never healthy. Quick/full checks can still be I/O intensive; use backup verification or offline checks when the online budget is insufficient.
+Checkpoint and retention default to dry-run. Apply is bounded and audited in the database maintenance sidecar. Retention deletes only eligible live_delta rows; chat messages, audit events, idempotency evidence, and referenced payloads are preserved.
+`);
+}

package/dist/debug/summary.js

@@ -42,6 +42,7 @@ export async function inspectDebugSummary(input, stores) {
             `pibo debug failures ${parsed.piboSessionId}`,
             `pibo debug messages ${parsed.piboSessionId} list`,
             `pibo debug trace ${parsed.piboSessionId} --check`,
+            `pibo debug cache ${parsed.piboSessionId}`,
             `pibo debug events ${parsed.piboSessionId} --limit 20`,
         ],
     };

package/dist/debug/trace.js

@@ -7,6 +7,8 @@ import { compareTraceNodes } from "../shared/trace-nodes.js";
 import { openReadOnlyDebugDatabase, withStorePath } from "./sql.js";
 import { formatNextCommands } from "./next-commands.js";
 import { resolveDebugTraceSessionStatus, summarizeDebugTraceStatus } from "./trace-status.js";
+import { cacheUsageWarningText } from "../shared/cache-observability.js";
+import { modelInferenceCacheReadRatio, modelInferenceCachedInputTokens, modelInferenceInputTokens, modelInferenceUncachedInputTokens, } from "../shared/model-inference-metrics.js";
 export async function inspectDebugTrace(piboSessionId, stores, options = {}) {
     if (!stores.sessions.exists)
         throw new Error(`Debug store "sessions" not found at ${stores.sessions.path}`);
@@ -113,6 +115,23 @@ export async function inspectDebugTraceNode(piboSessionId, stores, nodeId) {
         nextCommands: node ? buildNodeNextCommands(piboSessionId, node) : [`pibo debug trace ${piboSessionId}`],
     };
 }
+function formatMetric(value) {
+    return value === undefined ? "unknown" : String(value);
+}
+function formatRatio(value) {
+    return value === undefined ? "unknown" : `${(value * 100).toFixed(1)}%`;
+}
+function formatModelInference(record) {
+    return [
+        record.id,
+        `input=${formatMetric(modelInferenceInputTokens(record.metrics))}`,
+        `cacheRead=${formatMetric(modelInferenceCachedInputTokens(record.metrics))}`,
+        `uncached=${formatMetric(modelInferenceUncachedInputTokens(record.metrics))}`,
+        `cacheWrite=${formatMetric(record.metrics.cacheWriteTokens)}`,
+        `output=${formatMetric(record.metrics.outputTokens)}`,
+        `cacheReadRatio=${formatRatio(modelInferenceCacheReadRatio(record.metrics))}`,
+    ].join("\t");
+}
 export function formatDebugTrace(result, options = {}) {
     const lines = [
         `piboSessionId: ${result.piboSessionId}`,
@@ -152,6 +171,12 @@ export function formatDebugTrace(result, options = {}) {
             order: node.order,
         };
         lines.push(columns.map((column) => values[column] ?? "").join("\t"));
+        for (const inference of node.modelInferences ?? []) {
+            lines.push(`model-inference\t${formatModelInference(inference)}`);
+            const warning = inference.cacheObservation && cacheUsageWarningText(inference.cacheObservation);
+            if (warning)
+                lines.push(`cache-warning\t${inference.id}\t${warning}`);
+        }
     }
     lines.push(`nodes: ${result.nodes.length}${result.nodes.length !== result.rawNodeCount ? ` of ${result.rawNodeCount}` : ""}`);
     if (result.checks) {
@@ -194,6 +219,8 @@ export function formatDebugTraceNode(result) {
         lines.push(`runId: ${node.runId}`);
     if (node.toolCallId)
         lines.push(`toolCallId: ${node.toolCallId}`);
+    for (const inference of node.modelInferences ?? [])
+        lines.push(`modelInference: ${JSON.stringify(inference)}`);
     lines.push(...formatNextCommands(result.nextCommands));
     return lines.join("\n");
 }
@@ -214,6 +241,7 @@ function flattenTraceNodes(nodes, depth = 0) {
             startedAt: node.startedAt,
             completedAt: node.completedAt,
             childrenCount: node.children.length,
+            ...(node.modelInferences?.length ? { modelInferences: node.modelInferences } : {}),
             depth,
         },
         ...flattenTraceNodes(node.children, depth + 1),

package/dist/gateway/cli.js

@@ -231,14 +231,40 @@ function activeRun(value) {
         piboSessionId: stringValue(obj.controllerPiboSessionId) ?? stringValue(obj.piboSessionId),
     };
 }
+function runJobReliability(value) {
+    const obj = objectValue(value);
+    if (!obj || (obj.status !== "ok" && obj.status !== "degraded")
+        || !Number.isInteger(obj.expiredOrphanRunJobs) || Number(obj.expiredOrphanRunJobs) < 0
+        || !Number.isInteger(obj.orphanRunDeadLetters) || Number(obj.orphanRunDeadLetters) < 0)
+        return undefined;
+    return {
+        status: obj.status,
+        expiredOrphanRunJobs: Number(obj.expiredOrphanRunJobs),
+        orphanRunDeadLetters: Number(obj.orphanRunDeadLetters),
+    };
+}
+function durableMessageQueueStatus(value) {
+    const obj = objectValue(value);
+    if (!obj)
+        return undefined;
+    const status = obj.status === "healthy" || obj.status === "degraded" || obj.status === "ambiguous" ? obj.status : undefined;
+    const storage = objectValue(obj.storage);
+    const counts = Array.isArray(obj.counts) ? obj.counts.flatMap(value => { const row = objectValue(value); return row ? [{ state: stringValue(row.state), delivery: stringValue(row.delivery), count: numberValue(row.count), bytes: numberValue(row.bytes) }] : []; }) : undefined;
+    const affectedScopes = Array.isArray(obj.affectedScopes) ? obj.affectedScopes.flatMap(value => { const row = objectValue(value); return row ? [{ sessionId: stringValue(row.sessionId), roomId: stringValue(row.roomId), blockingCommandId: stringValue(row.blockingCommandId), blockedSince: numberValue(row.blockedSince), blockedSuccessors: numberValue(row.blockedSuccessors) }] : []; }) : undefined;
+    return { status, storage: storage ? { available: booleanValue(storage.available), error: stringValue(storage.error) } : undefined, counts, interruptedPredecessors: numberValue(obj.interruptedPredecessors), blockedSuccessors: numberValue(obj.blockedSuccessors), expiredOwnedLeases: numberValue(obj.expiredOwnedLeases), oldestDispatchableWaitMs: numberValue(obj.oldestDispatchableWaitMs), oldestBlockedWaitMs: numberValue(obj.oldestBlockedWaitMs), affectedScopes, degradedReasons: Array.isArray(obj.degradedReasons) && obj.degradedReasons.every(v => typeof v === "string") ? obj.degradedReasons : undefined, admissionCapacity: obj.admissionCapacity, bounded: obj.bounded };
+}
 function parseGatewaySafetyPayload(payload, reachable) {
     const obj = objectValue(payload);
     const mode = obj && (obj.mode === "dev" || obj.mode === "prod" || obj.mode === "fallback") ? obj.mode : "unknown";
     const runtimeStatuses = Array.isArray(obj?.runtimeStatuses) ? obj.runtimeStatuses.map(runtimeStatus).filter((item) => Boolean(item)) : [];
     const activeRuns = Array.isArray(obj?.activeRuns) ? obj.activeRuns.map(activeRun).filter((item) => Boolean(item)) : [];
+    const reliability = obj?.reliability === undefined ? undefined : runJobReliability(obj.reliability);
+    const durableMessageQueue = durableMessageQueueStatus(obj?.durableMessageQueue);
     const incomplete = !Array.isArray(obj?.runtimeStatuses) || !Array.isArray(obj?.activeRuns)
-        || runtimeStatuses.length !== obj.runtimeStatuses.length || activeRuns.length !== obj.activeRuns.length;
-    return { reachable, mode, generation: stringValue(obj?.generation), health: obj?.health, runtimeStatuses, activeRuns, ambiguous: incomplete || booleanValue(obj?.ambiguous) };
+        || runtimeStatuses.length !== obj.runtimeStatuses.length || activeRuns.length !== obj.activeRuns.length
+        || (obj?.reliability !== undefined && !reliability)
+        || (obj?.durableMessageQueue !== undefined && !durableMessageQueue?.status);
+    return { reachable, mode, generation: stringValue(obj?.generation), health: obj?.health, runtimeStatuses, activeRuns, reliability, durableMessageQueue, ambiguous: incomplete || booleanValue(obj?.ambiguous) };
 }
 export function checkActiveWork(status, target = "web") {
     const reasons = [];
@@ -337,7 +363,8 @@ function printSafetyStatus(target, status) {
     console.log(`  mode: ${status.mode}`);
     if (status.error)
         console.log(`  status error: ${status.error}`);
-    console.log(`  runtime sessions: ${status.runtimeStatuses.length}`);
+    console.log("  runtime queue layer:");
+    console.log(`    sessions: ${status.runtimeStatuses.length}`);
     for (const session of status.runtimeStatuses) {
         console.log(`    ${session.piboSessionId ?? "unknown"}: processing=${session.processing === true} streaming=${session.streaming === true} queued=${session.queuedMessages ?? 0}`);
         if (session.activeEventId)
@@ -359,6 +386,31 @@ function printSafetyStatus(target, status) {
     console.log(`  active yielded runs: ${status.activeRuns.length}`);
     for (const run of status.activeRuns)
         console.log(`    ${run.runId ?? "unknown"}: ${run.status ?? "active"}${run.toolName ? ` (${run.toolName})` : ""} session=${run.piboSessionId ?? "unknown"}`);
+    if (status.reliability) {
+        console.log(`  run-job reliability: ${status.reliability.status}`);
+        console.log(`    expired orphan jobs: ${status.reliability.expiredOrphanRunJobs}`);
+        console.log(`    orphan DLQ records: ${status.reliability.orphanRunDeadLetters}`);
+    }
+    console.log("  durable message queue layer:");
+    const durable = status.durableMessageQueue;
+    if (!durable)
+        console.log("    status: unavailable (gateway app did not report this layer)");
+    else {
+        console.log(`    status: ${durable.status ?? "ambiguous"}`);
+        console.log(`    storage: ${durable.storage?.available === true ? "available" : "unavailable"}${durable.storage?.error ? ` (${durable.storage.error})` : ""}`);
+        for (const row of durable.counts ?? [])
+            console.log(`    ${row.delivery ?? "unknown"}.${row.state ?? "unknown"}: count=${row.count ?? 0} bytes=${row.bytes ?? 0}`);
+        console.log(`    interrupted predecessors: ${durable.interruptedPredecessors ?? 0}`);
+        console.log(`    FIFO-blocked successors: ${durable.blockedSuccessors ?? 0}`);
+        console.log(`    expired owned leases: ${durable.expiredOwnedLeases ?? 0}`);
+        console.log(`    oldest dispatchable wait: ${durable.oldestDispatchableWaitMs ?? 0}ms`);
+        console.log(`    oldest blocked wait: ${durable.oldestBlockedWaitMs ?? 0}ms`);
+        for (const scope of durable.affectedScopes ?? [])
+            console.log(`    affected session=${scope.sessionId ?? "unknown"} room=${scope.roomId ?? "unknown"} blocker=${scope.blockingCommandId ?? "unknown"} successors=${scope.blockedSuccessors ?? 0}`);
+        for (const reason of durable.degradedReasons ?? [])
+            console.log(`    degraded: ${reason}`);
+    }
+    console.log("  next: pibo debug message-queue");
 }
 function managerRequiresShell(command) {
     return process.platform === "win32" && /\.(?:cmd|bat)$/i.test(command);
@@ -385,8 +437,11 @@ async function waitForManagedGatewayHealth(target) {
 async function runManagedGatewayCommand(target, command, args, argv = process.argv) {
     if (command === "status" || command === "doctor") {
         const status = await readGatewaySafetyStatus(target);
-        printSafetyStatus(target, status);
-        if (target === "web") {
+        if (args.includes("--json"))
+            console.log(JSON.stringify({ ...status, nextCommands: [`pibo gateway ${target} doctor`, `pibo debug message-queue`] }, null, 2));
+        else
+            printSafetyStatus(target, status);
+        if (target === "web" && !args.includes("--json")) {
             const active = checkActiveWork(status, target);
             if (active.unsafe) {
                 console.log("  restart safety: blocked");
@@ -397,8 +452,16 @@ async function runManagedGatewayCommand(target, command, args, argv = process.ar
                 console.log("  restart safety: idle");
             printRestartApproval(status);
         }
-        if (command === "doctor")
-            process.exitCode = status.reachable && !status.error && status.mode === expectedMode(target) ? 0 : 1;
+        if (command === "doctor") {
+            process.exitCode = status.reachable
+                && !status.error
+                && status.mode === expectedMode(target)
+                && status.reliability?.status !== "degraded"
+                && status.durableMessageQueue?.status === "healthy"
+                && status.durableMessageQueue.storage?.available === true
+                ? 0
+                : 1;
+        }
         return true;
     }
     if (command === "start") {
@@ -730,6 +793,7 @@ Commands:
   dev doctor       Check dev gateway health
 
 Options:
+  --json         Print status or doctor output as JSON with next discovery commands
   --force --confirm <snapshot-token>
                  Restart only the work explicitly approved from web status
 

package/dist/gateway/server.js

@@ -427,6 +427,7 @@ export class PiboGatewayServer {
             listSessionRuntimeStatuses: () => this.requireRouter().listSessionRuntimeStatuses(),
             getRuntimeCapacityStatus: () => this.requireRouter().getRuntimeCapacityStatus(),
             listRuns: (options) => this.requireRouter().listRuns(options),
+            getRunJobReliabilityStatus: () => this.requireRouter().getRunJobReliabilityStatus(),
             snapshotSignalSession: (piboSessionId) => this.requireRouter().snapshotSignalSession(piboSessionId),
             snapshotSignalTree: (rootPiboSessionId) => this.requireRouter().snapshotSignalTree(rootPiboSessionId),
             snapshotSignalStatuses: () => this.requireRouter().snapshotSignalStatuses(),

package/dist/reliability/store.js

@@ -44,12 +44,14 @@ function ensurePiboJobColumn(db, name, definition) {
         db.exec(`ALTER TABLE pibo_jobs ADD COLUMN ${name} ${definition}`);
 }
 export class PiboReliabilityStore {
+    options;
     db;
     appendEventStatement;
     getEventByStreamIdStatement;
     getEventByTopicEventIdStatement;
     getEventByTopicIdempotencyKeyStatement;
-    constructor(path = piboHomePath("pibo-events.sqlite")) {
+    constructor(path = piboHomePath("pibo-events.sqlite"), options = {}) {
+        this.options = options;
         const resolvedPath = path === ":memory:" ? path : resolve(path);
         const insidePiboHome = ensurePrivatePiboHomeForPath(resolvedPath);
         if (resolvedPath !== ":memory:")
@@ -571,6 +573,38 @@ export class PiboReliabilityStore {
         }
         return jobs;
     }
+    inspectJobs(input = {}) {
+        const timestamp = now();
+        return this.listJobs(input).map((job) => this.inspectJob(job, timestamp));
+    }
+    reconcileOrphanRunJobs(input) {
+        const checkedAt = asDate(input.now).toISOString();
+        return this.inImmediateTransaction(() => this.reconcileOrphanRunJobsInTransaction(checkedAt, input.apply));
+    }
+    getRunJobReliabilityStatus() {
+        const timestamp = now();
+        const expired = this.db.prepare(`
+			SELECT COUNT(*) AS count
+			FROM pibo_jobs AS job
+			LEFT JOIN pibo_runs AS run ON run.job_id = job.job_id
+			WHERE job.queue = 'runs'
+				AND job.state = 'running'
+				AND job.claim_expires_at IS NOT NULL
+				AND job.claim_expires_at <= ?
+				AND run.run_id IS NULL
+		`).get(timestamp);
+        const dead = this.db.prepare(`
+			SELECT COUNT(*) AS count FROM pibo_dead_jobs
+			WHERE queue = 'runs' AND dead_reason = 'orphan_run_job'
+		`).get();
+        const expiredOrphanRunJobs = Number(expired.count);
+        const orphanRunDeadLetters = Number(dead.count);
+        return {
+            status: expiredOrphanRunJobs > 0 || orphanRunDeadLetters > 0 ? "degraded" : "ok",
+            expiredOrphanRunJobs,
+            orphanRunDeadLetters,
+        };
+    }
     listRecoverableJobs(input = {}) {
         const timestamp = now();
         const clauses = [
@@ -683,28 +717,37 @@ export class PiboReliabilityStore {
         const runId = input.runId ?? `run_${randomUUID()}`;
         const maxAttempts = Math.max(1, input.maxAttempts ?? 1);
         const timeoutAt = input.timeoutMs === undefined ? undefined : new Date(Date.parse(timestamp) + input.timeoutMs).toISOString();
-        const job = this.enqueue({
-            queue: "runs",
-            payload: {
-                runId,
-                controllerPiboSessionId: input.controllerPiboSessionId,
-                toolName: input.toolName,
-                params: input.params,
-                timeoutMs: input.timeoutMs,
-            },
-            maxAttempts,
+        return this.inImmediateTransaction(() => {
+            const job = this.enqueue({
+                queue: "runs",
+                runAt: timestamp,
+                payload: {
+                    runId,
+                    controllerPiboSessionId: input.controllerPiboSessionId,
+                    toolName: input.toolName,
+                    params: input.params,
+                    timeoutMs: input.timeoutMs,
+                },
+                maxAttempts,
+            });
+            this.options.onRunCreationBoundary?.("after_enqueue");
+            const claimed = this.claimPendingJobInTransaction(job.jobId, input.workerId ?? `run-registry:${process.pid}`, timestamp, 24 * 60 * 60 * 1000);
+            if (!claimed)
+                throw new Error(`Failed to claim newly-created run job "${job.jobId}"`);
+            this.options.onRunCreationBoundary?.("after_claim");
+            this.options.onRunCreationBoundary?.("before_run_insert");
+            this.db
+                .prepare(`
+					INSERT INTO pibo_runs (
+						run_id, kind, controller_pibo_session_id, status, completion_policy, consumed, tool_name,
+						summary, result_json, error, notified_status, acknowledged_status, created_at, updated_at,
+						completed_at, job_id, retryable, max_attempts, timeout_ms, timeout_at, timeout_phase, service_warning, resource_json, origin_json
+					) VALUES (?, 'tool', ?, 'running', ?, 0, ?, ?, NULL, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?, ?, ?, NULL, ?, ?, ?)
+				`)
+                .run(runId, input.controllerPiboSessionId, input.completionPolicy, input.toolName, `${input.toolName} run is running.`, timestamp, timestamp, job.jobId, input.retryable ? 1 : 0, maxAttempts, input.timeoutMs ?? null, timeoutAt ?? null, input.serviceWarning ?? null, input.resources ? JSON.stringify(input.resources) : null, input.origin ? JSON.stringify(input.origin) : null);
+            this.options.onRunCreationBoundary?.("after_run_insert");
+            return this.requireRun(runId);
         });
-        this.db
-            .prepare(`
-				INSERT INTO pibo_runs (
-					run_id, kind, controller_pibo_session_id, status, completion_policy, consumed, tool_name,
-					summary, result_json, error, notified_status, acknowledged_status, created_at, updated_at,
-					completed_at, job_id, retryable, max_attempts, timeout_ms, timeout_at, timeout_phase, service_warning, resource_json, origin_json
-				) VALUES (?, 'tool', ?, 'running', ?, 0, ?, ?, NULL, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?, ?, ?, NULL, ?, ?, ?)
-			`)
-            .run(runId, input.controllerPiboSessionId, input.completionPolicy, input.toolName, `${input.toolName} run is running.`, timestamp, timestamp, job.jobId, input.retryable ? 1 : 0, maxAttempts, input.timeoutMs ?? null, timeoutAt ?? null, input.serviceWarning ?? null, input.resources ? JSON.stringify(input.resources) : null, input.origin ? JSON.stringify(input.origin) : null);
-        this.claimJob(job.jobId, input.workerId ?? `run-registry:${process.pid}`, 24 * 60 * 60 * 1000);
-        return this.requireRun(runId);
     }
     updateRun(runId, patch) {
         const existing = this.getRun(runId);
@@ -779,9 +822,10 @@ export class PiboReliabilityStore {
     }
     recoverInterruptedRuns(workerId = `run-registry:${process.pid}`) {
         return this.inImmediateTransaction(() => {
+            const timestamp = now();
+            this.reconcileOrphanRunJobsInTransaction(timestamp, true);
             const rows = this.db.prepare("SELECT * FROM pibo_runs WHERE status = 'running'").all();
             const recovered = [];
-            const timestamp = now();
             for (const row of rows) {
                 if (row.job_id && this.hasUnexpiredJobClaim(row.job_id, timestamp, workerId))
                     continue;
@@ -883,6 +927,58 @@ export class PiboReliabilityStore {
 			`)
             .get(jobId, workerId, claimToken ?? null, claimToken ?? null, timestamp);
     }
+    claimPendingJobInTransaction(jobId, workerId, timestamp, visibilityTimeoutMs) {
+        const claimExpiresAt = new Date(Date.parse(timestamp) + visibilityTimeoutMs).toISOString();
+        return this.db.prepare(`
+			UPDATE pibo_jobs
+			SET state = 'running',
+				worker_id = ?,
+				claim_expires_at = ?,
+				claim_token = claim_token + 1,
+				attempts = attempts + 1,
+				updated_at = ?
+			WHERE job_id = ?
+				AND state = 'pending'
+				AND run_at <= ?
+				AND attempts < max_attempts
+				AND (expires_at IS NULL OR expires_at > ?)
+			RETURNING *
+		`).get(workerId, claimExpiresAt, timestamp, jobId, timestamp, timestamp);
+    }
+    inspectJob(job, timestamp) {
+        const claimExpired = job.state === "running"
+            && (!job.claimExpiresAt || job.claimExpiresAt <= timestamp);
+        const missingRunRecord = job.queue === "runs"
+            && this.db.prepare("SELECT 1 AS found FROM pibo_runs WHERE job_id = ? LIMIT 1").get(job.jobId) === undefined;
+        let effectiveLiveness;
+        if (job.state === "pending")
+            effectiveLiveness = missingRunRecord ? "orphan_pending" : "pending";
+        else if (claimExpired)
+            effectiveLiveness = missingRunRecord ? "expired_orphan" : "expired_claim";
+        else
+            effectiveLiveness = missingRunRecord ? "live_orphan_claim" : "live_claim";
+        return { ...job, claimExpired, missingRunRecord, effectiveLiveness };
+    }
+    reconcileOrphanRunJobsInTransaction(timestamp, apply) {
+        const rows = this.db.prepare(`
+			SELECT job.*
+			FROM pibo_jobs AS job
+			LEFT JOIN pibo_runs AS run ON run.job_id = job.job_id
+			WHERE job.queue = 'runs'
+				AND job.state = 'running'
+				AND job.claim_expires_at IS NOT NULL
+				AND job.claim_expires_at <= ?
+				AND run.run_id IS NULL
+			ORDER BY job.created_at ASC, job.job_id ASC
+		`).all(timestamp);
+        const candidates = rows.map((row) => this.inspectJob(jobFromRow(row), timestamp));
+        if (apply) {
+            for (const row of rows) {
+                this.moveJobToDead(row, "Expired runs job has no matching pibo_runs record.", "orphan_run_job", timestamp);
+            }
+        }
+        return { checkedAt: timestamp, apply, candidates, moved: apply ? rows.length : 0 };
+    }
     moveExpiredJobs(timestamp) {
         const expired = this.db.prepare("SELECT * FROM pibo_jobs WHERE expires_at IS NOT NULL AND expires_at <= ?").all(timestamp);
         for (const row of expired)

package/dist/session-ui/terminalRows.js

@@ -23,8 +23,8 @@ export function buildCompactTerminalRows(traceView, options) {
     const candidates = syncThinkingToolRows(flatNodes.map((item) => createRowCandidate(item.node, item.turnId)));
     applyCompletedTurnTiming(candidates, turnById);
     const reconciled = reconcileConceptualRowCandidates(candidates);
-    const rows = !showToolDebugMetrics && (options.toolDisplayMode ?? "default") === "default"
-        ? groupRelatedToolCandidates(reconciled).map((candidate) => candidate.row)
+    const rows = (options.toolDisplayMode ?? "default") === "default"
+        ? groupRelatedToolCandidates(reconciled, showToolDebugMetrics).map((candidate) => candidate.row)
         : reconciled.map((candidate) => candidate.row);
     return applyToolDisplayMode(rows, options.toolDisplayMode ?? "default");
 }
@@ -942,12 +942,12 @@ function isThinkingOutput(value) {
 function isThinkingLevelSetOutput(value) {
     return isRecord(value) && value.action === "set_thinking_level";
 }
-function groupRelatedToolCandidates(candidates) {
+function groupRelatedToolCandidates(candidates, imagesOnly = false) {
     const grouped = [];
     for (let index = 0; index < candidates.length; index += 1) {
         const candidate = candidates[index];
         const groupKind = candidateGroupKind(candidate);
-        if (!groupKind) {
+        if (!groupKind || (imagesOnly && groupKind !== "images")) {
             grouped.push(candidate);
             continue;
         }
@@ -1028,21 +1028,20 @@ function createImageGroup(candidates) {
     const omittedDetailCount = Math.max(0, detailItems.length - visibleDetailItems.length);
     const firstRow = candidates[0]?.row;
     const firstId = firstRow?.id ?? "images";
-    const lastId = candidates[candidates.length - 1]?.row.id ?? firstId;
     const status = candidates.some((candidate) => candidate.row.status === "running")
         ? "running"
         : candidates.some((candidate) => candidate.row.status === "error")
             ? "error"
             : "done";
     return {
-        id: `group:images:${firstId}:${lastId}`,
+        id: firstId,
         kind: "tool.group.images",
         status,
         errorKind: status === "error" ? "tool" : undefined,
         lines: [
             {
                 prefix: "bullet",
-                tokens: [token(status === "running" ? "Viewing images" : status === "error" ? "Image reads failed" : "Viewed images", toneForStatus(status), "semibold")],
+                tokens: [token(status === "running" ? `Viewing ${detailItems.length} images` : status === "error" ? `${detailItems.length} image reads · error` : `${detailItems.length} ${detailItems.length === 1 ? "Image Viewed" : "Images Viewed"}`, toneForStatus(status), "semibold")],
             },
             ...visibleDetailItems.map((item, index) => ({
                 prefix: index === 0 ? "detail" : "continuation",
@@ -1062,7 +1061,7 @@ function createImageGroup(candidates) {
         orderStreamFrameIndex: firstRow?.orderStreamFrameIndex,
         detailItems,
         expandable: detailItems.some((item) => item.toolCallReference || item.input !== undefined || item.output !== undefined || Boolean(item.error)),
-        imagePreviews: detailItems.flatMap((item) => item.imagePreviews ?? []).slice(0, MAX_COMPACT_TERMINAL_IMAGE_PREVIEWS),
+        imagePreviews: detailItems.flatMap((item) => item.imagePreviews ?? []),
         previewOmission: omittedDetailCount > 0 ? {
             source: "details",
             visibleLineCount: visibleDetailItems.length,

package/dist/sessions/pibo-data-store.js

@@ -227,21 +227,21 @@ export class PiboDataSessionStore {
                 if (typeof index !== "number" || !Number.isSafeInteger(index) || index < 0)
                     continue;
                 maximum = Math.max(maximum, index);
-                persistedParts.push({ index, attributes });
+                persistedParts.push({ index, type: row.type, attributes });
             }
-            // An unfinished durable part may still belong to another process. Only a
-            // completed turn provides enough evidence to reattach an exact replay.
-            if (turnCompleted) {
-                const matchesReplay = ({ attributes }) => attributes.outputPartFingerprint === input.fingerprint
-                    || attributes.identityFingerprint === input.identityFingerprint;
-                const replay = (input.suppliedIndex === undefined
-                    ? undefined
-                    : persistedParts.find((part) => part.index === input.suppliedIndex && matchesReplay(part)))
-                    ?? persistedParts.find(matchesReplay);
-                if (replay) {
-                    this.observeOutputPartIndex(input, replay.index);
-                    return replay.index;
-                }
+            // A completed turn or an exact terminal-part fingerprint is durable
+            // evidence for replay. Nonterminal parts remain unattached while the turn
+            // is open because another producer may still own them.
+            const matchesReplay = ({ type, attributes }) => (turnCompleted || outputPartEventIsTerminal(type))
+                && (attributes.outputPartFingerprint === input.fingerprint
+                    || attributes.identityFingerprint === input.identityFingerprint);
+            const replay = (input.suppliedIndex === undefined
+                ? undefined
+                : persistedParts.find((part) => part.index === input.suppliedIndex && matchesReplay(part)))
+                ?? (turnCompleted ? persistedParts.find(matchesReplay) : undefined);
+            if (replay) {
+                this.observeOutputPartIndex(input, replay.index);
+                return replay.index;
             }
             const minimum = Math.max(input.proposedIndex, maximum + 1);
             const row = this.db.prepare(`
@@ -538,6 +538,7 @@ export class PiboDataSessionStore {
                         actorId: session.id,
                         event,
                         createdAt: at,
+                        persistenceProvenance: { producer: "runtime-recovery", projection: "product-history", phase: "startup-recovery" },
                     });
                 results.push({
                     turnId: recovered.turn.turnId,
@@ -726,6 +727,9 @@ function outputPartEventTypes(kind) {
         case "compaction": return ["compaction_start", "compaction_end"];
     }
 }
+function outputPartEventIsTerminal(type) {
+    return type === "assistant_message" || type === "thinking_finished" || type === "assistant_usage" || type === "compaction_end";
+}
 function parseJsonObject(json) {
     if (!json)
         return {};