@pasko70/pibo 3.1.3 → 3.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/chat/data/timeline-query-service.js +5 -2
- package/dist/apps/chat/output-compactor.js +33 -30
- package/dist/apps/chat/trace-v2.js +1 -0
- package/dist/apps/chat/trace.js +3 -1
- package/dist/apps/chat/web-app.js +106 -65
- package/dist/apps/chat-ui/assets/{dist-BNzch-mL.js → dist-BOqsX6_s.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-CoA9zNTP.js → dist-BcUbdOKJ.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-liw1S7HS.js → dist-DkpIJ_Pp.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-7DhHpLCA.js → dist-a0mykCz7.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-DDRlEGPR.js → dist-qDZ-CdlK.js} +1 -1
- package/dist/apps/chat-ui/assets/{index-BKp6zKIe.js → index-G2ic-FSG.js} +89 -89
- package/dist/apps/chat-ui/index.html +1 -1
- package/dist/apps/chat-vscode-web/assets/{index-CKC-46jZ.js → index-zQ1fNz5K.js} +1 -1
- package/dist/apps/chat-vscode-web/index.html +1 -1
- package/dist/cli-session/localSessionSource.js +26 -20
- package/dist/core/output-persistence-retry.js +23 -1
- package/dist/core/output-render-sequence.js +33 -9
- package/dist/data/ingest-service.js +8 -0
- package/dist/debug/index.js +294 -2
- package/dist/debug/output-integrity.js +608 -0
- package/dist/debug/output-repair.js +584 -0
- package/dist/debug/trace.js +2 -0
- package/dist/reliability/store.js +2 -2
- package/dist/sessions/pibo-data-store.js +26 -20
- package/dist/shared/trace-engine.js +5 -2
- package/dist/shared/trace-event-projection.js +74 -0
- package/dist/shared/trace-page-merge.js +31 -5
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<meta name="theme-color" content="#101d22" />
|
|
7
7
|
<title>Pibo</title>
|
|
8
|
-
<script type="module" crossorigin src="/apps/chat-vscode/assets/index-
|
|
8
|
+
<script type="module" crossorigin src="/apps/chat-vscode/assets/index-zQ1fNz5K.js"></script>
|
|
9
9
|
<link rel="stylesheet" crossorigin href="/apps/chat-vscode/assets/index-SluZr_-r.css">
|
|
10
10
|
</head>
|
|
11
11
|
<body>
|
|
@@ -7,7 +7,7 @@ import { OutputPersistenceRetryQueue, } from "../core/output-persistence-retry.j
|
|
|
7
7
|
import { getDefaultPiboWorkspace } from "../core/workspace.js";
|
|
8
8
|
import { buildTraceView } from "../apps/chat/trace.js";
|
|
9
9
|
import { storedPiboEventFromV2Row, } from "../apps/chat/data/chat-data-mappers.js";
|
|
10
|
-
import { ChatDataIngestService, outputIdempotencyKey, outputPersistenceDeliveryKey } from "../data/ingest-service.js";
|
|
10
|
+
import { ChatDataIngestService, outputIdempotencyKey, outputPersistenceDeliveryKey, outputPersistenceErrorIsRetryable } from "../data/ingest-service.js";
|
|
11
11
|
import { OutputCompactor } from "../apps/chat/output-compactor.js";
|
|
12
12
|
import { isPiboOutputEvent } from "../apps/chat/output-event-policy.js";
|
|
13
13
|
import { ChatRoomService } from "../apps/chat/data/room-service.js";
|
|
@@ -597,7 +597,9 @@ export class LocalCliSessionSource {
|
|
|
597
597
|
const session = this.sessionStore.get(event.piboSessionId);
|
|
598
598
|
if (!session)
|
|
599
599
|
return;
|
|
600
|
-
const positionedEvent =
|
|
600
|
+
const positionedEvent = validRenderSequence(event.renderSequence)
|
|
601
|
+
? event
|
|
602
|
+
: this.outputRenderSequencer.position(event);
|
|
601
603
|
const compacted = this.outputCompactor.prepare(positionedEvent);
|
|
602
604
|
if (compacted.persistedEvents.length === 0) {
|
|
603
605
|
compacted.ack();
|
|
@@ -608,7 +610,7 @@ export class LocalCliSessionSource {
|
|
|
608
610
|
piboSessionId: session.id,
|
|
609
611
|
deliveries: compacted.persistedEvents.map((persistedEvent) => ({ event: persistedEvent })),
|
|
610
612
|
};
|
|
611
|
-
this.outputPersistenceRetries.enqueue(this.createOutputPersistenceJob(persistenceState, () => compacted.ack(), () => compacted.
|
|
613
|
+
this.outputPersistenceRetries.enqueue(this.createOutputPersistenceJob(persistenceState, () => compacted.ack(), () => compacted.discard()));
|
|
612
614
|
}
|
|
613
615
|
for (const liveEvent of compacted.liveEvents)
|
|
614
616
|
this.recordOutputEvent(liveEvent);
|
|
@@ -621,6 +623,7 @@ export class LocalCliSessionSource {
|
|
|
621
623
|
eventId: firstEvent ? outputPersistenceDeliveryKey(firstEvent) : undefined,
|
|
622
624
|
payload: persistenceState,
|
|
623
625
|
run: (retryContext) => this.deliverOutputPersistenceState(retryContext),
|
|
626
|
+
isRetryable: outputPersistenceErrorIsRetryable,
|
|
624
627
|
onSuccess,
|
|
625
628
|
onDeadLetter,
|
|
626
629
|
};
|
|
@@ -632,15 +635,25 @@ export class LocalCliSessionSource {
|
|
|
632
635
|
const session = this.sessionStore.get(persistenceState.piboSessionId);
|
|
633
636
|
if (!session)
|
|
634
637
|
throw new Error(`No session available for ${persistenceState.piboSessionId}`);
|
|
638
|
+
const errors = [];
|
|
635
639
|
for (const delivery of persistenceState.deliveries) {
|
|
636
640
|
if (delivery.persisted)
|
|
637
641
|
continue;
|
|
638
|
-
|
|
639
|
-
|
|
642
|
+
try {
|
|
643
|
+
this.ingestOutputEvent(session, delivery.event);
|
|
644
|
+
}
|
|
645
|
+
catch (error) {
|
|
646
|
+
errors.push(error instanceof Error ? error : new Error(String(error)));
|
|
647
|
+
continue;
|
|
640
648
|
}
|
|
641
649
|
delivery.persisted = true;
|
|
642
650
|
retryContext.updatePayload(persistenceState);
|
|
643
651
|
}
|
|
652
|
+
if (errors.length === 1)
|
|
653
|
+
throw errors[0];
|
|
654
|
+
if (errors.length > 1) {
|
|
655
|
+
throw new AggregateError(errors, `Failed to persist one or more output deliveries: ${errors.map((error) => error.message).join("; ")}`);
|
|
656
|
+
}
|
|
644
657
|
}
|
|
645
658
|
recordOutputEvent(event) {
|
|
646
659
|
const positionedEvent = validRenderSequence(event.renderSequence)
|
|
@@ -741,21 +754,14 @@ export class LocalCliSessionSource {
|
|
|
741
754
|
}
|
|
742
755
|
ingestOutputEvent(session, event) {
|
|
743
756
|
if (!this.ingestService)
|
|
744
|
-
return
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
});
|
|
753
|
-
return true;
|
|
754
|
-
}
|
|
755
|
-
catch {
|
|
756
|
-
// The caller retains the prepared compaction and retries this exact event identity.
|
|
757
|
-
return false;
|
|
758
|
-
}
|
|
757
|
+
return;
|
|
758
|
+
this.ingestService.ingestOutputEvent({
|
|
759
|
+
session,
|
|
760
|
+
roomId: roomIdFromSession(session),
|
|
761
|
+
actorId: session.profile,
|
|
762
|
+
event,
|
|
763
|
+
createdAt: this.now(),
|
|
764
|
+
});
|
|
759
765
|
}
|
|
760
766
|
buildStatus(session) {
|
|
761
767
|
const runtime = this.router?.getSessionRuntimeStatus?.(session.id);
|
|
@@ -20,6 +20,7 @@ class LostOutputPersistenceClaimError extends Error {
|
|
|
20
20
|
export class OutputPersistenceRetryQueue {
|
|
21
21
|
pending = new Map();
|
|
22
22
|
deadLetters = [];
|
|
23
|
+
counters = { retriable: 0, permanent: 0, quarantined: 0 };
|
|
23
24
|
maxPending;
|
|
24
25
|
maxAttempts;
|
|
25
26
|
baseDelayMs;
|
|
@@ -96,6 +97,7 @@ export class OutputPersistenceRetryQueue {
|
|
|
96
97
|
quarantine(reasonCode) {
|
|
97
98
|
if (this.disposed)
|
|
98
99
|
return;
|
|
100
|
+
this.counters.quarantined += 1;
|
|
99
101
|
this.recordDeadLetter({ key: `quarantine:${randomUUID()}`, payload: null, run() { } }, new Error(reasonCode), 0, reasonCode, true);
|
|
100
102
|
}
|
|
101
103
|
recover(factory) {
|
|
@@ -164,6 +166,7 @@ export class OutputPersistenceRetryQueue {
|
|
|
164
166
|
pending: this.pending.size,
|
|
165
167
|
activeHeartbeats: [...this.pending.values()].filter((job) => job.heartbeatTimer !== undefined).length,
|
|
166
168
|
deadLetters: this.deadLetters.map((entry) => ({ ...entry })),
|
|
169
|
+
counters: { ...this.counters },
|
|
167
170
|
};
|
|
168
171
|
}
|
|
169
172
|
startAttempt(job, forcePending) {
|
|
@@ -209,6 +212,7 @@ export class OutputPersistenceRetryQueue {
|
|
|
209
212
|
const parsed = parseDurableEnvelope(claimed.payload);
|
|
210
213
|
if (!parsed.envelope) {
|
|
211
214
|
this.durableStore?.quarantineJob(claimed.jobId, parsed.reason, correlationFields(job));
|
|
215
|
+
this.counters.quarantined += 1;
|
|
212
216
|
this.pending.delete(job.key);
|
|
213
217
|
return;
|
|
214
218
|
}
|
|
@@ -253,8 +257,24 @@ export class OutputPersistenceRetryQueue {
|
|
|
253
257
|
if (this.disposed)
|
|
254
258
|
return;
|
|
255
259
|
job.durableClaimed = false;
|
|
260
|
+
if (error instanceof LostOutputPersistenceClaimError) {
|
|
261
|
+
this.reconcileClaimMiss(job);
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
const retryable = job.isRetryable?.(error) ?? true;
|
|
265
|
+
if (!retryable) {
|
|
266
|
+
if (job.durableJobId && !this.durableStore?.fail(job.durableJobId, this.workerId, errorMessage(error), job.durableClaimToken, "permanent")) {
|
|
267
|
+
this.reconcileClaimMiss(job);
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
this.counters.permanent += 1;
|
|
271
|
+
this.pending.delete(job.key);
|
|
272
|
+
this.recordDeadLetter(job, error, job.attempts, "permanent", false);
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
this.counters.retriable += 1;
|
|
256
276
|
if (job.durableJobId) {
|
|
257
|
-
const retried =
|
|
277
|
+
const retried = this.durableStore?.retry(job.durableJobId, this.workerId, {
|
|
258
278
|
error: errorMessage(error),
|
|
259
279
|
delayMs: retryDelay(job.attempts, this.baseDelayMs, this.maxDelayMs),
|
|
260
280
|
claimToken: job.durableClaimToken,
|
|
@@ -346,6 +366,7 @@ export class OutputPersistenceRetryQueue {
|
|
|
346
366
|
const parsed = parseDurableEnvelope(durableJob.payload);
|
|
347
367
|
if (!parsed.envelope) {
|
|
348
368
|
this.durableStore.quarantineJob(durableJob.jobId, parsed.reason);
|
|
369
|
+
this.counters.quarantined += 1;
|
|
349
370
|
continue;
|
|
350
371
|
}
|
|
351
372
|
const envelope = parsed.envelope;
|
|
@@ -364,6 +385,7 @@ export class OutputPersistenceRetryQueue {
|
|
|
364
385
|
}
|
|
365
386
|
catch {
|
|
366
387
|
this.durableStore.quarantineJob(durableJob.jobId, "payload_invalid");
|
|
388
|
+
this.counters.quarantined += 1;
|
|
367
389
|
continue;
|
|
368
390
|
}
|
|
369
391
|
const pending = {
|
|
@@ -106,25 +106,34 @@ export class OutputRenderSequencer {
|
|
|
106
106
|
const parts = state.outputParts.get(key) ?? [];
|
|
107
107
|
if (!state.outputParts.has(key))
|
|
108
108
|
state.outputParts.set(key, parts);
|
|
109
|
-
const
|
|
109
|
+
const suppliedPart = transition.suppliedIndex === undefined
|
|
110
|
+
? undefined
|
|
111
|
+
: parts.find((part) => part.index === transition.suppliedIndex);
|
|
112
|
+
if (transition.suppliedIndex !== undefined && (transition.canonicalIndex || suppliedPart)) {
|
|
113
|
+
this.highWaterStore?.observeOutputPart?.({ ...transition, index: transition.suppliedIndex });
|
|
114
|
+
this.recordOutputPart(parts, transition.suppliedIndex, transition.terminal);
|
|
115
|
+
this.trimOutputParts(state);
|
|
116
|
+
return event;
|
|
117
|
+
}
|
|
110
118
|
const latestOpen = [...parts].reverse().find((part) => !part.closed);
|
|
111
119
|
const localMaximum = parts.reduce((maximum, part) => Math.max(maximum, part.index), -1);
|
|
112
|
-
const localIndex =
|
|
113
|
-
?? latestOpen?.index
|
|
120
|
+
const localIndex = latestOpen?.index
|
|
114
121
|
?? (transition.proposedIndex > localMaximum ? transition.proposedIndex : localMaximum + 1);
|
|
115
|
-
const index =
|
|
122
|
+
const index = latestOpen
|
|
116
123
|
? localIndex
|
|
117
124
|
: this.highWaterStore?.claimOrAttachOutputPart?.({ ...transition, proposedIndex: localIndex }) ?? localIndex;
|
|
125
|
+
this.recordOutputPart(parts, index, transition.terminal);
|
|
126
|
+
this.trimOutputParts(state);
|
|
127
|
+
return withOutputPartIndex(event, transition.kind, index);
|
|
128
|
+
}
|
|
129
|
+
recordOutputPart(parts, index, terminal) {
|
|
118
130
|
let part = parts.find((candidate) => candidate.index === index);
|
|
119
131
|
if (!part) {
|
|
120
|
-
part = { index,
|
|
132
|
+
part = { index, closed: false };
|
|
121
133
|
parts.push(part);
|
|
122
134
|
}
|
|
123
|
-
|
|
124
|
-
if (transition.terminal)
|
|
135
|
+
if (terminal)
|
|
125
136
|
part.closed = true;
|
|
126
|
-
this.trimOutputParts(state);
|
|
127
|
-
return withOutputPartIndex(event, transition.kind, index);
|
|
128
137
|
}
|
|
129
138
|
trimOutputParts(state) {
|
|
130
139
|
let total = 0;
|
|
@@ -316,6 +325,9 @@ function outputPartTransition(event, activeEventId) {
|
|
|
316
325
|
eventId,
|
|
317
326
|
kind: "assistant",
|
|
318
327
|
proposedIndex: event.assistantIndex ?? event.contentIndex ?? 0,
|
|
328
|
+
...(validOutputPartIndex(event.assistantIndex)
|
|
329
|
+
? { suppliedIndex: event.assistantIndex, canonicalIndex: validRenderSequence(event.renderSequence) }
|
|
330
|
+
: {}),
|
|
319
331
|
fingerprint: outputPartFingerprint(event),
|
|
320
332
|
identityFingerprint: outputIdentityFingerprint(event),
|
|
321
333
|
terminal: event.type === "assistant_message",
|
|
@@ -327,6 +339,9 @@ function outputPartTransition(event, activeEventId) {
|
|
|
327
339
|
eventId,
|
|
328
340
|
kind: "thinking",
|
|
329
341
|
proposedIndex: event.thinkingIndex ?? event.contentIndex ?? 0,
|
|
342
|
+
...(validOutputPartIndex(event.thinkingIndex)
|
|
343
|
+
? { suppliedIndex: event.thinkingIndex, canonicalIndex: validRenderSequence(event.renderSequence) }
|
|
344
|
+
: {}),
|
|
330
345
|
fingerprint: outputPartFingerprint(event),
|
|
331
346
|
identityFingerprint: outputIdentityFingerprint(event),
|
|
332
347
|
terminal: event.type === "thinking_finished",
|
|
@@ -338,6 +353,9 @@ function outputPartTransition(event, activeEventId) {
|
|
|
338
353
|
eventId,
|
|
339
354
|
kind: "usage",
|
|
340
355
|
proposedIndex: event.usageIndex ?? 0,
|
|
356
|
+
...(validOutputPartIndex(event.usageIndex)
|
|
357
|
+
? { suppliedIndex: event.usageIndex, canonicalIndex: validRenderSequence(event.renderSequence) }
|
|
358
|
+
: {}),
|
|
341
359
|
fingerprint: outputPartFingerprint(event),
|
|
342
360
|
identityFingerprint: outputIdentityFingerprint(event),
|
|
343
361
|
terminal: true,
|
|
@@ -349,6 +367,9 @@ function outputPartTransition(event, activeEventId) {
|
|
|
349
367
|
eventId,
|
|
350
368
|
kind: "compaction",
|
|
351
369
|
proposedIndex: event.compactionIndex ?? 0,
|
|
370
|
+
...(validOutputPartIndex(event.compactionIndex)
|
|
371
|
+
? { suppliedIndex: event.compactionIndex, canonicalIndex: validRenderSequence(event.renderSequence) }
|
|
372
|
+
: {}),
|
|
352
373
|
fingerprint: outputPartFingerprint(event),
|
|
353
374
|
identityFingerprint: outputIdentityFingerprint(event),
|
|
354
375
|
terminal: event.type === "compaction_end",
|
|
@@ -356,6 +377,9 @@ function outputPartTransition(event, activeEventId) {
|
|
|
356
377
|
}
|
|
357
378
|
return undefined;
|
|
358
379
|
}
|
|
380
|
+
function validOutputPartIndex(value) {
|
|
381
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
382
|
+
}
|
|
359
383
|
export function outputIdentityFingerprint(event) {
|
|
360
384
|
const payload = { ...event };
|
|
361
385
|
delete payload.renderSequence;
|
|
@@ -14,6 +14,14 @@ export class PiboOutputIdentityCollisionError extends Error {
|
|
|
14
14
|
this.name = "PiboOutputIdentityCollisionError";
|
|
15
15
|
}
|
|
16
16
|
}
|
|
17
|
+
export function outputPersistenceErrorIsRetryable(error) {
|
|
18
|
+
if (error instanceof PiboOutputIdentityCollisionError)
|
|
19
|
+
return false;
|
|
20
|
+
if (error instanceof AggregateError) {
|
|
21
|
+
return error.errors.length === 0 || error.errors.some(outputPersistenceErrorIsRetryable);
|
|
22
|
+
}
|
|
23
|
+
return true;
|
|
24
|
+
}
|
|
17
25
|
const INLINE_MESSAGE_PAYLOAD_THRESHOLD_BYTES = 16 * 1024;
|
|
18
26
|
const INLINE_JSON_PAYLOAD_THRESHOLD_BYTES = 16 * 1024;
|
|
19
27
|
export class ChatDataIngestService {
|
package/dist/debug/index.js
CHANGED
|
@@ -53,6 +53,18 @@ export async function runDebugCli(argv = process.argv) {
|
|
|
53
53
|
await runDebugJobs(args.slice(1));
|
|
54
54
|
return;
|
|
55
55
|
}
|
|
56
|
+
if (args[0] === "integrity") {
|
|
57
|
+
await runDebugIntegrity(args.slice(1));
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
if (args[0] === "persistence") {
|
|
61
|
+
await runDebugPersistence(args.slice(1));
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
if (args[0] === "repair") {
|
|
65
|
+
await runDebugRepair(args.slice(1));
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
56
68
|
if (args[0] === "runs") {
|
|
57
69
|
await runDebugRuns(args.slice(1));
|
|
58
70
|
return;
|
|
@@ -86,6 +98,208 @@ export async function runDebugCli(argv = process.argv) {
|
|
|
86
98
|
process.exitCode = 1;
|
|
87
99
|
}
|
|
88
100
|
}
|
|
101
|
+
async function runDebugIntegrity(args) {
|
|
102
|
+
if (args.length === 0 || args[0] === "--help" || args[0] === "-h") {
|
|
103
|
+
printDebugIntegrityDiscovery();
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
if (args[0] !== "output") {
|
|
107
|
+
throw new Error(`Unknown pibo debug integrity command "${args[0]}". Run pibo debug integrity --help.`);
|
|
108
|
+
}
|
|
109
|
+
if (args[1] === "--help" || args[1] === "-h") {
|
|
110
|
+
printDebugIntegrityDiscovery();
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
await runOutputPersistenceInspection(args.slice(1), { allowPositionalSession: true, deadLettersOnly: false });
|
|
114
|
+
}
|
|
115
|
+
async function runDebugPersistence(args) {
|
|
116
|
+
if (args.length === 0 || args[0] === "--help" || args[0] === "-h") {
|
|
117
|
+
printDebugPersistenceDiscovery();
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
const command = args[0];
|
|
121
|
+
if (command !== "audit" && command !== "dead-letters") {
|
|
122
|
+
throw new Error(`Unknown pibo debug persistence command "${command}". Run pibo debug persistence --help.`);
|
|
123
|
+
}
|
|
124
|
+
if (args[1] === "--help" || args[1] === "-h") {
|
|
125
|
+
if (command === "audit")
|
|
126
|
+
printDebugPersistenceAuditHelp();
|
|
127
|
+
else
|
|
128
|
+
printDebugPersistenceDeadLettersHelp();
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
await runOutputPersistenceInspection(args.slice(1), {
|
|
132
|
+
allowPositionalSession: false,
|
|
133
|
+
deadLettersOnly: command === "dead-letters",
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
async function runOutputPersistenceInspection(args, mode) {
|
|
137
|
+
validateOutputPersistenceInspectionArgs(args, mode.allowPositionalSession);
|
|
138
|
+
const options = parseOptions(args);
|
|
139
|
+
if (options.after && !Number.isFinite(Date.parse(options.after)))
|
|
140
|
+
throw new Error("--since must be an ISO date");
|
|
141
|
+
if (options.before && !Number.isFinite(Date.parse(options.before)))
|
|
142
|
+
throw new Error("--before must be an ISO date");
|
|
143
|
+
if (options.after && options.before && Date.parse(options.after) >= Date.parse(options.before)) {
|
|
144
|
+
throw new Error("--since must be earlier than --before");
|
|
145
|
+
}
|
|
146
|
+
const { formatJson } = await import("./sql.js");
|
|
147
|
+
const { formatOutputIntegrityAudit, formatOutputPersistenceDeadLetters, inspectOutputIntegrity, outputPersistenceDeadLettersFromAudit, } = await import("./output-integrity.js");
|
|
148
|
+
const audit = inspectOutputIntegrity({
|
|
149
|
+
dataStore: resolveDebugStore("pibo-data"),
|
|
150
|
+
reliabilityStore: resolveDebugStore("reliability"),
|
|
151
|
+
piboSessionId: options.key ?? (mode.allowPositionalSession ? options.positionals[0] : undefined),
|
|
152
|
+
since: options.after,
|
|
153
|
+
before: options.before,
|
|
154
|
+
limit: options.limit,
|
|
155
|
+
findingMode: mode.deadLettersOnly ? "dead_letters" : "all",
|
|
156
|
+
});
|
|
157
|
+
if (mode.deadLettersOnly) {
|
|
158
|
+
const result = outputPersistenceDeadLettersFromAudit(audit);
|
|
159
|
+
if (options.json)
|
|
160
|
+
console.log(formatJson(result));
|
|
161
|
+
else
|
|
162
|
+
console.log(formatOutputPersistenceDeadLetters(result));
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
if (options.json)
|
|
166
|
+
console.log(formatJson(audit));
|
|
167
|
+
else
|
|
168
|
+
console.log(formatOutputIntegrityAudit(audit));
|
|
169
|
+
}
|
|
170
|
+
function validateOutputPersistenceInspectionArgs(args, allowPositionalSession) {
|
|
171
|
+
const valueOptions = new Set(["--session", "--since", "--before", "--limit"]);
|
|
172
|
+
let positionalCount = 0;
|
|
173
|
+
let hasSessionOption = false;
|
|
174
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
175
|
+
const arg = args[index];
|
|
176
|
+
if (arg === "--json")
|
|
177
|
+
continue;
|
|
178
|
+
if (arg === "--apply" || arg === "--dry-run") {
|
|
179
|
+
throw new Error(`Output persistence inspection is read-only; ${arg} is not supported`);
|
|
180
|
+
}
|
|
181
|
+
if (valueOptions.has(arg)) {
|
|
182
|
+
const value = args[index + 1];
|
|
183
|
+
if (!value || value.startsWith("-"))
|
|
184
|
+
throw new Error(`${arg} requires a value`);
|
|
185
|
+
if (arg === "--session")
|
|
186
|
+
hasSessionOption = true;
|
|
187
|
+
index += 1;
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
if (arg.startsWith("-")) {
|
|
191
|
+
throw new Error(`Unknown output persistence inspection option "${arg}"`);
|
|
192
|
+
}
|
|
193
|
+
positionalCount += 1;
|
|
194
|
+
}
|
|
195
|
+
if (!allowPositionalSession && positionalCount > 0) {
|
|
196
|
+
throw new Error("Use --session <pibo-session-id> to scope persistence inspection");
|
|
197
|
+
}
|
|
198
|
+
if (allowPositionalSession && positionalCount > 1) {
|
|
199
|
+
throw new Error("Use at most one positional Pibo Session ID");
|
|
200
|
+
}
|
|
201
|
+
if (allowPositionalSession && hasSessionOption && positionalCount > 0) {
|
|
202
|
+
throw new Error("Use either a positional Pibo Session ID or --session, not both");
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
async function runDebugRepair(args) {
|
|
206
|
+
if (args.length === 0 || args[0] === "--help" || args[0] === "-h") {
|
|
207
|
+
printDebugRepairDiscovery();
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
if (args[0] !== "output") {
|
|
211
|
+
throw new Error(`Unknown pibo debug repair command "${args[0]}". Run pibo debug repair --help.`);
|
|
212
|
+
}
|
|
213
|
+
if (args[1] === "--help" || args[1] === "-h") {
|
|
214
|
+
printDebugRepairDiscovery();
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
const repairArgs = args.slice(1);
|
|
218
|
+
validateOutputRepairArgs(repairArgs);
|
|
219
|
+
const options = parseOptions(repairArgs);
|
|
220
|
+
if (options.apply && options.dryRun)
|
|
221
|
+
throw new Error("Choose either --dry-run or --apply");
|
|
222
|
+
if (options.destructive)
|
|
223
|
+
throw new Error("Use --apply for output repair; --destructive is not supported");
|
|
224
|
+
if (options.after && !Number.isFinite(Date.parse(options.after)))
|
|
225
|
+
throw new Error("--since must be an ISO date");
|
|
226
|
+
if (options.before && !Number.isFinite(Date.parse(options.before)))
|
|
227
|
+
throw new Error("--before must be an ISO date");
|
|
228
|
+
if (options.after && options.before && Date.parse(options.after) >= Date.parse(options.before)) {
|
|
229
|
+
throw new Error("--since must be earlier than --before");
|
|
230
|
+
}
|
|
231
|
+
if (options.key && options.positionals.length > 1)
|
|
232
|
+
throw new Error("With --session, provide at most one positional <event-id>");
|
|
233
|
+
const piboSessionId = options.key ?? options.positionals[0];
|
|
234
|
+
const eventId = options.key ? options.positionals[0] : options.positionals[1];
|
|
235
|
+
if (!piboSessionId)
|
|
236
|
+
throw new Error("pibo debug repair output requires a session id or --session <pibo-session-id>");
|
|
237
|
+
const { formatJson } = await import("./sql.js");
|
|
238
|
+
const { formatOutputTurnRepair, readOutputRepairAdapterEvidence, repairOutputTurn, repairOutputTurns, } = await import("./output-repair.js");
|
|
239
|
+
const store = resolveDebugStore("pibo-data");
|
|
240
|
+
const reliabilityStore = resolveDebugStore("reliability");
|
|
241
|
+
const adapterEvidence = await readOutputRepairAdapterEvidence({ store, piboSessionId });
|
|
242
|
+
const result = eventId
|
|
243
|
+
? repairOutputTurn({ store, reliabilityStore, piboSessionId, eventId, adapterEvidence, apply: options.apply })
|
|
244
|
+
: repairOutputTurns({
|
|
245
|
+
store,
|
|
246
|
+
reliabilityStore,
|
|
247
|
+
piboSessionId,
|
|
248
|
+
since: options.after,
|
|
249
|
+
before: options.before,
|
|
250
|
+
limit: options.limit,
|
|
251
|
+
adapterEvidence,
|
|
252
|
+
apply: options.apply,
|
|
253
|
+
});
|
|
254
|
+
if (options.json)
|
|
255
|
+
console.log(formatJson(result));
|
|
256
|
+
else
|
|
257
|
+
console.log(formatOutputTurnRepair(result));
|
|
258
|
+
}
|
|
259
|
+
function validateOutputRepairArgs(args) {
|
|
260
|
+
const valueOptions = new Set(["--session", "--since", "--before", "--limit"]);
|
|
261
|
+
const flagOptions = new Set(["--json", "--dry-run", "--apply", "--destructive"]);
|
|
262
|
+
const seen = new Set();
|
|
263
|
+
const positionals = [];
|
|
264
|
+
let hasSessionOption = false;
|
|
265
|
+
let hasScopeOption = false;
|
|
266
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
267
|
+
const arg = args[index];
|
|
268
|
+
if (flagOptions.has(arg)) {
|
|
269
|
+
if (seen.has(arg))
|
|
270
|
+
throw new Error(`Output repair option ${arg} may only be provided once`);
|
|
271
|
+
seen.add(arg);
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
if (valueOptions.has(arg)) {
|
|
275
|
+
if (seen.has(arg))
|
|
276
|
+
throw new Error(`Output repair option ${arg} may only be provided once`);
|
|
277
|
+
const value = args[index + 1];
|
|
278
|
+
if (!value || value.startsWith("-"))
|
|
279
|
+
throw new Error(`${arg} requires a value`);
|
|
280
|
+
seen.add(arg);
|
|
281
|
+
if (arg === "--session")
|
|
282
|
+
hasSessionOption = true;
|
|
283
|
+
else
|
|
284
|
+
hasScopeOption = true;
|
|
285
|
+
index += 1;
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
if (arg.startsWith("-"))
|
|
289
|
+
throw new Error(`Unknown output repair option "${arg}"`);
|
|
290
|
+
positionals.push(arg);
|
|
291
|
+
}
|
|
292
|
+
if (hasSessionOption) {
|
|
293
|
+
if (positionals.length > 1)
|
|
294
|
+
throw new Error("With --session, provide at most one positional <event-id>");
|
|
295
|
+
}
|
|
296
|
+
else if (positionals.length !== 2) {
|
|
297
|
+
throw new Error("Exact output repair requires <pibo-session-id> <event-id>; use --session for scoped repair");
|
|
298
|
+
}
|
|
299
|
+
if (hasScopeOption && positionals.length > 0) {
|
|
300
|
+
throw new Error("--since, --before, and --limit are only supported for scoped repair without an <event-id>");
|
|
301
|
+
}
|
|
302
|
+
}
|
|
89
303
|
async function runDebugResources(args) {
|
|
90
304
|
if (args[0] === "--help" || args[0] === "-h") {
|
|
91
305
|
printDebugResourcesDiscovery();
|
|
@@ -853,10 +1067,10 @@ function parseOptions(args) {
|
|
|
853
1067
|
index += 1;
|
|
854
1068
|
continue;
|
|
855
1069
|
}
|
|
856
|
-
if (arg === "--after") {
|
|
1070
|
+
if (arg === "--after" || arg === "--since") {
|
|
857
1071
|
const value = args[index + 1];
|
|
858
1072
|
if (!value)
|
|
859
|
-
throw new Error(
|
|
1073
|
+
throw new Error(`${arg} requires a value`);
|
|
860
1074
|
parsed.after = value;
|
|
861
1075
|
index += 1;
|
|
862
1076
|
continue;
|
|
@@ -1103,6 +1317,9 @@ Commands:
|
|
|
1103
1317
|
tool Inspect one grouped tool call
|
|
1104
1318
|
failures List failed tool calls and trace/session errors
|
|
1105
1319
|
jobs Inspect durable Pibo jobs and DLQ
|
|
1320
|
+
persistence Audit output persistence and related dead letters
|
|
1321
|
+
integrity Compatibility alias for output persistence audit
|
|
1322
|
+
repair Dry-run or apply explicit persisted-output repairs
|
|
1106
1323
|
runs Inspect durable yielded runs
|
|
1107
1324
|
resources Show gateway memory reserve, related child processes, and heavy daemons
|
|
1108
1325
|
signals Inspect live session signal snapshots through Chat Web APIs
|
|
@@ -1117,6 +1334,8 @@ Next:
|
|
|
1117
1334
|
pibo debug messages <pibo-session-id> list
|
|
1118
1335
|
pibo debug trace <pibo-session-id> --running-only
|
|
1119
1336
|
pibo debug events stream --topic pibo.output
|
|
1337
|
+
pibo debug persistence
|
|
1338
|
+
pibo debug repair output <pibo-session-id> <event-id> --dry-run
|
|
1120
1339
|
pibo debug agents <parent-session-id> list
|
|
1121
1340
|
pibo debug resources --json
|
|
1122
1341
|
pibo debug signals tree ps_...
|
|
@@ -1125,6 +1344,79 @@ Next:
|
|
|
1125
1344
|
pibo debug pty run -- pibo tui:sessions --demo
|
|
1126
1345
|
`);
|
|
1127
1346
|
}
|
|
1347
|
+
function printDebugIntegrityDiscovery() {
|
|
1348
|
+
console.log(`pibo debug integrity - compatibility alias for output persistence audit
|
|
1349
|
+
|
|
1350
|
+
Usage:
|
|
1351
|
+
pibo debug integrity output [<pibo-session-id>] [--since <iso-date>] [--before <iso-date>] [--limit n] [--json]
|
|
1352
|
+
|
|
1353
|
+
Next:
|
|
1354
|
+
pibo debug persistence audit --help
|
|
1355
|
+
`);
|
|
1356
|
+
}
|
|
1357
|
+
function printDebugPersistenceDiscovery() {
|
|
1358
|
+
console.log(`pibo debug persistence - inspect persisted output without changing it
|
|
1359
|
+
|
|
1360
|
+
Commands:
|
|
1361
|
+
audit Audit output lifecycle, identity, queue, and trace-status integrity
|
|
1362
|
+
dead-letters List output-persistence dead letters with collision relationships
|
|
1363
|
+
|
|
1364
|
+
Next:
|
|
1365
|
+
pibo debug persistence audit --help
|
|
1366
|
+
pibo debug persistence dead-letters --help
|
|
1367
|
+
`);
|
|
1368
|
+
}
|
|
1369
|
+
function printDebugPersistenceAuditHelp() {
|
|
1370
|
+
console.log(`pibo debug persistence audit - read-only output integrity audit
|
|
1371
|
+
|
|
1372
|
+
Usage:
|
|
1373
|
+
pibo debug persistence audit [--session <pibo-session-id>] [--since <iso-date>] [--before <iso-date>] [--limit n] [--json]
|
|
1374
|
+
|
|
1375
|
+
Reports:
|
|
1376
|
+
Incomplete or invalid turn, reasoning, and tool lifecycles; identity collisions; reused output keys; session/trace-status mismatches; and pending or dead output-persistence jobs.
|
|
1377
|
+
|
|
1378
|
+
Next:
|
|
1379
|
+
pibo debug persistence audit --json
|
|
1380
|
+
pibo debug persistence audit --session ps_... --since 2026-08-01T00:00:00Z --json
|
|
1381
|
+
pibo debug persistence dead-letters
|
|
1382
|
+
`);
|
|
1383
|
+
}
|
|
1384
|
+
function printDebugPersistenceDeadLettersHelp() {
|
|
1385
|
+
console.log(`pibo debug persistence dead-letters - read-only output dead-letter inspection
|
|
1386
|
+
|
|
1387
|
+
Usage:
|
|
1388
|
+
pibo debug persistence dead-letters [--session <pibo-session-id>] [--since <iso-date>] [--before <iso-date>] [--limit n] [--json]
|
|
1389
|
+
|
|
1390
|
+
Reports:
|
|
1391
|
+
Output-persistence dead letters, permanent collision failures, and dead letters related to persisted collision diagnostics.
|
|
1392
|
+
|
|
1393
|
+
Next:
|
|
1394
|
+
pibo debug persistence dead-letters --json
|
|
1395
|
+
pibo debug persistence audit --json
|
|
1396
|
+
`);
|
|
1397
|
+
}
|
|
1398
|
+
function printDebugRepairDiscovery() {
|
|
1399
|
+
console.log(`pibo debug repair - explicitly repair persisted output
|
|
1400
|
+
|
|
1401
|
+
Usage:
|
|
1402
|
+
pibo debug repair output <pibo-session-id> <event-id> [--dry-run|--apply] [--json]
|
|
1403
|
+
pibo debug repair output --session <pibo-session-id> [--since <iso-date>] [--before <iso-date>] [--limit n] [--dry-run|--apply] [--json]
|
|
1404
|
+
|
|
1405
|
+
Behavior:
|
|
1406
|
+
Dry-run is the default. --apply is required for every mutation.
|
|
1407
|
+
Repair requires exactly one persisted message start, an inactive turn, no terminal event, and unambiguous evidence.
|
|
1408
|
+
Evidence is limited to Reliability payloads, Pibo Product History, and exact adapter-history turn matches.
|
|
1409
|
+
Repair never invents assistant content. Without an exact Reliability terminal, only a completed assistant record can justify message_finished.
|
|
1410
|
+
Every applied terminal writes a pibo.output.repair_applied audit event in the same transaction.
|
|
1411
|
+
Repair does not delete or replay pending or dead output-persistence jobs.
|
|
1412
|
+
|
|
1413
|
+
Next:
|
|
1414
|
+
pibo debug repair output ps_... <event-id> --dry-run --json
|
|
1415
|
+
pibo debug repair output --session ps_... --before 2026-08-31T00:00:00Z --dry-run --json
|
|
1416
|
+
pibo debug repair output ps_... <event-id> --apply --json
|
|
1417
|
+
pibo debug jobs dead --queue output-persistence
|
|
1418
|
+
`);
|
|
1419
|
+
}
|
|
1128
1420
|
function printDebugResourcesDiscovery() {
|
|
1129
1421
|
console.log(`pibo debug resources - inspect gateway resource guard state
|
|
1130
1422
|
|