@pasko70/pibo 3.1.3 → 3.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/chat/data/timeline-query-service.js +5 -2
- package/dist/apps/chat/output-compactor.js +33 -30
- package/dist/apps/chat/trace-v2.js +1 -0
- package/dist/apps/chat/trace.js +3 -1
- package/dist/apps/chat/web-app.js +106 -65
- package/dist/apps/chat-ui/assets/{dist-BNzch-mL.js → dist-BOqsX6_s.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-CoA9zNTP.js → dist-BcUbdOKJ.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-liw1S7HS.js → dist-DkpIJ_Pp.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-7DhHpLCA.js → dist-a0mykCz7.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-DDRlEGPR.js → dist-qDZ-CdlK.js} +1 -1
- package/dist/apps/chat-ui/assets/{index-BKp6zKIe.js → index-G2ic-FSG.js} +89 -89
- package/dist/apps/chat-ui/index.html +1 -1
- package/dist/apps/chat-vscode-web/assets/{index-CKC-46jZ.js → index-zQ1fNz5K.js} +1 -1
- package/dist/apps/chat-vscode-web/index.html +1 -1
- package/dist/cli-session/localSessionSource.js +26 -20
- package/dist/core/output-persistence-retry.js +23 -1
- package/dist/core/output-render-sequence.js +33 -9
- package/dist/data/ingest-service.js +8 -0
- package/dist/debug/index.js +294 -2
- package/dist/debug/output-integrity.js +608 -0
- package/dist/debug/output-repair.js +584 -0
- package/dist/debug/trace.js +2 -0
- package/dist/reliability/store.js +2 -2
- package/dist/sessions/pibo-data-store.js +26 -20
- package/dist/shared/trace-engine.js +5 -2
- package/dist/shared/trace-event-projection.js +74 -0
- package/dist/shared/trace-page-merge.js +31 -5
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
|
@@ -0,0 +1,608 @@
|
|
|
1
|
+
import { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import { normalizeLimit } from "./sql.js";
|
|
3
|
+
const THINKING_INDEX_SQL = "CASE WHEN json_valid(attributes_json) THEN COALESCE(json_extract(attributes_json, '$.thinkingIndex'), json_extract(attributes_json, '$.contentIndex'), 0) ELSE 0 END";
|
|
4
|
+
const TOOL_CALL_ID_SQL = "CASE WHEN tool_call_id IS NOT NULL THEN tool_call_id WHEN json_valid(attributes_json) THEN json_extract(attributes_json, '$.toolCallId') END";
|
|
5
|
+
const TOOL_ORDINAL_SQL = "CASE WHEN json_valid(attributes_json) THEN COALESCE(json_extract(attributes_json, '$.toolInvocationOrdinal'), 0) ELSE 0 END";
|
|
6
|
+
const COLLISION_KEY_SQL = "CASE WHEN json_valid(attributes_json) THEN json_extract(attributes_json, '$.outputIdempotencyKey') END";
|
|
7
|
+
const RELIABILITY_SESSION_ID_SQL = "CASE WHEN json_valid(payload_json) THEN COALESCE(json_extract(payload_json, '$.piboSessionId'), json_extract(payload_json, '$.state.piboSessionId')) END";
|
|
8
|
+
const RELIABILITY_EVENT_ID_SQL = "CASE WHEN json_valid(payload_json) THEN COALESCE(json_extract(payload_json, '$.eventId'), json_extract(payload_json, '$.state.eventId')) END";
|
|
9
|
+
export function inspectOutputIntegrity(input) {
|
|
10
|
+
const limit = normalizeLimit(input.limit);
|
|
11
|
+
const findings = [];
|
|
12
|
+
const collisionEventKeys = new Set();
|
|
13
|
+
let turnLifecycleIssues = 0;
|
|
14
|
+
let thinkingLifecycleIssues = 0;
|
|
15
|
+
let toolLifecycleIssues = 0;
|
|
16
|
+
let identityCollisions = 0;
|
|
17
|
+
let outputKeyReuses = 0;
|
|
18
|
+
let sessionTraceStatusMismatches = 0;
|
|
19
|
+
let pendingOutputJobs = 0;
|
|
20
|
+
let deadOutputJobs = 0;
|
|
21
|
+
let deadIdentityCollisions = 0;
|
|
22
|
+
if (input.dataStore.exists) {
|
|
23
|
+
const db = new DatabaseSync(input.dataStore.path, { readOnly: true });
|
|
24
|
+
db.exec("BEGIN");
|
|
25
|
+
try {
|
|
26
|
+
if (tableExists(db, "event_log")) {
|
|
27
|
+
const scope = eventScope(input.piboSessionId, input.since, input.before);
|
|
28
|
+
const lifecycle = lifecycleScope(input.piboSessionId, input.since, input.before);
|
|
29
|
+
turnLifecycleIssues = countRows(db, `
|
|
30
|
+
SELECT COUNT(*) AS count FROM (
|
|
31
|
+
SELECT session_id, event_id
|
|
32
|
+
FROM event_log
|
|
33
|
+
WHERE event_id IS NOT NULL
|
|
34
|
+
AND type IN ('message_started', 'assistant_message', 'message_finished', 'session_error')
|
|
35
|
+
${lifecycle.whereSql}
|
|
36
|
+
GROUP BY session_id, event_id
|
|
37
|
+
HAVING (
|
|
38
|
+
SUM(type = 'message_started') != 1
|
|
39
|
+
OR SUM(type IN ('message_finished', 'session_error')) != 1
|
|
40
|
+
OR (SUM(type = 'message_finished') = 1 AND SUM(type = 'assistant_message') = 0)
|
|
41
|
+
) ${lifecycle.havingSql}
|
|
42
|
+
)
|
|
43
|
+
`, lifecycle.params);
|
|
44
|
+
thinkingLifecycleIssues = countRows(db, `
|
|
45
|
+
SELECT COUNT(*) AS count FROM (
|
|
46
|
+
SELECT session_id, event_id,
|
|
47
|
+
${THINKING_INDEX_SQL} AS thinking_index
|
|
48
|
+
FROM event_log
|
|
49
|
+
WHERE event_id IS NOT NULL
|
|
50
|
+
AND type IN ('thinking_started', 'thinking_finished')
|
|
51
|
+
${lifecycle.whereSql}
|
|
52
|
+
GROUP BY session_id, event_id, thinking_index
|
|
53
|
+
HAVING (
|
|
54
|
+
SUM(type = 'thinking_started') != 1
|
|
55
|
+
OR SUM(type = 'thinking_finished') != 1
|
|
56
|
+
) ${lifecycle.havingSql}
|
|
57
|
+
)
|
|
58
|
+
`, lifecycle.params);
|
|
59
|
+
toolLifecycleIssues = countRows(db, `
|
|
60
|
+
SELECT COUNT(*) AS count FROM (
|
|
61
|
+
SELECT session_id, event_id,
|
|
62
|
+
${TOOL_CALL_ID_SQL} AS tool_call_id,
|
|
63
|
+
${TOOL_ORDINAL_SQL} AS tool_invocation_ordinal
|
|
64
|
+
FROM event_log
|
|
65
|
+
WHERE event_id IS NOT NULL
|
|
66
|
+
AND ${TOOL_CALL_ID_SQL} IS NOT NULL
|
|
67
|
+
AND type IN ('tool_call', 'tool_execution_started', 'tool_execution_finished')
|
|
68
|
+
${lifecycle.whereSql}
|
|
69
|
+
GROUP BY session_id, event_id, tool_call_id, tool_invocation_ordinal
|
|
70
|
+
HAVING (
|
|
71
|
+
SUM(type = 'tool_call') != 1
|
|
72
|
+
OR SUM(type = 'tool_execution_started') != 1
|
|
73
|
+
OR SUM(type = 'tool_execution_finished') != 1
|
|
74
|
+
) ${lifecycle.havingSql}
|
|
75
|
+
)
|
|
76
|
+
`, lifecycle.params);
|
|
77
|
+
identityCollisions = countRows(db, `
|
|
78
|
+
SELECT COUNT(*) AS count
|
|
79
|
+
FROM event_log
|
|
80
|
+
WHERE type = 'pibo.output.identity_collision' ${scope.sql}
|
|
81
|
+
`, scope.params);
|
|
82
|
+
for (const row of queryRows(db, `
|
|
83
|
+
SELECT DISTINCT session_id AS sessionId, event_id AS eventId
|
|
84
|
+
FROM event_log
|
|
85
|
+
WHERE type = 'pibo.output.identity_collision' AND event_id IS NOT NULL ${scope.sql}
|
|
86
|
+
`, scope.params))
|
|
87
|
+
collisionEventKeys.add(`${row.sessionId}\0${row.eventId}`);
|
|
88
|
+
outputKeyReuses = countRows(db, `
|
|
89
|
+
WITH output_keys AS (
|
|
90
|
+
SELECT session_id, event_id, idempotency_key AS output_key, created_at
|
|
91
|
+
FROM event_log
|
|
92
|
+
WHERE idempotency_key LIKE 'pibo.output:%' ${scope.sql}
|
|
93
|
+
UNION ALL
|
|
94
|
+
SELECT session_id, event_id, ${COLLISION_KEY_SQL} AS output_key, created_at
|
|
95
|
+
FROM event_log
|
|
96
|
+
WHERE type = 'pibo.output.identity_collision'
|
|
97
|
+
AND ${COLLISION_KEY_SQL} IS NOT NULL ${scope.sql}
|
|
98
|
+
)
|
|
99
|
+
SELECT COUNT(*) AS count FROM (
|
|
100
|
+
SELECT output_key FROM output_keys GROUP BY output_key HAVING COUNT(*) > 1
|
|
101
|
+
)
|
|
102
|
+
`, [...scope.params, ...scope.params]);
|
|
103
|
+
if (tableExists(db, "sessions")) {
|
|
104
|
+
const traceStatus = sessionTraceStatusSql(input.piboSessionId, input.since, input.before, true);
|
|
105
|
+
sessionTraceStatusMismatches = countRows(db, traceStatus.sql, traceStatus.params);
|
|
106
|
+
}
|
|
107
|
+
findings.push(...queryRows(db, `
|
|
108
|
+
SELECT session_id AS sessionId, event_id AS eventId,
|
|
109
|
+
SUM(type = 'message_started') AS started,
|
|
110
|
+
SUM(type IN ('message_finished', 'session_error')) AS finished,
|
|
111
|
+
SUM(type = 'message_finished') AS messageFinished,
|
|
112
|
+
SUM(type = 'session_error') AS sessionErrors,
|
|
113
|
+
SUM(type = 'assistant_message') AS assistantMessages,
|
|
114
|
+
MIN(created_at) AS firstAt, MAX(created_at) AS lastAt
|
|
115
|
+
FROM event_log
|
|
116
|
+
WHERE event_id IS NOT NULL
|
|
117
|
+
AND type IN ('message_started', 'assistant_message', 'message_finished', 'session_error')
|
|
118
|
+
${lifecycle.whereSql}
|
|
119
|
+
GROUP BY session_id, event_id
|
|
120
|
+
HAVING (started != 1 OR finished != 1 OR (messageFinished = 1 AND assistantMessages = 0)) ${lifecycle.havingSql}
|
|
121
|
+
ORDER BY lastAt DESC
|
|
122
|
+
LIMIT ?
|
|
123
|
+
`, [...lifecycle.params, limit]).map((row) => ({
|
|
124
|
+
kind: "turn_lifecycle",
|
|
125
|
+
piboSessionId: row.sessionId,
|
|
126
|
+
eventId: row.eventId,
|
|
127
|
+
firstAt: row.firstAt,
|
|
128
|
+
lastAt: row.lastAt,
|
|
129
|
+
started: row.started,
|
|
130
|
+
finished: row.finished,
|
|
131
|
+
messageFinished: row.messageFinished,
|
|
132
|
+
sessionErrors: row.sessionErrors,
|
|
133
|
+
assistantMessages: row.assistantMessages,
|
|
134
|
+
})));
|
|
135
|
+
findings.push(...queryRows(db, `
|
|
136
|
+
SELECT session_id AS sessionId, event_id AS eventId,
|
|
137
|
+
${THINKING_INDEX_SQL} AS thinkingIndex,
|
|
138
|
+
SUM(type = 'thinking_started') AS started,
|
|
139
|
+
SUM(type = 'thinking_finished') AS finished,
|
|
140
|
+
MIN(created_at) AS firstAt, MAX(created_at) AS lastAt
|
|
141
|
+
FROM event_log
|
|
142
|
+
WHERE event_id IS NOT NULL
|
|
143
|
+
AND type IN ('thinking_started', 'thinking_finished')
|
|
144
|
+
${lifecycle.whereSql}
|
|
145
|
+
GROUP BY session_id, event_id, thinkingIndex
|
|
146
|
+
HAVING (started != 1 OR finished != 1) ${lifecycle.havingSql}
|
|
147
|
+
ORDER BY lastAt DESC
|
|
148
|
+
LIMIT ?
|
|
149
|
+
`, [...lifecycle.params, limit]).map((row) => ({
|
|
150
|
+
kind: "thinking_lifecycle",
|
|
151
|
+
piboSessionId: row.sessionId,
|
|
152
|
+
eventId: row.eventId,
|
|
153
|
+
thinkingIndex: row.thinkingIndex,
|
|
154
|
+
firstAt: row.firstAt,
|
|
155
|
+
lastAt: row.lastAt,
|
|
156
|
+
started: row.started,
|
|
157
|
+
finished: row.finished,
|
|
158
|
+
})));
|
|
159
|
+
findings.push(...queryRows(db, `
|
|
160
|
+
SELECT session_id AS sessionId, event_id AS eventId,
|
|
161
|
+
${TOOL_CALL_ID_SQL} AS toolCallId,
|
|
162
|
+
${TOOL_ORDINAL_SQL} AS toolInvocationOrdinal,
|
|
163
|
+
SUM(type = 'tool_call') AS called,
|
|
164
|
+
SUM(type = 'tool_execution_started') AS started,
|
|
165
|
+
SUM(type = 'tool_execution_finished') AS finished,
|
|
166
|
+
MIN(created_at) AS firstAt, MAX(created_at) AS lastAt
|
|
167
|
+
FROM event_log
|
|
168
|
+
WHERE event_id IS NOT NULL
|
|
169
|
+
AND ${TOOL_CALL_ID_SQL} IS NOT NULL
|
|
170
|
+
AND type IN ('tool_call', 'tool_execution_started', 'tool_execution_finished')
|
|
171
|
+
${lifecycle.whereSql}
|
|
172
|
+
GROUP BY session_id, event_id, toolCallId, toolInvocationOrdinal
|
|
173
|
+
HAVING (called != 1 OR started != 1 OR finished != 1) ${lifecycle.havingSql}
|
|
174
|
+
ORDER BY lastAt DESC
|
|
175
|
+
LIMIT ?
|
|
176
|
+
`, [...lifecycle.params, limit]).map((row) => ({
|
|
177
|
+
kind: "tool_lifecycle",
|
|
178
|
+
piboSessionId: row.sessionId,
|
|
179
|
+
eventId: row.eventId,
|
|
180
|
+
toolCallId: row.toolCallId,
|
|
181
|
+
toolInvocationOrdinal: row.toolInvocationOrdinal,
|
|
182
|
+
called: row.called,
|
|
183
|
+
firstAt: row.firstAt,
|
|
184
|
+
lastAt: row.lastAt,
|
|
185
|
+
started: row.started,
|
|
186
|
+
finished: row.finished,
|
|
187
|
+
})));
|
|
188
|
+
findings.push(...queryRows(db, `
|
|
189
|
+
SELECT session_id AS sessionId, event_id AS eventId, stream_id AS streamId,
|
|
190
|
+
created_at AS createdAt,
|
|
191
|
+
${COLLISION_KEY_SQL} AS idempotencyKey
|
|
192
|
+
FROM event_log
|
|
193
|
+
WHERE type = 'pibo.output.identity_collision' ${scope.sql}
|
|
194
|
+
ORDER BY stream_id DESC
|
|
195
|
+
LIMIT ?
|
|
196
|
+
`, [...scope.params, limit]).map((row) => ({
|
|
197
|
+
kind: "identity_collision",
|
|
198
|
+
piboSessionId: row.sessionId,
|
|
199
|
+
...(row.eventId ? { eventId: row.eventId } : {}),
|
|
200
|
+
streamId: row.streamId,
|
|
201
|
+
lastAt: row.createdAt,
|
|
202
|
+
...(row.idempotencyKey ? { idempotencyKey: row.idempotencyKey } : {}),
|
|
203
|
+
})));
|
|
204
|
+
findings.push(...queryRows(db, `
|
|
205
|
+
WITH output_keys AS (
|
|
206
|
+
SELECT session_id, event_id, idempotency_key AS output_key, created_at
|
|
207
|
+
FROM event_log
|
|
208
|
+
WHERE idempotency_key LIKE 'pibo.output:%' ${scope.sql}
|
|
209
|
+
UNION ALL
|
|
210
|
+
SELECT session_id, event_id, ${COLLISION_KEY_SQL} AS output_key, created_at
|
|
211
|
+
FROM event_log
|
|
212
|
+
WHERE type = 'pibo.output.identity_collision'
|
|
213
|
+
AND ${COLLISION_KEY_SQL} IS NOT NULL ${scope.sql}
|
|
214
|
+
)
|
|
215
|
+
SELECT MIN(session_id) AS sessionId, MIN(event_id) AS eventId, output_key AS outputKey,
|
|
216
|
+
COUNT(*) AS uses, MIN(created_at) AS firstAt, MAX(created_at) AS lastAt
|
|
217
|
+
FROM output_keys
|
|
218
|
+
GROUP BY output_key
|
|
219
|
+
HAVING COUNT(*) > 1
|
|
220
|
+
ORDER BY lastAt DESC
|
|
221
|
+
LIMIT ?
|
|
222
|
+
`, [...scope.params, ...scope.params, limit]).map((row) => ({
|
|
223
|
+
kind: "output_key_reuse",
|
|
224
|
+
piboSessionId: row.sessionId,
|
|
225
|
+
...(row.eventId ? { eventId: row.eventId } : {}),
|
|
226
|
+
idempotencyKey: row.outputKey,
|
|
227
|
+
uses: Number(row.uses),
|
|
228
|
+
firstAt: row.firstAt,
|
|
229
|
+
lastAt: row.lastAt,
|
|
230
|
+
})));
|
|
231
|
+
if (tableExists(db, "sessions")) {
|
|
232
|
+
const traceStatus = sessionTraceStatusSql(input.piboSessionId, input.since, input.before, false);
|
|
233
|
+
findings.push(...queryRows(db, traceStatus.sql, [...traceStatus.params, limit]).map((row) => ({
|
|
234
|
+
kind: "session_trace_status",
|
|
235
|
+
piboSessionId: row.sessionId,
|
|
236
|
+
sessionStatus: row.sessionStatus,
|
|
237
|
+
projectedStatus: row.projectedStatus,
|
|
238
|
+
openTurns: Number(row.openTurns),
|
|
239
|
+
lastAt: row.lastAt,
|
|
240
|
+
})));
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
db.exec("COMMIT");
|
|
244
|
+
}
|
|
245
|
+
catch (error) {
|
|
246
|
+
if (db.isTransaction)
|
|
247
|
+
db.exec("ROLLBACK");
|
|
248
|
+
throw error;
|
|
249
|
+
}
|
|
250
|
+
finally {
|
|
251
|
+
db.close();
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
if (input.reliabilityStore.exists) {
|
|
255
|
+
const db = new DatabaseSync(input.reliabilityStore.path, { readOnly: true });
|
|
256
|
+
db.exec("BEGIN");
|
|
257
|
+
try {
|
|
258
|
+
const jobScope = reliabilityScope(input.piboSessionId, input.since, input.before, "updated_at");
|
|
259
|
+
const deadScope = reliabilityScope(input.piboSessionId, input.since, input.before, "dead_at");
|
|
260
|
+
if (tableExists(db, "pibo_jobs")) {
|
|
261
|
+
pendingOutputJobs = countRows(db, `
|
|
262
|
+
SELECT COUNT(*) AS count FROM pibo_jobs
|
|
263
|
+
WHERE queue IN ('output-persistence', 'output-persistence-cli') ${jobScope.sql}
|
|
264
|
+
`, jobScope.params);
|
|
265
|
+
findings.push(...queryRows(db, `
|
|
266
|
+
SELECT job_id AS jobId, queue, attempts, max_attempts AS maxAttempts,
|
|
267
|
+
${RELIABILITY_SESSION_ID_SQL} AS piboSessionId,
|
|
268
|
+
${RELIABILITY_EVENT_ID_SQL} AS eventId,
|
|
269
|
+
last_error AS lastError, json_valid(payload_json) AS payloadValid, updated_at AS updatedAt
|
|
270
|
+
FROM pibo_jobs
|
|
271
|
+
WHERE queue IN ('output-persistence', 'output-persistence-cli') ${jobScope.sql}
|
|
272
|
+
ORDER BY updated_at DESC
|
|
273
|
+
LIMIT ?
|
|
274
|
+
`, [...jobScope.params, limit]).map(pendingJobFinding));
|
|
275
|
+
}
|
|
276
|
+
if (tableExists(db, "pibo_dead_jobs")) {
|
|
277
|
+
deadOutputJobs = countRows(db, `
|
|
278
|
+
SELECT COUNT(*) AS count FROM pibo_dead_jobs
|
|
279
|
+
WHERE queue IN ('output-persistence', 'output-persistence-cli') ${deadScope.sql}
|
|
280
|
+
`, deadScope.params);
|
|
281
|
+
deadIdentityCollisions = countRows(db, `
|
|
282
|
+
SELECT COUNT(*) AS count FROM pibo_dead_jobs
|
|
283
|
+
WHERE queue IN ('output-persistence', 'output-persistence-cli')
|
|
284
|
+
AND last_error LIKE 'Pibo output identity collision for %' ${deadScope.sql}
|
|
285
|
+
`, deadScope.params);
|
|
286
|
+
findings.push(...queryRows(db, `
|
|
287
|
+
SELECT job_id AS jobId, queue, attempts, max_attempts AS maxAttempts,
|
|
288
|
+
${RELIABILITY_SESSION_ID_SQL} AS piboSessionId,
|
|
289
|
+
${RELIABILITY_EVENT_ID_SQL} AS eventId,
|
|
290
|
+
last_error AS lastError, dead_reason AS deadReason, json_valid(payload_json) AS payloadValid, dead_at AS updatedAt
|
|
291
|
+
FROM pibo_dead_jobs
|
|
292
|
+
WHERE queue IN ('output-persistence', 'output-persistence-cli') ${deadScope.sql}
|
|
293
|
+
ORDER BY dead_at DESC
|
|
294
|
+
LIMIT ?
|
|
295
|
+
`, [...deadScope.params, limit]).map((row) => deadJobFinding(row, collisionEventKeys)));
|
|
296
|
+
}
|
|
297
|
+
db.exec("COMMIT");
|
|
298
|
+
}
|
|
299
|
+
catch (error) {
|
|
300
|
+
if (db.isTransaction)
|
|
301
|
+
db.exec("ROLLBACK");
|
|
302
|
+
throw error;
|
|
303
|
+
}
|
|
304
|
+
finally {
|
|
305
|
+
db.close();
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
const visibleFindings = input.findingMode === "dead_letters"
|
|
309
|
+
? findings.filter((finding) => finding.kind === "dead_output_job")
|
|
310
|
+
: findings;
|
|
311
|
+
visibleFindings.sort((left, right) => (right.lastAt ?? "").localeCompare(left.lastAt ?? ""));
|
|
312
|
+
const returnedFindings = visibleFindings.slice(0, limit);
|
|
313
|
+
const findingCount = turnLifecycleIssues
|
|
314
|
+
+ thinkingLifecycleIssues
|
|
315
|
+
+ toolLifecycleIssues
|
|
316
|
+
+ identityCollisions
|
|
317
|
+
+ outputKeyReuses
|
|
318
|
+
+ sessionTraceStatusMismatches
|
|
319
|
+
+ pendingOutputJobs
|
|
320
|
+
+ deadOutputJobs;
|
|
321
|
+
const nextCommands = input.piboSessionId
|
|
322
|
+
? [
|
|
323
|
+
`pibo debug trace ${input.piboSessionId} --check`,
|
|
324
|
+
`pibo debug events ${input.piboSessionId} --limit 50`,
|
|
325
|
+
`pibo debug persistence dead-letters --session ${input.piboSessionId}`,
|
|
326
|
+
]
|
|
327
|
+
: [
|
|
328
|
+
"pibo debug persistence audit --session <pibo-session-id> --json",
|
|
329
|
+
"pibo debug persistence dead-letters",
|
|
330
|
+
"pibo debug jobs list --queue output-persistence",
|
|
331
|
+
];
|
|
332
|
+
return {
|
|
333
|
+
resultType: "debug.integrity.output",
|
|
334
|
+
readOnly: true,
|
|
335
|
+
scope: {
|
|
336
|
+
...(input.piboSessionId ? { piboSessionId: input.piboSessionId } : {}),
|
|
337
|
+
...(input.since ? { since: input.since } : {}),
|
|
338
|
+
...(input.before ? { before: input.before } : {}),
|
|
339
|
+
limit,
|
|
340
|
+
},
|
|
341
|
+
stores: {
|
|
342
|
+
data: { path: input.dataStore.path, exists: input.dataStore.exists },
|
|
343
|
+
reliability: { path: input.reliabilityStore.path, exists: input.reliabilityStore.exists },
|
|
344
|
+
},
|
|
345
|
+
summary: {
|
|
346
|
+
findingCount,
|
|
347
|
+
returnedFindings: returnedFindings.length,
|
|
348
|
+
turnLifecycleIssues,
|
|
349
|
+
thinkingLifecycleIssues,
|
|
350
|
+
toolLifecycleIssues,
|
|
351
|
+
identityCollisions,
|
|
352
|
+
outputKeyReuses,
|
|
353
|
+
sessionTraceStatusMismatches,
|
|
354
|
+
pendingOutputJobs,
|
|
355
|
+
deadOutputJobs,
|
|
356
|
+
deadIdentityCollisions,
|
|
357
|
+
},
|
|
358
|
+
findings: returnedFindings,
|
|
359
|
+
nextCommands,
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
export function outputPersistenceDeadLettersFromAudit(audit) {
|
|
363
|
+
const deadLetters = audit.findings.filter((finding) => finding.kind === "dead_output_job");
|
|
364
|
+
return {
|
|
365
|
+
resultType: "debug.persistence.dead-letters",
|
|
366
|
+
readOnly: true,
|
|
367
|
+
scope: audit.scope,
|
|
368
|
+
summary: {
|
|
369
|
+
deadOutputJobs: audit.summary.deadOutputJobs,
|
|
370
|
+
returnedDeadLetters: deadLetters.length,
|
|
371
|
+
identityCollisions: deadLetters.filter((finding) => finding.identityCollision).length,
|
|
372
|
+
relatedIdentityCollisions: deadLetters.filter((finding) => finding.relatedIdentityCollision).length,
|
|
373
|
+
},
|
|
374
|
+
deadLetters,
|
|
375
|
+
nextCommands: audit.scope.piboSessionId
|
|
376
|
+
? [`pibo debug persistence audit --session ${audit.scope.piboSessionId} --json`]
|
|
377
|
+
: ["pibo debug persistence audit --json"],
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
export function formatOutputPersistenceDeadLetters(result) {
|
|
381
|
+
const lines = [
|
|
382
|
+
"pibo debug persistence dead-letters",
|
|
383
|
+
`readOnly\t${result.readOnly}`,
|
|
384
|
+
`session\t${result.scope.piboSessionId ?? "all"}`,
|
|
385
|
+
...(result.scope.since ? [`since\t${result.scope.since}`] : []),
|
|
386
|
+
...(result.scope.before ? [`before\t${result.scope.before}`] : []),
|
|
387
|
+
`deadOutputJobs\t${result.summary.deadOutputJobs}`,
|
|
388
|
+
`returnedDeadLetters\t${result.summary.returnedDeadLetters}`,
|
|
389
|
+
`identityCollisions\t${result.summary.identityCollisions}`,
|
|
390
|
+
`relatedIdentityCollisions\t${result.summary.relatedIdentityCollisions}`,
|
|
391
|
+
];
|
|
392
|
+
if (result.deadLetters.length) {
|
|
393
|
+
lines.push("", "job\tsession\tevent\treason\tcollision\trelated\tlastAt");
|
|
394
|
+
for (const finding of result.deadLetters) {
|
|
395
|
+
lines.push([
|
|
396
|
+
finding.jobId ?? "-",
|
|
397
|
+
finding.piboSessionId ?? "-",
|
|
398
|
+
finding.eventId ?? "-",
|
|
399
|
+
finding.deadReason ?? "-",
|
|
400
|
+
finding.identityCollision ?? false,
|
|
401
|
+
finding.relatedIdentityCollision ?? false,
|
|
402
|
+
finding.lastAt ?? "-",
|
|
403
|
+
].join("\t"));
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
lines.push("", "Next:", ...result.nextCommands.map((command) => ` ${command}`));
|
|
407
|
+
return lines.join("\n");
|
|
408
|
+
}
|
|
409
|
+
export function formatOutputIntegrityAudit(audit) {
|
|
410
|
+
const scope = audit.scope.piboSessionId ?? "all";
|
|
411
|
+
const lines = [
|
|
412
|
+
`pibo debug integrity output`,
|
|
413
|
+
`readOnly\t${audit.readOnly}`,
|
|
414
|
+
`session\t${scope}`,
|
|
415
|
+
...(audit.scope.since ? [`since\t${audit.scope.since}`] : []),
|
|
416
|
+
...(audit.scope.before ? [`before\t${audit.scope.before}`] : []),
|
|
417
|
+
`findings\t${audit.summary.findingCount}`,
|
|
418
|
+
`turnLifecycle\t${audit.summary.turnLifecycleIssues}`,
|
|
419
|
+
`thinkingLifecycle\t${audit.summary.thinkingLifecycleIssues}`,
|
|
420
|
+
`toolLifecycle\t${audit.summary.toolLifecycleIssues}`,
|
|
421
|
+
`identityCollisions\t${audit.summary.identityCollisions}`,
|
|
422
|
+
`outputKeyReuses\t${audit.summary.outputKeyReuses}`,
|
|
423
|
+
`sessionTraceStatusMismatches\t${audit.summary.sessionTraceStatusMismatches}`,
|
|
424
|
+
`pendingOutputJobs\t${audit.summary.pendingOutputJobs}`,
|
|
425
|
+
`deadOutputJobs\t${audit.summary.deadOutputJobs}`,
|
|
426
|
+
`deadIdentityCollisions\t${audit.summary.deadIdentityCollisions}`,
|
|
427
|
+
];
|
|
428
|
+
if (audit.findings.length) {
|
|
429
|
+
lines.push("", "kind\tsession\tevent\tdetail\tlastAt");
|
|
430
|
+
for (const finding of audit.findings) {
|
|
431
|
+
lines.push([
|
|
432
|
+
finding.kind,
|
|
433
|
+
finding.piboSessionId ?? "-",
|
|
434
|
+
finding.eventId ?? finding.jobId ?? "-",
|
|
435
|
+
findingDetail(finding),
|
|
436
|
+
finding.lastAt ?? "-",
|
|
437
|
+
].join("\t"));
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
lines.push("", "Next:", ...audit.nextCommands.map((command) => ` ${command}`));
|
|
441
|
+
return lines.join("\n");
|
|
442
|
+
}
|
|
443
|
+
function eventScope(piboSessionId, since, before) {
|
|
444
|
+
const clauses = [];
|
|
445
|
+
const params = [];
|
|
446
|
+
if (piboSessionId) {
|
|
447
|
+
clauses.push("session_id = ?");
|
|
448
|
+
params.push(piboSessionId);
|
|
449
|
+
}
|
|
450
|
+
if (since) {
|
|
451
|
+
clauses.push("created_at >= ?");
|
|
452
|
+
params.push(since);
|
|
453
|
+
}
|
|
454
|
+
if (before) {
|
|
455
|
+
clauses.push("created_at < ?");
|
|
456
|
+
params.push(before);
|
|
457
|
+
}
|
|
458
|
+
return { sql: clauses.length ? `AND ${clauses.join(" AND ")}` : "", params };
|
|
459
|
+
}
|
|
460
|
+
function lifecycleScope(piboSessionId, since, before) {
|
|
461
|
+
const whereClauses = [];
|
|
462
|
+
const havingClauses = [];
|
|
463
|
+
const params = [];
|
|
464
|
+
if (piboSessionId) {
|
|
465
|
+
whereClauses.push("session_id = ?");
|
|
466
|
+
params.push(piboSessionId);
|
|
467
|
+
}
|
|
468
|
+
if (since) {
|
|
469
|
+
havingClauses.push("MAX(created_at) >= ?");
|
|
470
|
+
params.push(since);
|
|
471
|
+
}
|
|
472
|
+
if (before) {
|
|
473
|
+
havingClauses.push("MAX(created_at) < ?");
|
|
474
|
+
params.push(before);
|
|
475
|
+
}
|
|
476
|
+
return {
|
|
477
|
+
whereSql: whereClauses.length ? `AND ${whereClauses.join(" AND ")}` : "",
|
|
478
|
+
havingSql: havingClauses.length ? `AND ${havingClauses.join(" AND ")}` : "",
|
|
479
|
+
params,
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
function reliabilityScope(piboSessionId, since, before, timeColumn) {
|
|
483
|
+
const clauses = [];
|
|
484
|
+
const params = [];
|
|
485
|
+
if (piboSessionId) {
|
|
486
|
+
clauses.push(`${RELIABILITY_SESSION_ID_SQL} = ?`);
|
|
487
|
+
params.push(piboSessionId);
|
|
488
|
+
}
|
|
489
|
+
if (since) {
|
|
490
|
+
clauses.push(`${timeColumn} >= ?`);
|
|
491
|
+
params.push(since);
|
|
492
|
+
}
|
|
493
|
+
if (before) {
|
|
494
|
+
clauses.push(`${timeColumn} < ?`);
|
|
495
|
+
params.push(before);
|
|
496
|
+
}
|
|
497
|
+
return { sql: clauses.length ? `AND ${clauses.join(" AND ")}` : "", params };
|
|
498
|
+
}
|
|
499
|
+
function sessionTraceStatusSql(piboSessionId, since, before, countOnly) {
|
|
500
|
+
const clauses = [];
|
|
501
|
+
const params = [];
|
|
502
|
+
if (piboSessionId) {
|
|
503
|
+
clauses.push("latest.session_id = ?");
|
|
504
|
+
params.push(piboSessionId);
|
|
505
|
+
}
|
|
506
|
+
if (since) {
|
|
507
|
+
clauses.push("latest.created_at >= ?");
|
|
508
|
+
params.push(since);
|
|
509
|
+
}
|
|
510
|
+
if (before) {
|
|
511
|
+
clauses.push("latest.created_at < ?");
|
|
512
|
+
params.push(before);
|
|
513
|
+
}
|
|
514
|
+
const scopeSql = clauses.length ? `AND ${clauses.join(" AND ")}` : "";
|
|
515
|
+
const ctes = `
|
|
516
|
+
WITH ranked_status_events AS (
|
|
517
|
+
SELECT session_id, type, created_at,
|
|
518
|
+
ROW_NUMBER() OVER (PARTITION BY session_id ORDER BY stream_id DESC) AS rank
|
|
519
|
+
FROM event_log
|
|
520
|
+
WHERE session_id IS NOT NULL
|
|
521
|
+
AND type IN ('message_started', 'message_finished', 'session_error')
|
|
522
|
+
), latest AS (
|
|
523
|
+
SELECT session_id, type, created_at
|
|
524
|
+
FROM ranked_status_events
|
|
525
|
+
WHERE rank = 1
|
|
526
|
+
), mismatches AS (
|
|
527
|
+
SELECT sessions.id AS sessionId, sessions.status AS sessionStatus,
|
|
528
|
+
CASE latest.type
|
|
529
|
+
WHEN 'session_error' THEN 'error'
|
|
530
|
+
WHEN 'message_started' THEN 'running'
|
|
531
|
+
ELSE 'idle'
|
|
532
|
+
END AS projectedStatus,
|
|
533
|
+
CASE WHEN latest.type = 'message_started' THEN 1 ELSE 0 END AS openTurns,
|
|
534
|
+
latest.created_at AS lastAt
|
|
535
|
+
FROM sessions
|
|
536
|
+
JOIN latest ON latest.session_id = sessions.id
|
|
537
|
+
WHERE CASE WHEN sessions.status IN ('running', 'error') THEN sessions.status ELSE 'idle' END
|
|
538
|
+
!= CASE latest.type
|
|
539
|
+
WHEN 'session_error' THEN 'error'
|
|
540
|
+
WHEN 'message_started' THEN 'running'
|
|
541
|
+
ELSE 'idle'
|
|
542
|
+
END
|
|
543
|
+
${scopeSql}
|
|
544
|
+
)
|
|
545
|
+
`;
|
|
546
|
+
return {
|
|
547
|
+
sql: countOnly
|
|
548
|
+
? `${ctes} SELECT COUNT(*) AS count FROM mismatches`
|
|
549
|
+
: `${ctes} SELECT * FROM mismatches ORDER BY lastAt DESC LIMIT ?`,
|
|
550
|
+
params,
|
|
551
|
+
};
|
|
552
|
+
}
|
|
553
|
+
function queryRows(db, sql, params) {
|
|
554
|
+
return db.prepare(sql).all(...params);
|
|
555
|
+
}
|
|
556
|
+
function countRows(db, sql, params) {
|
|
557
|
+
const row = db.prepare(sql).get(...params);
|
|
558
|
+
return Number(row?.count ?? 0);
|
|
559
|
+
}
|
|
560
|
+
function tableExists(db, table) {
|
|
561
|
+
return Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(table));
|
|
562
|
+
}
|
|
563
|
+
function pendingJobFinding(row) {
|
|
564
|
+
return {
|
|
565
|
+
kind: "pending_output_job",
|
|
566
|
+
...(row.piboSessionId ? { piboSessionId: row.piboSessionId } : {}),
|
|
567
|
+
...(row.eventId ? { eventId: row.eventId } : {}),
|
|
568
|
+
jobId: row.jobId,
|
|
569
|
+
queue: row.queue,
|
|
570
|
+
attempts: row.attempts,
|
|
571
|
+
maxAttempts: row.maxAttempts,
|
|
572
|
+
payloadValid: row.payloadValid === 1,
|
|
573
|
+
lastAt: row.updatedAt,
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
function deadJobFinding(row, collisionEventKeys) {
|
|
577
|
+
return {
|
|
578
|
+
kind: "dead_output_job",
|
|
579
|
+
...(row.piboSessionId ? { piboSessionId: row.piboSessionId } : {}),
|
|
580
|
+
...(row.eventId ? { eventId: row.eventId } : {}),
|
|
581
|
+
jobId: row.jobId,
|
|
582
|
+
queue: row.queue,
|
|
583
|
+
attempts: row.attempts,
|
|
584
|
+
maxAttempts: row.maxAttempts,
|
|
585
|
+
...(row.deadReason ? { deadReason: row.deadReason } : {}),
|
|
586
|
+
identityCollision: row.lastError?.startsWith("Pibo output identity collision for ") ?? false,
|
|
587
|
+
relatedIdentityCollision: Boolean(row.piboSessionId && row.eventId && collisionEventKeys.has(`${row.piboSessionId}\0${row.eventId}`)),
|
|
588
|
+
payloadValid: row.payloadValid === 1,
|
|
589
|
+
lastAt: row.updatedAt,
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
function findingDetail(finding) {
|
|
593
|
+
if (finding.kind === "turn_lifecycle")
|
|
594
|
+
return `started=${finding.started ?? 0},assistant=${finding.assistantMessages ?? 0},messageFinished=${finding.messageFinished ?? 0},sessionErrors=${finding.sessionErrors ?? 0}`;
|
|
595
|
+
if (finding.kind === "thinking_lifecycle")
|
|
596
|
+
return `thinking=${finding.thinkingIndex ?? 0},started=${finding.started ?? 0},finished=${finding.finished ?? 0}`;
|
|
597
|
+
if (finding.kind === "tool_lifecycle")
|
|
598
|
+
return `tool=${finding.toolCallId ?? "-"},ordinal=${finding.toolInvocationOrdinal ?? 0},called=${finding.called ?? 0},started=${finding.started ?? 0},finished=${finding.finished ?? 0}`;
|
|
599
|
+
if (finding.kind === "identity_collision")
|
|
600
|
+
return finding.idempotencyKey ?? `stream=${finding.streamId ?? "-"}`;
|
|
601
|
+
if (finding.kind === "output_key_reuse")
|
|
602
|
+
return `uses=${finding.uses ?? 0},key=${finding.idempotencyKey ?? "-"}`;
|
|
603
|
+
if (finding.kind === "session_trace_status")
|
|
604
|
+
return `session=${finding.sessionStatus ?? "-"},projected=${finding.projectedStatus ?? "-"},openTurns=${finding.openTurns ?? 0}`;
|
|
605
|
+
if (finding.kind === "pending_output_job")
|
|
606
|
+
return `payloadValid=${finding.payloadValid ?? false},attempts=${finding.attempts ?? 0}/${finding.maxAttempts ?? 0}`;
|
|
607
|
+
return `reason=${finding.deadReason ?? "-"},collision=${finding.identityCollision ?? false},related=${finding.relatedIdentityCollision ?? false},payloadValid=${finding.payloadValid ?? false},attempts=${finding.attempts ?? 0}/${finding.maxAttempts ?? 0}`;
|
|
608
|
+
}
|