@pasko70/pibo 3.5.0 → 3.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-runtime/routed-session.js +80 -19
- package/dist/agent-runtimes/codex-native/turn.js +27 -3
- package/dist/apps/chat/data/chat-data-mappers.js +8 -1
- package/dist/apps/chat/data/read-state-service.js +24 -3
- package/dist/apps/chat/message-command-dispatcher.js +16 -1
- package/dist/apps/chat/web-app.js +40 -19
- package/dist/apps/chat-ui/assets/{dist-D79vyxSX.js → dist-B-auLrzD.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-DFZ8cwh0.js → dist-BA_dsINH.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-D6TjFhAm.js → dist-eJZar_0-.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-BG0n7zLd.js → dist-wNNR2Bci.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-cOjokPrK.js → dist-zcmEsIEp.js} +1 -1
- package/dist/apps/chat-ui/assets/{index-RMHUTJ62.js → index-DEkbN5Vo.js} +43 -43
- package/dist/apps/chat-ui/assets/index-DZK6Tzil.css +1 -0
- package/dist/apps/chat-ui/index.html +2 -2
- package/dist/apps/chat-vscode-web/assets/{index-xacbCyTx.js → index-DZgW1fCB.js} +11 -11
- package/dist/apps/chat-vscode-web/index.html +1 -1
- package/dist/cli-session/localSessionSource.js +3 -2
- package/dist/core/output-render-sequence.js +63 -6
- package/dist/core/session-router.js +99 -11
- package/dist/data/async-chat-storage.js +7 -2
- package/dist/data/bounded-worker-client.js +1 -1
- package/dist/data/chat-read-projections.js +4 -4
- package/dist/data/chat-storage-worker.js +24 -3
- package/dist/data/ingest-service.js +73 -4
- package/dist/data/message-command-store.js +153 -11
- package/dist/data/schema.js +24 -3
- package/dist/data/storage-maintenance.js +344 -0
- package/dist/data/storage-verification-worker.js +25 -0
- package/dist/debug/index.js +155 -1
- package/dist/debug/message-queue.js +108 -0
- package/dist/debug/output-collision-repair.js +140 -0
- package/dist/debug/output-integrity.js +38 -2
- package/dist/debug/output-repair.js +1 -0
- package/dist/debug/storage-backup.js +12 -4
- package/dist/debug/storage-maintenance.js +78 -0
- package/dist/gateway/cli.js +71 -7
- package/dist/gateway/server.js +1 -0
- package/dist/reliability/store.js +119 -23
- package/dist/session-ui/terminalRows.js +7 -8
- package/dist/sessions/pibo-data-store.js +18 -14
- package/dist/shared/trace-event-projection.js +13 -3
- package/dist/web/channel.js +114 -12
- package/dist/web/http.js +105 -44
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/dist/apps/chat-ui/assets/index-hEkrlRk-.css +0 -1
|
@@ -44,12 +44,14 @@ function ensurePiboJobColumn(db, name, definition) {
|
|
|
44
44
|
db.exec(`ALTER TABLE pibo_jobs ADD COLUMN ${name} ${definition}`);
|
|
45
45
|
}
|
|
46
46
|
export class PiboReliabilityStore {
|
|
47
|
+
options;
|
|
47
48
|
db;
|
|
48
49
|
appendEventStatement;
|
|
49
50
|
getEventByStreamIdStatement;
|
|
50
51
|
getEventByTopicEventIdStatement;
|
|
51
52
|
getEventByTopicIdempotencyKeyStatement;
|
|
52
|
-
constructor(path = piboHomePath("pibo-events.sqlite")) {
|
|
53
|
+
constructor(path = piboHomePath("pibo-events.sqlite"), options = {}) {
|
|
54
|
+
this.options = options;
|
|
53
55
|
const resolvedPath = path === ":memory:" ? path : resolve(path);
|
|
54
56
|
const insidePiboHome = ensurePrivatePiboHomeForPath(resolvedPath);
|
|
55
57
|
if (resolvedPath !== ":memory:")
|
|
@@ -571,6 +573,38 @@ export class PiboReliabilityStore {
|
|
|
571
573
|
}
|
|
572
574
|
return jobs;
|
|
573
575
|
}
|
|
576
|
+
inspectJobs(input = {}) {
|
|
577
|
+
const timestamp = now();
|
|
578
|
+
return this.listJobs(input).map((job) => this.inspectJob(job, timestamp));
|
|
579
|
+
}
|
|
580
|
+
reconcileOrphanRunJobs(input) {
|
|
581
|
+
const checkedAt = asDate(input.now).toISOString();
|
|
582
|
+
return this.inImmediateTransaction(() => this.reconcileOrphanRunJobsInTransaction(checkedAt, input.apply));
|
|
583
|
+
}
|
|
584
|
+
getRunJobReliabilityStatus() {
|
|
585
|
+
const timestamp = now();
|
|
586
|
+
const expired = this.db.prepare(`
|
|
587
|
+
SELECT COUNT(*) AS count
|
|
588
|
+
FROM pibo_jobs AS job
|
|
589
|
+
LEFT JOIN pibo_runs AS run ON run.job_id = job.job_id
|
|
590
|
+
WHERE job.queue = 'runs'
|
|
591
|
+
AND job.state = 'running'
|
|
592
|
+
AND job.claim_expires_at IS NOT NULL
|
|
593
|
+
AND job.claim_expires_at <= ?
|
|
594
|
+
AND run.run_id IS NULL
|
|
595
|
+
`).get(timestamp);
|
|
596
|
+
const dead = this.db.prepare(`
|
|
597
|
+
SELECT COUNT(*) AS count FROM pibo_dead_jobs
|
|
598
|
+
WHERE queue = 'runs' AND dead_reason = 'orphan_run_job'
|
|
599
|
+
`).get();
|
|
600
|
+
const expiredOrphanRunJobs = Number(expired.count);
|
|
601
|
+
const orphanRunDeadLetters = Number(dead.count);
|
|
602
|
+
return {
|
|
603
|
+
status: expiredOrphanRunJobs > 0 || orphanRunDeadLetters > 0 ? "degraded" : "ok",
|
|
604
|
+
expiredOrphanRunJobs,
|
|
605
|
+
orphanRunDeadLetters,
|
|
606
|
+
};
|
|
607
|
+
}
|
|
574
608
|
listRecoverableJobs(input = {}) {
|
|
575
609
|
const timestamp = now();
|
|
576
610
|
const clauses = [
|
|
@@ -683,28 +717,37 @@ export class PiboReliabilityStore {
|
|
|
683
717
|
const runId = input.runId ?? `run_${randomUUID()}`;
|
|
684
718
|
const maxAttempts = Math.max(1, input.maxAttempts ?? 1);
|
|
685
719
|
const timeoutAt = input.timeoutMs === undefined ? undefined : new Date(Date.parse(timestamp) + input.timeoutMs).toISOString();
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
720
|
+
return this.inImmediateTransaction(() => {
|
|
721
|
+
const job = this.enqueue({
|
|
722
|
+
queue: "runs",
|
|
723
|
+
runAt: timestamp,
|
|
724
|
+
payload: {
|
|
725
|
+
runId,
|
|
726
|
+
controllerPiboSessionId: input.controllerPiboSessionId,
|
|
727
|
+
toolName: input.toolName,
|
|
728
|
+
params: input.params,
|
|
729
|
+
timeoutMs: input.timeoutMs,
|
|
730
|
+
},
|
|
731
|
+
maxAttempts,
|
|
732
|
+
});
|
|
733
|
+
this.options.onRunCreationBoundary?.("after_enqueue");
|
|
734
|
+
const claimed = this.claimPendingJobInTransaction(job.jobId, input.workerId ?? `run-registry:${process.pid}`, timestamp, 24 * 60 * 60 * 1000);
|
|
735
|
+
if (!claimed)
|
|
736
|
+
throw new Error(`Failed to claim newly-created run job "${job.jobId}"`);
|
|
737
|
+
this.options.onRunCreationBoundary?.("after_claim");
|
|
738
|
+
this.options.onRunCreationBoundary?.("before_run_insert");
|
|
739
|
+
this.db
|
|
740
|
+
.prepare(`
|
|
741
|
+
INSERT INTO pibo_runs (
|
|
742
|
+
run_id, kind, controller_pibo_session_id, status, completion_policy, consumed, tool_name,
|
|
743
|
+
summary, result_json, error, notified_status, acknowledged_status, created_at, updated_at,
|
|
744
|
+
completed_at, job_id, retryable, max_attempts, timeout_ms, timeout_at, timeout_phase, service_warning, resource_json, origin_json
|
|
745
|
+
) VALUES (?, 'tool', ?, 'running', ?, 0, ?, ?, NULL, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?, ?, ?, NULL, ?, ?, ?)
|
|
746
|
+
`)
|
|
747
|
+
.run(runId, input.controllerPiboSessionId, input.completionPolicy, input.toolName, `${input.toolName} run is running.`, timestamp, timestamp, job.jobId, input.retryable ? 1 : 0, maxAttempts, input.timeoutMs ?? null, timeoutAt ?? null, input.serviceWarning ?? null, input.resources ? JSON.stringify(input.resources) : null, input.origin ? JSON.stringify(input.origin) : null);
|
|
748
|
+
this.options.onRunCreationBoundary?.("after_run_insert");
|
|
749
|
+
return this.requireRun(runId);
|
|
696
750
|
});
|
|
697
|
-
this.db
|
|
698
|
-
.prepare(`
|
|
699
|
-
INSERT INTO pibo_runs (
|
|
700
|
-
run_id, kind, controller_pibo_session_id, status, completion_policy, consumed, tool_name,
|
|
701
|
-
summary, result_json, error, notified_status, acknowledged_status, created_at, updated_at,
|
|
702
|
-
completed_at, job_id, retryable, max_attempts, timeout_ms, timeout_at, timeout_phase, service_warning, resource_json, origin_json
|
|
703
|
-
) VALUES (?, 'tool', ?, 'running', ?, 0, ?, ?, NULL, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?, ?, ?, NULL, ?, ?, ?)
|
|
704
|
-
`)
|
|
705
|
-
.run(runId, input.controllerPiboSessionId, input.completionPolicy, input.toolName, `${input.toolName} run is running.`, timestamp, timestamp, job.jobId, input.retryable ? 1 : 0, maxAttempts, input.timeoutMs ?? null, timeoutAt ?? null, input.serviceWarning ?? null, input.resources ? JSON.stringify(input.resources) : null, input.origin ? JSON.stringify(input.origin) : null);
|
|
706
|
-
this.claimJob(job.jobId, input.workerId ?? `run-registry:${process.pid}`, 24 * 60 * 60 * 1000);
|
|
707
|
-
return this.requireRun(runId);
|
|
708
751
|
}
|
|
709
752
|
updateRun(runId, patch) {
|
|
710
753
|
const existing = this.getRun(runId);
|
|
@@ -779,9 +822,10 @@ export class PiboReliabilityStore {
|
|
|
779
822
|
}
|
|
780
823
|
recoverInterruptedRuns(workerId = `run-registry:${process.pid}`) {
|
|
781
824
|
return this.inImmediateTransaction(() => {
|
|
825
|
+
const timestamp = now();
|
|
826
|
+
this.reconcileOrphanRunJobsInTransaction(timestamp, true);
|
|
782
827
|
const rows = this.db.prepare("SELECT * FROM pibo_runs WHERE status = 'running'").all();
|
|
783
828
|
const recovered = [];
|
|
784
|
-
const timestamp = now();
|
|
785
829
|
for (const row of rows) {
|
|
786
830
|
if (row.job_id && this.hasUnexpiredJobClaim(row.job_id, timestamp, workerId))
|
|
787
831
|
continue;
|
|
@@ -883,6 +927,58 @@ export class PiboReliabilityStore {
|
|
|
883
927
|
`)
|
|
884
928
|
.get(jobId, workerId, claimToken ?? null, claimToken ?? null, timestamp);
|
|
885
929
|
}
|
|
930
|
+
claimPendingJobInTransaction(jobId, workerId, timestamp, visibilityTimeoutMs) {
|
|
931
|
+
const claimExpiresAt = new Date(Date.parse(timestamp) + visibilityTimeoutMs).toISOString();
|
|
932
|
+
return this.db.prepare(`
|
|
933
|
+
UPDATE pibo_jobs
|
|
934
|
+
SET state = 'running',
|
|
935
|
+
worker_id = ?,
|
|
936
|
+
claim_expires_at = ?,
|
|
937
|
+
claim_token = claim_token + 1,
|
|
938
|
+
attempts = attempts + 1,
|
|
939
|
+
updated_at = ?
|
|
940
|
+
WHERE job_id = ?
|
|
941
|
+
AND state = 'pending'
|
|
942
|
+
AND run_at <= ?
|
|
943
|
+
AND attempts < max_attempts
|
|
944
|
+
AND (expires_at IS NULL OR expires_at > ?)
|
|
945
|
+
RETURNING *
|
|
946
|
+
`).get(workerId, claimExpiresAt, timestamp, jobId, timestamp, timestamp);
|
|
947
|
+
}
|
|
948
|
+
inspectJob(job, timestamp) {
|
|
949
|
+
const claimExpired = job.state === "running"
|
|
950
|
+
&& (!job.claimExpiresAt || job.claimExpiresAt <= timestamp);
|
|
951
|
+
const missingRunRecord = job.queue === "runs"
|
|
952
|
+
&& this.db.prepare("SELECT 1 AS found FROM pibo_runs WHERE job_id = ? LIMIT 1").get(job.jobId) === undefined;
|
|
953
|
+
let effectiveLiveness;
|
|
954
|
+
if (job.state === "pending")
|
|
955
|
+
effectiveLiveness = missingRunRecord ? "orphan_pending" : "pending";
|
|
956
|
+
else if (claimExpired)
|
|
957
|
+
effectiveLiveness = missingRunRecord ? "expired_orphan" : "expired_claim";
|
|
958
|
+
else
|
|
959
|
+
effectiveLiveness = missingRunRecord ? "live_orphan_claim" : "live_claim";
|
|
960
|
+
return { ...job, claimExpired, missingRunRecord, effectiveLiveness };
|
|
961
|
+
}
|
|
962
|
+
reconcileOrphanRunJobsInTransaction(timestamp, apply) {
|
|
963
|
+
const rows = this.db.prepare(`
|
|
964
|
+
SELECT job.*
|
|
965
|
+
FROM pibo_jobs AS job
|
|
966
|
+
LEFT JOIN pibo_runs AS run ON run.job_id = job.job_id
|
|
967
|
+
WHERE job.queue = 'runs'
|
|
968
|
+
AND job.state = 'running'
|
|
969
|
+
AND job.claim_expires_at IS NOT NULL
|
|
970
|
+
AND job.claim_expires_at <= ?
|
|
971
|
+
AND run.run_id IS NULL
|
|
972
|
+
ORDER BY job.created_at ASC, job.job_id ASC
|
|
973
|
+
`).all(timestamp);
|
|
974
|
+
const candidates = rows.map((row) => this.inspectJob(jobFromRow(row), timestamp));
|
|
975
|
+
if (apply) {
|
|
976
|
+
for (const row of rows) {
|
|
977
|
+
this.moveJobToDead(row, "Expired runs job has no matching pibo_runs record.", "orphan_run_job", timestamp);
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
return { checkedAt: timestamp, apply, candidates, moved: apply ? rows.length : 0 };
|
|
981
|
+
}
|
|
886
982
|
moveExpiredJobs(timestamp) {
|
|
887
983
|
const expired = this.db.prepare("SELECT * FROM pibo_jobs WHERE expires_at IS NOT NULL AND expires_at <= ?").all(timestamp);
|
|
888
984
|
for (const row of expired)
|
|
@@ -23,8 +23,8 @@ export function buildCompactTerminalRows(traceView, options) {
|
|
|
23
23
|
const candidates = syncThinkingToolRows(flatNodes.map((item) => createRowCandidate(item.node, item.turnId)));
|
|
24
24
|
applyCompletedTurnTiming(candidates, turnById);
|
|
25
25
|
const reconciled = reconcileConceptualRowCandidates(candidates);
|
|
26
|
-
const rows =
|
|
27
|
-
? groupRelatedToolCandidates(reconciled).map((candidate) => candidate.row)
|
|
26
|
+
const rows = (options.toolDisplayMode ?? "default") === "default"
|
|
27
|
+
? groupRelatedToolCandidates(reconciled, showToolDebugMetrics).map((candidate) => candidate.row)
|
|
28
28
|
: reconciled.map((candidate) => candidate.row);
|
|
29
29
|
return applyToolDisplayMode(rows, options.toolDisplayMode ?? "default");
|
|
30
30
|
}
|
|
@@ -942,12 +942,12 @@ function isThinkingOutput(value) {
|
|
|
942
942
|
function isThinkingLevelSetOutput(value) {
|
|
943
943
|
return isRecord(value) && value.action === "set_thinking_level";
|
|
944
944
|
}
|
|
945
|
-
function groupRelatedToolCandidates(candidates) {
|
|
945
|
+
function groupRelatedToolCandidates(candidates, imagesOnly = false) {
|
|
946
946
|
const grouped = [];
|
|
947
947
|
for (let index = 0; index < candidates.length; index += 1) {
|
|
948
948
|
const candidate = candidates[index];
|
|
949
949
|
const groupKind = candidateGroupKind(candidate);
|
|
950
|
-
if (!groupKind) {
|
|
950
|
+
if (!groupKind || (imagesOnly && groupKind !== "images")) {
|
|
951
951
|
grouped.push(candidate);
|
|
952
952
|
continue;
|
|
953
953
|
}
|
|
@@ -1028,21 +1028,20 @@ function createImageGroup(candidates) {
|
|
|
1028
1028
|
const omittedDetailCount = Math.max(0, detailItems.length - visibleDetailItems.length);
|
|
1029
1029
|
const firstRow = candidates[0]?.row;
|
|
1030
1030
|
const firstId = firstRow?.id ?? "images";
|
|
1031
|
-
const lastId = candidates[candidates.length - 1]?.row.id ?? firstId;
|
|
1032
1031
|
const status = candidates.some((candidate) => candidate.row.status === "running")
|
|
1033
1032
|
? "running"
|
|
1034
1033
|
: candidates.some((candidate) => candidate.row.status === "error")
|
|
1035
1034
|
? "error"
|
|
1036
1035
|
: "done";
|
|
1037
1036
|
return {
|
|
1038
|
-
id:
|
|
1037
|
+
id: firstId,
|
|
1039
1038
|
kind: "tool.group.images",
|
|
1040
1039
|
status,
|
|
1041
1040
|
errorKind: status === "error" ? "tool" : undefined,
|
|
1042
1041
|
lines: [
|
|
1043
1042
|
{
|
|
1044
1043
|
prefix: "bullet",
|
|
1045
|
-
tokens: [token(status === "running" ?
|
|
1044
|
+
tokens: [token(status === "running" ? `Viewing ${detailItems.length} images` : status === "error" ? `${detailItems.length} image reads · error` : `${detailItems.length} ${detailItems.length === 1 ? "Image Viewed" : "Images Viewed"}`, toneForStatus(status), "semibold")],
|
|
1046
1045
|
},
|
|
1047
1046
|
...visibleDetailItems.map((item, index) => ({
|
|
1048
1047
|
prefix: index === 0 ? "detail" : "continuation",
|
|
@@ -1062,7 +1061,7 @@ function createImageGroup(candidates) {
|
|
|
1062
1061
|
orderStreamFrameIndex: firstRow?.orderStreamFrameIndex,
|
|
1063
1062
|
detailItems,
|
|
1064
1063
|
expandable: detailItems.some((item) => item.toolCallReference || item.input !== undefined || item.output !== undefined || Boolean(item.error)),
|
|
1065
|
-
imagePreviews: detailItems.flatMap((item) => item.imagePreviews ?? [])
|
|
1064
|
+
imagePreviews: detailItems.flatMap((item) => item.imagePreviews ?? []),
|
|
1066
1065
|
previewOmission: omittedDetailCount > 0 ? {
|
|
1067
1066
|
source: "details",
|
|
1068
1067
|
visibleLineCount: visibleDetailItems.length,
|
|
@@ -227,21 +227,21 @@ export class PiboDataSessionStore {
|
|
|
227
227
|
if (typeof index !== "number" || !Number.isSafeInteger(index) || index < 0)
|
|
228
228
|
continue;
|
|
229
229
|
maximum = Math.max(maximum, index);
|
|
230
|
-
persistedParts.push({ index, attributes });
|
|
230
|
+
persistedParts.push({ index, type: row.type, attributes });
|
|
231
231
|
}
|
|
232
|
-
//
|
|
233
|
-
//
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
232
|
+
// A completed turn or an exact terminal-part fingerprint is durable
|
|
233
|
+
// evidence for replay. Nonterminal parts remain unattached while the turn
|
|
234
|
+
// is open because another producer may still own them.
|
|
235
|
+
const matchesReplay = ({ type, attributes }) => (turnCompleted || outputPartEventIsTerminal(type))
|
|
236
|
+
&& (attributes.outputPartFingerprint === input.fingerprint
|
|
237
|
+
|| attributes.identityFingerprint === input.identityFingerprint);
|
|
238
|
+
const replay = (input.suppliedIndex === undefined
|
|
239
|
+
? undefined
|
|
240
|
+
: persistedParts.find((part) => part.index === input.suppliedIndex && matchesReplay(part)))
|
|
241
|
+
?? (turnCompleted ? persistedParts.find(matchesReplay) : undefined);
|
|
242
|
+
if (replay) {
|
|
243
|
+
this.observeOutputPartIndex(input, replay.index);
|
|
244
|
+
return replay.index;
|
|
245
245
|
}
|
|
246
246
|
const minimum = Math.max(input.proposedIndex, maximum + 1);
|
|
247
247
|
const row = this.db.prepare(`
|
|
@@ -538,6 +538,7 @@ export class PiboDataSessionStore {
|
|
|
538
538
|
actorId: session.id,
|
|
539
539
|
event,
|
|
540
540
|
createdAt: at,
|
|
541
|
+
persistenceProvenance: { producer: "runtime-recovery", projection: "product-history", phase: "startup-recovery" },
|
|
541
542
|
});
|
|
542
543
|
results.push({
|
|
543
544
|
turnId: recovered.turn.turnId,
|
|
@@ -726,6 +727,9 @@ function outputPartEventTypes(kind) {
|
|
|
726
727
|
case "compaction": return ["compaction_start", "compaction_end"];
|
|
727
728
|
}
|
|
728
729
|
}
|
|
730
|
+
function outputPartEventIsTerminal(type) {
|
|
731
|
+
return type === "assistant_message" || type === "thinking_finished" || type === "assistant_usage" || type === "compaction_end";
|
|
732
|
+
}
|
|
729
733
|
function parseJsonObject(json) {
|
|
730
734
|
if (!json)
|
|
731
735
|
return {};
|
|
@@ -539,15 +539,25 @@ export function latestTraceStreamId(events, initial) {
|
|
|
539
539
|
function attachModelInferenceToLatestOutput(nodes, byId, event, storedEvent) {
|
|
540
540
|
const eventId = event.eventId;
|
|
541
541
|
const candidates = flattenTraceNodes(nodes)
|
|
542
|
-
.filter((node) => node.eventId === eventId && traceNodeStartedBeforeInference(node, storedEvent) && (node.type === "assistant.message"
|
|
542
|
+
.filter((node) => node.eventId === eventId && (event.inferenceTarget || traceNodeStartedBeforeInference(node, storedEvent)) && (node.type === "assistant.message"
|
|
543
543
|
|| node.type === "model.reasoning"
|
|
544
544
|
|| node.type === "tool.call"
|
|
545
545
|
|| node.type === "agent.delegation"))
|
|
546
546
|
.sort(compareTraceNodes);
|
|
547
|
-
const
|
|
547
|
+
const anchor = event.inferenceTarget;
|
|
548
|
+
const turnNode = eventId ? byId.get(messageTurnNodeId(eventId)) : undefined;
|
|
549
|
+
const target = anchor
|
|
550
|
+
? (anchor.type === "tool"
|
|
551
|
+
? candidates.find((node) => node.toolCallId === anchor.toolCallId)
|
|
552
|
+
: anchor.type === "assistant"
|
|
553
|
+
? candidates.find((node) => node.stableKey === `assistant:${eventId}:assistant:${anchor.assistantIndex}`)
|
|
554
|
+
: turnNode) ?? turnNode
|
|
555
|
+
: candidates.at(-1) ?? turnNode;
|
|
548
556
|
if (!target)
|
|
549
557
|
return;
|
|
550
|
-
const id =
|
|
558
|
+
const id = event.inferenceId
|
|
559
|
+
? `${eventId ?? storedEvent.piboSessionId}:inference:${event.inferenceId}`
|
|
560
|
+
: eventId ? `${eventId}:usage:${event.usageIndex ?? 0}` : storedEvent.id;
|
|
551
561
|
const record = {
|
|
552
562
|
id,
|
|
553
563
|
completedAt: storedEvent.createdAt,
|
package/dist/web/channel.js
CHANGED
|
@@ -191,16 +191,31 @@ function createGatewayRuntimeStatuses(channelContext) {
|
|
|
191
191
|
}
|
|
192
192
|
});
|
|
193
193
|
}
|
|
194
|
-
function createGatewayStatusResponse(channelContext, options, generation) {
|
|
194
|
+
async function createGatewayStatusResponse(channelContext, options, generation) {
|
|
195
195
|
const mode = gatewayMode(options);
|
|
196
|
+
const appStatuses = {};
|
|
197
|
+
for (const app of channelContext.getWebApps()) {
|
|
198
|
+
if (!app.gatewayStatus)
|
|
199
|
+
continue;
|
|
200
|
+
try {
|
|
201
|
+
Object.assign(appStatuses, await app.gatewayStatus());
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
appStatuses[`${app.name}Status`] = { status: "ambiguous", error: error instanceof Error ? error.message : "Status unavailable" };
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
const durable = appStatuses.durableMessageQueue;
|
|
196
208
|
return responseJson({
|
|
197
|
-
status: "ok",
|
|
209
|
+
status: durable?.status === "degraded" || durable?.status === "ambiguous" ? "degraded" : "ok",
|
|
198
210
|
mode,
|
|
199
211
|
generation,
|
|
200
|
-
health: { status: "ok", mode },
|
|
212
|
+
health: { status: durable?.status === "degraded" || durable?.status === "ambiguous" ? "degraded" : "ok", mode },
|
|
213
|
+
runtimeQueue: { layer: "runtime-session", statuses: createGatewayRuntimeStatuses(channelContext) },
|
|
201
214
|
runtimeStatuses: createGatewayRuntimeStatuses(channelContext),
|
|
202
215
|
...(channelContext.getRuntimeCapacityStatus ? { runtimeCapacity: channelContext.getRuntimeCapacityStatus() } : {}),
|
|
216
|
+
...(channelContext.getRunJobReliabilityStatus ? { reliability: channelContext.getRunJobReliabilityStatus() } : {}),
|
|
203
217
|
activeRuns: collectActiveRuns(channelContext),
|
|
218
|
+
...appStatuses,
|
|
204
219
|
});
|
|
205
220
|
}
|
|
206
221
|
function createCanonicalRedirect(request, canonicalBaseURL) {
|
|
@@ -218,6 +233,68 @@ function createCanonicalRedirect(request, canonicalBaseURL) {
|
|
|
218
233
|
function isEventStreamResponse(response) {
|
|
219
234
|
return response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream") === true;
|
|
220
235
|
}
|
|
236
|
+
function responseCanStart(response) {
|
|
237
|
+
return !response.destroyed && !response.writableEnded && !response.writableFinished && !response.headersSent;
|
|
238
|
+
}
|
|
239
|
+
function responseState(response) {
|
|
240
|
+
return [
|
|
241
|
+
`destroyed=${response.destroyed}`,
|
|
242
|
+
`writableEnded=${response.writableEnded}`,
|
|
243
|
+
`writableFinished=${response.writableFinished}`,
|
|
244
|
+
`headersSent=${response.headersSent}`,
|
|
245
|
+
].join(",");
|
|
246
|
+
}
|
|
247
|
+
function requestPath(request) {
|
|
248
|
+
return (request.url ?? "/").split(/[?#]/, 1)[0].slice(0, 256).replace(/[\r\n]/g, "_");
|
|
249
|
+
}
|
|
250
|
+
function errorIdentity(error) {
|
|
251
|
+
try {
|
|
252
|
+
if (!(error instanceof Error))
|
|
253
|
+
return typeof error;
|
|
254
|
+
const rawName = typeof error.name === "string" ? error.name : "Error";
|
|
255
|
+
const name = rawName.replace(/[^A-Za-z0-9_.-]/g, "").slice(0, 64) || "Error";
|
|
256
|
+
const code = "code" in error && typeof error.code === "string" ? error.code.replace(/[^A-Za-z0-9_-]/g, "").slice(0, 64) : undefined;
|
|
257
|
+
return code ? `${name}:${code}` : name;
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
return "Error";
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
function logHttpBoundaryFailure(phase, request, error, response) {
|
|
264
|
+
try {
|
|
265
|
+
const method = (request.method ?? "UNKNOWN").replace(/[^A-Z]/gi, "").slice(0, 16) || "UNKNOWN";
|
|
266
|
+
const state = response ? ` response={${responseState(response)}}` : "";
|
|
267
|
+
console.error(`[web-host] contained ${phase} failure method=${method} path=${requestPath(request)} error=${errorIdentity(error)}${state}`);
|
|
268
|
+
}
|
|
269
|
+
catch {
|
|
270
|
+
// Diagnostics are best-effort and must never reopen the request rejection path.
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
function terminateResponse(response, error) {
|
|
274
|
+
if (response.destroyed || response.writableEnded || response.writableFinished)
|
|
275
|
+
return;
|
|
276
|
+
try {
|
|
277
|
+
response.destroy(error instanceof Error ? error : undefined);
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
// A broken response implementation must not escape the request boundary.
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
function endUpgradeSocket(socket, statusLine) {
|
|
284
|
+
if (socket.destroyed)
|
|
285
|
+
return;
|
|
286
|
+
try {
|
|
287
|
+
socket.end(statusLine);
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
try {
|
|
291
|
+
socket.destroy();
|
|
292
|
+
}
|
|
293
|
+
catch {
|
|
294
|
+
// A broken socket implementation must not escape the upgrade boundary.
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
}
|
|
221
298
|
async function waitForServerClose(closePromise, timeoutMs) {
|
|
222
299
|
return await new Promise((resolve, reject) => {
|
|
223
300
|
let settled = false;
|
|
@@ -323,7 +400,7 @@ export function createWebHostChannel(options = {}) {
|
|
|
323
400
|
return;
|
|
324
401
|
}
|
|
325
402
|
if (url.pathname === "/gateway/status") {
|
|
326
|
-
await sendResponse(nodeResponse, createGatewayStatusResponse(requireContext(), options, generation));
|
|
403
|
+
await sendResponse(nodeResponse, await createGatewayStatusResponse(requireContext(), options, generation));
|
|
327
404
|
return;
|
|
328
405
|
}
|
|
329
406
|
if (url.pathname.startsWith("/api/auth/")) {
|
|
@@ -360,8 +437,18 @@ export function createWebHostChannel(options = {}) {
|
|
|
360
437
|
}
|
|
361
438
|
catch (error) {
|
|
362
439
|
const status = error instanceof PiboAuthError || error instanceof PiboWebHttpError ? error.statusCode : 500;
|
|
363
|
-
|
|
364
|
-
|
|
440
|
+
logHttpBoundaryFailure("request", nodeRequest, error, nodeResponse);
|
|
441
|
+
if (responseCanStart(nodeResponse)) {
|
|
442
|
+
try {
|
|
443
|
+
await sendResponse(nodeResponse, responseJson({ error: error instanceof Error ? error.message : String(error) }, { status }));
|
|
444
|
+
}
|
|
445
|
+
catch (responseError) {
|
|
446
|
+
logHttpBoundaryFailure("error-response", nodeRequest, responseError, nodeResponse);
|
|
447
|
+
terminateResponse(nodeResponse, responseError);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
else {
|
|
451
|
+
terminateResponse(nodeResponse, error);
|
|
365
452
|
}
|
|
366
453
|
}
|
|
367
454
|
finally {
|
|
@@ -384,10 +471,9 @@ export function createWebHostChannel(options = {}) {
|
|
|
384
471
|
await app.handleUpgrade(nodeRequest, socket, head, createAppContext(ctx), requestURL);
|
|
385
472
|
}
|
|
386
473
|
catch (error) {
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
}
|
|
474
|
+
logHttpBoundaryFailure("upgrade", nodeRequest, error);
|
|
475
|
+
const unauthorized = error instanceof PiboAuthError || error instanceof PiboWebHttpError;
|
|
476
|
+
endUpgradeSocket(socket, `HTTP/1.1 ${unauthorized ? 401 : 502} ${unauthorized ? "Unauthorized" : "Bad Gateway"}\r\nConnection: close\r\n\r\n`);
|
|
391
477
|
}
|
|
392
478
|
};
|
|
393
479
|
return {
|
|
@@ -405,10 +491,26 @@ export function createWebHostChannel(options = {}) {
|
|
|
405
491
|
for (const app of channelContext.getWebApps())
|
|
406
492
|
await app.initialize?.(createAppContext(channelContext));
|
|
407
493
|
server = createServer((request, response) => {
|
|
408
|
-
void handleRequest(request, response)
|
|
494
|
+
void handleRequest(request, response).catch((error) => {
|
|
495
|
+
try {
|
|
496
|
+
logHttpBoundaryFailure("request-terminal", request, error, response);
|
|
497
|
+
terminateResponse(response, error);
|
|
498
|
+
}
|
|
499
|
+
catch {
|
|
500
|
+
// The terminal request boundary itself is intentionally nonthrowing.
|
|
501
|
+
}
|
|
502
|
+
});
|
|
409
503
|
});
|
|
410
504
|
server.on("upgrade", (request, socket, head) => {
|
|
411
|
-
void handleUpgrade(request, socket, head)
|
|
505
|
+
void handleUpgrade(request, socket, head).catch((error) => {
|
|
506
|
+
try {
|
|
507
|
+
logHttpBoundaryFailure("upgrade-terminal", request, error);
|
|
508
|
+
endUpgradeSocket(socket);
|
|
509
|
+
}
|
|
510
|
+
catch {
|
|
511
|
+
// The terminal upgrade boundary itself is intentionally nonthrowing.
|
|
512
|
+
}
|
|
513
|
+
});
|
|
412
514
|
});
|
|
413
515
|
server.on("connection", (socket) => {
|
|
414
516
|
sockets.add(socket);
|