@pasko70/pibo 3.5.0 → 3.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/agent-runtime/routed-session.js +80 -19
  2. package/dist/agent-runtimes/codex-native/turn.js +27 -3
  3. package/dist/apps/chat/data/chat-data-mappers.js +8 -1
  4. package/dist/apps/chat/data/read-state-service.js +24 -3
  5. package/dist/apps/chat/message-command-dispatcher.js +16 -1
  6. package/dist/apps/chat/web-app.js +40 -19
  7. package/dist/apps/chat-ui/assets/{dist-D79vyxSX.js → dist-B-auLrzD.js} +1 -1
  8. package/dist/apps/chat-ui/assets/{dist-DFZ8cwh0.js → dist-BA_dsINH.js} +1 -1
  9. package/dist/apps/chat-ui/assets/{dist-D6TjFhAm.js → dist-eJZar_0-.js} +1 -1
  10. package/dist/apps/chat-ui/assets/{dist-BG0n7zLd.js → dist-wNNR2Bci.js} +1 -1
  11. package/dist/apps/chat-ui/assets/{dist-cOjokPrK.js → dist-zcmEsIEp.js} +1 -1
  12. package/dist/apps/chat-ui/assets/{index-RMHUTJ62.js → index-DEkbN5Vo.js} +43 -43
  13. package/dist/apps/chat-ui/assets/index-DZK6Tzil.css +1 -0
  14. package/dist/apps/chat-ui/index.html +2 -2
  15. package/dist/apps/chat-vscode-web/assets/{index-xacbCyTx.js → index-DZgW1fCB.js} +11 -11
  16. package/dist/apps/chat-vscode-web/index.html +1 -1
  17. package/dist/cli-session/localSessionSource.js +3 -2
  18. package/dist/core/output-render-sequence.js +63 -6
  19. package/dist/core/session-router.js +99 -11
  20. package/dist/data/async-chat-storage.js +7 -2
  21. package/dist/data/bounded-worker-client.js +1 -1
  22. package/dist/data/chat-read-projections.js +4 -4
  23. package/dist/data/chat-storage-worker.js +24 -3
  24. package/dist/data/ingest-service.js +73 -4
  25. package/dist/data/message-command-store.js +153 -11
  26. package/dist/data/schema.js +24 -3
  27. package/dist/data/storage-maintenance.js +344 -0
  28. package/dist/data/storage-verification-worker.js +25 -0
  29. package/dist/debug/index.js +155 -1
  30. package/dist/debug/message-queue.js +108 -0
  31. package/dist/debug/output-collision-repair.js +140 -0
  32. package/dist/debug/output-integrity.js +38 -2
  33. package/dist/debug/output-repair.js +1 -0
  34. package/dist/debug/storage-backup.js +12 -4
  35. package/dist/debug/storage-maintenance.js +78 -0
  36. package/dist/gateway/cli.js +71 -7
  37. package/dist/gateway/server.js +1 -0
  38. package/dist/reliability/store.js +119 -23
  39. package/dist/session-ui/terminalRows.js +7 -8
  40. package/dist/sessions/pibo-data-store.js +18 -14
  41. package/dist/shared/trace-event-projection.js +13 -3
  42. package/dist/web/channel.js +114 -12
  43. package/dist/web/http.js +105 -44
  44. package/npm-shrinkwrap.json +2 -2
  45. package/package.json +1 -1
  46. package/dist/apps/chat-ui/assets/index-hEkrlRk-.css +0 -1
@@ -44,12 +44,14 @@ function ensurePiboJobColumn(db, name, definition) {
44
44
  db.exec(`ALTER TABLE pibo_jobs ADD COLUMN ${name} ${definition}`);
45
45
  }
46
46
  export class PiboReliabilityStore {
47
+ options;
47
48
  db;
48
49
  appendEventStatement;
49
50
  getEventByStreamIdStatement;
50
51
  getEventByTopicEventIdStatement;
51
52
  getEventByTopicIdempotencyKeyStatement;
52
- constructor(path = piboHomePath("pibo-events.sqlite")) {
53
+ constructor(path = piboHomePath("pibo-events.sqlite"), options = {}) {
54
+ this.options = options;
53
55
  const resolvedPath = path === ":memory:" ? path : resolve(path);
54
56
  const insidePiboHome = ensurePrivatePiboHomeForPath(resolvedPath);
55
57
  if (resolvedPath !== ":memory:")
@@ -571,6 +573,38 @@ export class PiboReliabilityStore {
571
573
  }
572
574
  return jobs;
573
575
  }
576
+ inspectJobs(input = {}) {
577
+ const timestamp = now();
578
+ return this.listJobs(input).map((job) => this.inspectJob(job, timestamp));
579
+ }
580
+ reconcileOrphanRunJobs(input) {
581
+ const checkedAt = asDate(input.now).toISOString();
582
+ return this.inImmediateTransaction(() => this.reconcileOrphanRunJobsInTransaction(checkedAt, input.apply));
583
+ }
584
+ getRunJobReliabilityStatus() {
585
+ const timestamp = now();
586
+ const expired = this.db.prepare(`
587
+ SELECT COUNT(*) AS count
588
+ FROM pibo_jobs AS job
589
+ LEFT JOIN pibo_runs AS run ON run.job_id = job.job_id
590
+ WHERE job.queue = 'runs'
591
+ AND job.state = 'running'
592
+ AND job.claim_expires_at IS NOT NULL
593
+ AND job.claim_expires_at <= ?
594
+ AND run.run_id IS NULL
595
+ `).get(timestamp);
596
+ const dead = this.db.prepare(`
597
+ SELECT COUNT(*) AS count FROM pibo_dead_jobs
598
+ WHERE queue = 'runs' AND dead_reason = 'orphan_run_job'
599
+ `).get();
600
+ const expiredOrphanRunJobs = Number(expired.count);
601
+ const orphanRunDeadLetters = Number(dead.count);
602
+ return {
603
+ status: expiredOrphanRunJobs > 0 || orphanRunDeadLetters > 0 ? "degraded" : "ok",
604
+ expiredOrphanRunJobs,
605
+ orphanRunDeadLetters,
606
+ };
607
+ }
574
608
  listRecoverableJobs(input = {}) {
575
609
  const timestamp = now();
576
610
  const clauses = [
@@ -683,28 +717,37 @@ export class PiboReliabilityStore {
683
717
  const runId = input.runId ?? `run_${randomUUID()}`;
684
718
  const maxAttempts = Math.max(1, input.maxAttempts ?? 1);
685
719
  const timeoutAt = input.timeoutMs === undefined ? undefined : new Date(Date.parse(timestamp) + input.timeoutMs).toISOString();
686
- const job = this.enqueue({
687
- queue: "runs",
688
- payload: {
689
- runId,
690
- controllerPiboSessionId: input.controllerPiboSessionId,
691
- toolName: input.toolName,
692
- params: input.params,
693
- timeoutMs: input.timeoutMs,
694
- },
695
- maxAttempts,
720
+ return this.inImmediateTransaction(() => {
721
+ const job = this.enqueue({
722
+ queue: "runs",
723
+ runAt: timestamp,
724
+ payload: {
725
+ runId,
726
+ controllerPiboSessionId: input.controllerPiboSessionId,
727
+ toolName: input.toolName,
728
+ params: input.params,
729
+ timeoutMs: input.timeoutMs,
730
+ },
731
+ maxAttempts,
732
+ });
733
+ this.options.onRunCreationBoundary?.("after_enqueue");
734
+ const claimed = this.claimPendingJobInTransaction(job.jobId, input.workerId ?? `run-registry:${process.pid}`, timestamp, 24 * 60 * 60 * 1000);
735
+ if (!claimed)
736
+ throw new Error(`Failed to claim newly-created run job "${job.jobId}"`);
737
+ this.options.onRunCreationBoundary?.("after_claim");
738
+ this.options.onRunCreationBoundary?.("before_run_insert");
739
+ this.db
740
+ .prepare(`
741
+ INSERT INTO pibo_runs (
742
+ run_id, kind, controller_pibo_session_id, status, completion_policy, consumed, tool_name,
743
+ summary, result_json, error, notified_status, acknowledged_status, created_at, updated_at,
744
+ completed_at, job_id, retryable, max_attempts, timeout_ms, timeout_at, timeout_phase, service_warning, resource_json, origin_json
745
+ ) VALUES (?, 'tool', ?, 'running', ?, 0, ?, ?, NULL, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?, ?, ?, NULL, ?, ?, ?)
746
+ `)
747
+ .run(runId, input.controllerPiboSessionId, input.completionPolicy, input.toolName, `${input.toolName} run is running.`, timestamp, timestamp, job.jobId, input.retryable ? 1 : 0, maxAttempts, input.timeoutMs ?? null, timeoutAt ?? null, input.serviceWarning ?? null, input.resources ? JSON.stringify(input.resources) : null, input.origin ? JSON.stringify(input.origin) : null);
748
+ this.options.onRunCreationBoundary?.("after_run_insert");
749
+ return this.requireRun(runId);
696
750
  });
697
- this.db
698
- .prepare(`
699
- INSERT INTO pibo_runs (
700
- run_id, kind, controller_pibo_session_id, status, completion_policy, consumed, tool_name,
701
- summary, result_json, error, notified_status, acknowledged_status, created_at, updated_at,
702
- completed_at, job_id, retryable, max_attempts, timeout_ms, timeout_at, timeout_phase, service_warning, resource_json, origin_json
703
- ) VALUES (?, 'tool', ?, 'running', ?, 0, ?, ?, NULL, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?, ?, ?, NULL, ?, ?, ?)
704
- `)
705
- .run(runId, input.controllerPiboSessionId, input.completionPolicy, input.toolName, `${input.toolName} run is running.`, timestamp, timestamp, job.jobId, input.retryable ? 1 : 0, maxAttempts, input.timeoutMs ?? null, timeoutAt ?? null, input.serviceWarning ?? null, input.resources ? JSON.stringify(input.resources) : null, input.origin ? JSON.stringify(input.origin) : null);
706
- this.claimJob(job.jobId, input.workerId ?? `run-registry:${process.pid}`, 24 * 60 * 60 * 1000);
707
- return this.requireRun(runId);
708
751
  }
709
752
  updateRun(runId, patch) {
710
753
  const existing = this.getRun(runId);
@@ -779,9 +822,10 @@ export class PiboReliabilityStore {
779
822
  }
780
823
  recoverInterruptedRuns(workerId = `run-registry:${process.pid}`) {
781
824
  return this.inImmediateTransaction(() => {
825
+ const timestamp = now();
826
+ this.reconcileOrphanRunJobsInTransaction(timestamp, true);
782
827
  const rows = this.db.prepare("SELECT * FROM pibo_runs WHERE status = 'running'").all();
783
828
  const recovered = [];
784
- const timestamp = now();
785
829
  for (const row of rows) {
786
830
  if (row.job_id && this.hasUnexpiredJobClaim(row.job_id, timestamp, workerId))
787
831
  continue;
@@ -883,6 +927,58 @@ export class PiboReliabilityStore {
883
927
  `)
884
928
  .get(jobId, workerId, claimToken ?? null, claimToken ?? null, timestamp);
885
929
  }
930
+ claimPendingJobInTransaction(jobId, workerId, timestamp, visibilityTimeoutMs) {
931
+ const claimExpiresAt = new Date(Date.parse(timestamp) + visibilityTimeoutMs).toISOString();
932
+ return this.db.prepare(`
933
+ UPDATE pibo_jobs
934
+ SET state = 'running',
935
+ worker_id = ?,
936
+ claim_expires_at = ?,
937
+ claim_token = claim_token + 1,
938
+ attempts = attempts + 1,
939
+ updated_at = ?
940
+ WHERE job_id = ?
941
+ AND state = 'pending'
942
+ AND run_at <= ?
943
+ AND attempts < max_attempts
944
+ AND (expires_at IS NULL OR expires_at > ?)
945
+ RETURNING *
946
+ `).get(workerId, claimExpiresAt, timestamp, jobId, timestamp, timestamp);
947
+ }
948
+ inspectJob(job, timestamp) {
949
+ const claimExpired = job.state === "running"
950
+ && (!job.claimExpiresAt || job.claimExpiresAt <= timestamp);
951
+ const missingRunRecord = job.queue === "runs"
952
+ && this.db.prepare("SELECT 1 AS found FROM pibo_runs WHERE job_id = ? LIMIT 1").get(job.jobId) === undefined;
953
+ let effectiveLiveness;
954
+ if (job.state === "pending")
955
+ effectiveLiveness = missingRunRecord ? "orphan_pending" : "pending";
956
+ else if (claimExpired)
957
+ effectiveLiveness = missingRunRecord ? "expired_orphan" : "expired_claim";
958
+ else
959
+ effectiveLiveness = missingRunRecord ? "live_orphan_claim" : "live_claim";
960
+ return { ...job, claimExpired, missingRunRecord, effectiveLiveness };
961
+ }
962
+ reconcileOrphanRunJobsInTransaction(timestamp, apply) {
963
+ const rows = this.db.prepare(`
964
+ SELECT job.*
965
+ FROM pibo_jobs AS job
966
+ LEFT JOIN pibo_runs AS run ON run.job_id = job.job_id
967
+ WHERE job.queue = 'runs'
968
+ AND job.state = 'running'
969
+ AND job.claim_expires_at IS NOT NULL
970
+ AND job.claim_expires_at <= ?
971
+ AND run.run_id IS NULL
972
+ ORDER BY job.created_at ASC, job.job_id ASC
973
+ `).all(timestamp);
974
+ const candidates = rows.map((row) => this.inspectJob(jobFromRow(row), timestamp));
975
+ if (apply) {
976
+ for (const row of rows) {
977
+ this.moveJobToDead(row, "Expired runs job has no matching pibo_runs record.", "orphan_run_job", timestamp);
978
+ }
979
+ }
980
+ return { checkedAt: timestamp, apply, candidates, moved: apply ? rows.length : 0 };
981
+ }
886
982
  moveExpiredJobs(timestamp) {
887
983
  const expired = this.db.prepare("SELECT * FROM pibo_jobs WHERE expires_at IS NOT NULL AND expires_at <= ?").all(timestamp);
888
984
  for (const row of expired)
@@ -23,8 +23,8 @@ export function buildCompactTerminalRows(traceView, options) {
23
23
  const candidates = syncThinkingToolRows(flatNodes.map((item) => createRowCandidate(item.node, item.turnId)));
24
24
  applyCompletedTurnTiming(candidates, turnById);
25
25
  const reconciled = reconcileConceptualRowCandidates(candidates);
26
- const rows = !showToolDebugMetrics && (options.toolDisplayMode ?? "default") === "default"
27
- ? groupRelatedToolCandidates(reconciled).map((candidate) => candidate.row)
26
+ const rows = (options.toolDisplayMode ?? "default") === "default"
27
+ ? groupRelatedToolCandidates(reconciled, showToolDebugMetrics).map((candidate) => candidate.row)
28
28
  : reconciled.map((candidate) => candidate.row);
29
29
  return applyToolDisplayMode(rows, options.toolDisplayMode ?? "default");
30
30
  }
@@ -942,12 +942,12 @@ function isThinkingOutput(value) {
942
942
  function isThinkingLevelSetOutput(value) {
943
943
  return isRecord(value) && value.action === "set_thinking_level";
944
944
  }
945
- function groupRelatedToolCandidates(candidates) {
945
+ function groupRelatedToolCandidates(candidates, imagesOnly = false) {
946
946
  const grouped = [];
947
947
  for (let index = 0; index < candidates.length; index += 1) {
948
948
  const candidate = candidates[index];
949
949
  const groupKind = candidateGroupKind(candidate);
950
- if (!groupKind) {
950
+ if (!groupKind || (imagesOnly && groupKind !== "images")) {
951
951
  grouped.push(candidate);
952
952
  continue;
953
953
  }
@@ -1028,21 +1028,20 @@ function createImageGroup(candidates) {
1028
1028
  const omittedDetailCount = Math.max(0, detailItems.length - visibleDetailItems.length);
1029
1029
  const firstRow = candidates[0]?.row;
1030
1030
  const firstId = firstRow?.id ?? "images";
1031
- const lastId = candidates[candidates.length - 1]?.row.id ?? firstId;
1032
1031
  const status = candidates.some((candidate) => candidate.row.status === "running")
1033
1032
  ? "running"
1034
1033
  : candidates.some((candidate) => candidate.row.status === "error")
1035
1034
  ? "error"
1036
1035
  : "done";
1037
1036
  return {
1038
- id: `group:images:${firstId}:${lastId}`,
1037
+ id: firstId,
1039
1038
  kind: "tool.group.images",
1040
1039
  status,
1041
1040
  errorKind: status === "error" ? "tool" : undefined,
1042
1041
  lines: [
1043
1042
  {
1044
1043
  prefix: "bullet",
1045
- tokens: [token(status === "running" ? "Viewing images" : status === "error" ? "Image reads failed" : "Viewed images", toneForStatus(status), "semibold")],
1044
+ tokens: [token(status === "running" ? `Viewing ${detailItems.length} images` : status === "error" ? `${detailItems.length} image reads · error` : `${detailItems.length} ${detailItems.length === 1 ? "Image Viewed" : "Images Viewed"}`, toneForStatus(status), "semibold")],
1046
1045
  },
1047
1046
  ...visibleDetailItems.map((item, index) => ({
1048
1047
  prefix: index === 0 ? "detail" : "continuation",
@@ -1062,7 +1061,7 @@ function createImageGroup(candidates) {
1062
1061
  orderStreamFrameIndex: firstRow?.orderStreamFrameIndex,
1063
1062
  detailItems,
1064
1063
  expandable: detailItems.some((item) => item.toolCallReference || item.input !== undefined || item.output !== undefined || Boolean(item.error)),
1065
- imagePreviews: detailItems.flatMap((item) => item.imagePreviews ?? []).slice(0, MAX_COMPACT_TERMINAL_IMAGE_PREVIEWS),
1064
+ imagePreviews: detailItems.flatMap((item) => item.imagePreviews ?? []),
1066
1065
  previewOmission: omittedDetailCount > 0 ? {
1067
1066
  source: "details",
1068
1067
  visibleLineCount: visibleDetailItems.length,
@@ -227,21 +227,21 @@ export class PiboDataSessionStore {
227
227
  if (typeof index !== "number" || !Number.isSafeInteger(index) || index < 0)
228
228
  continue;
229
229
  maximum = Math.max(maximum, index);
230
- persistedParts.push({ index, attributes });
230
+ persistedParts.push({ index, type: row.type, attributes });
231
231
  }
232
- // An unfinished durable part may still belong to another process. Only a
233
- // completed turn provides enough evidence to reattach an exact replay.
234
- if (turnCompleted) {
235
- const matchesReplay = ({ attributes }) => attributes.outputPartFingerprint === input.fingerprint
236
- || attributes.identityFingerprint === input.identityFingerprint;
237
- const replay = (input.suppliedIndex === undefined
238
- ? undefined
239
- : persistedParts.find((part) => part.index === input.suppliedIndex && matchesReplay(part)))
240
- ?? persistedParts.find(matchesReplay);
241
- if (replay) {
242
- this.observeOutputPartIndex(input, replay.index);
243
- return replay.index;
244
- }
232
+ // A completed turn or an exact terminal-part fingerprint is durable
233
+ // evidence for replay. Nonterminal parts remain unattached while the turn
234
+ // is open because another producer may still own them.
235
+ const matchesReplay = ({ type, attributes }) => (turnCompleted || outputPartEventIsTerminal(type))
236
+ && (attributes.outputPartFingerprint === input.fingerprint
237
+ || attributes.identityFingerprint === input.identityFingerprint);
238
+ const replay = (input.suppliedIndex === undefined
239
+ ? undefined
240
+ : persistedParts.find((part) => part.index === input.suppliedIndex && matchesReplay(part)))
241
+ ?? (turnCompleted ? persistedParts.find(matchesReplay) : undefined);
242
+ if (replay) {
243
+ this.observeOutputPartIndex(input, replay.index);
244
+ return replay.index;
245
245
  }
246
246
  const minimum = Math.max(input.proposedIndex, maximum + 1);
247
247
  const row = this.db.prepare(`
@@ -538,6 +538,7 @@ export class PiboDataSessionStore {
538
538
  actorId: session.id,
539
539
  event,
540
540
  createdAt: at,
541
+ persistenceProvenance: { producer: "runtime-recovery", projection: "product-history", phase: "startup-recovery" },
541
542
  });
542
543
  results.push({
543
544
  turnId: recovered.turn.turnId,
@@ -726,6 +727,9 @@ function outputPartEventTypes(kind) {
726
727
  case "compaction": return ["compaction_start", "compaction_end"];
727
728
  }
728
729
  }
730
+ function outputPartEventIsTerminal(type) {
731
+ return type === "assistant_message" || type === "thinking_finished" || type === "assistant_usage" || type === "compaction_end";
732
+ }
729
733
  function parseJsonObject(json) {
730
734
  if (!json)
731
735
  return {};
@@ -539,15 +539,25 @@ export function latestTraceStreamId(events, initial) {
539
539
  function attachModelInferenceToLatestOutput(nodes, byId, event, storedEvent) {
540
540
  const eventId = event.eventId;
541
541
  const candidates = flattenTraceNodes(nodes)
542
- .filter((node) => node.eventId === eventId && traceNodeStartedBeforeInference(node, storedEvent) && (node.type === "assistant.message"
542
+ .filter((node) => node.eventId === eventId && (event.inferenceTarget || traceNodeStartedBeforeInference(node, storedEvent)) && (node.type === "assistant.message"
543
543
  || node.type === "model.reasoning"
544
544
  || node.type === "tool.call"
545
545
  || node.type === "agent.delegation"))
546
546
  .sort(compareTraceNodes);
547
- const target = candidates.at(-1) ?? (eventId ? byId.get(messageTurnNodeId(eventId)) : undefined);
547
+ const anchor = event.inferenceTarget;
548
+ const turnNode = eventId ? byId.get(messageTurnNodeId(eventId)) : undefined;
549
+ const target = anchor
550
+ ? (anchor.type === "tool"
551
+ ? candidates.find((node) => node.toolCallId === anchor.toolCallId)
552
+ : anchor.type === "assistant"
553
+ ? candidates.find((node) => node.stableKey === `assistant:${eventId}:assistant:${anchor.assistantIndex}`)
554
+ : turnNode) ?? turnNode
555
+ : candidates.at(-1) ?? turnNode;
548
556
  if (!target)
549
557
  return;
550
- const id = eventId ? `${eventId}:usage:${event.usageIndex ?? 0}` : storedEvent.id;
558
+ const id = event.inferenceId
559
+ ? `${eventId ?? storedEvent.piboSessionId}:inference:${event.inferenceId}`
560
+ : eventId ? `${eventId}:usage:${event.usageIndex ?? 0}` : storedEvent.id;
551
561
  const record = {
552
562
  id,
553
563
  completedAt: storedEvent.createdAt,
@@ -191,16 +191,31 @@ function createGatewayRuntimeStatuses(channelContext) {
191
191
  }
192
192
  });
193
193
  }
194
- function createGatewayStatusResponse(channelContext, options, generation) {
194
+ async function createGatewayStatusResponse(channelContext, options, generation) {
195
195
  const mode = gatewayMode(options);
196
+ const appStatuses = {};
197
+ for (const app of channelContext.getWebApps()) {
198
+ if (!app.gatewayStatus)
199
+ continue;
200
+ try {
201
+ Object.assign(appStatuses, await app.gatewayStatus());
202
+ }
203
+ catch (error) {
204
+ appStatuses[`${app.name}Status`] = { status: "ambiguous", error: error instanceof Error ? error.message : "Status unavailable" };
205
+ }
206
+ }
207
+ const durable = appStatuses.durableMessageQueue;
196
208
  return responseJson({
197
- status: "ok",
209
+ status: durable?.status === "degraded" || durable?.status === "ambiguous" ? "degraded" : "ok",
198
210
  mode,
199
211
  generation,
200
- health: { status: "ok", mode },
212
+ health: { status: durable?.status === "degraded" || durable?.status === "ambiguous" ? "degraded" : "ok", mode },
213
+ runtimeQueue: { layer: "runtime-session", statuses: createGatewayRuntimeStatuses(channelContext) },
201
214
  runtimeStatuses: createGatewayRuntimeStatuses(channelContext),
202
215
  ...(channelContext.getRuntimeCapacityStatus ? { runtimeCapacity: channelContext.getRuntimeCapacityStatus() } : {}),
216
+ ...(channelContext.getRunJobReliabilityStatus ? { reliability: channelContext.getRunJobReliabilityStatus() } : {}),
203
217
  activeRuns: collectActiveRuns(channelContext),
218
+ ...appStatuses,
204
219
  });
205
220
  }
206
221
  function createCanonicalRedirect(request, canonicalBaseURL) {
@@ -218,6 +233,68 @@ function createCanonicalRedirect(request, canonicalBaseURL) {
218
233
  function isEventStreamResponse(response) {
219
234
  return response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream") === true;
220
235
  }
236
+ function responseCanStart(response) {
237
+ return !response.destroyed && !response.writableEnded && !response.writableFinished && !response.headersSent;
238
+ }
239
+ function responseState(response) {
240
+ return [
241
+ `destroyed=${response.destroyed}`,
242
+ `writableEnded=${response.writableEnded}`,
243
+ `writableFinished=${response.writableFinished}`,
244
+ `headersSent=${response.headersSent}`,
245
+ ].join(",");
246
+ }
247
+ function requestPath(request) {
248
+ return (request.url ?? "/").split(/[?#]/, 1)[0].slice(0, 256).replace(/[\r\n]/g, "_");
249
+ }
250
+ function errorIdentity(error) {
251
+ try {
252
+ if (!(error instanceof Error))
253
+ return typeof error;
254
+ const rawName = typeof error.name === "string" ? error.name : "Error";
255
+ const name = rawName.replace(/[^A-Za-z0-9_.-]/g, "").slice(0, 64) || "Error";
256
+ const code = "code" in error && typeof error.code === "string" ? error.code.replace(/[^A-Za-z0-9_-]/g, "").slice(0, 64) : undefined;
257
+ return code ? `${name}:${code}` : name;
258
+ }
259
+ catch {
260
+ return "Error";
261
+ }
262
+ }
263
+ function logHttpBoundaryFailure(phase, request, error, response) {
264
+ try {
265
+ const method = (request.method ?? "UNKNOWN").replace(/[^A-Z]/gi, "").slice(0, 16) || "UNKNOWN";
266
+ const state = response ? ` response={${responseState(response)}}` : "";
267
+ console.error(`[web-host] contained ${phase} failure method=${method} path=${requestPath(request)} error=${errorIdentity(error)}${state}`);
268
+ }
269
+ catch {
270
+ // Diagnostics are best-effort and must never reopen the request rejection path.
271
+ }
272
+ }
273
+ function terminateResponse(response, error) {
274
+ if (response.destroyed || response.writableEnded || response.writableFinished)
275
+ return;
276
+ try {
277
+ response.destroy(error instanceof Error ? error : undefined);
278
+ }
279
+ catch {
280
+ // A broken response implementation must not escape the request boundary.
281
+ }
282
+ }
283
+ function endUpgradeSocket(socket, statusLine) {
284
+ if (socket.destroyed)
285
+ return;
286
+ try {
287
+ socket.end(statusLine);
288
+ }
289
+ catch {
290
+ try {
291
+ socket.destroy();
292
+ }
293
+ catch {
294
+ // A broken socket implementation must not escape the upgrade boundary.
295
+ }
296
+ }
297
+ }
221
298
  async function waitForServerClose(closePromise, timeoutMs) {
222
299
  return await new Promise((resolve, reject) => {
223
300
  let settled = false;
@@ -323,7 +400,7 @@ export function createWebHostChannel(options = {}) {
323
400
  return;
324
401
  }
325
402
  if (url.pathname === "/gateway/status") {
326
- await sendResponse(nodeResponse, createGatewayStatusResponse(requireContext(), options, generation));
403
+ await sendResponse(nodeResponse, await createGatewayStatusResponse(requireContext(), options, generation));
327
404
  return;
328
405
  }
329
406
  if (url.pathname.startsWith("/api/auth/")) {
@@ -360,8 +437,18 @@ export function createWebHostChannel(options = {}) {
360
437
  }
361
438
  catch (error) {
362
439
  const status = error instanceof PiboAuthError || error instanceof PiboWebHttpError ? error.statusCode : 500;
363
- if (!nodeResponse.destroyed) {
364
- await sendResponse(nodeResponse, responseJson({ error: error instanceof Error ? error.message : String(error) }, { status }));
440
+ logHttpBoundaryFailure("request", nodeRequest, error, nodeResponse);
441
+ if (responseCanStart(nodeResponse)) {
442
+ try {
443
+ await sendResponse(nodeResponse, responseJson({ error: error instanceof Error ? error.message : String(error) }, { status }));
444
+ }
445
+ catch (responseError) {
446
+ logHttpBoundaryFailure("error-response", nodeRequest, responseError, nodeResponse);
447
+ terminateResponse(nodeResponse, responseError);
448
+ }
449
+ }
450
+ else {
451
+ terminateResponse(nodeResponse, error);
365
452
  }
366
453
  }
367
454
  finally {
@@ -384,10 +471,9 @@ export function createWebHostChannel(options = {}) {
384
471
  await app.handleUpgrade(nodeRequest, socket, head, createAppContext(ctx), requestURL);
385
472
  }
386
473
  catch (error) {
387
- if (!socket.destroyed) {
388
- const unauthorized = error instanceof PiboAuthError || error instanceof PiboWebHttpError;
389
- socket.end(`HTTP/1.1 ${unauthorized ? 401 : 502} ${unauthorized ? "Unauthorized" : "Bad Gateway"}\r\nConnection: close\r\n\r\n`);
390
- }
474
+ logHttpBoundaryFailure("upgrade", nodeRequest, error);
475
+ const unauthorized = error instanceof PiboAuthError || error instanceof PiboWebHttpError;
476
+ endUpgradeSocket(socket, `HTTP/1.1 ${unauthorized ? 401 : 502} ${unauthorized ? "Unauthorized" : "Bad Gateway"}\r\nConnection: close\r\n\r\n`);
391
477
  }
392
478
  };
393
479
  return {
@@ -405,10 +491,26 @@ export function createWebHostChannel(options = {}) {
405
491
  for (const app of channelContext.getWebApps())
406
492
  await app.initialize?.(createAppContext(channelContext));
407
493
  server = createServer((request, response) => {
408
- void handleRequest(request, response);
494
+ void handleRequest(request, response).catch((error) => {
495
+ try {
496
+ logHttpBoundaryFailure("request-terminal", request, error, response);
497
+ terminateResponse(response, error);
498
+ }
499
+ catch {
500
+ // The terminal request boundary itself is intentionally nonthrowing.
501
+ }
502
+ });
409
503
  });
410
504
  server.on("upgrade", (request, socket, head) => {
411
- void handleUpgrade(request, socket, head);
505
+ void handleUpgrade(request, socket, head).catch((error) => {
506
+ try {
507
+ logHttpBoundaryFailure("upgrade-terminal", request, error);
508
+ endUpgradeSocket(socket);
509
+ }
510
+ catch {
511
+ // The terminal upgrade boundary itself is intentionally nonthrowing.
512
+ }
513
+ });
412
514
  });
413
515
  server.on("connection", (socket) => {
414
516
  sockets.add(socket);