@sema-agent/server 7.86.0 → 7.87.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/USAGE.md +1 -1
  2. package/dist/approval-hmac.d.ts +62 -14
  3. package/dist/approval-hmac.js +15 -1
  4. package/dist/boot/stage-01-config.js +2 -2
  5. package/dist/fleet/fleet-bus.d.ts +23 -3
  6. package/dist/fleet/fleet-reconciler.d.ts +35 -0
  7. package/dist/fleet/fleet-reconciler.js +50 -7
  8. package/dist/http/approval-sweeps.js +1 -1
  9. package/dist/http/resume-legs.js +3 -2
  10. package/dist/http/routes/a2a-serve.js +1 -1
  11. package/dist/http/routes/approvals-assistant.d.ts +35 -23
  12. package/dist/http/routes/approvals-assistant.js +107 -30
  13. package/dist/http/routes/runs.js +2 -2
  14. package/dist/http/routes/tasks.js +2 -2
  15. package/dist/http/routes/workflows.js +1 -1
  16. package/dist/http/server.d.ts +2 -2
  17. package/dist/http/server.js +3 -3
  18. package/dist/leader/diffout.js +1 -1
  19. package/dist/leader/endpoint.js +1 -1
  20. package/dist/leader/fanout.js +1 -1
  21. package/dist/leader/leader.js +1 -1
  22. package/dist/leader/merge.js +1 -1
  23. package/dist/leader/planner.js +1 -1
  24. package/dist/leader/repair-oracle.js +1 -1
  25. package/dist/leader/repair-wire.js +1 -1
  26. package/dist/leader/wire.js +1 -1
  27. package/dist/observability/fail-open.d.ts +8 -0
  28. package/dist/observability/fail-open.js +8 -0
  29. package/dist/observability/metrics.d.ts +12 -0
  30. package/dist/observability/metrics.js +17 -0
  31. package/dist/observability/{run-terminal-log.d.ts → run-terminal.d.ts} +50 -10
  32. package/dist/observability/{run-terminal-log.js → run-terminal.js} +27 -8
  33. package/dist/plugins/file-run-store.js +2 -2
  34. package/dist/plugins/memory-run-store.js +2 -2
  35. package/dist/plugins/remote-scratchpad.js +1 -1
  36. package/dist/plugins/run-store-sql.js +8 -7
  37. package/dist/plugins/store-contracts.d.ts +2 -2
  38. package/dist/plugins/workflow-run-store-sql.js +1 -1
  39. package/dist/runs.js +1 -1
  40. package/dist/session-titler.js +9 -5
  41. package/dist/tool-approval.d.ts +22 -2
  42. package/dist/tool-approval.js +2 -0
  43. package/dist/trace/core-keyset-guard.d.ts +1 -1
  44. package/dist/trace/ledger-events.d.ts +1 -1
  45. package/dist/trace/project.d.ts +1 -1
  46. package/dist/trace/project.js +1 -1
  47. package/dist/trace/redact.d.ts +1 -1
  48. package/package.json +2 -2
@@ -6,11 +6,11 @@ import { APPROVAL_GATE_KINDS } from "../../tool-approval.js";
6
6
  import { isParkedRunStatus } from "../../plugins/store-contracts.js";
7
7
  import { principalFrom, decodeCheckpointScope, PRINCIPAL_TOKEN_HEADER, APPROVAL_MAC_HEADER, APPROVAL_MAC_KID_HEADER } from "../../security.js";
8
8
  import { editedPlanBinding, isWellFormedUnicode, verifyDirectDoorProof } from "../../principal-jwt.js";
9
- import { MAX_APPROVAL_REASON_CHARS } from "../../approval-hmac.js";
9
+ import { MAX_APPROVAL_REASON_CHARS, projectQuestionAnswer, questionAnswerFromProjection } from "../../approval-hmac.js";
10
10
  import { PLAN_APPROVE_PERMISSION_MODES_AFTER, isPlanApprovePermissionModeAfter, storedBodyPermissionMode } from "../../task-settings.js";
11
11
  import { MAX_APPROVAL_NONCE_CHARS } from "../../plugins/approval-nonce-store.js";
12
12
  import { redactedPreview, redactSecrets } from "../../trace/redact.js";
13
- import { redactRiskDescriptor } from "../../observability/run-terminal-log.js";
13
+ import { redactRiskDescriptor } from "../../observability/run-terminal.js";
14
14
  import { fleetRunLabels } from "../../fleet/fleet-bus.js";
15
15
  import { sleep } from "../sse-log.js";
16
16
  import { bindSseLifecycle } from "../sse-lifecycle.js";
@@ -23,6 +23,8 @@ import { MAX_KEY_ECHO, keyClosureClause, scanKeyClosure } from "../../request-ke
23
23
  const DECIDE_BODY_DECLARED_KEYS = [
24
24
  "answer", "boundCallId", "boundInputHash", "checkpointToken", "decision", "reason", "remember", "updatedInput",
25
25
  ];
26
+ const DECIDE_PROOF_COVERED_KEYS = ["answer", "boundCallId", "boundInputHash", "decision", "reason"];
27
+ const DECIDE_NOT_IN_PROOF_KEYS = ["checkpointToken", "remember", "updatedInput"];
26
28
  const ACCEPTED_DECIDE_BODY = new Set(DECIDE_BODY_DECLARED_KEYS);
27
29
  const UNSUPPORTED_DECIDE_BODY = new Set(["editedPlan", "sessionId", "taskId"]);
28
30
  export function decideBodyKeyIssue(raw) {
@@ -50,6 +52,23 @@ export function decideBodyKeyIssue(raw) {
50
52
  const PLAN_REVIEW_BODY_DECLARED_KEYS = [
51
53
  "boundCallId", "decision", "editedPlan", "permissionModeAfter", "reason",
52
54
  ];
55
+ const PLAN_REVIEW_PROOF_COVERED_KEYS = ["boundCallId", "decision", "editedPlan", "permissionModeAfter", "reason"];
56
+ const PLAN_REVIEW_NOT_IN_PROOF_KEYS = [];
57
+ const PROOF_SLOT_OF = {
58
+ answer: "answer",
59
+ boundCallId: "boundCallId",
60
+ boundInputHash: "boundInputHash",
61
+ decision: "decision",
62
+ editedPlan: "boundInputHash",
63
+ permissionModeAfter: "permissionModeAfter",
64
+ reason: "reason",
65
+ };
66
+ function buildApprovalEnvelope(sessionId, covered, reading) {
67
+ const envelope = { sessionId, decision: reading.decision };
68
+ for (const key of covered)
69
+ Object.assign(envelope, { [PROOF_SLOT_OF[key]]: reading[key] });
70
+ return envelope;
71
+ }
53
72
  const ACCEPTED_PLAN_REVIEW_BODY = new Set(PLAN_REVIEW_BODY_DECLARED_KEYS);
54
73
  const UNSUPPORTED_PLAN_REVIEW_BODY = new Set([
55
74
  "answer", "boundInputHash", "checkpointToken", "permissionMode", "remember", "sessionId", "taskId", "updatedInput",
@@ -79,13 +98,27 @@ export function planReviewBodyKeyIssue(raw) {
79
98
  export function decisionReasonOf(body) {
80
99
  return typeof body.reason === "string" && body.reason !== "" ? body.reason : undefined;
81
100
  }
82
- export function isQuestionAnswer(v) {
101
+ const QUESTION_ANSWER_TOP_KEYS = ["answers"];
102
+ const QUESTION_ANSWER_ITEM_KEYS = ["header", "selected", "note"];
103
+ const ACCEPTED_QUESTION_ANSWER_TOP = new Set(QUESTION_ANSWER_TOP_KEYS);
104
+ const ACCEPTED_QUESTION_ANSWER_ITEM = new Set(QUESTION_ANSWER_ITEM_KEYS);
105
+ const NO_TOLERATED_KEYS = new Set();
106
+ export function checkQuestionAnswer(v) {
83
107
  if (!v || typeof v !== "object" || Array.isArray(v))
84
- return false;
108
+ return { ok: false, kind: "shape" };
85
109
  const a = v.answers;
86
110
  if (!Array.isArray(a) || a.length === 0)
87
- return false;
88
- return a.every((it) => {
111
+ return { ok: false, kind: "shape" };
112
+ const buckets = { unknown: [], unsupported: [] };
113
+ scanKeyClosure(v, "answer.", ACCEPTED_QUESTION_ANSWER_TOP, NO_TOLERATED_KEYS, buckets);
114
+ a.forEach((it, i) => {
115
+ if (!it || typeof it !== "object" || Array.isArray(it))
116
+ return;
117
+ scanKeyClosure(it, `answer.answers[${i}].`, ACCEPTED_QUESTION_ANSWER_ITEM, NO_TOLERATED_KEYS, buckets);
118
+ });
119
+ if (buckets.unknown.length > 0)
120
+ return { ok: false, kind: "unknown_key", keyPaths: buckets.unknown.sort().slice(0, MAX_KEY_ECHO) };
121
+ const shapeOk = a.every((it) => {
89
122
  if (!it || typeof it !== "object" || Array.isArray(it))
90
123
  return false;
91
124
  const { header, selected, note } = it;
@@ -95,6 +128,7 @@ export function isQuestionAnswer(v) {
95
128
  selected.every((s) => typeof s === "string") &&
96
129
  (note === undefined || typeof note === "string"));
97
130
  });
131
+ return shapeOk ? { ok: true, value: v } : { ok: false, kind: "shape" };
98
132
  }
99
133
  async function decideIdempotentReplay(cs, args) {
100
134
  const { sessionId, out, binding, decision, payload, deciderPrincipal, explicitOperator, warn } = args;
@@ -173,7 +207,7 @@ export async function handleApprovalsAssistant(req, res, match, ctx) {
173
207
  const scope = operator
174
208
  ? (new URLSearchParams(match.search).get("owner") ?? undefined)
175
209
  : (principal ?? "__none__");
176
- await streamApprovals(req, res, cs, scope, deps.config);
210
+ await streamApprovals(req, res, cs, deps.toolApproval, scope, deps.config);
177
211
  return;
178
212
  }
179
213
  if (match.decl.id === "approvals-list") {
@@ -464,7 +498,13 @@ export async function handleApprovalsAssistant(req, res, match, ctx) {
464
498
  }
465
499
  const storeNowMs = await store.nowMs();
466
500
  const hdr = (n) => { const h = req.headers[n]; return Array.isArray(h) ? h[0] : h; };
467
- const proof = verifyDirectDoorProof({ jwt: hdr(PRINCIPAL_TOKEN_HEADER), mac: hdr(APPROVAL_MAC_HEADER), kid: hdr(APPROVAL_MAC_KID_HEADER) }, { sessionId: run.sessionId, boundCallId: planNonce ?? null, boundInputHash: decision === "edit" ? editedPlanBinding(body.editedPlan) : null, decision, reason: typeof body.reason === "string" ? body.reason : null, ...(isPlanApprovePermissionModeAfter(body.permissionModeAfter) ? { permissionModeAfter: body.permissionModeAfter } : {}) }, deps.config, { actionBinding: false, nowSec: Math.floor(storeNowMs / 1000) });
501
+ const proof = verifyDirectDoorProof({ jwt: hdr(PRINCIPAL_TOKEN_HEADER), mac: hdr(APPROVAL_MAC_HEADER), kid: hdr(APPROVAL_MAC_KID_HEADER) }, buildApprovalEnvelope(run.sessionId, PLAN_REVIEW_PROOF_COVERED_KEYS, {
502
+ boundCallId: planNonce ?? null,
503
+ decision,
504
+ editedPlan: decision === "edit" ? editedPlanBinding(body.editedPlan) : null,
505
+ permissionModeAfter: isPlanApprovePermissionModeAfter(body.permissionModeAfter) ? body.permissionModeAfter : undefined,
506
+ reason: typeof body.reason === "string" ? body.reason : null,
507
+ }), deps.config, { actionBinding: false, nowSec: Math.floor(storeNowMs / 1000) });
468
508
  if (!proof.ok) {
469
509
  sendError(res, proof.status, proof.errorCode, proof.error);
470
510
  return;
@@ -556,12 +596,19 @@ export async function handleApprovalsAssistant(req, res, match, ctx) {
556
596
  remember = true;
557
597
  }
558
598
  let answer;
599
+ let answerProjection;
559
600
  if (body.answer !== undefined) {
560
- if (!isQuestionAnswer(body.answer)) {
561
- sendError(res, 400, "request.body_shape", "malformed answer — expected { answers: [{ header, selected: string[], note? }] }");
601
+ const checked = checkQuestionAnswer(body.answer);
602
+ if (!checked.ok) {
603
+ if (checked.kind === "shape") {
604
+ sendError(res, 400, "request.body_shape", "malformed answer — expected { answers: [{ header, selected: string[], note? }] }");
605
+ return;
606
+ }
607
+ sendError(res, 400, "request.field_invalid", `answer: unrecognized key(s) — refused rather than dropped, because a dropped key reads as "in effect" to the caller (a misspelled selected key silently drops part of the operator's answer). ${keyClosureClause([...checked.keyPaths], "unknown (not a question-answer key — check the spelling)")}. Accepted: answers, answers[].header, answers[].selected, answers[].note`, { unknownKeys: [...checked.keyPaths] });
562
608
  return;
563
609
  }
564
- answer = body.answer;
610
+ answerProjection = projectQuestionAnswer(checked.value);
611
+ answer = questionAnswerFromProjection(answerProjection);
565
612
  }
566
613
  for (const f of ["checkpointToken", "boundCallId", "boundInputHash"]) {
567
614
  if (body[f] !== undefined && typeof body[f] !== "string") {
@@ -573,6 +620,10 @@ export async function handleApprovalsAssistant(req, res, match, ctx) {
573
620
  sendError(res, 400, "updated_input_not_in_proof", "updatedInput is not supported on a direct-door worker (not covered by the decision proof)");
574
621
  return;
575
622
  }
623
+ if (deps.config.directDoorActive && body.checkpointToken !== undefined) {
624
+ sendError(res, 400, "checkpoint_token_not_in_proof", "checkpointToken is not supported on a direct-door worker (not covered by the decision proof); bind the decision with boundCallId and boundInputHash instead");
625
+ return;
626
+ }
576
627
  const binding = {
577
628
  checkpointToken: body.checkpointToken,
578
629
  boundCallId: body.boundCallId,
@@ -585,7 +636,13 @@ export async function handleApprovalsAssistant(req, res, match, ctx) {
585
636
  const h = req.headers[n];
586
637
  return Array.isArray(h) ? h[0] : h;
587
638
  };
588
- const proof = verifyDirectDoorProof({ jwt: hdr(PRINCIPAL_TOKEN_HEADER), mac: hdr(APPROVAL_MAC_HEADER), kid: hdr(APPROVAL_MAC_KID_HEADER) }, { sessionId, boundCallId: binding.boundCallId, boundInputHash: binding.boundInputHash, decision, reason: typeof body.reason === "string" ? body.reason : null }, deps.config);
639
+ const proof = verifyDirectDoorProof({ jwt: hdr(PRINCIPAL_TOKEN_HEADER), mac: hdr(APPROVAL_MAC_HEADER), kid: hdr(APPROVAL_MAC_KID_HEADER) }, buildApprovalEnvelope(sessionId, DECIDE_PROOF_COVERED_KEYS, {
640
+ answer: answerProjection,
641
+ boundCallId: binding.boundCallId,
642
+ boundInputHash: binding.boundInputHash,
643
+ decision,
644
+ reason: typeof body.reason === "string" ? body.reason : null,
645
+ }), deps.config);
589
646
  if (!proof.ok) {
590
647
  sendError(res, proof.status, proof.errorCode, proof.error);
591
648
  return;
@@ -664,7 +721,24 @@ function projectPendingForWire(row, governance) {
664
721
  const governanceForced = governanceOriginOf({ riskDescriptor: row.riskDescriptor ?? undefined }, governance);
665
722
  return governanceForced ? { ...redacted, governanceForced } : redacted;
666
723
  }
667
- export async function streamApprovals(req, res, cs, scope, governance, pollMs = APPROVALS_STREAM_POLL_MS) {
724
+ function createSseDeltaLane(res, spec) {
725
+ let prev = new Map();
726
+ return {
727
+ tick(rows) {
728
+ const cur = new Map(rows.map((r) => [spec.keyOf(r), { row: r, bytes: spec.bytesOf(r) }]));
729
+ for (const [k, v] of cur) {
730
+ if (prev.get(k)?.bytes !== v.bytes)
731
+ res.write(`event: ${spec.upsertEvent}\ndata: ${JSON.stringify(spec.upsertData(v.row, Date.now()))}\n\n`);
732
+ }
733
+ for (const [k, v] of prev) {
734
+ if (!cur.has(k))
735
+ res.write(`event: ${spec.exitEvent}\ndata: ${JSON.stringify(spec.exitData(v.row, Date.now()))}\n\n`);
736
+ }
737
+ prev = cur;
738
+ },
739
+ };
740
+ }
741
+ export async function streamApprovals(req, res, cs, live, scope, governance, pollMs = APPROVALS_STREAM_POLL_MS) {
668
742
  const keyOf = (p) => JSON.stringify([p.sessionId, p.toolCallId ?? null]);
669
743
  sseHeaders(res);
670
744
  res.write(`event: meta\ndata: ${JSON.stringify({ type: "meta", version: 1, mode: "approvals-delta", pollMs: APPROVALS_STREAM_POLL_MS })}\n\n`);
@@ -672,13 +746,29 @@ export async function streamApprovals(req, res, cs, scope, governance, pollMs =
672
746
  bindSseLifecycle(req, res, () => { closed = true; });
673
747
  const start = Date.now();
674
748
  let lastBeat = Date.now();
675
- let prev = new Map();
676
- let prevFrames = new Map();
749
+ const durableLane = createSseDeltaLane(res, {
750
+ upsertEvent: "pending",
751
+ exitEvent: "resolved",
752
+ keyOf,
753
+ bytesOf: (p) => JSON.stringify({ type: "pending", ...p }),
754
+ upsertData: (p, nowMs) => ({ type: "pending", ...p, serverNowMs: nowMs }),
755
+ exitData: (p, nowMs) => ({ type: "resolved", sessionId: p.sessionId, toolCallId: p.toolCallId, serverNowMs: nowMs }),
756
+ });
757
+ const liveLane = createSseDeltaLane(res, {
758
+ upsertEvent: "live_pending",
759
+ exitEvent: "live_resolved",
760
+ keyOf: (r) => r.approvalId,
761
+ bytesOf: (r) => JSON.stringify({ type: "live_pending", ...r }),
762
+ upsertData: (r, nowMs) => ({ type: "live_pending", ...r, serverNowMs: nowMs }),
763
+ exitData: (r, nowMs) => ({ type: "live_resolved", approvalId: r.approvalId, serverNowMs: nowMs }),
764
+ });
677
765
  let first = true;
678
766
  while (!closed) {
679
767
  let pending;
768
+ let livePending;
680
769
  try {
681
770
  pending = (await cs.listPending(scope)).map((r) => projectPendingForWire(r, governance));
771
+ livePending = live ? live.listLivePending(scope) : [];
682
772
  }
683
773
  catch {
684
774
  if (!res.writableEnded)
@@ -686,25 +776,12 @@ export async function streamApprovals(req, res, cs, scope, governance, pollMs =
686
776
  await sleep(pollMs);
687
777
  continue;
688
778
  }
689
- const cur = new Map(pending.map((p) => [keyOf(p), p]));
690
- const curFrames = new Map([...cur].map(([k, p]) => [k, JSON.stringify({ type: "pending", ...p })]));
691
- const pendingFrame = (k) => JSON.stringify({ type: "pending", ...cur.get(k), serverNowMs: Date.now() });
779
+ durableLane.tick(pending);
780
+ liveLane.tick(livePending);
692
781
  if (first) {
693
- for (const k of curFrames.keys())
694
- res.write(`event: pending\ndata: ${pendingFrame(k)}\n\n`);
695
782
  res.write(`event: synced\ndata: ${JSON.stringify({ type: "synced", count: pending.length })}\n\n`);
696
783
  first = false;
697
784
  }
698
- else {
699
- for (const [k, frame] of curFrames)
700
- if (prevFrames.get(k) !== frame)
701
- res.write(`event: pending\ndata: ${pendingFrame(k)}\n\n`);
702
- for (const [k, p] of prev)
703
- if (!cur.has(k))
704
- res.write(`event: resolved\ndata: ${JSON.stringify({ type: "resolved", sessionId: p.sessionId, toolCallId: p.toolCallId, serverNowMs: Date.now() })}\n\n`);
705
- }
706
- prev = cur;
707
- prevFrames = curFrames;
708
785
  if (Date.now() - start > SSE_MAX_STREAM_MS) {
709
786
  res.write(`event: error\ndata: ${JSON.stringify({ type: "error", errorCode: "STREAM_MAX_DURATION", message: "approvals stream reached its 15-minute cap — reconnect to continue" })}\n\n`);
710
787
  break;
@@ -1,5 +1,5 @@
1
1
  import { persistedPlaneOf } from "../../terminal.js";
2
- import { memoRedactedRead, memoRedactTerminalResult, redactErrorMessage, redactHead, redactLedgerEventData, redactTerminalResult } from "../../observability/run-terminal-log.js";
2
+ import { memoRedactedRead, memoRedactTerminalResult, redactErrorMessage, redactHead, redactLedgerEventData, redactTerminalResult } from "../../observability/run-terminal.js";
3
3
  import { isModelTextLedgerEventType } from "../../trace/ledger-events.js";
4
4
  import { z } from "zod";
5
5
  import { uuidv7, mintCheckpointToken, CheckpointError, validatePendingSteer } from "@sema-agent/core";
@@ -89,7 +89,7 @@ async function subagentReadGate(req, res, ctx, runId, search) {
89
89
  }
90
90
  function sendHandleMiss(res, run, thisInstanceId) {
91
91
  if (isTerminalRunStatus(run.status)) {
92
- sendError(res, 404, "not_found.task_owner_settled", "no such background task handle here. Routing hint about its OWNER (not about this handle): the run has already settled, and background handles live in the replica-local registry for the parent's lifetime, so none of its handles are readable any more. Read the run's own outcome via GET /v1/runs/:id");
92
+ sendError(res, 404, "not_found.task_owner_settled", "no such background task handle here. Routing hint about its OWNER (not about this handle): the run has already settled. Read the run's own outcome via GET /v1/runs/:id");
93
93
  return true;
94
94
  }
95
95
  if (run.status === "running" && runProvenElsewhere(run, thisInstanceId)) {
@@ -1,4 +1,4 @@
1
- import { redactErrorMessage, redactHead, redactTerminalResult } from "../../observability/run-terminal-log.js";
1
+ import { redactErrorMessage, redactHead, redactTerminalResult } from "../../observability/run-terminal.js";
2
2
  import { runWithVerification, runCascade, uuidv7, terminalProjection } from "@sema-agent/core";
3
3
  import { graftParkToolCallId, markChildrenStoppedByUserOnAbort, resumeAtHttpStatus, HEARTBEAT_MS } from "../../runs.js";
4
4
  import { stripResumeToken } from "../../terminal.js";
@@ -414,7 +414,7 @@ export async function handleTasks(req, res, match, ctx) {
414
414
  finalizeTaskResult(ev.result, principal, prepared.spec.objective, prepared.spec.sessionId);
415
415
  noteRunOutcome(finalResult);
416
416
  putRewindAnchor();
417
- settleFleet(finalResult ? terminalProjection(finalResult.terminal).status : "completed", fleetRunResiduals(finalResult));
417
+ settleFleet(terminalProjection(finalResult.terminal).status, fleetRunResiduals(finalResult));
418
418
  if (ledgerSink && finalResult) {
419
419
  if (usageKey && deps.modelUsage && deps.runStore && durableTaskId) {
420
420
  const rs2 = deps.runStore;
@@ -5,7 +5,7 @@ import { redactSteerIn, STEER_IN_MAX_CHARS, STEER_IN_MAX_REQUEST_CHARS } from ".
5
5
  import { journalEntryOfRow, journalEntryPayload, journalRowPayloadOf } from "../../orchestration/workflow-journal-entry.js";
6
6
  import { redactSecrets } from "../../trace/redact.js";
7
7
  import { createHash } from "node:crypto";
8
- import { redactErrorMessage, redactHead } from "../../observability/run-terminal-log.js";
8
+ import { redactErrorMessage, redactHead } from "../../observability/run-terminal.js";
9
9
  const JOURNAL_PROJECTION_MEMO_CAP = 1024;
10
10
  const journalProjectionMemo = new Map();
11
11
  import { sendJson, sendError, sseHeaders } from "../send.js";
@@ -28,8 +28,8 @@ import type { ModelUsageTracker, PromptManifestTracker } from "../budget.js";
28
28
  import { scopedIdempotencyKey } from "./idempotency.js";
29
29
  export { scopedIdempotencyKey };
30
30
  import type { ConsolidationSeatLabel } from "../boot/memory-consolidation.js";
31
- import { streamApprovals, isQuestionAnswer } from "./routes/approvals-assistant.js";
32
- export { streamApprovals, isQuestionAnswer };
31
+ import { streamApprovals, checkQuestionAnswer } from "./routes/approvals-assistant.js";
32
+ export { streamApprovals, checkQuestionAnswer };
33
33
  import { cascadeConfig } from "./run-meta.js";
34
34
  export { cascadeConfig };
35
35
  import { coarseStatusForState, errorCodeForExit } from "./routes/images.js";
@@ -1,5 +1,5 @@
1
1
  import http from "node:http";
2
- import { redactHead } from "../observability/run-terminal-log.js";
2
+ import { redactHead } from "../observability/run-terminal.js";
3
3
  import { validatePendingSteer, terminalProjection } from "@sema-agent/core";
4
4
  import {} from "../config-center/facade.js";
5
5
  import { HttpError } from "../security.js";
@@ -12,8 +12,8 @@ import {} from "../tool-approval.js";
12
12
  import {} from "../observability/rate-limit.js";
13
13
  import { IdempotencyCache, scopedIdempotencyKey } from "./idempotency.js";
14
14
  export { scopedIdempotencyKey };
15
- import { streamApprovals, isQuestionAnswer } from "./routes/approvals-assistant.js";
16
- export { streamApprovals, isQuestionAnswer };
15
+ import { streamApprovals, checkQuestionAnswer } from "./routes/approvals-assistant.js";
16
+ export { streamApprovals, checkQuestionAnswer };
17
17
  import { cascadeConfig } from "./run-meta.js";
18
18
  export { cascadeConfig };
19
19
  import { coarseStatusForState, errorCodeForExit } from "./routes/images.js";
@@ -1,4 +1,4 @@
1
- import { redactHead } from "../observability/run-terminal-log.js";
1
+ import { redactHead } from "../observability/run-terminal.js";
2
2
  export const DEFAULT_MAX_DIFF_BYTES = 5 * 1024 * 1024;
3
3
  const DEFAULT_READ_TIMEOUT_MS = 30_000;
4
4
  const SAFE_REV = /^[A-Za-z0-9._/-]{1,200}$/;
@@ -1,5 +1,5 @@
1
1
  import { uuidv7 } from "@sema-agent/core";
2
- import { redactErrorMessage, redactHead, redactLeaderResult, redactLeaderResultWithFlag } from "../observability/run-terminal-log.js";
2
+ import { redactErrorMessage, redactHead, redactLeaderResult, redactLeaderResultWithFlag } from "../observability/run-terminal.js";
3
3
  import { withPrincipal } from "../observability/principal-context.js";
4
4
  import { leaderIdemKey } from "../plugins/leader-run-store-sql.js";
5
5
  function statusForLeaderResult(result) {
@@ -1,5 +1,5 @@
1
1
  import { terminalProjection } from "@sema-agent/core";
2
- import { redactErrorMessage, redactWorkerReportError } from "../observability/run-terminal-log.js";
2
+ import { redactErrorMessage, redactWorkerReportError } from "../observability/run-terminal.js";
3
3
  import { pausedCauseOf } from "../terminal.js";
4
4
  class Semaphore {
5
5
  max;
@@ -1,5 +1,5 @@
1
1
  import { fanOut } from "./fanout.js";
2
- import { redactErrorMessage, redactHead } from "../observability/run-terminal-log.js";
2
+ import { redactErrorMessage, redactHead } from "../observability/run-terminal.js";
3
3
  import { pullDiff } from "./diffout.js";
4
4
  import { mergeBranches } from "./merge.js";
5
5
  import { Runner, runWithVerification, resumeWithVerification, terminalProjection } from "@sema-agent/core";
@@ -1,5 +1,5 @@
1
1
  import { delimitUntrusted } from "@sema-agent/core";
2
- import { OVERSIZED_DIAG_PLACEHOLDER, redactArtifactText, redactErrorMessage, redactHead, redactTail } from "../observability/run-terminal-log.js";
2
+ import { OVERSIZED_DIAG_PLACEHOLDER, redactArtifactText, redactErrorMessage, redactHead, redactTail } from "../observability/run-terminal.js";
3
3
  import { applyAndGrade, changedFilesOutOfAllowlist } from "./grader-env-factory.js";
4
4
  import { shellQuote as Q } from "../plugins/remote-shell.js";
5
5
  const CONFLICT_FILES_MAX = 200;
@@ -1,4 +1,4 @@
1
- import { redactErrorMessage, redactHead, redactTail } from "../observability/run-terminal-log.js";
1
+ import { redactErrorMessage, redactHead, redactTail } from "../observability/run-terminal.js";
2
2
  export function economicGuard(objective, opts = {}) {
3
3
  if (opts.fanoutEnabled === false)
4
4
  return { single: true, reason: "fan-out disabled by deployment (A6)" };
@@ -1,7 +1,7 @@
1
1
  import { runExecGate, } from "@sema-agent/core";
2
2
  import { createHash } from "node:crypto";
3
3
  import { shellQuote as sq } from "../plugins/remote-shell.js";
4
- import { redactHead, redactTail } from "../observability/run-terminal-log.js";
4
+ import { redactHead, redactTail } from "../observability/run-terminal.js";
5
5
  const sha = (s) => createHash("sha256").update(s).digest("hex").slice(0, 16);
6
6
  function changedFilesOf(patch) {
7
7
  const out = new Set();
@@ -1,7 +1,7 @@
1
1
  import { runRepairLoop, delimitUntrusted, } from "@sema-agent/core";
2
2
  import { mkRepairOracle } from "./repair-oracle.js";
3
3
  import { terminalProjection } from "@sema-agent/core";
4
- import { redactHead } from "../observability/run-terminal-log.js";
4
+ import { redactHead } from "../observability/run-terminal.js";
5
5
  function statusForTerminal(t) {
6
6
  return t === "fixed" || t === "candidate_only" ? "completed" : "failed";
7
7
  }
@@ -16,7 +16,7 @@ import { attachRepairLoopDeps } from "./repair-wire.js";
16
16
  import { routePlanWithFallback, validateSubtasks } from "./planner.js";
17
17
  import { createEngineNoticeSeat } from "../boot/runner-deps.js";
18
18
  import { currentPrincipal } from "../observability/principal-context.js";
19
- import { redactHead } from "../observability/run-terminal-log.js";
19
+ import { redactHead } from "../observability/run-terminal.js";
20
20
  const REPAIR_LOOP_WIRED = true;
21
21
  const sh = (env) => async (cmd) => {
22
22
  const r = await env.exec(`bash -lc ${JSON.stringify(cmd)}`);
@@ -141,6 +141,10 @@ export declare const FAIL_OPEN_TAGS: {
141
141
  readonly cls: "F";
142
142
  readonly note: "#261 fleet 存活对账腿的一次 durable 读失败/超预算(2s)⇒ **本周期整轮跳过**并开熔断,不做任何投影。放行的最坏后果 = 幽灵行(发布方已死、行僵在内存 Map 里)在面板上多活几个周期,与本腿存在之前的行为等同(那时它永远活着),不是新损失;真源不受影响(`GET /v1/runs/:id` 照常)。方向刻意 fail-open 且**只向留行一侧**:读不出来 ≠ 该退休——库抖一下就把一屏在跑的任务从面板上抹掉,比多留一条幽灵行伤得多。必须留痕:静默吞掉之后「对账腿为什么一直没生效」在遥测里没有任何痕迹,而本腿的全部价值就是「幽灵行会自己走」。";
143
143
  };
144
+ readonly "server.fleet.terminal-notify": {
145
+ readonly cls: "F";
146
+ readonly note: "S-450:**两条臂共用**(detail 里 `read-failed` 分得开)—— ① `observability/run-terminal.ts` 的**行为订阅者**扇出(唯一铸点 `notifyRunTerminal`,三只 store 的 CAS 赢分支各调一次):某个订阅者(今天只有 fleet 对账腿的候补窗升级腿)在 `setTerminal` 的调用栈上抛错 ⇒ 吞掉并计数,终局写照常成功;② `fleet/fleet-reconciler.ts` 的事件腿在收到事件后那一次 durable 点读失败 ⇒ 这条行留在候补窗里等周期臂。⚠️ 同一只铸点上的**日志腿**(`setRunTerminalLogObserver`)走的是**相反**契约(刻意不 try、抛出让 `setTerminal` reject),**不计本 tag** —— 读遥测别把两席混成一件。方向承重:这条腿的全部价值是「把候补窗行的成对终帧从『下一个对账周期(≤60s)』提前到『durable 写落地的同一 tick』」,它是一格**投影提速**;把它的 bug 外溢成终局写 reject 会让 run 卡 running 到 reaper、claim 悬挂([868] 指纹),代价高出几个量级。放行的最坏后果 = 这条行的成对终帧退回到周期臂补发,**与本件存在之前的行为逐字等同**,不是新损失;故 F 类。必须留痕:静默吞掉之后「事件腿为什么一直没生效」在遥测里没有任何痕迹,而面板上看到的只是『偶尔慢一分钟』这种没人会立案的症状。";
147
+ };
144
148
  readonly "server.fleet.run-status-unmapped": {
145
149
  readonly cls: "F";
146
150
  readonly note: "A-032 P1-④:`runStatusToFleet` 收到**词表外**的 run 状态词(闭集签名之外的唯一来路 = `rowToRun` 把无约束的 status 文本列裸 cast,滚动升级里更新的副本写的新词被旧副本读到)⇒ 折成 `idle`。放行的最坏后果 = 该行在 fleet 面板上**永久滞留**(removal 集 {completed,failed,killed} 收不到 idle),纯展示面、不影响 run 本身与任何执法判据,故 F 类。方向刻意不改(把 miss 折成终局会把一条还在跑的任务从面板上摘掉,误摘比多一行更伤)。留痕是承重的:2026-06-28 on-box e2e 逮到的 `blocked` 泄漏当年只能靠人眼在面板上发现——计数让下一次词表漂移在遥测里当场显形。编译期一侧已由闭集入参 + `never` 臂执法。";
@@ -189,6 +193,10 @@ export declare const FAIL_OPEN_TAGS: {
189
193
  readonly cls: "F";
190
194
  readonly note: "S-02 codex R1-[medium](验真后修):decide 的 409 `approval_stale` 臂对**调用方回显的旧 token** 做归属判定(`cs.get(binding.checkpointToken)`),那次读**抛了**(store 抖动 / 版本守卫 / 行恰被清理)。修前这里是静默 `catch(()=>null)`,「读不出」被折成「确认不在库」——叠上 S-02 的 404 收口后,一次 store 抖动就会把合法旧 token 的 409+terminal 谎报成 404「没有这条审批」。修后:unreadable ⇒ **保持 409**(`terminal` 不披露——证不出归属就不说,与 2026-07-26 纪律同向;404 折叠只在读成功且未命中时走,探测方无法自控 store 故障 ⇒ 谕示封闭不破)。放行的最坏后果 = 该次拒体 `terminal` 诚实缺席(`currentPending` 由它自己的读口与属主复核另判,不受本臂牵连),壳退回重拉列表,judgment 面 fail-safe,故 F 类。留痕理由:这次读是 terminal 披露与 404 折叠两个判定的输入,长期抖动必须在遥测显形。";
191
195
  };
196
+ readonly "server.resume.reopen-confirm-unreadable": {
197
+ readonly cls: "F";
198
+ readonly note: "S-447(#60 M2 自查项,`http/resume-legs.ts` 的 `confirmCheckpointReopened`):续跑腿在 core 给出 reopen 族失败后按 **token** 复核「这张 checkpoint 是不是又 pending 了」,这一读抛(库拖/版本太新/店不可达)。**方向本来就是 fail-closed**:读不出 ⇒ `return true` ⇒ 「不写任何终局、重新 park」(`:1206-1212` 头注已把取舍与有界代价写死:错 park 由 `reapSuspended` 兑,错终局会孤儿化一张活 park 到 TTL)—— 所以这条腿不是 P 类,缺的只是**留痕**。放行的最坏后果 = 多一次多余的 re-park(一张已经被决掉的卡重新挂到 reaper 窗里),人面上是「这条续跑没动」而不是谁被放行了。必须登记的理由是 **uniformity**:同文件三只同形读(`:249` `server.approvals.stale-echoed-token-unreadable` · `:288` `server.approvals.stale-current-pending-unreadable` · 本腿)中只有它是黑的,于是「店一直读不出所以每条续跑都在白白 re-park」与「一切正常」在遥测上同形。计数非零 = 这台机器的 checkpoint 店在拖,运维要去看库。";
199
+ };
192
200
  readonly "server.approvals.stale-current-pending-unreadable": {
193
201
  readonly cls: "F";
194
202
  readonly note: "S-02 件2([5522]④→[5525]):409 `approval_stale` 拒体铸 additive 指路键 `currentPending`(本会话当前 pending 的 D-1 坐标,壳一跳重定位)时,当前 pending 行的 `cs.get(token)` 读**抛了**(store 抖动 / 滚动升级期 checkpoint 格式 version guard / 行恰被并发消费)⇒ 本次拒体**缺 `currentPending` 键**,409 本体与 `terminal` 判定照旧。放行的最坏后果 = 壳退回该键到货前的行为(重拉 GET /v1/approvals 自行重定位),纯指路/便利面,不参与任何门/CAS/resume 判定,故 F 类(同族先例:`server.park.toolcallid-read-failed`,同为 cs.get 投影读失败臂)。必须留痕:键缺席与「此刻确实没有 pending」在 wire 上同形,不计数则「指路为什么总缺」在遥测里永不显形。";
@@ -132,6 +132,10 @@ export const FAIL_OPEN_TAGS = {
132
132
  cls: "F",
133
133
  note: "#261 fleet 存活对账腿的一次 durable 读失败/超预算(2s)⇒ **本周期整轮跳过**并开熔断,不做任何投影。放行的最坏后果 = 幽灵行(发布方已死、行僵在内存 Map 里)在面板上多活几个周期,与本腿存在之前的行为等同(那时它永远活着),不是新损失;真源不受影响(`GET /v1/runs/:id` 照常)。方向刻意 fail-open 且**只向留行一侧**:读不出来 ≠ 该退休——库抖一下就把一屏在跑的任务从面板上抹掉,比多留一条幽灵行伤得多。必须留痕:静默吞掉之后「对账腿为什么一直没生效」在遥测里没有任何痕迹,而本腿的全部价值就是「幽灵行会自己走」。",
134
134
  },
135
+ "server.fleet.terminal-notify": {
136
+ cls: "F",
137
+ note: "S-450:**两条臂共用**(detail 里 `read-failed` 分得开)—— ① `observability/run-terminal.ts` 的**行为订阅者**扇出(唯一铸点 `notifyRunTerminal`,三只 store 的 CAS 赢分支各调一次):某个订阅者(今天只有 fleet 对账腿的候补窗升级腿)在 `setTerminal` 的调用栈上抛错 ⇒ 吞掉并计数,终局写照常成功;② `fleet/fleet-reconciler.ts` 的事件腿在收到事件后那一次 durable 点读失败 ⇒ 这条行留在候补窗里等周期臂。⚠️ 同一只铸点上的**日志腿**(`setRunTerminalLogObserver`)走的是**相反**契约(刻意不 try、抛出让 `setTerminal` reject),**不计本 tag** —— 读遥测别把两席混成一件。方向承重:这条腿的全部价值是「把候补窗行的成对终帧从『下一个对账周期(≤60s)』提前到『durable 写落地的同一 tick』」,它是一格**投影提速**;把它的 bug 外溢成终局写 reject 会让 run 卡 running 到 reaper、claim 悬挂([868] 指纹),代价高出几个量级。放行的最坏后果 = 这条行的成对终帧退回到周期臂补发,**与本件存在之前的行为逐字等同**,不是新损失;故 F 类。必须留痕:静默吞掉之后「事件腿为什么一直没生效」在遥测里没有任何痕迹,而面板上看到的只是『偶尔慢一分钟』这种没人会立案的症状。",
138
+ },
135
139
  "server.fleet.run-status-unmapped": {
136
140
  cls: "F",
137
141
  note: "A-032 P1-④:`runStatusToFleet` 收到**词表外**的 run 状态词(闭集签名之外的唯一来路 = `rowToRun` 把无约束的 status 文本列裸 cast,滚动升级里更新的副本写的新词被旧副本读到)⇒ 折成 `idle`。放行的最坏后果 = 该行在 fleet 面板上**永久滞留**(removal 集 {completed,failed,killed} 收不到 idle),纯展示面、不影响 run 本身与任何执法判据,故 F 类。方向刻意不改(把 miss 折成终局会把一条还在跑的任务从面板上摘掉,误摘比多一行更伤)。留痕是承重的:2026-06-28 on-box e2e 逮到的 `blocked` 泄漏当年只能靠人眼在面板上发现——计数让下一次词表漂移在遥测里当场显形。编译期一侧已由闭集入参 + `never` 臂执法。",
@@ -180,6 +184,10 @@ export const FAIL_OPEN_TAGS = {
180
184
  cls: "F",
181
185
  note: "S-02 codex R1-[medium](验真后修):decide 的 409 `approval_stale` 臂对**调用方回显的旧 token** 做归属判定(`cs.get(binding.checkpointToken)`),那次读**抛了**(store 抖动 / 版本守卫 / 行恰被清理)。修前这里是静默 `catch(()=>null)`,「读不出」被折成「确认不在库」——叠上 S-02 的 404 收口后,一次 store 抖动就会把合法旧 token 的 409+terminal 谎报成 404「没有这条审批」。修后:unreadable ⇒ **保持 409**(`terminal` 不披露——证不出归属就不说,与 2026-07-26 纪律同向;404 折叠只在读成功且未命中时走,探测方无法自控 store 故障 ⇒ 谕示封闭不破)。放行的最坏后果 = 该次拒体 `terminal` 诚实缺席(`currentPending` 由它自己的读口与属主复核另判,不受本臂牵连),壳退回重拉列表,judgment 面 fail-safe,故 F 类。留痕理由:这次读是 terminal 披露与 404 折叠两个判定的输入,长期抖动必须在遥测显形。",
182
186
  },
187
+ "server.resume.reopen-confirm-unreadable": {
188
+ cls: "F",
189
+ note: "S-447(#60 M2 自查项,`http/resume-legs.ts` 的 `confirmCheckpointReopened`):续跑腿在 core 给出 reopen 族失败后按 **token** 复核「这张 checkpoint 是不是又 pending 了」,这一读抛(库拖/版本太新/店不可达)。**方向本来就是 fail-closed**:读不出 ⇒ `return true` ⇒ 「不写任何终局、重新 park」(`:1206-1212` 头注已把取舍与有界代价写死:错 park 由 `reapSuspended` 兑,错终局会孤儿化一张活 park 到 TTL)—— 所以这条腿不是 P 类,缺的只是**留痕**。放行的最坏后果 = 多一次多余的 re-park(一张已经被决掉的卡重新挂到 reaper 窗里),人面上是「这条续跑没动」而不是谁被放行了。必须登记的理由是 **uniformity**:同文件三只同形读(`:249` `server.approvals.stale-echoed-token-unreadable` · `:288` `server.approvals.stale-current-pending-unreadable` · 本腿)中只有它是黑的,于是「店一直读不出所以每条续跑都在白白 re-park」与「一切正常」在遥测上同形。计数非零 = 这台机器的 checkpoint 店在拖,运维要去看库。",
190
+ },
183
191
  "server.approvals.stale-current-pending-unreadable": {
184
192
  cls: "F",
185
193
  note: "S-02 件2([5522]④→[5525]):409 `approval_stale` 拒体铸 additive 指路键 `currentPending`(本会话当前 pending 的 D-1 坐标,壳一跳重定位)时,当前 pending 行的 `cs.get(token)` 读**抛了**(store 抖动 / 滚动升级期 checkpoint 格式 version guard / 行恰被并发消费)⇒ 本次拒体**缺 `currentPending` 键**,409 本体与 `terminal` 判定照旧。放行的最坏后果 = 壳退回该键到货前的行为(重拉 GET /v1/approvals 自行重定位),纯指路/便利面,不参与任何门/CAS/resume 判定,故 F 类(同族先例:`server.park.toolcallid-read-failed`,同为 cs.get 投影读失败臂)。必须留痕:键缺席与「此刻确实没有 pending」在 wire 上同形,不计数则「指路为什么总缺」在遥测里永不显形。",
@@ -28,6 +28,9 @@ export declare class Metrics {
28
28
  * needed). Counters are cumulative totals; histograms are reported as averages (sum/count). */
29
29
  summarize(): MetricsSummary;
30
30
  }
31
+ /** S-451:`/metrics/summary` 上「失败最多的路由」保留几条。看板要的是**头部**,不是全表;
32
+ * `routeLabel` 的值域本来就有界(路由声明表 + `"other"`),这个上限只是让这张脸的大小与路由表长解耦。 */
33
+ export declare const HTTP_ERROR_ROUTE_TOP_N = 10;
31
34
  export interface MetricsSummary {
32
35
  runsActive: number;
33
36
  tasks: Record<string, number>;
@@ -61,6 +64,15 @@ export interface MetricsSummary {
61
64
  verifications: Record<string, number>;
62
65
  councilRuns: Record<string, number>;
63
66
  toolErrors: Record<string, number>;
67
+ /** S-451(additive,7.87.0):窗内 HTTP **失败**请求数,按状态码**类**聚合 —— 键只有 `"4xx"` / `"5xx"`
68
+ * 两种,2xx/3xx 一格不进。⚠️ `"4xx"` 里**含 `499`**(本仓给「客户端在响应结束前断连」铸的标签,
69
+ * SSE 长连接上是常态)—— 看一场真风暴时按 {@link httpErrorsByRoute} 分辨是哪条腿。
70
+ * 空对象 = 窗内零失败(与 `tasks`/`toolErrors` 同律:键恒在场,不用缺席表示零)。 */
71
+ httpErrorsByClass: Record<string, number>;
72
+ /** S-451(additive,7.87.0):失败(4xx+5xx 合计)最多的前 {@link HTTP_ERROR_ROUTE_TOP_N} 条 **route 标签**
73
+ * (`routeLabel` 的低基数形,id 已折成 `:id`;未命中路由表的落 `"other"`)。按计数降序。
74
+ * 这是「180 次 404 到底打在哪」的那一格。 */
75
+ httpErrorsByRoute: Record<string, number>;
64
76
  }
65
77
  export interface MetricSnapshot {
66
78
  counters: Array<{
@@ -188,9 +188,25 @@ export class Metrics {
188
188
  toolErrors: Object.fromEntries([...(this.counters.get("tool_calls_total")?.series.values() ?? [])]
189
189
  .filter((s) => s.labels.ok === "false")
190
190
  .map((s) => [s.labels.name ?? "?", s.value])),
191
+ ...(() => {
192
+ const failures = [...(this.counters.get("http_requests_total")?.series.values() ?? [])]
193
+ .map((s) => ({ cls: /^([45])\d\d$/.exec(s.labels.status ?? "")?.[1], route: s.labels.route ?? "?", value: s.value }))
194
+ .filter((x) => x.cls !== undefined);
195
+ const byClass = {};
196
+ const byRoute = {};
197
+ for (const f of failures) {
198
+ byClass[`${f.cls}xx`] = (byClass[`${f.cls}xx`] ?? 0) + f.value;
199
+ byRoute[f.route] = (byRoute[f.route] ?? 0) + f.value;
200
+ }
201
+ return {
202
+ httpErrorsByClass: byClass,
203
+ httpErrorsByRoute: Object.fromEntries(Object.entries(byRoute).sort((a, b) => b[1] - a[1]).slice(0, HTTP_ERROR_ROUTE_TOP_N)),
204
+ };
205
+ })(),
191
206
  };
192
207
  }
193
208
  }
209
+ export const HTTP_ERROR_ROUTE_TOP_N = 10;
194
210
  export function createMetrics() {
195
211
  const m = new Metrics();
196
212
  m.counter("metric_gauge_eval_errors_total", "Dynamic gauge eval errors by gauge name (an erroring gauge emits NO sample that scrape — absence means unknown, not 0)");
@@ -270,6 +286,7 @@ export function createMetrics() {
270
286
  m.counter("compaction_events_total", "Compaction lifecycle events by outcome (started/completed/failed/skipped) and trigger");
271
287
  m.counter("hook_llm_calls_total", "prompt/agent hook entries that completed a model call (hooks 3b), by type");
272
288
  m.counter("auth_bridge_verify_total", "Registry SSO JWT verifications (auth-bridge), by outcome");
289
+ m.counter("fleet_rows_reconciled_total", "Fleet rows the #261 liveness-reconcile leg acted on, by `disposition`: 'retired' = durable was terminal, the leg projected the settled pair (terminal task frame → task_remove); 'retired_deferred' = a row that had already left declaratively (removeReason:\"orphaned\") got its settled pair BACKFILLED once durable turned terminal; 'orphaned' / 'orphaned_child' = declarative exit, NO terminal was minted (the reaper owns the death ruling); 'workflow_retired' = same projection for a workflow row. `trigger` rides ONLY the retired_deferred series (S-450): 'event' = the backfill fired on the same tick a terminal write landed through the run-store write口 (single replica), 'cycle' = it fired on the 60s reconcile cycle (cross-replica writes, and terminals flipped by the BATCH reapers, which never pass through the write口). A retired_deferred series that is 100% trigger=cycle means the event leg is not wired or this deployment\'s terminals all come from those two families");
273
290
  m.counter("fleet_lease_issued_total", "Fleet trace-lease tokens issued");
274
291
  m.counter("fleet_lease_rejected_total", "Fleet trace-lease requests rejected");
275
292
  m.counter("fleet_lease_exhausted_total", "Fleet trace-lease requests rejected because the pool was exhausted");
@@ -3,8 +3,14 @@ import { type RunRecord } from "../plugins/store-contracts.js";
3
3
  export interface RunTerminalRecord {
4
4
  /** 这条 run 的账本 id = `GET /v1/runs/:id` 的 id(本仓 run 行主键名为 taskId,同一个值)。 */
5
5
  runId: string;
6
- sessionId: string;
7
- owner: string | null;
6
+ /**
7
+ * 🔴 **只在「这条终局要出声」时在场**(S-450):三只 store 只在 `runTerminalLogLevel(...) !== undefined`
8
+ * 时才多付那一次 PK 读去装它(成功终局零额外查询 —— 那条注在 `run-store-sql.ts` 的 `setTerminal` 里)。
9
+ * 日志腿恰好只在同一形下运行,所以**它读到的恒是在场的**;行为订阅者不得依赖它。
10
+ */
11
+ sessionId?: string;
12
+ /** 同 {@link RunTerminalRecord.sessionId}:只在要出声的那一形在场。`null` = 行上没有 owner。 */
13
+ owner?: string | null;
8
14
  status: RunRecord["status"];
9
15
  /** 终局结果里的引擎侧 taskId —— 只在与 {@link RunTerminalRecord.runId} **不同**时进日志(sync 腿上 core 会把它
10
16
  * 铸成 sessionId,两者同值时重复一遍只是噪音)。 */
@@ -15,8 +21,16 @@ export interface RunTerminalRecord {
15
21
  /** 行创建到终局的墙钟毫秒(store 侧用行的 createdAt 算;算不出=不传,键缺席)。 */
16
22
  elapsedMs?: number | undefined;
17
23
  }
18
- /** 观测者收到的是**已判级 + 已脱敏 + 已截断**的成品——装配层只负责把它送进 logger。 */
19
- export type RunTerminalObserver = (level: "error" | "info", fields: Record<string, unknown>) => void;
24
+ /** **日志腿**收到的是**已判级 + 已脱敏 + 已截断**的成品——装配层只负责把它送进 logger。 */
25
+ export type RunTerminalLogObserver = (level: "error" | "info", fields: Record<string, unknown>) => void;
26
+ /**
27
+ * **行为订阅者**(S-450):CAS 赢的那一刻收到整只 {@link RunTerminalRecord}。
28
+ *
29
+ * 🔴 载荷是**这次写自己的视角**,不是回读的 durable 行 —— 只有 `runId` / `status` 两位在**任何**形下
30
+ * 都为真(CAS 赢 ⇒ 行就是这个词)。日志腿专用的那几位在成功终局上**刻意缺席**(store 不为它们多付一次
31
+ * PK 往返,见 {@link RunTerminalRecord.sessionId})。要更多事实的订阅者请自己按 `runId` 读 durable。
32
+ */
33
+ export type RunTerminalSubscriber = (rec: RunTerminalRecord) => void;
20
34
  /**
21
35
  * 终局行的级别判据。`undefined` = 不打行。
22
36
  *
@@ -25,13 +39,39 @@ export type RunTerminalObserver = (level: "error" | "info", fields: Record<strin
25
39
  * 穷举 switch,并在 `blocked` 那一格给出与 fleet 读面互斥的答案)。表的编译期双向钉在属主处。
26
40
  */
27
41
  export declare function runTerminalLogLevel(status: RunRecord["status"], errorCode: string | undefined): "error" | "info" | undefined;
28
- /** 装配点(`main.ts` 唯一)。传 `undefined` 卸载(测试隔离)。 */
29
- export declare function setRunTerminalObserver(fn: RunTerminalObserver | undefined): void;
42
+ /** **日志腿**的装配点(`boot/stage-01-config.ts` 唯一)。传 `undefined` 卸载(测试隔离)。 */
43
+ export declare function setRunTerminalLogObserver(fn: RunTerminalLogObserver | undefined): void;
44
+ /**
45
+ * **行为订阅者**的登记口(S-450)。返回解除登记的句柄。
46
+ *
47
+ * 与 {@link setRunTerminalLogObserver} 是**两个语义不同的席**,不是一个带旗的列表:
48
+ * · 日志腿**恰一只**、装配层亲接、**不隔离**(抛出会让 `setTerminal` reject —— 见 {@link notifyRunTerminal});
49
+ * · 行为订阅者**可多只**、运行期来去、**逐只隔离**(抛出吞掉 + `recordFailOpen`)。
50
+ * 两条失败契约相反,所以它们必须是两个名字;混进一个列表再加一个 `isolate` 位,就是把「这条腿出事
51
+ * 该不该把 run 拖下水」这个**语义**降级成一个**配置**。
52
+ */
53
+ export declare function subscribeRunTerminal(fn: RunTerminalSubscriber): () => void;
54
+ /** test-only:清空行为订阅者(用例之间不串台;日志腿用 `setRunTerminalLogObserver(undefined)`)。 */
55
+ export declare function resetRunTerminalSubscribersForTest(): void;
30
56
  /**
31
- * store 的终局写口在 **CAS 赢了之后**调用它。未装配观测者 ⇒ 无操作(纯内存/无 boot 的单测车道)。
57
+ * 🔴 **run 行翻终局的唯一铸点** —— 三个 run store 的 `setTerminal` 在 **CAS 赢了之后**各调它一次
58
+ * (SQL 腿 `w.affected > 0`,file / memory 腿在同判据的 CAS 分支内)。「CAS 赢了才发」是**恰一次**的来源:
59
+ * 调用侧的兜底腿有意会对同一行重复写(cancel × leg-throw × reaper 三向),打空的那几次一个字都不发。
60
+ *
61
+ * 两条出口,**失败契约相反**,顺序承重:
62
+ * ① **行为订阅者先**({@link subscribeRunTerminal},逐只 try)—— 它们要在日志腿万一违约抛出之前跑完;
63
+ * ② **日志腿后**({@link setRunTerminalLogObserver},级别筛 + 脱敏 + 不 try)。
64
+ *
65
+ * 为什么日志腿**刻意不 try**(原文保留,[ref] 的成文理由):静默吞掉观测者异常 = 又造一条「走了兜底与
66
+ * 一切正常同形」的路,方向与本模块的存在理由相反;生产装配的那一只只调 `Logger`(`createLogger` 的 emit
67
+ * 内部已把 JSON.stringify 与 write 全部 try 住,结构上不抛)。
68
+ * 为什么行为订阅者**必须 try**(S-450 新加的那一半):它们是**行为腿**,不是一只结构上不抛的 logger;
69
+ * 而本函数跑在 `setTerminal` 的调用栈上、**CAS 已经赢了、行已经落盘**,此时把订阅者的 bug 放大成
70
+ * 「终局写 reject」= run 卡 `running` 到 reaper + claim 悬挂([ref] 指纹)。吞但**不静默**:
71
+ * `recordFailOpen("server.fleet.terminal-notify")`(F 类,普查登记在案)。
32
72
  *
33
- * 脱敏:`errorMessage` 过 `redactSecrets` 再截断到 {@link MESSAGE_CAP} —— provider 的错误体
34
- * 会原样回显请求片段(取证里的 403 体带订阅链接;别的 provider 会回显 Authorization 头前缀),
73
+ * 脱敏(只在日志腿那一格):`errorMessage` 过 `redactSecrets` 再截断到 {@link MESSAGE_CAP} —— provider 的
74
+ * 错误体会原样回显请求片段(取证里的 403 体带订阅链接;别的 provider 会回显 Authorization 头前缀),
35
75
  * 而引擎日志是运维面、常被整条采集走。`errorCode` / `model` 是闭集式短标量,verbatim。
36
76
  */
37
77
  export declare function notifyRunTerminal(rec: RunTerminalRecord): void;
@@ -164,4 +204,4 @@ export declare function memoRedactTerminalResult<T extends Parameters<typeof red
164
204
  export declare function diffScanShadowCountForTest(): number;
165
205
  export declare function resetDiffScanShadowCountForTest(): void;
166
206
  export declare function resetReadRedactionMemoForTest(): void;
167
- //# sourceMappingURL=run-terminal-log.d.ts.map
207
+ //# sourceMappingURL=run-terminal.d.ts.map