@sema-agent/server 7.86.0 → 7.87.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/USAGE.md +1 -1
- package/dist/approval-hmac.d.ts +62 -14
- package/dist/approval-hmac.js +15 -1
- package/dist/boot/stage-01-config.js +2 -2
- package/dist/fleet/fleet-bus.d.ts +23 -3
- package/dist/fleet/fleet-reconciler.d.ts +35 -0
- package/dist/fleet/fleet-reconciler.js +50 -7
- package/dist/http/approval-sweeps.js +1 -1
- package/dist/http/resume-legs.js +3 -2
- package/dist/http/routes/a2a-serve.js +1 -1
- package/dist/http/routes/approvals-assistant.d.ts +35 -23
- package/dist/http/routes/approvals-assistant.js +107 -30
- package/dist/http/routes/runs.js +2 -2
- package/dist/http/routes/tasks.js +2 -2
- package/dist/http/routes/workflows.js +1 -1
- package/dist/http/server.d.ts +2 -2
- package/dist/http/server.js +3 -3
- package/dist/leader/diffout.js +1 -1
- package/dist/leader/endpoint.js +1 -1
- package/dist/leader/fanout.js +1 -1
- package/dist/leader/leader.js +1 -1
- package/dist/leader/merge.js +1 -1
- package/dist/leader/planner.js +1 -1
- package/dist/leader/repair-oracle.js +1 -1
- package/dist/leader/repair-wire.js +1 -1
- package/dist/leader/wire.js +1 -1
- package/dist/observability/fail-open.d.ts +8 -0
- package/dist/observability/fail-open.js +8 -0
- package/dist/observability/metrics.d.ts +12 -0
- package/dist/observability/metrics.js +17 -0
- package/dist/observability/{run-terminal-log.d.ts → run-terminal.d.ts} +50 -10
- package/dist/observability/{run-terminal-log.js → run-terminal.js} +27 -8
- package/dist/plugins/file-run-store.js +2 -2
- package/dist/plugins/memory-run-store.js +2 -2
- package/dist/plugins/remote-scratchpad.js +1 -1
- package/dist/plugins/run-store-sql.js +8 -7
- package/dist/plugins/store-contracts.d.ts +2 -2
- package/dist/plugins/workflow-run-store-sql.js +1 -1
- package/dist/runs.js +1 -1
- package/dist/session-titler.js +9 -5
- package/dist/tool-approval.d.ts +22 -2
- package/dist/tool-approval.js +2 -0
- package/dist/trace/core-keyset-guard.d.ts +1 -1
- package/dist/trace/ledger-events.d.ts +1 -1
- package/dist/trace/project.d.ts +1 -1
- package/dist/trace/project.js +1 -1
- package/dist/trace/redact.d.ts +1 -1
- package/package.json +2 -2
|
@@ -6,11 +6,11 @@ import { APPROVAL_GATE_KINDS } from "../../tool-approval.js";
|
|
|
6
6
|
import { isParkedRunStatus } from "../../plugins/store-contracts.js";
|
|
7
7
|
import { principalFrom, decodeCheckpointScope, PRINCIPAL_TOKEN_HEADER, APPROVAL_MAC_HEADER, APPROVAL_MAC_KID_HEADER } from "../../security.js";
|
|
8
8
|
import { editedPlanBinding, isWellFormedUnicode, verifyDirectDoorProof } from "../../principal-jwt.js";
|
|
9
|
-
import { MAX_APPROVAL_REASON_CHARS } from "../../approval-hmac.js";
|
|
9
|
+
import { MAX_APPROVAL_REASON_CHARS, projectQuestionAnswer, questionAnswerFromProjection } from "../../approval-hmac.js";
|
|
10
10
|
import { PLAN_APPROVE_PERMISSION_MODES_AFTER, isPlanApprovePermissionModeAfter, storedBodyPermissionMode } from "../../task-settings.js";
|
|
11
11
|
import { MAX_APPROVAL_NONCE_CHARS } from "../../plugins/approval-nonce-store.js";
|
|
12
12
|
import { redactedPreview, redactSecrets } from "../../trace/redact.js";
|
|
13
|
-
import { redactRiskDescriptor } from "../../observability/run-terminal
|
|
13
|
+
import { redactRiskDescriptor } from "../../observability/run-terminal.js";
|
|
14
14
|
import { fleetRunLabels } from "../../fleet/fleet-bus.js";
|
|
15
15
|
import { sleep } from "../sse-log.js";
|
|
16
16
|
import { bindSseLifecycle } from "../sse-lifecycle.js";
|
|
@@ -23,6 +23,8 @@ import { MAX_KEY_ECHO, keyClosureClause, scanKeyClosure } from "../../request-ke
|
|
|
23
23
|
const DECIDE_BODY_DECLARED_KEYS = [
|
|
24
24
|
"answer", "boundCallId", "boundInputHash", "checkpointToken", "decision", "reason", "remember", "updatedInput",
|
|
25
25
|
];
|
|
26
|
+
const DECIDE_PROOF_COVERED_KEYS = ["answer", "boundCallId", "boundInputHash", "decision", "reason"];
|
|
27
|
+
const DECIDE_NOT_IN_PROOF_KEYS = ["checkpointToken", "remember", "updatedInput"];
|
|
26
28
|
const ACCEPTED_DECIDE_BODY = new Set(DECIDE_BODY_DECLARED_KEYS);
|
|
27
29
|
const UNSUPPORTED_DECIDE_BODY = new Set(["editedPlan", "sessionId", "taskId"]);
|
|
28
30
|
export function decideBodyKeyIssue(raw) {
|
|
@@ -50,6 +52,23 @@ export function decideBodyKeyIssue(raw) {
|
|
|
50
52
|
const PLAN_REVIEW_BODY_DECLARED_KEYS = [
|
|
51
53
|
"boundCallId", "decision", "editedPlan", "permissionModeAfter", "reason",
|
|
52
54
|
];
|
|
55
|
+
const PLAN_REVIEW_PROOF_COVERED_KEYS = ["boundCallId", "decision", "editedPlan", "permissionModeAfter", "reason"];
|
|
56
|
+
const PLAN_REVIEW_NOT_IN_PROOF_KEYS = [];
|
|
57
|
+
const PROOF_SLOT_OF = {
|
|
58
|
+
answer: "answer",
|
|
59
|
+
boundCallId: "boundCallId",
|
|
60
|
+
boundInputHash: "boundInputHash",
|
|
61
|
+
decision: "decision",
|
|
62
|
+
editedPlan: "boundInputHash",
|
|
63
|
+
permissionModeAfter: "permissionModeAfter",
|
|
64
|
+
reason: "reason",
|
|
65
|
+
};
|
|
66
|
+
function buildApprovalEnvelope(sessionId, covered, reading) {
|
|
67
|
+
const envelope = { sessionId, decision: reading.decision };
|
|
68
|
+
for (const key of covered)
|
|
69
|
+
Object.assign(envelope, { [PROOF_SLOT_OF[key]]: reading[key] });
|
|
70
|
+
return envelope;
|
|
71
|
+
}
|
|
53
72
|
const ACCEPTED_PLAN_REVIEW_BODY = new Set(PLAN_REVIEW_BODY_DECLARED_KEYS);
|
|
54
73
|
const UNSUPPORTED_PLAN_REVIEW_BODY = new Set([
|
|
55
74
|
"answer", "boundInputHash", "checkpointToken", "permissionMode", "remember", "sessionId", "taskId", "updatedInput",
|
|
@@ -79,13 +98,27 @@ export function planReviewBodyKeyIssue(raw) {
|
|
|
79
98
|
export function decisionReasonOf(body) {
|
|
80
99
|
return typeof body.reason === "string" && body.reason !== "" ? body.reason : undefined;
|
|
81
100
|
}
|
|
82
|
-
|
|
101
|
+
const QUESTION_ANSWER_TOP_KEYS = ["answers"];
|
|
102
|
+
const QUESTION_ANSWER_ITEM_KEYS = ["header", "selected", "note"];
|
|
103
|
+
const ACCEPTED_QUESTION_ANSWER_TOP = new Set(QUESTION_ANSWER_TOP_KEYS);
|
|
104
|
+
const ACCEPTED_QUESTION_ANSWER_ITEM = new Set(QUESTION_ANSWER_ITEM_KEYS);
|
|
105
|
+
const NO_TOLERATED_KEYS = new Set();
|
|
106
|
+
export function checkQuestionAnswer(v) {
|
|
83
107
|
if (!v || typeof v !== "object" || Array.isArray(v))
|
|
84
|
-
return false;
|
|
108
|
+
return { ok: false, kind: "shape" };
|
|
85
109
|
const a = v.answers;
|
|
86
110
|
if (!Array.isArray(a) || a.length === 0)
|
|
87
|
-
return false;
|
|
88
|
-
|
|
111
|
+
return { ok: false, kind: "shape" };
|
|
112
|
+
const buckets = { unknown: [], unsupported: [] };
|
|
113
|
+
scanKeyClosure(v, "answer.", ACCEPTED_QUESTION_ANSWER_TOP, NO_TOLERATED_KEYS, buckets);
|
|
114
|
+
a.forEach((it, i) => {
|
|
115
|
+
if (!it || typeof it !== "object" || Array.isArray(it))
|
|
116
|
+
return;
|
|
117
|
+
scanKeyClosure(it, `answer.answers[${i}].`, ACCEPTED_QUESTION_ANSWER_ITEM, NO_TOLERATED_KEYS, buckets);
|
|
118
|
+
});
|
|
119
|
+
if (buckets.unknown.length > 0)
|
|
120
|
+
return { ok: false, kind: "unknown_key", keyPaths: buckets.unknown.sort().slice(0, MAX_KEY_ECHO) };
|
|
121
|
+
const shapeOk = a.every((it) => {
|
|
89
122
|
if (!it || typeof it !== "object" || Array.isArray(it))
|
|
90
123
|
return false;
|
|
91
124
|
const { header, selected, note } = it;
|
|
@@ -95,6 +128,7 @@ export function isQuestionAnswer(v) {
|
|
|
95
128
|
selected.every((s) => typeof s === "string") &&
|
|
96
129
|
(note === undefined || typeof note === "string"));
|
|
97
130
|
});
|
|
131
|
+
return shapeOk ? { ok: true, value: v } : { ok: false, kind: "shape" };
|
|
98
132
|
}
|
|
99
133
|
async function decideIdempotentReplay(cs, args) {
|
|
100
134
|
const { sessionId, out, binding, decision, payload, deciderPrincipal, explicitOperator, warn } = args;
|
|
@@ -173,7 +207,7 @@ export async function handleApprovalsAssistant(req, res, match, ctx) {
|
|
|
173
207
|
const scope = operator
|
|
174
208
|
? (new URLSearchParams(match.search).get("owner") ?? undefined)
|
|
175
209
|
: (principal ?? "__none__");
|
|
176
|
-
await streamApprovals(req, res, cs, scope, deps.config);
|
|
210
|
+
await streamApprovals(req, res, cs, deps.toolApproval, scope, deps.config);
|
|
177
211
|
return;
|
|
178
212
|
}
|
|
179
213
|
if (match.decl.id === "approvals-list") {
|
|
@@ -464,7 +498,13 @@ export async function handleApprovalsAssistant(req, res, match, ctx) {
|
|
|
464
498
|
}
|
|
465
499
|
const storeNowMs = await store.nowMs();
|
|
466
500
|
const hdr = (n) => { const h = req.headers[n]; return Array.isArray(h) ? h[0] : h; };
|
|
467
|
-
const proof = verifyDirectDoorProof({ jwt: hdr(PRINCIPAL_TOKEN_HEADER), mac: hdr(APPROVAL_MAC_HEADER), kid: hdr(APPROVAL_MAC_KID_HEADER) },
|
|
501
|
+
const proof = verifyDirectDoorProof({ jwt: hdr(PRINCIPAL_TOKEN_HEADER), mac: hdr(APPROVAL_MAC_HEADER), kid: hdr(APPROVAL_MAC_KID_HEADER) }, buildApprovalEnvelope(run.sessionId, PLAN_REVIEW_PROOF_COVERED_KEYS, {
|
|
502
|
+
boundCallId: planNonce ?? null,
|
|
503
|
+
decision,
|
|
504
|
+
editedPlan: decision === "edit" ? editedPlanBinding(body.editedPlan) : null,
|
|
505
|
+
permissionModeAfter: isPlanApprovePermissionModeAfter(body.permissionModeAfter) ? body.permissionModeAfter : undefined,
|
|
506
|
+
reason: typeof body.reason === "string" ? body.reason : null,
|
|
507
|
+
}), deps.config, { actionBinding: false, nowSec: Math.floor(storeNowMs / 1000) });
|
|
468
508
|
if (!proof.ok) {
|
|
469
509
|
sendError(res, proof.status, proof.errorCode, proof.error);
|
|
470
510
|
return;
|
|
@@ -556,12 +596,19 @@ export async function handleApprovalsAssistant(req, res, match, ctx) {
|
|
|
556
596
|
remember = true;
|
|
557
597
|
}
|
|
558
598
|
let answer;
|
|
599
|
+
let answerProjection;
|
|
559
600
|
if (body.answer !== undefined) {
|
|
560
|
-
|
|
561
|
-
|
|
601
|
+
const checked = checkQuestionAnswer(body.answer);
|
|
602
|
+
if (!checked.ok) {
|
|
603
|
+
if (checked.kind === "shape") {
|
|
604
|
+
sendError(res, 400, "request.body_shape", "malformed answer — expected { answers: [{ header, selected: string[], note? }] }");
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
607
|
+
sendError(res, 400, "request.field_invalid", `answer: unrecognized key(s) — refused rather than dropped, because a dropped key reads as "in effect" to the caller (a misspelled selected key silently drops part of the operator's answer). ${keyClosureClause([...checked.keyPaths], "unknown (not a question-answer key — check the spelling)")}. Accepted: answers, answers[].header, answers[].selected, answers[].note`, { unknownKeys: [...checked.keyPaths] });
|
|
562
608
|
return;
|
|
563
609
|
}
|
|
564
|
-
|
|
610
|
+
answerProjection = projectQuestionAnswer(checked.value);
|
|
611
|
+
answer = questionAnswerFromProjection(answerProjection);
|
|
565
612
|
}
|
|
566
613
|
for (const f of ["checkpointToken", "boundCallId", "boundInputHash"]) {
|
|
567
614
|
if (body[f] !== undefined && typeof body[f] !== "string") {
|
|
@@ -573,6 +620,10 @@ export async function handleApprovalsAssistant(req, res, match, ctx) {
|
|
|
573
620
|
sendError(res, 400, "updated_input_not_in_proof", "updatedInput is not supported on a direct-door worker (not covered by the decision proof)");
|
|
574
621
|
return;
|
|
575
622
|
}
|
|
623
|
+
if (deps.config.directDoorActive && body.checkpointToken !== undefined) {
|
|
624
|
+
sendError(res, 400, "checkpoint_token_not_in_proof", "checkpointToken is not supported on a direct-door worker (not covered by the decision proof); bind the decision with boundCallId and boundInputHash instead");
|
|
625
|
+
return;
|
|
626
|
+
}
|
|
576
627
|
const binding = {
|
|
577
628
|
checkpointToken: body.checkpointToken,
|
|
578
629
|
boundCallId: body.boundCallId,
|
|
@@ -585,7 +636,13 @@ export async function handleApprovalsAssistant(req, res, match, ctx) {
|
|
|
585
636
|
const h = req.headers[n];
|
|
586
637
|
return Array.isArray(h) ? h[0] : h;
|
|
587
638
|
};
|
|
588
|
-
const proof = verifyDirectDoorProof({ jwt: hdr(PRINCIPAL_TOKEN_HEADER), mac: hdr(APPROVAL_MAC_HEADER), kid: hdr(APPROVAL_MAC_KID_HEADER) },
|
|
639
|
+
const proof = verifyDirectDoorProof({ jwt: hdr(PRINCIPAL_TOKEN_HEADER), mac: hdr(APPROVAL_MAC_HEADER), kid: hdr(APPROVAL_MAC_KID_HEADER) }, buildApprovalEnvelope(sessionId, DECIDE_PROOF_COVERED_KEYS, {
|
|
640
|
+
answer: answerProjection,
|
|
641
|
+
boundCallId: binding.boundCallId,
|
|
642
|
+
boundInputHash: binding.boundInputHash,
|
|
643
|
+
decision,
|
|
644
|
+
reason: typeof body.reason === "string" ? body.reason : null,
|
|
645
|
+
}), deps.config);
|
|
589
646
|
if (!proof.ok) {
|
|
590
647
|
sendError(res, proof.status, proof.errorCode, proof.error);
|
|
591
648
|
return;
|
|
@@ -664,7 +721,24 @@ function projectPendingForWire(row, governance) {
|
|
|
664
721
|
const governanceForced = governanceOriginOf({ riskDescriptor: row.riskDescriptor ?? undefined }, governance);
|
|
665
722
|
return governanceForced ? { ...redacted, governanceForced } : redacted;
|
|
666
723
|
}
|
|
667
|
-
|
|
724
|
+
function createSseDeltaLane(res, spec) {
|
|
725
|
+
let prev = new Map();
|
|
726
|
+
return {
|
|
727
|
+
tick(rows) {
|
|
728
|
+
const cur = new Map(rows.map((r) => [spec.keyOf(r), { row: r, bytes: spec.bytesOf(r) }]));
|
|
729
|
+
for (const [k, v] of cur) {
|
|
730
|
+
if (prev.get(k)?.bytes !== v.bytes)
|
|
731
|
+
res.write(`event: ${spec.upsertEvent}\ndata: ${JSON.stringify(spec.upsertData(v.row, Date.now()))}\n\n`);
|
|
732
|
+
}
|
|
733
|
+
for (const [k, v] of prev) {
|
|
734
|
+
if (!cur.has(k))
|
|
735
|
+
res.write(`event: ${spec.exitEvent}\ndata: ${JSON.stringify(spec.exitData(v.row, Date.now()))}\n\n`);
|
|
736
|
+
}
|
|
737
|
+
prev = cur;
|
|
738
|
+
},
|
|
739
|
+
};
|
|
740
|
+
}
|
|
741
|
+
export async function streamApprovals(req, res, cs, live, scope, governance, pollMs = APPROVALS_STREAM_POLL_MS) {
|
|
668
742
|
const keyOf = (p) => JSON.stringify([p.sessionId, p.toolCallId ?? null]);
|
|
669
743
|
sseHeaders(res);
|
|
670
744
|
res.write(`event: meta\ndata: ${JSON.stringify({ type: "meta", version: 1, mode: "approvals-delta", pollMs: APPROVALS_STREAM_POLL_MS })}\n\n`);
|
|
@@ -672,13 +746,29 @@ export async function streamApprovals(req, res, cs, scope, governance, pollMs =
|
|
|
672
746
|
bindSseLifecycle(req, res, () => { closed = true; });
|
|
673
747
|
const start = Date.now();
|
|
674
748
|
let lastBeat = Date.now();
|
|
675
|
-
|
|
676
|
-
|
|
749
|
+
const durableLane = createSseDeltaLane(res, {
|
|
750
|
+
upsertEvent: "pending",
|
|
751
|
+
exitEvent: "resolved",
|
|
752
|
+
keyOf,
|
|
753
|
+
bytesOf: (p) => JSON.stringify({ type: "pending", ...p }),
|
|
754
|
+
upsertData: (p, nowMs) => ({ type: "pending", ...p, serverNowMs: nowMs }),
|
|
755
|
+
exitData: (p, nowMs) => ({ type: "resolved", sessionId: p.sessionId, toolCallId: p.toolCallId, serverNowMs: nowMs }),
|
|
756
|
+
});
|
|
757
|
+
const liveLane = createSseDeltaLane(res, {
|
|
758
|
+
upsertEvent: "live_pending",
|
|
759
|
+
exitEvent: "live_resolved",
|
|
760
|
+
keyOf: (r) => r.approvalId,
|
|
761
|
+
bytesOf: (r) => JSON.stringify({ type: "live_pending", ...r }),
|
|
762
|
+
upsertData: (r, nowMs) => ({ type: "live_pending", ...r, serverNowMs: nowMs }),
|
|
763
|
+
exitData: (r, nowMs) => ({ type: "live_resolved", approvalId: r.approvalId, serverNowMs: nowMs }),
|
|
764
|
+
});
|
|
677
765
|
let first = true;
|
|
678
766
|
while (!closed) {
|
|
679
767
|
let pending;
|
|
768
|
+
let livePending;
|
|
680
769
|
try {
|
|
681
770
|
pending = (await cs.listPending(scope)).map((r) => projectPendingForWire(r, governance));
|
|
771
|
+
livePending = live ? live.listLivePending(scope) : [];
|
|
682
772
|
}
|
|
683
773
|
catch {
|
|
684
774
|
if (!res.writableEnded)
|
|
@@ -686,25 +776,12 @@ export async function streamApprovals(req, res, cs, scope, governance, pollMs =
|
|
|
686
776
|
await sleep(pollMs);
|
|
687
777
|
continue;
|
|
688
778
|
}
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
const pendingFrame = (k) => JSON.stringify({ type: "pending", ...cur.get(k), serverNowMs: Date.now() });
|
|
779
|
+
durableLane.tick(pending);
|
|
780
|
+
liveLane.tick(livePending);
|
|
692
781
|
if (first) {
|
|
693
|
-
for (const k of curFrames.keys())
|
|
694
|
-
res.write(`event: pending\ndata: ${pendingFrame(k)}\n\n`);
|
|
695
782
|
res.write(`event: synced\ndata: ${JSON.stringify({ type: "synced", count: pending.length })}\n\n`);
|
|
696
783
|
first = false;
|
|
697
784
|
}
|
|
698
|
-
else {
|
|
699
|
-
for (const [k, frame] of curFrames)
|
|
700
|
-
if (prevFrames.get(k) !== frame)
|
|
701
|
-
res.write(`event: pending\ndata: ${pendingFrame(k)}\n\n`);
|
|
702
|
-
for (const [k, p] of prev)
|
|
703
|
-
if (!cur.has(k))
|
|
704
|
-
res.write(`event: resolved\ndata: ${JSON.stringify({ type: "resolved", sessionId: p.sessionId, toolCallId: p.toolCallId, serverNowMs: Date.now() })}\n\n`);
|
|
705
|
-
}
|
|
706
|
-
prev = cur;
|
|
707
|
-
prevFrames = curFrames;
|
|
708
785
|
if (Date.now() - start > SSE_MAX_STREAM_MS) {
|
|
709
786
|
res.write(`event: error\ndata: ${JSON.stringify({ type: "error", errorCode: "STREAM_MAX_DURATION", message: "approvals stream reached its 15-minute cap — reconnect to continue" })}\n\n`);
|
|
710
787
|
break;
|
package/dist/http/routes/runs.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { persistedPlaneOf } from "../../terminal.js";
|
|
2
|
-
import { memoRedactedRead, memoRedactTerminalResult, redactErrorMessage, redactHead, redactLedgerEventData, redactTerminalResult } from "../../observability/run-terminal
|
|
2
|
+
import { memoRedactedRead, memoRedactTerminalResult, redactErrorMessage, redactHead, redactLedgerEventData, redactTerminalResult } from "../../observability/run-terminal.js";
|
|
3
3
|
import { isModelTextLedgerEventType } from "../../trace/ledger-events.js";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
import { uuidv7, mintCheckpointToken, CheckpointError, validatePendingSteer } from "@sema-agent/core";
|
|
@@ -89,7 +89,7 @@ async function subagentReadGate(req, res, ctx, runId, search) {
|
|
|
89
89
|
}
|
|
90
90
|
function sendHandleMiss(res, run, thisInstanceId) {
|
|
91
91
|
if (isTerminalRunStatus(run.status)) {
|
|
92
|
-
sendError(res, 404, "not_found.task_owner_settled", "no such background task handle here. Routing hint about its OWNER (not about this handle): the run has already settled
|
|
92
|
+
sendError(res, 404, "not_found.task_owner_settled", "no such background task handle here. Routing hint about its OWNER (not about this handle): the run has already settled. Read the run's own outcome via GET /v1/runs/:id");
|
|
93
93
|
return true;
|
|
94
94
|
}
|
|
95
95
|
if (run.status === "running" && runProvenElsewhere(run, thisInstanceId)) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { redactErrorMessage, redactHead, redactTerminalResult } from "../../observability/run-terminal
|
|
1
|
+
import { redactErrorMessage, redactHead, redactTerminalResult } from "../../observability/run-terminal.js";
|
|
2
2
|
import { runWithVerification, runCascade, uuidv7, terminalProjection } from "@sema-agent/core";
|
|
3
3
|
import { graftParkToolCallId, markChildrenStoppedByUserOnAbort, resumeAtHttpStatus, HEARTBEAT_MS } from "../../runs.js";
|
|
4
4
|
import { stripResumeToken } from "../../terminal.js";
|
|
@@ -414,7 +414,7 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
414
414
|
finalizeTaskResult(ev.result, principal, prepared.spec.objective, prepared.spec.sessionId);
|
|
415
415
|
noteRunOutcome(finalResult);
|
|
416
416
|
putRewindAnchor();
|
|
417
|
-
settleFleet(
|
|
417
|
+
settleFleet(terminalProjection(finalResult.terminal).status, fleetRunResiduals(finalResult));
|
|
418
418
|
if (ledgerSink && finalResult) {
|
|
419
419
|
if (usageKey && deps.modelUsage && deps.runStore && durableTaskId) {
|
|
420
420
|
const rs2 = deps.runStore;
|
|
@@ -5,7 +5,7 @@ import { redactSteerIn, STEER_IN_MAX_CHARS, STEER_IN_MAX_REQUEST_CHARS } from ".
|
|
|
5
5
|
import { journalEntryOfRow, journalEntryPayload, journalRowPayloadOf } from "../../orchestration/workflow-journal-entry.js";
|
|
6
6
|
import { redactSecrets } from "../../trace/redact.js";
|
|
7
7
|
import { createHash } from "node:crypto";
|
|
8
|
-
import { redactErrorMessage, redactHead } from "../../observability/run-terminal
|
|
8
|
+
import { redactErrorMessage, redactHead } from "../../observability/run-terminal.js";
|
|
9
9
|
const JOURNAL_PROJECTION_MEMO_CAP = 1024;
|
|
10
10
|
const journalProjectionMemo = new Map();
|
|
11
11
|
import { sendJson, sendError, sseHeaders } from "../send.js";
|
package/dist/http/server.d.ts
CHANGED
|
@@ -28,8 +28,8 @@ import type { ModelUsageTracker, PromptManifestTracker } from "../budget.js";
|
|
|
28
28
|
import { scopedIdempotencyKey } from "./idempotency.js";
|
|
29
29
|
export { scopedIdempotencyKey };
|
|
30
30
|
import type { ConsolidationSeatLabel } from "../boot/memory-consolidation.js";
|
|
31
|
-
import { streamApprovals,
|
|
32
|
-
export { streamApprovals,
|
|
31
|
+
import { streamApprovals, checkQuestionAnswer } from "./routes/approvals-assistant.js";
|
|
32
|
+
export { streamApprovals, checkQuestionAnswer };
|
|
33
33
|
import { cascadeConfig } from "./run-meta.js";
|
|
34
34
|
export { cascadeConfig };
|
|
35
35
|
import { coarseStatusForState, errorCodeForExit } from "./routes/images.js";
|
package/dist/http/server.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import http from "node:http";
|
|
2
|
-
import { redactHead } from "../observability/run-terminal
|
|
2
|
+
import { redactHead } from "../observability/run-terminal.js";
|
|
3
3
|
import { validatePendingSteer, terminalProjection } from "@sema-agent/core";
|
|
4
4
|
import {} from "../config-center/facade.js";
|
|
5
5
|
import { HttpError } from "../security.js";
|
|
@@ -12,8 +12,8 @@ import {} from "../tool-approval.js";
|
|
|
12
12
|
import {} from "../observability/rate-limit.js";
|
|
13
13
|
import { IdempotencyCache, scopedIdempotencyKey } from "./idempotency.js";
|
|
14
14
|
export { scopedIdempotencyKey };
|
|
15
|
-
import { streamApprovals,
|
|
16
|
-
export { streamApprovals,
|
|
15
|
+
import { streamApprovals, checkQuestionAnswer } from "./routes/approvals-assistant.js";
|
|
16
|
+
export { streamApprovals, checkQuestionAnswer };
|
|
17
17
|
import { cascadeConfig } from "./run-meta.js";
|
|
18
18
|
export { cascadeConfig };
|
|
19
19
|
import { coarseStatusForState, errorCodeForExit } from "./routes/images.js";
|
package/dist/leader/diffout.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { redactHead } from "../observability/run-terminal
|
|
1
|
+
import { redactHead } from "../observability/run-terminal.js";
|
|
2
2
|
export const DEFAULT_MAX_DIFF_BYTES = 5 * 1024 * 1024;
|
|
3
3
|
const DEFAULT_READ_TIMEOUT_MS = 30_000;
|
|
4
4
|
const SAFE_REV = /^[A-Za-z0-9._/-]{1,200}$/;
|
package/dist/leader/endpoint.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { uuidv7 } from "@sema-agent/core";
|
|
2
|
-
import { redactErrorMessage, redactHead, redactLeaderResult, redactLeaderResultWithFlag } from "../observability/run-terminal
|
|
2
|
+
import { redactErrorMessage, redactHead, redactLeaderResult, redactLeaderResultWithFlag } from "../observability/run-terminal.js";
|
|
3
3
|
import { withPrincipal } from "../observability/principal-context.js";
|
|
4
4
|
import { leaderIdemKey } from "../plugins/leader-run-store-sql.js";
|
|
5
5
|
function statusForLeaderResult(result) {
|
package/dist/leader/fanout.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { terminalProjection } from "@sema-agent/core";
|
|
2
|
-
import { redactErrorMessage, redactWorkerReportError } from "../observability/run-terminal
|
|
2
|
+
import { redactErrorMessage, redactWorkerReportError } from "../observability/run-terminal.js";
|
|
3
3
|
import { pausedCauseOf } from "../terminal.js";
|
|
4
4
|
class Semaphore {
|
|
5
5
|
max;
|
package/dist/leader/leader.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { fanOut } from "./fanout.js";
|
|
2
|
-
import { redactErrorMessage, redactHead } from "../observability/run-terminal
|
|
2
|
+
import { redactErrorMessage, redactHead } from "../observability/run-terminal.js";
|
|
3
3
|
import { pullDiff } from "./diffout.js";
|
|
4
4
|
import { mergeBranches } from "./merge.js";
|
|
5
5
|
import { Runner, runWithVerification, resumeWithVerification, terminalProjection } from "@sema-agent/core";
|
package/dist/leader/merge.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { delimitUntrusted } from "@sema-agent/core";
|
|
2
|
-
import { OVERSIZED_DIAG_PLACEHOLDER, redactArtifactText, redactErrorMessage, redactHead, redactTail } from "../observability/run-terminal
|
|
2
|
+
import { OVERSIZED_DIAG_PLACEHOLDER, redactArtifactText, redactErrorMessage, redactHead, redactTail } from "../observability/run-terminal.js";
|
|
3
3
|
import { applyAndGrade, changedFilesOutOfAllowlist } from "./grader-env-factory.js";
|
|
4
4
|
import { shellQuote as Q } from "../plugins/remote-shell.js";
|
|
5
5
|
const CONFLICT_FILES_MAX = 200;
|
package/dist/leader/planner.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { redactErrorMessage, redactHead, redactTail } from "../observability/run-terminal
|
|
1
|
+
import { redactErrorMessage, redactHead, redactTail } from "../observability/run-terminal.js";
|
|
2
2
|
export function economicGuard(objective, opts = {}) {
|
|
3
3
|
if (opts.fanoutEnabled === false)
|
|
4
4
|
return { single: true, reason: "fan-out disabled by deployment (A6)" };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { runExecGate, } from "@sema-agent/core";
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { shellQuote as sq } from "../plugins/remote-shell.js";
|
|
4
|
-
import { redactHead, redactTail } from "../observability/run-terminal
|
|
4
|
+
import { redactHead, redactTail } from "../observability/run-terminal.js";
|
|
5
5
|
const sha = (s) => createHash("sha256").update(s).digest("hex").slice(0, 16);
|
|
6
6
|
function changedFilesOf(patch) {
|
|
7
7
|
const out = new Set();
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { runRepairLoop, delimitUntrusted, } from "@sema-agent/core";
|
|
2
2
|
import { mkRepairOracle } from "./repair-oracle.js";
|
|
3
3
|
import { terminalProjection } from "@sema-agent/core";
|
|
4
|
-
import { redactHead } from "../observability/run-terminal
|
|
4
|
+
import { redactHead } from "../observability/run-terminal.js";
|
|
5
5
|
function statusForTerminal(t) {
|
|
6
6
|
return t === "fixed" || t === "candidate_only" ? "completed" : "failed";
|
|
7
7
|
}
|
package/dist/leader/wire.js
CHANGED
|
@@ -16,7 +16,7 @@ import { attachRepairLoopDeps } from "./repair-wire.js";
|
|
|
16
16
|
import { routePlanWithFallback, validateSubtasks } from "./planner.js";
|
|
17
17
|
import { createEngineNoticeSeat } from "../boot/runner-deps.js";
|
|
18
18
|
import { currentPrincipal } from "../observability/principal-context.js";
|
|
19
|
-
import { redactHead } from "../observability/run-terminal
|
|
19
|
+
import { redactHead } from "../observability/run-terminal.js";
|
|
20
20
|
const REPAIR_LOOP_WIRED = true;
|
|
21
21
|
const sh = (env) => async (cmd) => {
|
|
22
22
|
const r = await env.exec(`bash -lc ${JSON.stringify(cmd)}`);
|
|
@@ -141,6 +141,10 @@ export declare const FAIL_OPEN_TAGS: {
|
|
|
141
141
|
readonly cls: "F";
|
|
142
142
|
readonly note: "#261 fleet 存活对账腿的一次 durable 读失败/超预算(2s)⇒ **本周期整轮跳过**并开熔断,不做任何投影。放行的最坏后果 = 幽灵行(发布方已死、行僵在内存 Map 里)在面板上多活几个周期,与本腿存在之前的行为等同(那时它永远活着),不是新损失;真源不受影响(`GET /v1/runs/:id` 照常)。方向刻意 fail-open 且**只向留行一侧**:读不出来 ≠ 该退休——库抖一下就把一屏在跑的任务从面板上抹掉,比多留一条幽灵行伤得多。必须留痕:静默吞掉之后「对账腿为什么一直没生效」在遥测里没有任何痕迹,而本腿的全部价值就是「幽灵行会自己走」。";
|
|
143
143
|
};
|
|
144
|
+
readonly "server.fleet.terminal-notify": {
|
|
145
|
+
readonly cls: "F";
|
|
146
|
+
readonly note: "S-450:**两条臂共用**(detail 里 `read-failed` 分得开)—— ① `observability/run-terminal.ts` 的**行为订阅者**扇出(唯一铸点 `notifyRunTerminal`,三只 store 的 CAS 赢分支各调一次):某个订阅者(今天只有 fleet 对账腿的候补窗升级腿)在 `setTerminal` 的调用栈上抛错 ⇒ 吞掉并计数,终局写照常成功;② `fleet/fleet-reconciler.ts` 的事件腿在收到事件后那一次 durable 点读失败 ⇒ 这条行留在候补窗里等周期臂。⚠️ 同一只铸点上的**日志腿**(`setRunTerminalLogObserver`)走的是**相反**契约(刻意不 try、抛出让 `setTerminal` reject),**不计本 tag** —— 读遥测别把两席混成一件。方向承重:这条腿的全部价值是「把候补窗行的成对终帧从『下一个对账周期(≤60s)』提前到『durable 写落地的同一 tick』」,它是一格**投影提速**;把它的 bug 外溢成终局写 reject 会让 run 卡 running 到 reaper、claim 悬挂([868] 指纹),代价高出几个量级。放行的最坏后果 = 这条行的成对终帧退回到周期臂补发,**与本件存在之前的行为逐字等同**,不是新损失;故 F 类。必须留痕:静默吞掉之后「事件腿为什么一直没生效」在遥测里没有任何痕迹,而面板上看到的只是『偶尔慢一分钟』这种没人会立案的症状。";
|
|
147
|
+
};
|
|
144
148
|
readonly "server.fleet.run-status-unmapped": {
|
|
145
149
|
readonly cls: "F";
|
|
146
150
|
readonly note: "A-032 P1-④:`runStatusToFleet` 收到**词表外**的 run 状态词(闭集签名之外的唯一来路 = `rowToRun` 把无约束的 status 文本列裸 cast,滚动升级里更新的副本写的新词被旧副本读到)⇒ 折成 `idle`。放行的最坏后果 = 该行在 fleet 面板上**永久滞留**(removal 集 {completed,failed,killed} 收不到 idle),纯展示面、不影响 run 本身与任何执法判据,故 F 类。方向刻意不改(把 miss 折成终局会把一条还在跑的任务从面板上摘掉,误摘比多一行更伤)。留痕是承重的:2026-06-28 on-box e2e 逮到的 `blocked` 泄漏当年只能靠人眼在面板上发现——计数让下一次词表漂移在遥测里当场显形。编译期一侧已由闭集入参 + `never` 臂执法。";
|
|
@@ -189,6 +193,10 @@ export declare const FAIL_OPEN_TAGS: {
|
|
|
189
193
|
readonly cls: "F";
|
|
190
194
|
readonly note: "S-02 codex R1-[medium](验真后修):decide 的 409 `approval_stale` 臂对**调用方回显的旧 token** 做归属判定(`cs.get(binding.checkpointToken)`),那次读**抛了**(store 抖动 / 版本守卫 / 行恰被清理)。修前这里是静默 `catch(()=>null)`,「读不出」被折成「确认不在库」——叠上 S-02 的 404 收口后,一次 store 抖动就会把合法旧 token 的 409+terminal 谎报成 404「没有这条审批」。修后:unreadable ⇒ **保持 409**(`terminal` 不披露——证不出归属就不说,与 2026-07-26 纪律同向;404 折叠只在读成功且未命中时走,探测方无法自控 store 故障 ⇒ 谕示封闭不破)。放行的最坏后果 = 该次拒体 `terminal` 诚实缺席(`currentPending` 由它自己的读口与属主复核另判,不受本臂牵连),壳退回重拉列表,judgment 面 fail-safe,故 F 类。留痕理由:这次读是 terminal 披露与 404 折叠两个判定的输入,长期抖动必须在遥测显形。";
|
|
191
195
|
};
|
|
196
|
+
readonly "server.resume.reopen-confirm-unreadable": {
|
|
197
|
+
readonly cls: "F";
|
|
198
|
+
readonly note: "S-447(#60 M2 自查项,`http/resume-legs.ts` 的 `confirmCheckpointReopened`):续跑腿在 core 给出 reopen 族失败后按 **token** 复核「这张 checkpoint 是不是又 pending 了」,这一读抛(库拖/版本太新/店不可达)。**方向本来就是 fail-closed**:读不出 ⇒ `return true` ⇒ 「不写任何终局、重新 park」(`:1206-1212` 头注已把取舍与有界代价写死:错 park 由 `reapSuspended` 兑,错终局会孤儿化一张活 park 到 TTL)—— 所以这条腿不是 P 类,缺的只是**留痕**。放行的最坏后果 = 多一次多余的 re-park(一张已经被决掉的卡重新挂到 reaper 窗里),人面上是「这条续跑没动」而不是谁被放行了。必须登记的理由是 **uniformity**:同文件三只同形读(`:249` `server.approvals.stale-echoed-token-unreadable` · `:288` `server.approvals.stale-current-pending-unreadable` · 本腿)中只有它是黑的,于是「店一直读不出所以每条续跑都在白白 re-park」与「一切正常」在遥测上同形。计数非零 = 这台机器的 checkpoint 店在拖,运维要去看库。";
|
|
199
|
+
};
|
|
192
200
|
readonly "server.approvals.stale-current-pending-unreadable": {
|
|
193
201
|
readonly cls: "F";
|
|
194
202
|
readonly note: "S-02 件2([5522]④→[5525]):409 `approval_stale` 拒体铸 additive 指路键 `currentPending`(本会话当前 pending 的 D-1 坐标,壳一跳重定位)时,当前 pending 行的 `cs.get(token)` 读**抛了**(store 抖动 / 滚动升级期 checkpoint 格式 version guard / 行恰被并发消费)⇒ 本次拒体**缺 `currentPending` 键**,409 本体与 `terminal` 判定照旧。放行的最坏后果 = 壳退回该键到货前的行为(重拉 GET /v1/approvals 自行重定位),纯指路/便利面,不参与任何门/CAS/resume 判定,故 F 类(同族先例:`server.park.toolcallid-read-failed`,同为 cs.get 投影读失败臂)。必须留痕:键缺席与「此刻确实没有 pending」在 wire 上同形,不计数则「指路为什么总缺」在遥测里永不显形。";
|
|
@@ -132,6 +132,10 @@ export const FAIL_OPEN_TAGS = {
|
|
|
132
132
|
cls: "F",
|
|
133
133
|
note: "#261 fleet 存活对账腿的一次 durable 读失败/超预算(2s)⇒ **本周期整轮跳过**并开熔断,不做任何投影。放行的最坏后果 = 幽灵行(发布方已死、行僵在内存 Map 里)在面板上多活几个周期,与本腿存在之前的行为等同(那时它永远活着),不是新损失;真源不受影响(`GET /v1/runs/:id` 照常)。方向刻意 fail-open 且**只向留行一侧**:读不出来 ≠ 该退休——库抖一下就把一屏在跑的任务从面板上抹掉,比多留一条幽灵行伤得多。必须留痕:静默吞掉之后「对账腿为什么一直没生效」在遥测里没有任何痕迹,而本腿的全部价值就是「幽灵行会自己走」。",
|
|
134
134
|
},
|
|
135
|
+
"server.fleet.terminal-notify": {
|
|
136
|
+
cls: "F",
|
|
137
|
+
note: "S-450:**两条臂共用**(detail 里 `read-failed` 分得开)—— ① `observability/run-terminal.ts` 的**行为订阅者**扇出(唯一铸点 `notifyRunTerminal`,三只 store 的 CAS 赢分支各调一次):某个订阅者(今天只有 fleet 对账腿的候补窗升级腿)在 `setTerminal` 的调用栈上抛错 ⇒ 吞掉并计数,终局写照常成功;② `fleet/fleet-reconciler.ts` 的事件腿在收到事件后那一次 durable 点读失败 ⇒ 这条行留在候补窗里等周期臂。⚠️ 同一只铸点上的**日志腿**(`setRunTerminalLogObserver`)走的是**相反**契约(刻意不 try、抛出让 `setTerminal` reject),**不计本 tag** —— 读遥测别把两席混成一件。方向承重:这条腿的全部价值是「把候补窗行的成对终帧从『下一个对账周期(≤60s)』提前到『durable 写落地的同一 tick』」,它是一格**投影提速**;把它的 bug 外溢成终局写 reject 会让 run 卡 running 到 reaper、claim 悬挂([868] 指纹),代价高出几个量级。放行的最坏后果 = 这条行的成对终帧退回到周期臂补发,**与本件存在之前的行为逐字等同**,不是新损失;故 F 类。必须留痕:静默吞掉之后「事件腿为什么一直没生效」在遥测里没有任何痕迹,而面板上看到的只是『偶尔慢一分钟』这种没人会立案的症状。",
|
|
138
|
+
},
|
|
135
139
|
"server.fleet.run-status-unmapped": {
|
|
136
140
|
cls: "F",
|
|
137
141
|
note: "A-032 P1-④:`runStatusToFleet` 收到**词表外**的 run 状态词(闭集签名之外的唯一来路 = `rowToRun` 把无约束的 status 文本列裸 cast,滚动升级里更新的副本写的新词被旧副本读到)⇒ 折成 `idle`。放行的最坏后果 = 该行在 fleet 面板上**永久滞留**(removal 集 {completed,failed,killed} 收不到 idle),纯展示面、不影响 run 本身与任何执法判据,故 F 类。方向刻意不改(把 miss 折成终局会把一条还在跑的任务从面板上摘掉,误摘比多一行更伤)。留痕是承重的:2026-06-28 on-box e2e 逮到的 `blocked` 泄漏当年只能靠人眼在面板上发现——计数让下一次词表漂移在遥测里当场显形。编译期一侧已由闭集入参 + `never` 臂执法。",
|
|
@@ -180,6 +184,10 @@ export const FAIL_OPEN_TAGS = {
|
|
|
180
184
|
cls: "F",
|
|
181
185
|
note: "S-02 codex R1-[medium](验真后修):decide 的 409 `approval_stale` 臂对**调用方回显的旧 token** 做归属判定(`cs.get(binding.checkpointToken)`),那次读**抛了**(store 抖动 / 版本守卫 / 行恰被清理)。修前这里是静默 `catch(()=>null)`,「读不出」被折成「确认不在库」——叠上 S-02 的 404 收口后,一次 store 抖动就会把合法旧 token 的 409+terminal 谎报成 404「没有这条审批」。修后:unreadable ⇒ **保持 409**(`terminal` 不披露——证不出归属就不说,与 2026-07-26 纪律同向;404 折叠只在读成功且未命中时走,探测方无法自控 store 故障 ⇒ 谕示封闭不破)。放行的最坏后果 = 该次拒体 `terminal` 诚实缺席(`currentPending` 由它自己的读口与属主复核另判,不受本臂牵连),壳退回重拉列表,judgment 面 fail-safe,故 F 类。留痕理由:这次读是 terminal 披露与 404 折叠两个判定的输入,长期抖动必须在遥测显形。",
|
|
182
186
|
},
|
|
187
|
+
"server.resume.reopen-confirm-unreadable": {
|
|
188
|
+
cls: "F",
|
|
189
|
+
note: "S-447(#60 M2 自查项,`http/resume-legs.ts` 的 `confirmCheckpointReopened`):续跑腿在 core 给出 reopen 族失败后按 **token** 复核「这张 checkpoint 是不是又 pending 了」,这一读抛(库拖/版本太新/店不可达)。**方向本来就是 fail-closed**:读不出 ⇒ `return true` ⇒ 「不写任何终局、重新 park」(`:1206-1212` 头注已把取舍与有界代价写死:错 park 由 `reapSuspended` 兑,错终局会孤儿化一张活 park 到 TTL)—— 所以这条腿不是 P 类,缺的只是**留痕**。放行的最坏后果 = 多一次多余的 re-park(一张已经被决掉的卡重新挂到 reaper 窗里),人面上是「这条续跑没动」而不是谁被放行了。必须登记的理由是 **uniformity**:同文件三只同形读(`:249` `server.approvals.stale-echoed-token-unreadable` · `:288` `server.approvals.stale-current-pending-unreadable` · 本腿)中只有它是黑的,于是「店一直读不出所以每条续跑都在白白 re-park」与「一切正常」在遥测上同形。计数非零 = 这台机器的 checkpoint 店在拖,运维要去看库。",
|
|
190
|
+
},
|
|
183
191
|
"server.approvals.stale-current-pending-unreadable": {
|
|
184
192
|
cls: "F",
|
|
185
193
|
note: "S-02 件2([5522]④→[5525]):409 `approval_stale` 拒体铸 additive 指路键 `currentPending`(本会话当前 pending 的 D-1 坐标,壳一跳重定位)时,当前 pending 行的 `cs.get(token)` 读**抛了**(store 抖动 / 滚动升级期 checkpoint 格式 version guard / 行恰被并发消费)⇒ 本次拒体**缺 `currentPending` 键**,409 本体与 `terminal` 判定照旧。放行的最坏后果 = 壳退回该键到货前的行为(重拉 GET /v1/approvals 自行重定位),纯指路/便利面,不参与任何门/CAS/resume 判定,故 F 类(同族先例:`server.park.toolcallid-read-failed`,同为 cs.get 投影读失败臂)。必须留痕:键缺席与「此刻确实没有 pending」在 wire 上同形,不计数则「指路为什么总缺」在遥测里永不显形。",
|
|
@@ -28,6 +28,9 @@ export declare class Metrics {
|
|
|
28
28
|
* needed). Counters are cumulative totals; histograms are reported as averages (sum/count). */
|
|
29
29
|
summarize(): MetricsSummary;
|
|
30
30
|
}
|
|
31
|
+
/** S-451:`/metrics/summary` 上「失败最多的路由」保留几条。看板要的是**头部**,不是全表;
|
|
32
|
+
* `routeLabel` 的值域本来就有界(路由声明表 + `"other"`),这个上限只是让这张脸的大小与路由表长解耦。 */
|
|
33
|
+
export declare const HTTP_ERROR_ROUTE_TOP_N = 10;
|
|
31
34
|
export interface MetricsSummary {
|
|
32
35
|
runsActive: number;
|
|
33
36
|
tasks: Record<string, number>;
|
|
@@ -61,6 +64,15 @@ export interface MetricsSummary {
|
|
|
61
64
|
verifications: Record<string, number>;
|
|
62
65
|
councilRuns: Record<string, number>;
|
|
63
66
|
toolErrors: Record<string, number>;
|
|
67
|
+
/** S-451(additive,7.87.0):窗内 HTTP **失败**请求数,按状态码**类**聚合 —— 键只有 `"4xx"` / `"5xx"`
|
|
68
|
+
* 两种,2xx/3xx 一格不进。⚠️ `"4xx"` 里**含 `499`**(本仓给「客户端在响应结束前断连」铸的标签,
|
|
69
|
+
* SSE 长连接上是常态)—— 看一场真风暴时按 {@link httpErrorsByRoute} 分辨是哪条腿。
|
|
70
|
+
* 空对象 = 窗内零失败(与 `tasks`/`toolErrors` 同律:键恒在场,不用缺席表示零)。 */
|
|
71
|
+
httpErrorsByClass: Record<string, number>;
|
|
72
|
+
/** S-451(additive,7.87.0):失败(4xx+5xx 合计)最多的前 {@link HTTP_ERROR_ROUTE_TOP_N} 条 **route 标签**
|
|
73
|
+
* (`routeLabel` 的低基数形,id 已折成 `:id`;未命中路由表的落 `"other"`)。按计数降序。
|
|
74
|
+
* 这是「180 次 404 到底打在哪」的那一格。 */
|
|
75
|
+
httpErrorsByRoute: Record<string, number>;
|
|
64
76
|
}
|
|
65
77
|
export interface MetricSnapshot {
|
|
66
78
|
counters: Array<{
|
|
@@ -188,9 +188,25 @@ export class Metrics {
|
|
|
188
188
|
toolErrors: Object.fromEntries([...(this.counters.get("tool_calls_total")?.series.values() ?? [])]
|
|
189
189
|
.filter((s) => s.labels.ok === "false")
|
|
190
190
|
.map((s) => [s.labels.name ?? "?", s.value])),
|
|
191
|
+
...(() => {
|
|
192
|
+
const failures = [...(this.counters.get("http_requests_total")?.series.values() ?? [])]
|
|
193
|
+
.map((s) => ({ cls: /^([45])\d\d$/.exec(s.labels.status ?? "")?.[1], route: s.labels.route ?? "?", value: s.value }))
|
|
194
|
+
.filter((x) => x.cls !== undefined);
|
|
195
|
+
const byClass = {};
|
|
196
|
+
const byRoute = {};
|
|
197
|
+
for (const f of failures) {
|
|
198
|
+
byClass[`${f.cls}xx`] = (byClass[`${f.cls}xx`] ?? 0) + f.value;
|
|
199
|
+
byRoute[f.route] = (byRoute[f.route] ?? 0) + f.value;
|
|
200
|
+
}
|
|
201
|
+
return {
|
|
202
|
+
httpErrorsByClass: byClass,
|
|
203
|
+
httpErrorsByRoute: Object.fromEntries(Object.entries(byRoute).sort((a, b) => b[1] - a[1]).slice(0, HTTP_ERROR_ROUTE_TOP_N)),
|
|
204
|
+
};
|
|
205
|
+
})(),
|
|
191
206
|
};
|
|
192
207
|
}
|
|
193
208
|
}
|
|
209
|
+
export const HTTP_ERROR_ROUTE_TOP_N = 10;
|
|
194
210
|
export function createMetrics() {
|
|
195
211
|
const m = new Metrics();
|
|
196
212
|
m.counter("metric_gauge_eval_errors_total", "Dynamic gauge eval errors by gauge name (an erroring gauge emits NO sample that scrape — absence means unknown, not 0)");
|
|
@@ -270,6 +286,7 @@ export function createMetrics() {
|
|
|
270
286
|
m.counter("compaction_events_total", "Compaction lifecycle events by outcome (started/completed/failed/skipped) and trigger");
|
|
271
287
|
m.counter("hook_llm_calls_total", "prompt/agent hook entries that completed a model call (hooks 3b), by type");
|
|
272
288
|
m.counter("auth_bridge_verify_total", "Registry SSO JWT verifications (auth-bridge), by outcome");
|
|
289
|
+
m.counter("fleet_rows_reconciled_total", "Fleet rows the #261 liveness-reconcile leg acted on, by `disposition`: 'retired' = durable was terminal, the leg projected the settled pair (terminal task frame → task_remove); 'retired_deferred' = a row that had already left declaratively (removeReason:\"orphaned\") got its settled pair BACKFILLED once durable turned terminal; 'orphaned' / 'orphaned_child' = declarative exit, NO terminal was minted (the reaper owns the death ruling); 'workflow_retired' = same projection for a workflow row. `trigger` rides ONLY the retired_deferred series (S-450): 'event' = the backfill fired on the same tick a terminal write landed through the run-store write口 (single replica), 'cycle' = it fired on the 60s reconcile cycle (cross-replica writes, and terminals flipped by the BATCH reapers, which never pass through the write口). A retired_deferred series that is 100% trigger=cycle means the event leg is not wired or this deployment\'s terminals all come from those two families");
|
|
273
290
|
m.counter("fleet_lease_issued_total", "Fleet trace-lease tokens issued");
|
|
274
291
|
m.counter("fleet_lease_rejected_total", "Fleet trace-lease requests rejected");
|
|
275
292
|
m.counter("fleet_lease_exhausted_total", "Fleet trace-lease requests rejected because the pool was exhausted");
|
|
@@ -3,8 +3,14 @@ import { type RunRecord } from "../plugins/store-contracts.js";
|
|
|
3
3
|
export interface RunTerminalRecord {
|
|
4
4
|
/** 这条 run 的账本 id = `GET /v1/runs/:id` 的 id(本仓 run 行主键名为 taskId,同一个值)。 */
|
|
5
5
|
runId: string;
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
/**
|
|
7
|
+
* 🔴 **只在「这条终局要出声」时在场**(S-450):三只 store 只在 `runTerminalLogLevel(...) !== undefined`
|
|
8
|
+
* 时才多付那一次 PK 读去装它(成功终局零额外查询 —— 那条注在 `run-store-sql.ts` 的 `setTerminal` 里)。
|
|
9
|
+
* 日志腿恰好只在同一形下运行,所以**它读到的恒是在场的**;行为订阅者不得依赖它。
|
|
10
|
+
*/
|
|
11
|
+
sessionId?: string;
|
|
12
|
+
/** 同 {@link RunTerminalRecord.sessionId}:只在要出声的那一形在场。`null` = 行上没有 owner。 */
|
|
13
|
+
owner?: string | null;
|
|
8
14
|
status: RunRecord["status"];
|
|
9
15
|
/** 终局结果里的引擎侧 taskId —— 只在与 {@link RunTerminalRecord.runId} **不同**时进日志(sync 腿上 core 会把它
|
|
10
16
|
* 铸成 sessionId,两者同值时重复一遍只是噪音)。 */
|
|
@@ -15,8 +21,16 @@ export interface RunTerminalRecord {
|
|
|
15
21
|
/** 行创建到终局的墙钟毫秒(store 侧用行的 createdAt 算;算不出=不传,键缺席)。 */
|
|
16
22
|
elapsedMs?: number | undefined;
|
|
17
23
|
}
|
|
18
|
-
/**
|
|
19
|
-
export type
|
|
24
|
+
/** **日志腿**收到的是**已判级 + 已脱敏 + 已截断**的成品——装配层只负责把它送进 logger。 */
|
|
25
|
+
export type RunTerminalLogObserver = (level: "error" | "info", fields: Record<string, unknown>) => void;
|
|
26
|
+
/**
|
|
27
|
+
* **行为订阅者**(S-450):CAS 赢的那一刻收到整只 {@link RunTerminalRecord}。
|
|
28
|
+
*
|
|
29
|
+
* 🔴 载荷是**这次写自己的视角**,不是回读的 durable 行 —— 只有 `runId` / `status` 两位在**任何**形下
|
|
30
|
+
* 都为真(CAS 赢 ⇒ 行就是这个词)。日志腿专用的那几位在成功终局上**刻意缺席**(store 不为它们多付一次
|
|
31
|
+
* PK 往返,见 {@link RunTerminalRecord.sessionId})。要更多事实的订阅者请自己按 `runId` 读 durable。
|
|
32
|
+
*/
|
|
33
|
+
export type RunTerminalSubscriber = (rec: RunTerminalRecord) => void;
|
|
20
34
|
/**
|
|
21
35
|
* 终局行的级别判据。`undefined` = 不打行。
|
|
22
36
|
*
|
|
@@ -25,13 +39,39 @@ export type RunTerminalObserver = (level: "error" | "info", fields: Record<strin
|
|
|
25
39
|
* 穷举 switch,并在 `blocked` 那一格给出与 fleet 读面互斥的答案)。表的编译期双向钉在属主处。
|
|
26
40
|
*/
|
|
27
41
|
export declare function runTerminalLogLevel(status: RunRecord["status"], errorCode: string | undefined): "error" | "info" | undefined;
|
|
28
|
-
/**
|
|
29
|
-
export declare function
|
|
42
|
+
/** **日志腿**的装配点(`boot/stage-01-config.ts` 唯一)。传 `undefined` 卸载(测试隔离)。 */
|
|
43
|
+
export declare function setRunTerminalLogObserver(fn: RunTerminalLogObserver | undefined): void;
|
|
44
|
+
/**
|
|
45
|
+
* **行为订阅者**的登记口(S-450)。返回解除登记的句柄。
|
|
46
|
+
*
|
|
47
|
+
* 与 {@link setRunTerminalLogObserver} 是**两个语义不同的席**,不是一个带旗的列表:
|
|
48
|
+
* · 日志腿**恰一只**、装配层亲接、**不隔离**(抛出会让 `setTerminal` reject —— 见 {@link notifyRunTerminal});
|
|
49
|
+
* · 行为订阅者**可多只**、运行期来去、**逐只隔离**(抛出吞掉 + `recordFailOpen`)。
|
|
50
|
+
* 两条失败契约相反,所以它们必须是两个名字;混进一个列表再加一个 `isolate` 位,就是把「这条腿出事
|
|
51
|
+
* 该不该把 run 拖下水」这个**语义**降级成一个**配置**。
|
|
52
|
+
*/
|
|
53
|
+
export declare function subscribeRunTerminal(fn: RunTerminalSubscriber): () => void;
|
|
54
|
+
/** test-only:清空行为订阅者(用例之间不串台;日志腿用 `setRunTerminalLogObserver(undefined)`)。 */
|
|
55
|
+
export declare function resetRunTerminalSubscribersForTest(): void;
|
|
30
56
|
/**
|
|
31
|
-
*
|
|
57
|
+
* 🔴 **run 行翻终局的唯一铸点** —— 三个 run store 的 `setTerminal` 在 **CAS 赢了之后**各调它一次
|
|
58
|
+
* (SQL 腿 `w.affected > 0`,file / memory 腿在同判据的 CAS 分支内)。「CAS 赢了才发」是**恰一次**的来源:
|
|
59
|
+
* 调用侧的兜底腿有意会对同一行重复写(cancel × leg-throw × reaper 三向),打空的那几次一个字都不发。
|
|
60
|
+
*
|
|
61
|
+
* 两条出口,**失败契约相反**,顺序承重:
|
|
62
|
+
* ① **行为订阅者先**({@link subscribeRunTerminal},逐只 try)—— 它们要在日志腿万一违约抛出之前跑完;
|
|
63
|
+
* ② **日志腿后**({@link setRunTerminalLogObserver},级别筛 + 脱敏 + 不 try)。
|
|
64
|
+
*
|
|
65
|
+
* 为什么日志腿**刻意不 try**(原文保留,[ref] 的成文理由):静默吞掉观测者异常 = 又造一条「走了兜底与
|
|
66
|
+
* 一切正常同形」的路,方向与本模块的存在理由相反;生产装配的那一只只调 `Logger`(`createLogger` 的 emit
|
|
67
|
+
* 内部已把 JSON.stringify 与 write 全部 try 住,结构上不抛)。
|
|
68
|
+
* 为什么行为订阅者**必须 try**(S-450 新加的那一半):它们是**行为腿**,不是一只结构上不抛的 logger;
|
|
69
|
+
* 而本函数跑在 `setTerminal` 的调用栈上、**CAS 已经赢了、行已经落盘**,此时把订阅者的 bug 放大成
|
|
70
|
+
* 「终局写 reject」= run 卡 `running` 到 reaper + claim 悬挂([ref] 指纹)。吞但**不静默**:
|
|
71
|
+
* `recordFailOpen("server.fleet.terminal-notify")`(F 类,普查登记在案)。
|
|
32
72
|
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
73
|
+
* 脱敏(只在日志腿那一格):`errorMessage` 过 `redactSecrets` 再截断到 {@link MESSAGE_CAP} —— provider 的
|
|
74
|
+
* 错误体会原样回显请求片段(取证里的 403 体带订阅链接;别的 provider 会回显 Authorization 头前缀),
|
|
35
75
|
* 而引擎日志是运维面、常被整条采集走。`errorCode` / `model` 是闭集式短标量,verbatim。
|
|
36
76
|
*/
|
|
37
77
|
export declare function notifyRunTerminal(rec: RunTerminalRecord): void;
|
|
@@ -164,4 +204,4 @@ export declare function memoRedactTerminalResult<T extends Parameters<typeof red
|
|
|
164
204
|
export declare function diffScanShadowCountForTest(): number;
|
|
165
205
|
export declare function resetDiffScanShadowCountForTest(): void;
|
|
166
206
|
export declare function resetReadRedactionMemoForTest(): void;
|
|
167
|
-
//# sourceMappingURL=run-terminal
|
|
207
|
+
//# sourceMappingURL=run-terminal.d.ts.map
|