@sema-agent/server 7.1.0 → 7.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/README.zh-CN.md +1 -1
- package/USAGE.md +6 -1
- package/dist/approval-ask-machine.d.ts +39 -0
- package/dist/approval-ask-machine.js +101 -0
- package/dist/approval-card.d.ts +244 -0
- package/dist/approval-card.js +237 -0
- package/dist/approval-deny-reasons.d.ts +56 -0
- package/dist/approval-deny-reasons.js +54 -0
- package/dist/approval-reconciler.d.ts +167 -0
- package/dist/approval-reconciler.js +307 -0
- package/dist/boot/coordinators.d.ts +1 -0
- package/dist/boot/coordinators.js +36 -3
- package/dist/boot/lexical-path-env.d.ts +14 -0
- package/dist/boot/lexical-path-env.js +116 -0
- package/dist/boot/org-memory.d.ts +8 -5
- package/dist/boot/org-memory.js +23 -12
- package/dist/boot/reapers.d.ts +34 -0
- package/dist/boot/reapers.js +198 -23
- package/dist/boot/resolve-spec.d.ts +16 -1
- package/dist/boot/resolve-spec.js +104 -46
- package/dist/config-center/facade.d.ts +1 -1
- package/dist/config-center/facade.js +1 -1
- package/dist/config-center/http-client.d.ts +19 -0
- package/dist/config-center/http-client.js +81 -32
- package/dist/config-types.d.ts +76 -7
- package/dist/config.d.ts +1 -0
- package/dist/config.js +134 -4
- package/dist/elicitation.d.ts +4 -0
- package/dist/elicitation.js +2 -2
- package/dist/http/routes/capabilities.js +18 -2
- package/dist/http/routes/runs.d.ts +1 -0
- package/dist/http/routes/runs.js +548 -16
- package/dist/http/routes/tasks.js +190 -12
- package/dist/http/server.d.ts +1 -1
- package/dist/http/server.js +124 -8
- package/dist/http/sse-log.d.ts +51 -0
- package/dist/http/sse-log.js +64 -0
- package/dist/http/wire-types.d.ts +20 -5
- package/dist/leader/wire.js +4 -0
- package/dist/main.js +7 -4
- package/dist/org-memory-admission.d.ts +2 -1
- package/dist/org-memory-admission.js +10 -4
- package/dist/parked-decide.js +5 -2
- package/dist/plugins/approval-ask-store-memory.d.ts +38 -0
- package/dist/plugins/approval-ask-store-memory.js +299 -0
- package/dist/plugins/approval-ask-store-sql.d.ts +341 -0
- package/dist/plugins/approval-ask-store-sql.js +705 -0
- package/dist/plugins/background-agent-store-sql.js +20 -1
- package/dist/plugins/checkpoint-store-sql.d.ts +84 -9
- package/dist/plugins/checkpoint-store-sql.js +297 -16
- package/dist/plugins/local-checkpoint-store.d.ts +6 -5
- package/dist/plugins/local-checkpoint-store.js +4 -0
- package/dist/plugins/pg-pool.js +11 -0
- package/dist/plugins/store-backend.d.ts +18 -0
- package/dist/plugins/store-backend.js +10 -0
- package/dist/plugins/tidb-pool.js +27 -4
- package/dist/question.d.ts +18 -3
- package/dist/question.js +20 -6
- package/dist/runs.d.ts +16 -1
- package/dist/runs.js +61 -3
- package/dist/runtime-caps-resolver.d.ts +7 -1
- package/dist/runtime-caps-resolver.js +65 -3
- package/dist/runtime-governance.d.ts +11 -4
- package/dist/runtime-governance.js +16 -5
- package/dist/spec-fields.d.ts +4 -0
- package/dist/spec-fields.js +6 -0
- package/dist/task-settings.d.ts +35 -15
- package/dist/task-settings.js +19 -5
- package/dist/tool-approval.d.ts +296 -3
- package/dist/tool-approval.js +1066 -50
- package/dist/trace/core-keyset-guard.d.ts +2 -2
- package/dist/trace/ledger-sink.js +14 -1
- package/dist/trace/project.d.ts +55 -0
- package/dist/trace/project.js +135 -0
- package/package.json +4 -3
package/dist/boot/reapers.d.ts
CHANGED
|
@@ -21,6 +21,36 @@ import type { Metrics } from "../observability/metrics.js";
|
|
|
21
21
|
import type { WorkflowNotifyGate, WorkflowNotifyJournalStore } from "../orchestration/workflow-notify-journal.js";
|
|
22
22
|
import type { TaskAttachmentStore } from "../plugins/task-attachment-store.js";
|
|
23
23
|
import type { CheckpointStoreFull, CostQuotaStore, ImageBake, RateLimiterStore, ServiceWorkflowJournalStore, StoreBackend, ToolResultStoreFull } from "../plugins/store-backend.js";
|
|
24
|
+
import type { ToolApprovalCoordinator } from "../tool-approval.js";
|
|
25
|
+
/** #157 / [2762]§五(sema-comms `audits/failopen-governance-176.md` §7 "Z 类 1 条"):default consecutive-failure
|
|
26
|
+
* count before a reaper leg's swallowed error gets a `warn` line. The ~15 `.catch(() => undefined)` sites
|
|
27
|
+
* below are correct in DIRECTION (a hiccuping periodic sweep must never crash the whole tick loop — one bad
|
|
28
|
+
* leg cannot be allowed to starve its ~15 siblings), but were zero-observability: an operator watching only
|
|
29
|
+
* `warn`/`error` would never learn a leg had been silently broken for days. `5` (not `1`) is deliberate —
|
|
30
|
+
* a transient blip (one DB reconnect, one timeout) is exactly what fail-open exists to absorb without noise;
|
|
31
|
+
* only a STREAK crosses into "diagnosable, not transient". Exported so a future per-leg override is a
|
|
32
|
+
* constructor-arg change, not a magic-number hunt. */
|
|
33
|
+
export declare const REAPER_FAILURE_WARN_THRESHOLD = 5;
|
|
34
|
+
/** The pair of hooks {@link createThrottledReaperCatch} returns — `onSuccess` resets the streak (call on the
|
|
35
|
+
* chain's fulfilled path), `onError` is the `.catch()`-equivalent rejection handler (throttled, never rethrows). */
|
|
36
|
+
export interface ThrottledReaperCatch {
|
|
37
|
+
onSuccess: () => void;
|
|
38
|
+
onError: (err: unknown) => void;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Builds a NAMED, stateful throttle for one reaper leg: attach as `.then(() => guard.onSuccess(), guard.onError)`
|
|
42
|
+
* in place of a bare `.catch(() => undefined)` (the two-arg `.then` form still catches a rejection from any
|
|
43
|
+
* earlier link in the chain — same coverage as the `.catch()` it replaces — while also observing the FULFILLED
|
|
44
|
+
* path, which a plain `.catch()` cannot). Behavior this closure owns, unchanged reaper scheduling/cleanup:
|
|
45
|
+
* - a successful run resets the consecutive-failure streak to 0 (one hiccup mid-outage doesn't inflate the count);
|
|
46
|
+
* - a failure increments the streak; only once the streak reaches `threshold` (default
|
|
47
|
+
* {@link REAPER_FAILURE_WARN_THRESHOLD}) does ONE `warn` line fire (reaper name + streak length + the most
|
|
48
|
+
* recent error's message) — then the streak resets to 0 so a STILL-broken leg reports again after another
|
|
49
|
+
* full `threshold` run, rather than going silent forever after its first report.
|
|
50
|
+
* One instance must be created ONCE per named leg (outside the tick's `setInterval` callback) so the streak
|
|
51
|
+
* survives across ticks — a fresh instance per tick would never accumulate past 1.
|
|
52
|
+
*/
|
|
53
|
+
export declare function createThrottledReaperCatch(name: string, logger: Logger, threshold?: number): ThrottledReaperCatch;
|
|
24
54
|
export interface ReapersCtx {
|
|
25
55
|
config: ServiceConfig;
|
|
26
56
|
logger: Logger;
|
|
@@ -44,6 +74,10 @@ export interface ReapersCtx {
|
|
|
44
74
|
rosterStore: RunnerDeps["rosterStore"];
|
|
45
75
|
backgroundAgentStore: RunnerDeps["backgroundAgentStore"];
|
|
46
76
|
mailboxStore: RunnerDeps["mailboxStore"];
|
|
77
|
+
/** #151 车6:流内审批撤卡帧的 live 投递口(收敛器/孤儿代打腿产出的帧经它反查 broker 的活连接)。
|
|
78
|
+
* `TOOL_APPROVAL_ENABLED=false` 的部署恒 undefined ⇒ 收敛器照常收敛,只是不发通知帧(壳侧靠
|
|
79
|
+
* 重连 preamble 对账,见 approval-card.ts 的 `ApprovalRevokeFrame` 顶注)。 */
|
|
80
|
+
toolApproval: ToolApprovalCoordinator | undefined;
|
|
47
81
|
/** 晚绑(server 造出来才有)——见文件头「位置即契约」①。 */
|
|
48
82
|
getRunDenySweep: () => ((now: number) => Promise<void>) | undefined;
|
|
49
83
|
}
|
package/dist/boot/reapers.js
CHANGED
|
@@ -15,9 +15,66 @@
|
|
|
15
15
|
import { Runner, defaultTaskRegistry } from "@sema-agent/core";
|
|
16
16
|
import { sweepStaleScratchpads } from "../env-facts.js";
|
|
17
17
|
import { CostQuota } from "../observability/cost-quota.js";
|
|
18
|
+
import { createApprovalReconciler } from "../approval-reconciler.js";
|
|
19
|
+
/** #157 / [2762]§五(sema-comms `audits/failopen-governance-176.md` §7 "Z 类 1 条"):default consecutive-failure
|
|
20
|
+
* count before a reaper leg's swallowed error gets a `warn` line. The ~15 `.catch(() => undefined)` sites
|
|
21
|
+
* below are correct in DIRECTION (a hiccuping periodic sweep must never crash the whole tick loop — one bad
|
|
22
|
+
* leg cannot be allowed to starve its ~15 siblings), but were zero-observability: an operator watching only
|
|
23
|
+
* `warn`/`error` would never learn a leg had been silently broken for days. `5` (not `1`) is deliberate —
|
|
24
|
+
* a transient blip (one DB reconnect, one timeout) is exactly what fail-open exists to absorb without noise;
|
|
25
|
+
* only a STREAK crosses into "diagnosable, not transient". Exported so a future per-leg override is a
|
|
26
|
+
* constructor-arg change, not a magic-number hunt. */
|
|
27
|
+
export const REAPER_FAILURE_WARN_THRESHOLD = 5;
|
|
28
|
+
/**
|
|
29
|
+
* Builds a NAMED, stateful throttle for one reaper leg: attach as `.then(() => guard.onSuccess(), guard.onError)`
|
|
30
|
+
* in place of a bare `.catch(() => undefined)` (the two-arg `.then` form still catches a rejection from any
|
|
31
|
+
* earlier link in the chain — same coverage as the `.catch()` it replaces — while also observing the FULFILLED
|
|
32
|
+
* path, which a plain `.catch()` cannot). Behavior this closure owns, unchanged reaper scheduling/cleanup:
|
|
33
|
+
* - a successful run resets the consecutive-failure streak to 0 (one hiccup mid-outage doesn't inflate the count);
|
|
34
|
+
* - a failure increments the streak; only once the streak reaches `threshold` (default
|
|
35
|
+
* {@link REAPER_FAILURE_WARN_THRESHOLD}) does ONE `warn` line fire (reaper name + streak length + the most
|
|
36
|
+
* recent error's message) — then the streak resets to 0 so a STILL-broken leg reports again after another
|
|
37
|
+
* full `threshold` run, rather than going silent forever after its first report.
|
|
38
|
+
* One instance must be created ONCE per named leg (outside the tick's `setInterval` callback) so the streak
|
|
39
|
+
* survives across ticks — a fresh instance per tick would never accumulate past 1.
|
|
40
|
+
*/
|
|
41
|
+
export function createThrottledReaperCatch(name, logger, threshold = REAPER_FAILURE_WARN_THRESHOLD) {
|
|
42
|
+
let streak = 0;
|
|
43
|
+
return {
|
|
44
|
+
onSuccess: () => {
|
|
45
|
+
streak = 0;
|
|
46
|
+
},
|
|
47
|
+
onError: (err) => {
|
|
48
|
+
streak += 1;
|
|
49
|
+
if (streak < threshold)
|
|
50
|
+
return;
|
|
51
|
+
try {
|
|
52
|
+
// #157 补痕交叉复审真发现(codex,2026-08-05):`String(err)` on a pathological rejection reason
|
|
53
|
+
// (a throwing `Symbol.toPrimitive`/`toString`) or a throwing `logger.warn` transport would otherwise
|
|
54
|
+
// make THIS handler throw — and since onError is wired as a `.then(fulfilled, onError)` REJECTED
|
|
55
|
+
// branch with no further `.catch` downstream (every reapers.ts call site), that throw becomes a
|
|
56
|
+
// floating rejected promise (an unhandled rejection under Node's default handling can crash the
|
|
57
|
+
// process). The ORIGINAL `.catch(() => undefined)` this replaces could never do that — a failure
|
|
58
|
+
// in the REPORTING of a swallowed leg failure must not be worse than the silence it replaces, so
|
|
59
|
+
// this whole block is wrapped total/non-throwing.
|
|
60
|
+
logger.warn("reaper_leg_repeated_failure", {
|
|
61
|
+
reaper: name,
|
|
62
|
+
consecutiveFailures: streak,
|
|
63
|
+
lastError: err instanceof Error ? err.message : String(err),
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
// observability must never itself become the outage — see comment above.
|
|
68
|
+
}
|
|
69
|
+
finally {
|
|
70
|
+
streak = 0; // re-open the window — a leg that stays broken reports again every `threshold` failures, not once ever.
|
|
71
|
+
}
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
}
|
|
18
75
|
/** 起后台维护 tick,返回定时器 handle(收尾段 clearInterval 用)。 */
|
|
19
76
|
export function startReapers(ctx) {
|
|
20
|
-
const { config, logger, metrics, localRoot, backend, subRunner, runStore, checkpointStore, rateLimiter, costQuota, toolResultStore, fileSnapshotStore, taskAttachmentStore, imageBakes, worktreeReap, workflowNotifyGate, workflowJournalStore, sqlWorkflowRunStore, workflowNotifyJournal, rosterStore, backgroundAgentStore, mailboxStore, getRunDenySweep, } = ctx;
|
|
77
|
+
const { config, logger, metrics, localRoot, backend, subRunner, runStore, checkpointStore, rateLimiter, costQuota, toolResultStore, fileSnapshotStore, taskAttachmentStore, imageBakes, worktreeReap, workflowNotifyGate, workflowJournalStore, sqlWorkflowRunStore, workflowNotifyJournal, rosterStore, backgroundAgentStore, mailboxStore, toolApproval, getRunDenySweep, } = ctx;
|
|
21
78
|
// S7 (SILENT-FALLBACK P0-d): the sweeps' return counts were discarded — an instance death that batch-fails
|
|
22
79
|
// N orphans was indistinguishable from organic failures. Count + log ONLY when a sweep flipped rows (>0),
|
|
23
80
|
// so healthy ticks stay silent. Tolerant of void-returning stores (typeof guard).
|
|
@@ -31,6 +88,54 @@ export function startReapers(ctx) {
|
|
|
31
88
|
let attachmentSweepInFlight = false; // 复审 F1:附件孤儿 sweep 的重入守卫(同上)
|
|
32
89
|
let bgAgentReapInFlight = false; // core 1.364: same serialization for the durable bg-agent joint reap (per-scope serial loop)
|
|
33
90
|
let worktreeReapInFlight = false; // A10 留档发现①(2026-07-29):worktreeReap 重入守卫(同上三个邻居——见下方调用点注释)
|
|
91
|
+
// #157 / [2762]§五 Z 类补痕:one NAMED throttled-catch guard per periodic leg below (created ONCE here, not
|
|
92
|
+
// per-tick, so each leg's consecutive-failure streak survives across ticks — see createThrottledReaperCatch).
|
|
93
|
+
const probeClockSkewGuard = createThrottledReaperCatch("probe_clock_skew", logger);
|
|
94
|
+
const runsReapStaleGuard = createThrottledReaperCatch("runs_reap_stale", logger);
|
|
95
|
+
const checkpointsReapExpiredGuard = createThrottledReaperCatch("checkpoints_reap_expired", logger);
|
|
96
|
+
const runsFailSuspendedExpiredCheckpointGuard = createThrottledReaperCatch("runs_fail_suspended_expired_checkpoint", logger);
|
|
97
|
+
const runDenySweepGuard = createThrottledReaperCatch("run_deny_sweep", logger);
|
|
98
|
+
const runsReapSuspendedGuard = createThrottledReaperCatch("runs_reap_suspended", logger);
|
|
99
|
+
const checkpointReapCtxGuard = createThrottledReaperCatch("checkpoint_reap_ctx", logger);
|
|
100
|
+
const toolResultReapOlderThanGuard = createThrottledReaperCatch("tool_result_reap_older_than", logger);
|
|
101
|
+
const fileSnapshotSweepOrphanBlobsGuard = createThrottledReaperCatch("file_snapshot_sweep_orphan_blobs", logger);
|
|
102
|
+
const attachmentReapUnboundGuard = createThrottledReaperCatch("attachment_reap_unbound", logger);
|
|
103
|
+
const sessionSweepStagingGuard = createThrottledReaperCatch("session_sweep_staging", logger);
|
|
104
|
+
const imageBakesReapStaleGuard = createThrottledReaperCatch("image_bakes_reap_stale", logger);
|
|
105
|
+
const worktreeReapGuard = createThrottledReaperCatch("worktree_reap", logger);
|
|
106
|
+
const workflowNotifyRecoverGuard = createThrottledReaperCatch("workflow_notify_recover", logger);
|
|
107
|
+
const workflowJournalReapExpiredGuard = createThrottledReaperCatch("workflow_journal_reap_expired", logger);
|
|
108
|
+
const workflowRunReapAllScopesGuard = createThrottledReaperCatch("workflow_run_reap_all_scopes", logger);
|
|
109
|
+
const workflowNotifyJournalReapAckedGuard = createThrottledReaperCatch("workflow_notify_journal_reap_acked", logger);
|
|
110
|
+
const bgAgentsReapGuard = createThrottledReaperCatch("bg_agents_reap", logger);
|
|
111
|
+
const rosterReapOlderThanGuard = createThrottledReaperCatch("roster_reap_older_than", logger);
|
|
112
|
+
const scratchpadSweepStaleGuard = createThrottledReaperCatch("scratchpad_sweep_stale", logger);
|
|
113
|
+
const approvalReconcileGuard = createThrottledReaperCatch("approval_reconcile", logger);
|
|
114
|
+
let approvalReconcileInFlight = false; // 车5 §8 C-8:重入守卫(邻居四先例同形——本腿时长随 PARKING 行数增长)
|
|
115
|
+
// #151 车5(design/172 §3.0 对账收敛器 + 崩溃恢复扫描):**总开关默认 OFF ⇒ 本腿根本不注册**
|
|
116
|
+
// (`STREAM_APPROVAL_ENABLED=false` 的部署一行库都不扫,零新增查询)。开且有 durable backend 时才建。
|
|
117
|
+
// 判据 1/2 的两个读口按能力面接:checkpoint 店的窄读口只有 SQL twins 有(local 车道的店没有这个面),
|
|
118
|
+
// 缺席 ⇒ 判据 1 恒不命中(诚实降级,行落 ②③④⑤),不是「假装没 checkpoint 就该 DENY」。
|
|
119
|
+
const approvalReconciler = (() => {
|
|
120
|
+
if (!config.streamApproval.enabled || !backend)
|
|
121
|
+
return undefined;
|
|
122
|
+
// duck probe(邻居同形:本文件 `sweepStagingSessions` / `reapAllScopes` 都是这个姿势)——
|
|
123
|
+
// `CheckpointStoreFull` 的 local 成员没有这个窄读口,它是 SQL twins 的扩展面。
|
|
124
|
+
const cpPort = checkpointStore;
|
|
125
|
+
return createApprovalReconciler({
|
|
126
|
+
askStore: backend.approvalAsk(),
|
|
127
|
+
checkpoints: typeof cpPort?.findCheckpointCandidatesForAsk === "function" ? cpPort : undefined,
|
|
128
|
+
// RunStore 三形(TiDB/Pg/File)都带 `getRun`,故直接接、无 probe。
|
|
129
|
+
runs: runStore,
|
|
130
|
+
logger,
|
|
131
|
+
metrics,
|
|
132
|
+
batchLimit: config.streamApproval.reconcileBatch,
|
|
133
|
+
pendingGraceMs: config.streamApproval.pendingGraceMs,
|
|
134
|
+
adhocGraceMs: config.streamApproval.adhocGraceMs,
|
|
135
|
+
orphanTtlMs: config.streamApproval.orphanTtlMs,
|
|
136
|
+
...(toolApproval ? { emitRevoke: (frame, target) => void toolApproval.emitRevoke(frame, target) } : {}),
|
|
137
|
+
});
|
|
138
|
+
})();
|
|
34
139
|
// core 1.364([1503] 提货单③):retention **只走** reapDurableAgents(联合 reap:条件删赢了才 release
|
|
35
140
|
// 转录 session;裸 store.reap 会 strand 转录)。staleRunning 翻转在 core 编排内先行(store.reap 一步)。
|
|
36
141
|
// scope 枚举:core 1.368 `BackgroundAgentStore.listScopes?()`([1516]② 交付,SQL twins/File 双实现
|
|
@@ -87,7 +192,17 @@ export function startReapers(ctx) {
|
|
|
87
192
|
}
|
|
88
193
|
catch (err) {
|
|
89
194
|
failedScopes++;
|
|
90
|
-
|
|
195
|
+
// codex 交叉复审第三轮真发现(2026-08-05):a poison `err` (throwing `Symbol.toPrimitive`/`toString`)
|
|
196
|
+
// or a throwing `logger.warn` must not escape THIS catch — the outer `bgAgentsReapGuard` (this
|
|
197
|
+
// car's own wiring) already stops it from becoming an unhandled rejection, but an uncaught throw
|
|
198
|
+
// HERE would abort the `for` loop early and skip every remaining scope, defeating the per-scope
|
|
199
|
+
// isolation this block exists for ("[1522] LOW3" above — one bad scope must not starve the rest).
|
|
200
|
+
try {
|
|
201
|
+
logger.warn("bg_agent_reap_scope_failed", { scope, err: err instanceof Error ? err.message : String(err) });
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
// observability must never break the per-scope loop — see comment above.
|
|
205
|
+
}
|
|
91
206
|
}
|
|
92
207
|
}
|
|
93
208
|
if (failedScopes > 0)
|
|
@@ -116,8 +231,11 @@ export function startReapers(ctx) {
|
|
|
116
231
|
}
|
|
117
232
|
};
|
|
118
233
|
const reaper = setInterval(() => {
|
|
119
|
-
void probeClockSkew().
|
|
120
|
-
void runStore
|
|
234
|
+
void probeClockSkew().then(() => probeClockSkewGuard.onSuccess(), probeClockSkewGuard.onError); // S10
|
|
235
|
+
void runStore
|
|
236
|
+
?.reapStale(config.runStaleSec * 1000)
|
|
237
|
+
.then(reapCount("runs_reaped_total", { kind: "stale" }))
|
|
238
|
+
.then(() => runsReapStaleGuard.onSuccess(), runsReapStaleGuard.onError);
|
|
121
239
|
// Durable F4 (design/45) + design/80 §3 inv#3 crash-safe backstop: CAS-expire checkpoints past their
|
|
122
240
|
// deadline OR their absolute terminal_at backstop (≈ deny), then fail the suspended run rows whose
|
|
123
241
|
// checkpoint was thereby expired (release task_active = unlock the session). These run EVERY tick,
|
|
@@ -131,32 +249,51 @@ export function startReapers(ctx) {
|
|
|
131
249
|
// 真 pin 只存在于 in-memory TtlSessionStore;本部署 durable checkpoint 恒配 durable session 后端
|
|
132
250
|
// (pin=no-op),deny-sweep 走 resumeCheckpoint=core 内部 unpin ✓。SESSION_BACKEND=memory+durable
|
|
133
251
|
// checkpoint 的 niche dev 组合下 abort-expire 臂会把被钉会话泄到进程终——显式接受,不为 dev 形加腿。
|
|
134
|
-
void checkpointStore
|
|
135
|
-
|
|
252
|
+
void checkpointStore
|
|
253
|
+
.reapExpired(Date.now())
|
|
254
|
+
.then(reapCount("checkpoints_reaped_total", {}))
|
|
255
|
+
.then(() => checkpointsReapExpiredGuard.onSuccess(), checkpointsReapExpiredGuard.onError);
|
|
256
|
+
void runStore
|
|
257
|
+
?.failSuspendedWithExpiredCheckpoint()
|
|
258
|
+
.then(reapCount("runs_reaped_total", { kind: "expired_checkpoint" }))
|
|
259
|
+
.then(() => runsFailSuspendedExpiredCheckpointGuard.onSuccess(), runsFailSuspendedExpiredCheckpointGuard.onError);
|
|
136
260
|
// D-D SLA-timer: resolve-DENY human/irreversible_ask gates past their deadline (graceful — the model
|
|
137
261
|
// continues with the denial), vs the abort reapExpired gives the other kinds. Bounded per tick.
|
|
138
|
-
void getRunDenySweep()?.(Date.now())
|
|
262
|
+
void getRunDenySweep()?.(Date.now())
|
|
263
|
+
.then(() => runDenySweepGuard.onSuccess(), runDenySweepGuard.onError); // A10 搬运改写:晚绑取值(原 `runDenySweep?.`)
|
|
139
264
|
}
|
|
140
265
|
// The finer, OPT-IN time-based suspended-run reaper (APPROVAL_TIMEOUT_SEC). Stays gated — reapSuspended
|
|
141
266
|
// with a 0 TTL would nuke ALL suspended rows (cutoff = now). The absolute backstop above is the safety
|
|
142
267
|
// floor; this is the operator-chosen deadline. ([2354] A-5:F4 轮询店的 expireStale 腿随车道退役。)
|
|
143
268
|
if (config.approvalTimeoutSec > 0) {
|
|
144
|
-
void runStore
|
|
269
|
+
void runStore
|
|
270
|
+
?.reapSuspended(config.approvalTimeoutSec * 1000)
|
|
271
|
+
.then(reapCount("runs_reaped_total", { kind: "suspended" }))
|
|
272
|
+
.then(() => runsReapSuspendedGuard.onSuccess(), runsReapSuspendedGuard.onError);
|
|
145
273
|
}
|
|
146
274
|
// GC checkpoint_ctx rows whose checkpoint is gone (bound to the checkpoint lifecycle).
|
|
147
|
-
void checkpointStore
|
|
275
|
+
void checkpointStore
|
|
276
|
+
?.reapCtx(Date.now() - config.runStaleSec * 1000)
|
|
277
|
+
.then(() => checkpointReapCtxGuard.onSuccess(), checkpointReapCtxGuard.onError);
|
|
148
278
|
rateLimiter?.sweep();
|
|
149
279
|
if (costQuota instanceof CostQuota)
|
|
150
280
|
costQuota.reap(); // TiDB variant self-reaps in its flush loop
|
|
151
|
-
void toolResultStore
|
|
281
|
+
void toolResultStore
|
|
282
|
+
?.reapOlderThan?.(Date.now() - config.toolResultTtlSec * 1000)
|
|
283
|
+
?.then(() => toolResultReapOlderThanGuard.onSuccess(), toolResultReapOlderThanGuard.onError); // optional extra: SQL twins only (the local FileToolResultStore persists like transcripts)
|
|
152
284
|
// 2c session-sync: GLOBALLY GC grace-window orphan blobs (standalone /sync/blobs PUTs that were never
|
|
153
285
|
// imported are otherwise collected only on reap()/deleteBySession(), which never fire for a never-imported scope).
|
|
154
286
|
// Bounds the standalone-PUT orphan-blob exhaustion (review finding) to the grace window. Durable twins only (local
|
|
155
287
|
// omits the seam → optional-chained no-op).
|
|
156
|
-
void fileSnapshotStore
|
|
288
|
+
void fileSnapshotStore
|
|
289
|
+
?.sweepOrphanBlobs?.()
|
|
290
|
+
.then(() => fileSnapshotSweepOrphanBlobsGuard.onSuccess(), fileSnapshotSweepOrphanBlobsGuard.onError);
|
|
157
291
|
// D-1 附件 TTL:上传后从未被任何 task 引用(session_id NULL)且超过 attachmentUnboundTtlMs 的行收割
|
|
158
292
|
// (绑定行不在此收——随会话 E21 级联删)。best-effort,与其余 reaper 腿同姿。
|
|
159
|
-
void taskAttachmentStore
|
|
293
|
+
void taskAttachmentStore
|
|
294
|
+
?.reapUnbound(Date.now() - config.attachmentUnboundTtlMs)
|
|
295
|
+
.then(reapCount("attachments_reaped_total", {}))
|
|
296
|
+
.then(() => attachmentReapUnboundGuard.onSuccess(), attachmentReapUnboundGuard.onError);
|
|
160
297
|
// D-1 孤儿**对象**彻底 GC(clay 拍 2026-07-28):对象存储 × meta 行对账,grace 默认 1h(上传先行窗
|
|
161
298
|
// 保护;`ATTACHMENT_ORPHAN_GRACE_MS=0` 关掉本腿——同 tick 邻居都有旋钮,复审 F10)。
|
|
162
299
|
// 列举失败=本轮 warn 跳过(「列不出来」绝不当「没有孤儿」),下轮再试。
|
|
@@ -171,18 +308,35 @@ export function startReapers(ctx) {
|
|
|
171
308
|
metrics.inc("attachment_orphan_objects_swept_total", {}, n);
|
|
172
309
|
logger.info("attachment_orphan_objects_swept", { removed: n });
|
|
173
310
|
} })
|
|
174
|
-
.catch((err) =>
|
|
311
|
+
.catch((err) => {
|
|
312
|
+
// codex 交叉复审第二轮真发现(2026-08-05):`String(err)` on a pathological rejection reason or a
|
|
313
|
+
// throwing `logger.warn` transport must not escape THIS handler — this `.catch` sits ahead of a
|
|
314
|
+
// `.finally()` with no further `.catch` downstream, so an uncaught throw here becomes a floating
|
|
315
|
+
// rejected promise (unhandled rejection under Node's default handling). Same invariant as
|
|
316
|
+
// createThrottledReaperCatch.onError above: a reaper leg's failure REPORTING must never itself
|
|
317
|
+
// crash the tick loop.
|
|
318
|
+
try {
|
|
319
|
+
logger.warn("attachment_orphan_sweep_failed", { err: err instanceof Error ? err.message : String(err) });
|
|
320
|
+
}
|
|
321
|
+
catch {
|
|
322
|
+
// observability must never crash the tick — see comment above.
|
|
323
|
+
}
|
|
324
|
+
})
|
|
175
325
|
.finally(() => { attachmentSweepInFlight = false; });
|
|
176
326
|
}
|
|
177
327
|
// P1d-β 2c session-sync: GC ABANDONED staged imports — staging-id session_event rows (`%#stg-%`) with NO
|
|
178
328
|
// session_meta whose oldest row is older than the grace window (an in-flight stream stays fresh → never reaped).
|
|
179
329
|
// Bounds the orphan-staging-row growth (a Phase B that opened a staging then never committed). Durable session
|
|
180
330
|
// stores only (the local backend stages in memory → no durable rows; the seam is absent → optional-chained no-op).
|
|
181
|
-
void backend?.session()?.sweepStagingSessions?.()
|
|
331
|
+
void backend?.session()?.sweepStagingSessions?.()
|
|
332
|
+
.then(() => sessionSweepStagingGuard.onSuccess(), sessionSweepStagingGuard.onError);
|
|
182
333
|
// Bake-runner backstop (§P2.7): fail any `running` bake whose lease went stale (the runner crashed mid-build),
|
|
183
334
|
// append a synthetic terminal `done{failed}` so SSE readers settle, and force-release the single-flight lease.
|
|
184
335
|
// staleMs (≈3× the 30s heartbeat) bounds a healthy slow build so it is never wrongly reaped.
|
|
185
|
-
void imageBakes
|
|
336
|
+
void imageBakes
|
|
337
|
+
?.reapStaleBakes(config.imageBakes.staleMs)
|
|
338
|
+
.then(reapCount("bakes_reaped_total", {}))
|
|
339
|
+
.then(() => imageBakesReapStaleGuard.onSuccess(), imageBakesReapStaleGuard.onError);
|
|
186
340
|
// SVC-3 worktree isolation: deregister worktrees orphaned by a process crash (the Runner never reached
|
|
187
341
|
// destroy → `git worktree remove` never ran). `git worktree prune` cleans registrations whose dirs are
|
|
188
342
|
// already gone. Best-effort, userland (core ships no post-kill Runner hook); never throws. Unset = no-op.
|
|
@@ -192,16 +346,22 @@ export function startReapers(ctx) {
|
|
|
192
346
|
// 照抄 bgAgentReap 的 in-flight + finally 释放形。
|
|
193
347
|
if (worktreeReap && !worktreeReapInFlight) {
|
|
194
348
|
worktreeReapInFlight = true;
|
|
195
|
-
void worktreeReap()
|
|
349
|
+
void worktreeReap()
|
|
350
|
+
.then(() => worktreeReapGuard.onSuccess(), worktreeReapGuard.onError)
|
|
351
|
+
.finally(() => (worktreeReapInFlight = false));
|
|
196
352
|
}
|
|
197
353
|
// SVC-1 (adversarial-review HIGH): PERIODIC notify-recovery sweep (not just at boot) — re-delivers a terminal
|
|
198
354
|
// run whose in-process notify was lost, AND finalizes-as-abandoned a `running` run orphaned past the grace
|
|
199
355
|
// window (core never resumes/reaps a prior `running` row, so nothing else would). Idempotent; best-effort.
|
|
200
|
-
void workflowNotifyGate
|
|
356
|
+
void workflowNotifyGate
|
|
357
|
+
?.recover({ orphanGraceMs: config.workflowOrphanGraceMs })
|
|
358
|
+
.then(() => workflowNotifyRecoverGuard.onSuccess(), workflowNotifyRecoverGuard.onError);
|
|
201
359
|
// SVC-2 (adversarial-review HIGH): time-based GC of the workflow_journal table (the heaviest, TaskResult-bearing
|
|
202
360
|
// one) — the per-run deleteByRun has no run-store reap hook, so this bounded sweep is what stops unbounded
|
|
203
361
|
// growth. A resume of a journal older than the retention window re-runs live (resume is an optimization).
|
|
204
|
-
void workflowJournalStore
|
|
362
|
+
void workflowJournalStore
|
|
363
|
+
?.reapExpired?.(Date.now(), config.workflowJournalRetentionMs)
|
|
364
|
+
.then(() => workflowJournalReapExpiredGuard.onSuccess(), workflowJournalReapExpiredGuard.onError);
|
|
205
365
|
// Retention (WorkflowRunStore never auto-purges, reap is explicit):
|
|
206
366
|
// age out TERMINAL workflow_run rows across ALL scopes. SQL twins only (reapAllScopes is their DISTINCT-scope
|
|
207
367
|
// extension; the contract itself has no cross-scope enumeration, and the File store keeps the transcripts-like
|
|
@@ -212,37 +372,52 @@ export function startReapers(ctx) {
|
|
|
212
372
|
const sweep = sqlWorkflowRunStore?.reapAllScopes?.(Date.now(), { maxAgeMs: config.workflowRunRetentionMs });
|
|
213
373
|
if (sweep) {
|
|
214
374
|
wfRunReapInFlight = true;
|
|
215
|
-
void sweep
|
|
375
|
+
void sweep
|
|
376
|
+
.then(() => workflowRunReapAllScopesGuard.onSuccess(), workflowRunReapAllScopesGuard.onError)
|
|
377
|
+
.finally(() => (wfRunReapInFlight = false));
|
|
216
378
|
}
|
|
217
379
|
// 1.108: SQL notify-journal retention rides the same knob — ACKED rows are pure history (pending rows are
|
|
218
380
|
// the recovery backlog and are NEVER reaped; the orphan-grace sweep retires a stuck pending run).
|
|
219
381
|
void workflowNotifyJournal
|
|
220
382
|
?.reapAcked?.(Date.now() - config.workflowRunRetentionMs)
|
|
221
|
-
.
|
|
383
|
+
.then(() => workflowNotifyJournalReapAckedGuard.onSuccess(), workflowNotifyJournalReapAckedGuard.onError);
|
|
222
384
|
}
|
|
223
385
|
// core 1.364 durable bg-agent joint reap(定义在 interval 上方,契约注释在彼)。in-flight 守卫同
|
|
224
386
|
// wfRun sweep(per-scope 串行循环,短 tick + 多 scope 下不叠罗汉)。
|
|
225
387
|
if (reapBgAgents && !bgAgentReapInFlight) {
|
|
226
388
|
bgAgentReapInFlight = true;
|
|
227
|
-
void reapBgAgents()
|
|
389
|
+
void reapBgAgents()
|
|
390
|
+
.then(() => bgAgentsReapGuard.onSuccess(), bgAgentsReapGuard.onError)
|
|
391
|
+
.finally(() => (bgAgentReapInFlight = false));
|
|
228
392
|
}
|
|
229
393
|
// [1522] MED2:agent_roster TTL 清理(core RB-23②③ 派给部署的半场)——SQL twins 扩展面
|
|
230
394
|
// (reapOlderThan,duck probe;File/Memory 店 core 自带 maxAgeMs,无此面=no-op)。
|
|
231
395
|
void rosterStore
|
|
232
396
|
?.reapOlderThan?.(Date.now() - config.rosterRetentionMs)
|
|
233
397
|
.then(reapCount("roster_rows_reaped_total", {}))
|
|
234
|
-
.
|
|
398
|
+
.then(() => rosterReapOlderThanGuard.onSuccess(), rosterReapOlderThanGuard.onError);
|
|
235
399
|
// 修8(三路复审 absorb-2,接线 (b)):periodic scratchpad sweep — 无 E21 purge 兜到的孤儿目录(session 从未
|
|
236
400
|
// DELETE、purge 当次失败、local 后端无 purge coordinator)按 mtime 过期回收。SCRATCHPAD_SWEEP_TTL_MS
|
|
237
401
|
// (default 7d,0=禁用)。activeSessionIds 不传:活跃判据的诚实边界在 env-facts.ts 的 sweep 文档——TTL 7d
|
|
238
402
|
// 远大于任何在飞任务;一个 7 天零写入的 scratchpad 被回收是可接受的(它本就是临时区,fact 文案即如此宣示)。
|
|
403
|
+
// #151 车5:流内审批的对账收敛器 + 崩溃恢复扫描(两段都在 `runOnce` 内,见 approval-reconciler.ts
|
|
404
|
+
// 判据表 v2)。**不新起独立定时器**——复用本 tick(收敛延迟上界 = 一个 reaper tick;壳侧的体感由
|
|
405
|
+
// 车4 的 425 `Retry-After` 覆盖)。in-flight 守卫同邻居四腿:本腿时长随 PARKING 行数与两个读口的
|
|
406
|
+
// 库延迟增长,短 REAP_INTERVAL_SEC 下无守卫会逐 tick 叠罗汉压同一批行。
|
|
407
|
+
if (approvalReconciler && !approvalReconcileInFlight) {
|
|
408
|
+
approvalReconcileInFlight = true;
|
|
409
|
+
void approvalReconciler
|
|
410
|
+
.runOnce(Date.now())
|
|
411
|
+
.then(() => approvalReconcileGuard.onSuccess(), approvalReconcileGuard.onError)
|
|
412
|
+
.finally(() => (approvalReconcileInFlight = false));
|
|
413
|
+
}
|
|
239
414
|
if (config.scratchpadSweepTtlMs > 0) {
|
|
240
415
|
void sweepStaleScratchpads(config.localDataRoot ?? localRoot, { olderThanMs: config.scratchpadSweepTtlMs })
|
|
241
416
|
.then((removed) => {
|
|
242
417
|
if (removed > 0)
|
|
243
418
|
logger.info("scratchpads_swept", { removed });
|
|
244
419
|
})
|
|
245
|
-
.
|
|
420
|
+
.then(() => scratchpadSweepStaleGuard.onSuccess(), scratchpadSweepStaleGuard.onError);
|
|
246
421
|
}
|
|
247
422
|
}, config.reapIntervalSec * 1000);
|
|
248
423
|
reaper.unref?.();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type MemoryBackend, type RunnerDeps } from "@sema-agent/core";
|
|
1
|
+
import { type MemoryBackend, type RunnerDeps, type ToolPolicy } from "@sema-agent/core";
|
|
2
2
|
import type { PromptsDomainFaces } from "../prompts-domain-validate.js";
|
|
3
3
|
import { SessionEnvironmentSelection, selectEnvironmentTool } from "../capabilities/select-environment-tool.js";
|
|
4
4
|
import { sendUserFileTool } from "../capabilities/send-user-file-tool.js";
|
|
@@ -69,6 +69,21 @@ export interface ResolveSpecCtx {
|
|
|
69
69
|
perTaskImage: PerTaskImageRegistry;
|
|
70
70
|
sessionEnvSelection: SessionEnvironmentSelection;
|
|
71
71
|
}
|
|
72
|
+
/** #152([2703] 案二):durable 部署上的 AskUserQuestion 门。活体面(QuestionCoordinator)缺席 ⇒ 原形
|
|
73
|
+
* `createDurableQuestionPolicy()`(恒 ask ⇒ 恒 durable park)。在场 ⇒ **判决时**按活流上下文分腿:
|
|
74
|
+
* 活流腿(bg/SSE,coordinator.runWithContext 包裹且投递面此刻可达,ALS 判)allow——工具执行落到
|
|
75
|
+
* RunnerDeps.onQuestion 的 coordinator,问正在 tail 流的活人;无活流腿(sync /v1/tasks、verify/cascade、
|
|
76
|
+
* durable resume 驱动、断连后的 detach 腿)ask——durable park 原语义逐字保留(此时执行只会拿到 headless
|
|
77
|
+
* 空答,park 才是把问题送到人面前的那条腿)。
|
|
78
|
+
* 工具名字面量与 server.ts 的 pre-CAS 守卫同源("AskUserQuestion",core 未根导出常量)。
|
|
79
|
+
* ⚠️ **单一属主**(复审 A2):AskUserQuestion 的 durable 判决只有这一处。任何需要「同参重建」这条判决的
|
|
80
|
+
* 地方(main.ts 的 parkedReviveInheritedGate 父约束链)必须调本工厂,不得自折 core 原形——两份拷贝里
|
|
81
|
+
* 只改一份正是本条 finding 的成因。**登记豁免一处**:leader worker 腿(src/leader/wire.ts provisionWorker)
|
|
82
|
+
* 自折 core 原形——该腿无活体问答面可装且 leader 不 import boot 层(分层),core 原形+sentinel 即其完整
|
|
83
|
+
* 语义;豁免注在彼处互指,接活体面之日必须并回本工厂。 */
|
|
84
|
+
export declare function createDurableQuestionGate(live: {
|
|
85
|
+
hasLiveContext(): boolean;
|
|
86
|
+
} | undefined): ToolPolicy;
|
|
72
87
|
export declare function createResolveSpec(ctx: ResolveSpecCtx): ServiceDeps["resolveSpec"];
|
|
73
88
|
export {};
|
|
74
89
|
//# sourceMappingURL=resolve-spec.d.ts.map
|