@sema-agent/server 7.23.0 → 7.24.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -118,6 +118,24 @@ export const REBIND_LEGS = [
118
118
  residualKey: undefined, // PK(id);UNIQUE(repo, digest) 不含 tenant_id
119
119
  why: "镜像可见性的租户列(tenant-scoped 镜像只对该 principal 可见)—— 活的可见性轴,不是史实",
120
120
  },
121
+ {
122
+ // #193 车4 件1 同批:`leader_run` 这张表一落地,收编的身份轴普查门(adoption-db-integration)当场逮到
123
+ // 它的 `owner` 列未表态 —— 正是那道门的存在理由。判定=**活的可见性轴**(GET /v1/leader/:id 的 404
124
+ // 属主门读它),与 `task_run#owner` 同形,故真迁而不是登记进 declared。
125
+ // 🔴 `idem_key` 不随迁(也无腿):它是 `sha256(owner \x1f 调用方原始键)`,重绑后旧行的哈希对新身份
126
+ // 不再命中 ⇒ 提交幂等窗在收编那一刻失效(重投会起一条新 run)。这是**有意的**:收编按 183 I1 要求
127
+ // 停机进行,窗内没有在飞的重投;而反过来去重算哈希会把「这行当初是用哪个键提交的」这条史实改写,
128
+ // 且新旧哈希不等 ⇒ 结构上也撞不了 UNIQUE(故 residualKey 仍是 undefined)。
129
+ leg: "leader_run#owner",
130
+ table: "leader_run",
131
+ kind: "bulk-rebind",
132
+ action: "row-rewrite",
133
+ columns: ["owner"],
134
+ matchColumn: "owner",
135
+ encoding: "verbatim",
136
+ residualKey: undefined, // PK(leader_run_id);UNIQUE(idem_key) 不含 owner
137
+ why: "leader run 的属主列(GET /v1/leader/:id 的 404 属主门读它)—— 不迁 = 收编后这些 run 对新身份全部读作「不存在」",
138
+ },
121
139
  {
122
140
  leg: "workflow_completion_inbox#owner",
123
141
  table: "workflow_completion_inbox",
@@ -25,6 +25,8 @@ export interface LeaderCtx {
25
25
  /** design/170 件B/C/D(#252,codex R2-F1):部署治理三座席 —— leader 车道的每一只 Runner 都要带上它,
26
26
  * 否则开了 leader 端点的部署就有一条在部署治理**之外**的执行面(合规否决/锁都够不着)。 */
27
27
  governanceSeams: import("./governance-seams.js").GovernanceSeams;
28
+ /** #193 车4 件1:durable store 后端(leader-run 登记表从这里取)。缺席 = 无 DB 的 env-only worker。 */
29
+ backend?: StoreBackend | undefined;
28
30
  }
29
31
  /**
30
32
  * 鲁棒性批5 A6(2026-08-05):k8s 腿要求 MinIO 三件套(ENDPOINT/ACCESS_KEY/SECRET_KEY)全在——半开(一件缺)
@@ -37,5 +39,19 @@ export interface LeaderCtx {
37
39
  * 返回 null = 不适用该警告(未开启/非 k8s 腿/三件套齐全);非 null = 该报警,携带具体缺的变量名列表。
38
40
  */
39
41
  export declare function leaderK8sMinioGap(leaderEnabled: boolean, leaderProvider: string | undefined, hasS3: boolean): string[] | null;
42
+ /**
43
+ * #193 车4 件2 的**能力面**门(codex 对抗复审 R1-high①,验真后修)。
44
+ *
45
+ * config 层的门②判的是 `DB_BACKEND` 的字面值 —— 那句话回答的是「运维要没要 durable」。它回答不了
46
+ * 「这次启动**真的拿到**了 durable 吗」:`openStoreBackendWithFallback` 在 `SESSION_BACKEND=auto`
47
+ * (默认)下,SQL 连不上 / 建表失败 ⇒ 返回 `backend=undefined` + 一条 warn 就继续启动。那次启动里
48
+ * `leaderRun()` 缺席,leader 端点会静默落进单副本内存臂:多副本各自受理同一个 Idempotency-Key、
49
+ * 跨副本 GET 404、重启失忆 —— 而这条腿的产物是**真 git push**。
50
+ *
51
+ * ⇒ 判据锚**真拿到了没有**。纯谓词、只吃两个原语(与 {@link leaderK8sMinioGap} 同姿势),便于单测。
52
+ * 抛而不是 warn:这正是 #157「安全/执行车道禁静默 fail-open」说的那一类 —— 降级的代价由用户承担,
53
+ * 而承担的方式是重复的真实推送。
54
+ */
55
+ export declare function assertLeaderDurableStore(leaderEnabled: boolean, hasLeaderRunStore: boolean): void;
40
56
  export declare function createLeaderFace(ctx: LeaderCtx): ReturnType<typeof createLeaderEndpoint> | undefined;
41
57
  //# sourceMappingURL=leader.d.ts.map
@@ -19,6 +19,29 @@ export function leaderK8sMinioGap(leaderEnabled, leaderProvider, hasS3) {
19
19
  !process.env.MINIO_SECRET_KEY && "MINIO_SECRET_KEY",
20
20
  ].filter((v) => typeof v === "string");
21
21
  }
22
+ /**
23
+ * #193 车4 件2 的**能力面**门(codex 对抗复审 R1-high①,验真后修)。
24
+ *
25
+ * config 层的门②判的是 `DB_BACKEND` 的字面值 —— 那句话回答的是「运维要没要 durable」。它回答不了
26
+ * 「这次启动**真的拿到**了 durable 吗」:`openStoreBackendWithFallback` 在 `SESSION_BACKEND=auto`
27
+ * (默认)下,SQL 连不上 / 建表失败 ⇒ 返回 `backend=undefined` + 一条 warn 就继续启动。那次启动里
28
+ * `leaderRun()` 缺席,leader 端点会静默落进单副本内存臂:多副本各自受理同一个 Idempotency-Key、
29
+ * 跨副本 GET 404、重启失忆 —— 而这条腿的产物是**真 git push**。
30
+ *
31
+ * ⇒ 判据锚**真拿到了没有**。纯谓词、只吃两个原语(与 {@link leaderK8sMinioGap} 同姿势),便于单测。
32
+ * 抛而不是 warn:这正是 #157「安全/执行车道禁静默 fail-open」说的那一类 —— 降级的代价由用户承担,
33
+ * 而承担的方式是重复的真实推送。
34
+ */
35
+ export function assertLeaderDurableStore(leaderEnabled, hasLeaderRunStore) {
36
+ if (!leaderEnabled || hasLeaderRunStore)
37
+ return;
38
+ throw new Error("LEADER_ENABLED=true but no durable leader-run store is available at boot — the SQL backend is configured " +
39
+ "yet this replica did not get one (an unreachable DB / failed DDL degrades to the in-memory fallback when " +
40
+ "SESSION_BACKEND=auto). Refusing to serve the leader endpoint from a single-replica in-memory registry: " +
41
+ "the `leader_run` table is what makes a retried submit idempotent and a cross-replica GET answerable, and " +
42
+ "without it a retry starts a SECOND real git push. Fix the database (or unset LEADER_ENABLED) and restart " +
43
+ "(#193 车4 件2 capability gate; the DB_BACKEND literal check in config.ts is the intent half of the same rule)");
44
+ }
22
45
  export function createLeaderFace(ctx) {
23
46
  const { config, logger, brain, pricing, executionEnvFactory, toolResultStore, sessionStore, checkpointStore, governanceSeams } = ctx;
24
47
  // v2 leader endpoint (design/50 + design/68): wire when LEADER_ENABLED + an isolated remote-exec backend
@@ -38,12 +61,29 @@ export function createLeaderFace(ctx) {
38
61
  }
39
62
  : {};
40
63
  const leaderProvider = config.remoteExec?.provider;
64
+ // #193 车4 件1:durable leader-run 登记表(SQL 后端专有;`backend` 本身可缺席=无 DB 的 env-only worker)。
65
+ // 件2 能力面门(codex R1-high①)紧随其后:开了 leader 却没真拿到店 ⇒ 拒启,绝不静默落内存臂。
66
+ const leaderRunStore = ctx.backend?.leaderRun?.();
67
+ assertLeaderDurableStore(config.leaderEnabled, leaderRunStore !== undefined);
41
68
  const minioGap = leaderK8sMinioGap(config.leaderEnabled, leaderProvider, "s3" in leaderMinio);
42
69
  if (minioGap)
43
70
  logger.warn("leader_k8s_minio_incomplete", { missing: minioGap });
44
71
  // DUAL-MODE §5: orchestration is an ENGINE capability, not fleet-only — the TOC `host` lane runs leader fan-out
45
- // bounded by ONE box (isolation=none, NON-durable: no snapshot, so the durable sub-worker suspend block below is
46
- // skipped — host workers run to completion in-process-adjacent). e2b/k8s keep their isolated/suspendable posture.
72
+ // bounded by ONE box (`remote-env-host.ts` declares `capabilities = { isolation:false, suspendable:false }`).
73
+ // e2b/k8s keep their isolated/suspendable posture.
74
+ //
75
+ // 🔴 #193 车4 件4(台账 P2-10)核实更正 —— 这里原先断言 host 腿「既不 durable 也没有快照,所以下面那个
76
+ // durable 子 worker 挂起块会被跳过」。**没有任何判别式这样做**:下面的 `durable` 块只 gate 在
77
+ // `checkpointStore && approvalRequire.length > 0` 上,与 provider 无关,host 腿照样拿到它。core 侧亲读
78
+ // (5.35.0)也不是「跳过」而是**park-only 降级**:一个 `suspendable:false` 的 remote env 仍然 durable
79
+ // suspend,被跳过的只有 `suspendVM` 那一步 —— 见 `@sema-agent/core` `dist/core/remote-env.d.ts` 的
80
+ // `WorkspaceHandle.restoreMode:"park_only"`(「non-suspendable env;workspace persists on the target,
81
+ // resume skips resumeVM」)与 `dist/core/types.d.ts` 的 `TaskResult.workspaceRestoreMode:"snapshot"|"park_only"`
82
+ // (「the env declared itself non-suspendable …, so NOTHING was paused: the machine keeps running (and keeps
83
+ // costing) … resume simply reconnects to the still-present workspace」)。
84
+ // 运维面的实义差别(也是原注为什么危险):park-only 的 host worker 在挂起期间**继续占着这台机器、继续
85
+ // 计费**,而 `suspendable:false` 同时**断言工作区在目标机上外部持久**(core 那条 LOAD-BEARING INVARIANT)
86
+ // —— 所以谁清理这台机器上的残留由部署方负责,不能指望「反正被跳过了」。
47
87
  const leaderEndpoint = config.leaderEnabled &&
48
88
  (leaderProvider === "e2b" ||
49
89
  (leaderProvider === "k8s" && "s3" in leaderMinio && executionEnvFactory) ||
@@ -100,6 +140,10 @@ export function createLeaderFace(ctx) {
100
140
  // 非空,此前同步门只查 objective,不完整请求会先拿到 202 再在后台必然失败——同步前移,让错误
101
141
  // 尽早暴露而不是靠后续 GET 才发现。
102
142
  requiredFields: ["objective", "durableRemote", "testCmd", "seedCmd", "baseSha"],
143
+ // #193 车4 件1:durable 登记表(跨副本 GET + 提交幂等)。SQL 后端专有;缺席 ⇒ endpoint 落到
144
+ // 显式的单副本内存降级臂。生产上这条缺席路径够不着 —— config 层已把「LEADER_ENABLED=true 且
145
+ // 无外部 SQL 店」整个组合拒启(件2 门②),所以这里的可选链只服务单机试跑与测试装配。
146
+ ...(leaderRunStore ? { store: leaderRunStore } : {}),
103
147
  })
104
148
  : undefined;
105
149
  if (leaderEndpoint)
@@ -135,6 +135,10 @@ export function startReapers(ctx) {
135
135
  const workflowNotifyJournalReapAckedGuard = createThrottledReaperCatch("workflow_notify_journal_reap_acked", logger);
136
136
  const bgAgentsReapGuard = createThrottledReaperCatch("bg_agents_reap", logger);
137
137
  const rosterReapOlderThanGuard = createThrottledReaperCatch("roster_reap_older_than", logger);
138
+ const leaderRunStaleGuard = createThrottledReaperCatch("leader_run_reap_stale", logger); // #193 车4 件1
139
+ // #193 车4 件1:陈旧 running 清算腿的店。合取 `config.leaderEnabled` —— 默认关的部署不注册这条腿
140
+ // (取不到店也一样,`?.` 让整条腿成为 no-op)。
141
+ const leaderRunStore = config.leaderEnabled ? backend?.leaderRun?.() : undefined;
138
142
  const scratchpadSweepStaleGuard = createThrottledReaperCatch("scratchpad_sweep_stale", logger);
139
143
  const approvalReconcileGuard = createThrottledReaperCatch("approval_reconcile", logger);
140
144
  const permissionRuleReapGuard = createThrottledReaperCatch("permission_rule_reap_expired", logger);
@@ -451,6 +455,15 @@ export function startReapers(ctx) {
451
455
  ?.reapOlderThan?.(Date.now() - config.rosterRetentionMs)
452
456
  .then(reapCount("roster_rows_reaped_total", {}))
453
457
  .then(() => rosterReapOlderThanGuard.onSuccess(), rosterReapOlderThanGuard.onError));
458
+ // #193 车4 件1(codex 对抗复审 R1-high②):陈旧 `running` leader run 的清算 —— 驱动它的副本死了,
459
+ // 行会永远停在 `running`(跨副本读面显示「在跑」而没有任何东西在跑)。到期迁 `failed` 并写清 WHY。
460
+ // **不是接管**(理由见 leader-run-store-sql.ts 的 reapStaleRunning 头注)。腿只在开了 leader 的
461
+ // 部署上注册:`leaderRunStore` 是 `config.leaderEnabled && backend?.leaderRun?.()` 的合取(见上面
462
+ // 的取值处),默认关的部署零扫描。
463
+ leg(leaderRunStore
464
+ ?.reapStaleRunning(config.leaderRunStaleMs, Date.now())
465
+ .then(reapCount("leader_runs_stale_reaped_total", {}))
466
+ .then(() => leaderRunStaleGuard.onSuccess(), leaderRunStaleGuard.onError));
454
467
  // 修8(三路复审 absorb-2,接线 (b)):periodic scratchpad sweep — 无 E21 purge 兜到的孤儿目录(session 从未
455
468
  // DELETE、purge 当次失败、local 后端无 purge coordinator)按 mtime 过期回收。SCRATCHPAD_SWEEP_TTL_MS
456
469
  // (default 7d,0=禁用)。activeSessionIds 不传:活跃判据的诚实边界在 env-facts.ts 的 sweep 文档——TTL 7d
@@ -861,6 +861,11 @@ export interface ServiceConfigFlat {
861
861
  * 每 run 一新行,无清理=线性无界)。按行的 recorded_at_ms(最后 upsert 触碰)删;SQL twins 扩展面,
862
862
  * File/Memory 店 core 自带 maxAgeMs。Default 30 days, floor 1m。`ROSTER_RETENTION_MS`。 */
863
863
  rosterRetentionMs: number;
864
+ /** #193 车4 件1(codex R1-high②):`leader_run` 行停在 `running` 超过这个时长即被清算成 `failed`
865
+ * 并写清 WHY —— 驱动它的副本已经死了,没有任何东西在跑。**不是接管**(见 store 的
866
+ * `reapStaleRunning` 头注)。默认 26h(leader run 的文档上限 24h + 余量),下限 1m。
867
+ * Env: `LEADER_RUN_STALE_MS`。`LEADER_ENABLED=false` 的部署上这条腿根本不注册。 */
868
+ leaderRunStaleMs: number;
864
869
  /** 修8(三路复审 absorb-2):periodic scratchpad sweep TTL — the reaper removes per-session scratchpad dirs
865
870
  * ([820]③ ensureScratchpadDir) whose mtime is older than this (env-facts.ts `sweepStaleScratchpads`; the E21
866
871
  * session-DELETE purge is the primary lifecycle, this sweep is the orphan backstop). Default 7 days, floor 1m
@@ -1212,7 +1217,7 @@ export type ServiceMemoryConfig = Pick<ServiceConfigFlat, "memoryEngineEnabled"
1212
1217
  * config 这一层它只是一句声明。 */
1213
1218
  export type ServiceAuthConfig = Pick<ServiceConfigFlat, "authToken" | "authTokens" | "allowUnauthedWrites" | "corsOrigins" | "principalHeader" | "requirePrincipal" | "autonomy" | "commandPolicy" | "compliancePosture" | "lockedConfigKeys" | "retentionPolicy" | "operatorPrincipals" | "principalJwtPubkeys" | "principalJwtIss" | "principalJwtAud" | "principalJwtMaxTtlSec" | "bindHost" | "bindHostSource">;
1214
1219
  /** 组:orchestration(编排 + 执行车道 + 沙箱面 + workflow/后台 agent 存留)。 */
1215
- export type ServiceOrchestrationConfig = Pick<ServiceConfigFlat, "remoteExec" | "worktreeIsolation" | "leaderEnabled" | "leaderFanoutEnabled" | "routerEnabled" | "selfOrchestrationEnabled" | "selfOrchestrationModels" | "selfOrchestrationWorkerIsolation" | "forkEnabled" | "experimentalObserverAgents" | "schedulerEnabled" | "schedulerSessionWakeup" | "schedulerSessionLifetime" | "schedulerStorePath" | "planModeEnabled" | "workflowRunStoreBackend" | "workflowOrphanGraceMs" | "workflowJournalRetentionMs" | "workflowRunRetentionMs" | "usageWindows" | "workflowAgentsReadOnly" | "workflowSizeGuideline" | "backgroundAgentRetentionMs" | "backgroundAgentStaleRunningMs" | "backgroundAgentParkClaimStaleMs" | "rosterRetentionMs" | "scratchpadSweepTtlMs" | "sandboxPkgSource" | "sessionAutoTitle" | "selectEnvironmentTool" | "envFactsEnabled" | "toolDeferLongtail" | "lspEnabled" | "lspHostEnabled" | "imageBakes" | "readFace" | "readDenyPatterns" | "readDenyBuiltinTiers" | "readDenyBuiltinExclude">;
1220
+ export type ServiceOrchestrationConfig = Pick<ServiceConfigFlat, "remoteExec" | "worktreeIsolation" | "leaderEnabled" | "leaderFanoutEnabled" | "routerEnabled" | "selfOrchestrationEnabled" | "selfOrchestrationModels" | "selfOrchestrationWorkerIsolation" | "forkEnabled" | "experimentalObserverAgents" | "schedulerEnabled" | "schedulerSessionWakeup" | "schedulerSessionLifetime" | "schedulerStorePath" | "planModeEnabled" | "workflowRunStoreBackend" | "workflowOrphanGraceMs" | "workflowJournalRetentionMs" | "workflowRunRetentionMs" | "usageWindows" | "workflowAgentsReadOnly" | "workflowSizeGuideline" | "backgroundAgentRetentionMs" | "backgroundAgentStaleRunningMs" | "backgroundAgentParkClaimStaleMs" | "rosterRetentionMs" | "leaderRunStaleMs" | "scratchpadSweepTtlMs" | "sandboxPkgSource" | "sessionAutoTitle" | "selectEnvironmentTool" | "envFactsEnabled" | "toolDeferLongtail" | "lspEnabled" | "lspHostEnabled" | "imageBakes" | "readFace" | "readDenyPatterns" | "readDenyBuiltinTiers" | "readDenyBuiltinExclude">;
1216
1221
  /** 组:limitsHttp(HTTP 面 + 各类上限/配额/回收窗)。 */
1217
1222
  export type ServiceLimitsHttpConfig = Pick<ServiceConfigFlat, "port" | "attachmentOrphanGraceMs" | "workspaceFileMaxBytes" | "attachmentMaxBytes" | "attachmentMimeAllowlist" | "attachmentUnboundTtlMs" | "infraCostRates" | "drainGraceMs" | "sighupIdleGraceMs" | "parentPid" | "rateLimitPerMin" | "maxTaskCostUsd" | "maxTaskTokens" | "maxPrincipalCostUsd" | "costQuotaWindowSec" | "reapIntervalSec" | "runStaleSec" | "toolResultTtlSec">;
1218
1223
  /** 组:observability(可观测)。 */
package/dist/config.js CHANGED
@@ -1960,6 +1960,7 @@ function parseOrchestrationDomain(ctx) {
1960
1960
  backgroundAgentStaleRunningMs: clampEnvWithWarn("BG_AGENT_STALE_RUNNING_MS", String(15 * 60_000), 10 * 60_000), // core 契约 ≥10min 硬钳(60s 写者心跳)
1961
1961
  backgroundAgentParkClaimStaleMs: clampEnvWithWarn("BG_AGENT_PARK_CLAIM_STALE_MS", String(60 * 60_000), 60_000), // [1575] F1 修:default 1h,floor 1m(零/负值都不该让整块清算失活)
1962
1962
  rosterRetentionMs: clampEnvWithWarn("ROSTER_RETENTION_MS", String(30 * 24 * 60 * 60 * 1000), 60_000), // [1522] MED2: 30d default, floor 1m; SQL twins only
1963
+ leaderRunStaleMs: clampEnvWithWarn("LEADER_RUN_STALE_MS", String(26 * 60 * 60 * 1000), 60_000), // #193 车4 件1(codex R1-high②): 26h default(24h 文档上限+余量), floor 1m
1963
1964
  // 修8: scratchpad sweep TTL — 0(or negative)= disabled; enabled values floor at 1m (a sub-minute TTL would
1964
1965
  // race in-flight tasks' scratchpad writes for no operational gain).
1965
1966
  // #210 B 档,只对**下限夹取**点名:`≤0 ⇒ 0` 是上面那条设计注写明的「显式关本腿」,是 operator 的
@@ -2186,7 +2187,7 @@ const ORCHESTRATION_GROUP_KEYS = [
2186
2187
  "schedulerEnabled", "schedulerSessionWakeup", "schedulerSessionLifetime", "schedulerStorePath", "planModeEnabled", "workflowRunStoreBackend",
2187
2188
  "workflowOrphanGraceMs", "workflowJournalRetentionMs", "workflowRunRetentionMs", "usageWindows", "workflowAgentsReadOnly",
2188
2189
  "workflowSizeGuideline", "backgroundAgentRetentionMs", "backgroundAgentStaleRunningMs",
2189
- "backgroundAgentParkClaimStaleMs", "rosterRetentionMs", "scratchpadSweepTtlMs", "sandboxPkgSource",
2190
+ "backgroundAgentParkClaimStaleMs", "rosterRetentionMs", "leaderRunStaleMs", "scratchpadSweepTtlMs", "sandboxPkgSource",
2190
2191
  "sessionAutoTitle", "selectEnvironmentTool", "envFactsEnabled", "toolDeferLongtail", "lspEnabled", "lspHostEnabled",
2191
2192
  "imageBakes", "readFace", "readDenyPatterns", "readDenyBuiltinTiers", "readDenyBuiltinExclude",
2192
2193
  ];
@@ -2352,6 +2353,35 @@ export function loadConfig() {
2352
2353
  if (imageBakesEnabled && !Object.values(authTokens).includes(bakeRunnerPrincipal)) {
2353
2354
  throw new Error(`IMAGE_BAKES_ENABLED=true but no SERVICE_AUTH_TOKENS entry maps to the runner principal '${bakeRunnerPrincipal}' — the bake-runner authenticates by its credential (token-derived source), so a '<BAKE_RUNNER_TOKEN>=${bakeRunnerPrincipal}' SERVICE_AUTH_TOKENS entry is required or claim/ingest/heartbeat all 403 (IMAGE-API-DESIGN.md §P2.12 boot invariant)`);
2354
2355
  }
2356
+ // 🔴 boot invariant #193 车4 件2(台账 P1-10 / docs/DESIGN-193-CAR4-leader-failclosed.md 件2):leader
2357
+ // 车道的两条「启用即损坏」组合,在 config 层 fail-closed。两门都只在 `LEADER_ENABLED=true` 时可能触发
2358
+ // —— 默认 false ⇒ 现行为逐位不变(负控在 config-validation.test.ts 里钉着)。
2359
+ //
2360
+ // 门①(host 腿 per-worker workspace 未接线):`leader/wire.ts` 里 worker 环境的 workspace 基准是
2361
+ // `cfg.workspace ?? "/home/user"` 这一条**绝对路径**,并贯穿 stageWorkerEnv / provisionIntegrationSandbox /
2362
+ // mkRepair / mkConflictResolver;而 `plugins/remote-env-host.ts` 给每个 host env 分的是**各不相同**的随机
2363
+ // 目录(`sema-host-<id>-<rand>`)——绝对路径不随之走。于是 N 个并发 host worker 全部落在同一个物理目录里
2364
+ // 竞态写 git 状态:不是报错,是**静默数据损坏**。真修(provisionWorker 给每个 worker 传唯一 workspace)
2365
+ // 不在本车;门先行,消灭「启用即损坏」的窗口。
2366
+ if (orchestration.leaderEnabled && orchestration.remoteExec?.provider === "host") {
2367
+ throw new Error("LEADER_ENABLED=true + REMOTE_EXEC=host is refused: the leader wire pins ONE absolute workspace path " +
2368
+ "(LEADER_WORKSPACE / the /home/user default) for every sub-worker, while the host adapter gives each env a " +
2369
+ "DIFFERENT random directory — so N concurrent host workers would share one physical git worktree and " +
2370
+ "corrupt each other's state SILENTLY (per-worker workspace is not wired on this lane yet). Run the leader " +
2371
+ "on an isolated lane (REMOTE_EXEC=e2b or k8s), or turn the leader off (unset LEADER_ENABLED) " +
2372
+ "(#193 车4 件2 / staleness ledger P1-10 boot invariant)");
2373
+ }
2374
+ // 门②(无外部 SQL 持久层):仓级前提是「durable state lives in an external SQL store」。没有 SQL 后端时
2375
+ // leader run 的登记表退化成进程内 Map ⇒ 跨副本 GET 404、重启即失忆、POST 重试直接起**第二次真 git push**。
2376
+ // 「多副本」这件事在 config 层判不了(副本数不是本进程的知识),可判的**代理**就是 durable 前提本身:
2377
+ // 一个没有外部 SQL 店的部署,与 leader run(N worker、最长 24h、真 push)的语义结构性不相容。
2378
+ if (orchestration.leaderEnabled && store.dbBackend !== "mysql" && store.dbBackend !== "pg") {
2379
+ throw new Error(`LEADER_ENABLED=true but DB_BACKEND=${store.dbBackend} (no external SQL store) is refused: a leader run is an ` +
2380
+ "N-worker fan-out lasting up to 24h that ends in a REAL git push, and without a durable store its registry is " +
2381
+ "a per-process Map — a cross-replica GET 404s, a restart forgets the run, and a retried POST starts a SECOND " +
2382
+ "push. Point the service at a durable store (DB_BACKEND=mysql|pg), or turn the leader off (unset LEADER_ENABLED) " +
2383
+ "(#193 车4 件2 / staleness ledger P1-9+P1-10 boot invariant)");
2384
+ }
2355
2385
  // #157 F 类观测:执行车道落点记账放在这里 —— 九域解析与上面全部跨域不变量都过了之后,即「这一轮
2356
2386
  // 确实启得起来」才成立的位置(写在 REMOTE_EXEC 解析途中的话,后置拒启会留下一条没启起来的落点)。
2357
2387
  EXEC_LANE_NOTICE = buildExecLaneNotice(orchestration.remoteExec);
@@ -1,5 +1,5 @@
1
1
  import { sendJson, sendError } from "../send.js";
2
- import { gatedPrincipal } from "../principal-gate.js";
2
+ import { gatedPrincipal, headerStr } from "../principal-gate.js";
3
3
  export async function handleLeader(req, res, url, ctx) {
4
4
  const miss = { fell: false };
5
5
  await handleLeaderBody(req, res, url, ctx, miss);
@@ -38,14 +38,17 @@ async function handleLeaderBody(req, res, url, ctx, miss) {
38
38
  sendError(res, 400, "request.invalid_json", "invalid JSON body");
39
39
  return;
40
40
  }
41
- const r = deps.leaderEndpoint.handle("POST", url, body, requester);
41
+ // #193 车4 件1:`Idempotency-Key` 头接到 leader 的 durable 提交幂等上(POST /v1/runs、
42
+ // POST /v1/images/bakes 同一条头名)。原始头值原样递给 endpoint —— 按提交者 scope + 定宽
43
+ // hash 都发生在那一层(`leaderIdemKey`),路由层不做语义加工。
44
+ const r = await deps.leaderEndpoint.handle("POST", url, body, requester, headerStr(req.headers["idempotency-key"]) ?? null);
42
45
  if (r) {
43
46
  sendJson(res, r.status, r.body);
44
47
  return;
45
48
  }
46
49
  }
47
50
  else {
48
- const r = deps.leaderEndpoint.handle("GET", url, undefined, requester);
51
+ const r = await deps.leaderEndpoint.handle("GET", url, undefined, requester);
49
52
  if (r) {
50
53
  sendJson(res, r.status, r.body);
51
54
  return;
@@ -1,4 +1,17 @@
1
1
  import type { LeaderResult } from "./leader.js";
2
+ import { type LeaderRunRecord, type LeaderRunStatus, type NewLeaderRun, type LeaderRunTerminal } from "../plugins/leader-run-store-sql.js";
3
+ /**
4
+ * endpoint 消费的 store 面(结构型,不是 nominal union):真实现是 `SqlLeaderRunStore` 的两个方言孪生,
5
+ * 测试用进程内假件。窄到只有 endpoint 真调的三个方法 —— 端点不该看见店的其余口。
6
+ */
7
+ export interface LeaderRunStore {
8
+ createRun(input: NewLeaderRun): Promise<{
9
+ created: boolean;
10
+ run: LeaderRunRecord;
11
+ }>;
12
+ getRun(id: string): Promise<LeaderRunRecord | undefined>;
13
+ finishRun(id: string, patch: LeaderRunTerminal): Promise<boolean>;
14
+ }
2
15
  export interface LeaderRequestBody {
3
16
  /** The task to decompose + execute (the Planner turns it into N disjoint sub-tasks). */
4
17
  objective: string;
@@ -12,7 +25,7 @@ export interface LeaderRun {
12
25
  * push chokepoint ABSTAINED (a `candidate_only`/`needs_human_oracle`/`conflict` repair terminal), held the push,
13
26
  * and surfaced a candidate awaiting a human `/decide` accept — distinct from "broke" (`failed`). Default runs
14
27
  * never reach it (no `repairTerminal` ⇒ `completed`/`failed` exactly as before). */
15
- status: "running" | "completed" | "failed" | "needs_human";
28
+ status: LeaderRunStatus;
16
29
  result?: LeaderResult;
17
30
  error?: string;
18
31
  startedAt: number;
@@ -24,15 +37,21 @@ export interface LeaderRun {
24
37
  export interface LeaderEndpoint {
25
38
  /** Returns a {status,body} to send, or null if the (method,url) is not a leader route. Kicks the background
26
39
  * run for POST. `body` is the already-parsed request body (server reads it; undefined for GET). `requester`
27
- * (BL-4) is the authenticated principal: recorded as the run owner on POST and owner-checked on GET. */
28
- handle(method: string, url: string, body: unknown, requester?: string | null): {
40
+ * (BL-4) is the authenticated principal: recorded as the run owner on POST and owner-checked on GET.
41
+ * `idempotencyKey` is the caller's RAW `Idempotency-Key` header (POST only; scoped by owner before it
42
+ * reaches the store — see {@link leaderIdemKey}). ASYNC because the durable arm reads/writes SQL. */
43
+ handle(method: string, url: string, body: unknown, requester?: string | null, idempotencyKey?: string | null): Promise<{
29
44
  status: number;
30
45
  body: object;
31
- } | null;
46
+ } | null>;
32
47
  /** 对抗复查 B-1(CRITICAL):the number of leader runs still RUNNING in the background — the drain gate
33
- * counts them (a SIGTERM used to see 0 and hard-shut mid-leader-run). */
48
+ * counts them (a SIGTERM used to see 0 and hard-shut mid-leader-run).
49
+ *
50
+ * 🔴 刻意**只数本副本**(即使 store 在场):drain 门问的是「这个进程还在驱动几条 run」,不是「全集群
51
+ * 还有几条」。读库会把别的副本正在跑的 run 也算进来,于是本副本永远排不干净。 */
34
52
  inflight(): number;
35
- /** In-memory run registry (exposed for tests / a future reaper). */
53
+ /** In-memory run registry — the DEGRADED arm's storage, and (on both arms) the drain gate's inflight set.
54
+ * With a store in place this Map holds only the runs THIS replica is driving. Exposed for tests. */
36
55
  runs: Map<string, LeaderRun>;
37
56
  }
38
57
  export declare function createLeaderEndpoint(runLeader: (body: LeaderRequestBody) => Promise<LeaderResult>, opts?: {
@@ -47,5 +66,7 @@ export declare function createLeaderEndpoint(runLeader: (body: LeaderRequestBody
47
66
  * runLeader 的内部实现细节,依旧保持 pure/testable。不传 = 向后兼容旧行为(只查 objective) ——
48
67
  * 本文件其余全部既有用例都用最小 {objective} payload,不能被这个修复打红。 */
49
68
  requiredFields?: readonly string[];
69
+ /** #193 车4 件1:durable 登记表。缺席 ⇒ 显式的单副本内存降级臂(见文件头注)。 */
70
+ store?: LeaderRunStore;
50
71
  }): LeaderEndpoint;
51
72
  //# sourceMappingURL=endpoint.d.ts.map
@@ -4,10 +4,22 @@
4
4
  * GET /v1/leader/:id → { id, status, result?, error? }
5
5
  * A leader run is long (minutes: provision N E2B workers → real model coding → merge → push), so it is async
6
6
  * (background + poll), mirroring /v1/runs. `runLeader` is injected (the real wiring lives in wire.ts /
7
- * main.ts) so this handler is pure + mock-tested. MVP tracking is in-memory (single-replica); a TiDB-backed
8
- * leader-run store is the cross-replica follow-up (same shape as run-store).
7
+ * main.ts) so this handler is pure + mock-tested.
8
+ *
9
+ * ── 登记表的两条腿(#193 车4 件1,台账 P1-9)────────────────────────────────────────────────────────
10
+ * **默认腿 = durable SQL 店**(`opts.store`,`plugins/leader-run-store-sql.ts` 的双方言孪生)。它一次解掉
11
+ * 三条同源缺口:跨副本 GET(行在库里)、POST 提交幂等(`idem_key` UNIQUE 的原子赢或观察)、以及
12
+ * 「重复 spawn」这件事本身(准入处只放一条进去,worker 层就不需要那道由构造即无效的并发去重面 ——
13
+ * 它的删除理由见 `fanout.ts` 头注)。
14
+ *
15
+ * **降级腿 = 进程内 `Map`**(`opts.store` 缺席时)。它是**显式的单副本限定**形,不是「以后再补」:
16
+ * · 跨副本 GET 404、进程重启即失忆、幂等作用域只到本进程;
17
+ * · 生产部署不会落到这条腿上 —— config 层已把「`LEADER_ENABLED=true` 且无外部 SQL 店」整个组合拒启
18
+ * (#193 车4 件2 门②),所以这条腿的真实用户只有单元测试与手工试跑。
19
+ * 两腿的**可观察行为**除上述三条外逐位一致(同一套 4xx 码、同一套属主门、同一份 202/200 体)。
9
20
  */
10
21
  import { uuidv7 } from "@sema-agent/core";
22
+ import { leaderIdemKey } from "../plugins/leader-run-store-sql.js";
11
23
  /**
12
24
  * LEADER-REPAIRLOOP-INTEGRATION §5/§10.6 — map a finished `LeaderResult` to the run status. The repair terminal
13
25
  * (set only when LEADER_REPAIR_LOOP/LEADER_MEASURE_GATES drove a single-agent path or the merge push chokepoint
@@ -29,20 +41,100 @@ function statusForLeaderResult(result) {
29
41
  }
30
42
  }
31
43
  const LEADER_ID_RE = /^\/v1\/leader\/([^/]+)$/;
32
- /** How long a FINISHED leader run stays pollable before it's evicted from the in-memory registry. */
44
+ /** How long a FINISHED leader run stays pollable before it's evicted from the in-memory registry.
45
+ * Bounds PROCESS MEMORY on the degraded arm — it is not a retention policy (the durable table keeps its rows;
46
+ * see the store's header for why it has no reaper leg). */
33
47
  const RUN_RETENTION_MS = 60 * 60_000;
48
+ /** 终局写库的有界重试(codex R1-high③):一次瞬断不该永久丢掉一条真 run 的终局。 */
49
+ const TERMINAL_WRITE_ATTEMPTS = 3;
50
+ const TERMINAL_WRITE_RETRY_MS = 200;
51
+ /** `Idempotency-Key` 的形:1–255 且无首尾空白(与 `/v1/runs` 的 zod `.max(255)` + 店门口
52
+ * `assertIdempotencyKeyShape` 同规)。坏形 ⇒ 400,不静默忽略 —— 忽略一个去重键 = 悄悄放行第二次真 push。 */
53
+ function idemShapeError(raw) {
54
+ if (raw.length === 0 || raw.length > 255 || raw.trim() !== raw) {
55
+ return `invalid Idempotency-Key: must be 1-255 chars with no leading/trailing whitespace (got length ${raw.length})`;
56
+ }
57
+ return undefined;
58
+ }
34
59
  export function createLeaderEndpoint(runLeader, opts = {}) {
35
60
  const runs = new Map();
61
+ /** 降级臂的进程内幂等索引(scope 过的键 → run id)。store 在场时不使用(库上的 UNIQUE 才是执法者)。 */
62
+ const memIdem = new Map();
36
63
  // Bound the in-memory registry: a finished leader run stays pollable for RUN_RETENTION_MS, then is evicted.
37
64
  // Swept on each POST (amortized, no timer) so the map can't grow without limit over a long-lived process.
38
65
  function reap() {
39
66
  const cutoff = Date.now() - RUN_RETENTION_MS;
40
67
  for (const [id, run] of runs) {
41
- if (run.status !== "running" && (run.finishedAt ?? run.startedAt) < cutoff)
68
+ if (run.status !== "running" && (run.finishedAt ?? run.startedAt) < cutoff) {
42
69
  runs.delete(id);
70
+ for (const [k, v] of memIdem)
71
+ if (v === id)
72
+ memIdem.delete(k);
73
+ }
43
74
  }
44
75
  }
45
- function handle(method, url, body, requester) {
76
+ /** 后台驱动一条已准入的 run。两腿共用:终局既写本地 Map(drain + 降级臂的读面),又写库(若在场)。 */
77
+ function drive(id, body) {
78
+ void (async () => {
79
+ let terminal;
80
+ try {
81
+ const result = await runLeader(body);
82
+ const status = statusForLeaderResult(result); // §10.6: ok→completed/failed PLUS needs_human for a held candidate
83
+ terminal = { status: status, result };
84
+ opts.logger?.info?.("leader_run_done", { id, ok: result.ok, status, ...(result.repairTerminal ? { repairTerminal: result.repairTerminal } : {}), merged: result.merge && result.merge.ok ? result.merge.merged : undefined });
85
+ }
86
+ catch (e) {
87
+ terminal = { status: "failed", error: e instanceof Error ? e.message : String(e) };
88
+ opts.logger?.error?.("leader_run_error", { id, err: terminal.error });
89
+ }
90
+ // 🔴 codex R1-high③(验真后修):**先落库,后改本地记录**。`inflight()` 数的是本地 `running`,
91
+ // 而 drain 门读的正是它 —— 反过来写(先改本地再 await 落库)会让 SIGTERM 在终局 SQL 还在飞时看到 0,
92
+ // hardShutdown 关掉连接池 ⇒ 终局写丢 ⇒ 别的副本永远读到 `running`,而代码可能已经真 push 过了。
93
+ // 顺序反过来之后,这条 run 直到库写**尘埃落定**(成功或彻底失败)才退出 inflight 集。
94
+ if (opts.store) {
95
+ // 有界重试:一次瞬断(连接被回收/主从切换)不该让一条真 run 的终局永久丢失。重试是**有界**的
96
+ // ——停机窗里无限重试只会把 drain 拖死,而那时该做的是响亮记一笔让运维手工对账。
97
+ for (let attempt = 1;; attempt += 1) {
98
+ try {
99
+ const moved = await opts.store.finishRun(id, terminal);
100
+ if (!moved)
101
+ opts.logger?.error?.("leader_run_terminal_not_applied", { id, status: terminal.status, why: "row missing or already terminal — another writer settled it (e.g. the stale-running sweep), or the row was deleted" });
102
+ break;
103
+ }
104
+ catch (e) {
105
+ if (attempt >= TERMINAL_WRITE_ATTEMPTS) {
106
+ opts.logger?.error?.("leader_run_terminal_write_failed", { id, status: terminal.status, attempts: attempt, err: e instanceof Error ? e.message : String(e) });
107
+ break;
108
+ }
109
+ await new Promise((r) => setTimeout(r, TERMINAL_WRITE_RETRY_MS));
110
+ }
111
+ }
112
+ }
113
+ const local = runs.get(id);
114
+ if (local) {
115
+ local.status = terminal.status;
116
+ if (terminal.result !== undefined)
117
+ local.result = terminal.result;
118
+ if (terminal.error !== undefined)
119
+ local.error = terminal.error;
120
+ local.finishedAt = Date.now();
121
+ }
122
+ })();
123
+ }
124
+ /** GET 的公共出口形(两腿同一份体)。 */
125
+ function getBody(run) {
126
+ return {
127
+ status: 200,
128
+ body: {
129
+ id: run.id,
130
+ status: run.status,
131
+ ...(run.result ? { result: run.result } : {}),
132
+ ...(run.error ? { error: run.error } : {}),
133
+ },
134
+ };
135
+ }
136
+ const NOT_FOUND = { status: 404, body: { error: "leader run not found", errorCode: "not_found.leader_run" } };
137
+ async function handle(method, url, body, requester, idempotencyKey) {
46
138
  if (method === "POST" && url === "/v1/leader") {
47
139
  const b = body;
48
140
  if (!b || typeof b.objective !== "string" || b.objective.trim() === "") {
@@ -54,49 +146,65 @@ export function createLeaderEndpoint(runLeader, opts = {}) {
54
146
  return { status: 400, body: { error: `missing required field(s): ${missing.join(", ")}`, errorCode: "request.body_shape" } };
55
147
  }
56
148
  }
149
+ const owner = requester ?? null;
150
+ let scopedIdem = null;
151
+ if (idempotencyKey != null) {
152
+ const bad = idemShapeError(idempotencyKey);
153
+ if (bad)
154
+ return { status: 400, body: { error: bad, errorCode: "request.body_shape" } };
155
+ // 🔴 按提交者 scope:两个 principal 用了同一个字面键,不该撞进同一条 run —— 那既是一次别人的
156
+ // run id 泄漏(存在性预言机),也会让 B 的提交被 A 的 run 静默吞掉。
157
+ scopedIdem = leaderIdemKey(idempotencyKey, owner);
158
+ }
57
159
  reap();
58
- const id = uuidv7();
59
- runs.set(id, { id, status: "running", startedAt: Date.now(), owner: requester ?? null }); // BL-4: record owner
60
- // fire-and-forget; status polled via GET. Never throws out of the handler.
61
- void (async () => {
62
- try {
63
- const result = await runLeader(b);
64
- const run = runs.get(id);
65
- run.status = statusForLeaderResult(result); // §10.6: ok→completed/failed PLUS needs_human for a held candidate
66
- run.result = result;
67
- run.finishedAt = Date.now();
68
- opts.logger?.info?.("leader_run_done", { id, ok: result.ok, status: run.status, ...(result.repairTerminal ? { repairTerminal: result.repairTerminal } : {}), merged: result.merge && result.merge.ok ? result.merge.merged : undefined });
160
+ if (opts.store) {
161
+ const { created, run } = await opts.store.createRun({ objective: b.objective, owner, idemKey: scopedIdem });
162
+ // 幂等重投:回既有行,**不**起第二条后台腿(第二次 = 第二次真 git push)。
163
+ if (!created) {
164
+ // 🔴 codex R1-high④(验真后修):重放**必须校属主**。`idem_key` 编码的是提交时的 owner,而
165
+ // `owner` 列会被身份收编(design/183)重绑 —— 于是旧身份拿旧键重投,可以命中一条**现属他人**的行。
166
+ // 直接回显 = 把别人的 leaderRunId 与状态泄漏出去。姿势照 `/v1/runs` 的 clientTaskId 重放先例:
167
+ // 属主不符 409 冲突,不回显任何标识(不给跨租户预言机)。
168
+ if (run.owner != null && run.owner !== owner) {
169
+ return { status: 409, body: { error: "idempotency key already used by another principal", errorCode: "conflict.leader_run_exists" } };
170
+ }
171
+ return { status: 202, body: { leaderRunId: run.id, status: run.status } };
69
172
  }
70
- catch (e) {
71
- const run = runs.get(id);
72
- run.status = "failed";
73
- run.error = e instanceof Error ? e.message : String(e);
74
- run.finishedAt = Date.now();
75
- opts.logger?.error?.("leader_run_error", { id, err: run.error });
76
- }
77
- })();
173
+ runs.set(run.id, { id: run.id, status: "running", startedAt: run.startedAtMs, owner });
174
+ drive(run.id, b);
175
+ return { status: 202, body: { leaderRunId: run.id, status: "running" } };
176
+ }
177
+ // 降级臂(无 store):作用域=本进程的同一套语义。
178
+ if (scopedIdem) {
179
+ const prior = memIdem.get(scopedIdem);
180
+ const priorRun = prior ? runs.get(prior) : undefined;
181
+ if (priorRun)
182
+ return { status: 202, body: { leaderRunId: priorRun.id, status: priorRun.status } };
183
+ }
184
+ const id = uuidv7();
185
+ runs.set(id, { id, status: "running", startedAt: Date.now(), owner }); // BL-4: record owner
186
+ if (scopedIdem)
187
+ memIdem.set(scopedIdem, id);
188
+ drive(id, b);
78
189
  return { status: 202, body: { leaderRunId: id, status: "running" } };
79
190
  }
80
191
  const m = method === "GET" ? LEADER_ID_RE.exec(url) : null;
81
192
  if (m) {
82
- const run = runs.get(m[1]);
83
- if (!run)
84
- return { status: 404, body: { error: "leader run not found", errorCode: "not_found.leader_run" } };
193
+ const id = m[1];
85
194
  // BL-4 owner check: an owned run is visible ONLY to its owner — 404 (not 403) to anyone else, so a
86
195
  // non-owner can't even confirm the id exists. owner null/undefined = anonymous/single-tenant run,
87
196
  // visible to any authenticated caller (back-compat; mirrors GET /v1/runs/:id's runOwnerOk).
88
- if (run.owner != null && run.owner !== (requester ?? null)) {
89
- return { status: 404, body: { error: "leader run not found", errorCode: "not_found.leader_run" } };
197
+ const ownerOk = (owner) => owner == null || owner === (requester ?? null);
198
+ if (opts.store) {
199
+ const run = await opts.store.getRun(id);
200
+ if (!run || !ownerOk(run.owner))
201
+ return NOT_FOUND;
202
+ return getBody(run);
90
203
  }
91
- return {
92
- status: 200,
93
- body: {
94
- id: run.id,
95
- status: run.status,
96
- ...(run.result ? { result: run.result } : {}),
97
- ...(run.error ? { error: run.error } : {}),
98
- },
99
- };
204
+ const run = runs.get(id);
205
+ if (!run || !ownerOk(run.owner))
206
+ return NOT_FOUND;
207
+ return getBody(run);
100
208
  }
101
209
  return null;
102
210
  }