@sema-agent/server 7.16.0-rc.3 → 7.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/reapers.js +29 -0
- package/dist/config-types.d.ts +3 -0
- package/dist/config.js +14 -11
- package/dist/observability/fail-open.d.ts +4 -0
- package/dist/observability/fail-open.js +4 -0
- package/dist/plugins/e2b-orphan-reclaim.d.ts +74 -0
- package/dist/plugins/e2b-orphan-reclaim.js +135 -0
- package/dist/plugins/remote-env-e2b.js +2 -1
- package/dist/tool-approval.js +20 -3
- package/dist/trace/core-keyset-guard.d.ts +1 -1
- package/package.json +2 -2
package/dist/boot/reapers.js
CHANGED
|
@@ -17,6 +17,7 @@ import { sweepStaleScratchpads } from "../env-facts.js";
|
|
|
17
17
|
import { CostQuota } from "../observability/cost-quota.js";
|
|
18
18
|
import { isAdoptionQuiescing, trackMaintenanceLeg } from "../adoption/quiesce.js";
|
|
19
19
|
import { createApprovalReconciler } from "../approval-reconciler.js";
|
|
20
|
+
import { createE2bOrphanSandboxProvider, reclaimOrphanE2bSandboxes, E2B_ORPHAN_GRACE_MS } from "../plugins/e2b-orphan-reclaim.js";
|
|
20
21
|
/** #157 / [2762]§五(sema-comms `audits/failopen-governance-176.md` §7 "Z 类 1 条"):default consecutive-failure
|
|
21
22
|
* count before a reaper leg's swallowed error gets a `warn` line. The ~15 `.catch(() => undefined)` sites
|
|
22
23
|
* below are correct in DIRECTION (a hiccuping periodic sweep must never crash the whole tick loop — one bad
|
|
@@ -97,6 +98,17 @@ export function startReapers(ctx) {
|
|
|
97
98
|
logger.info("reaper_swept", { metric, ...labels, count: n });
|
|
98
99
|
}
|
|
99
100
|
};
|
|
101
|
+
// #242:E2B 孤儿回收腿的三件座——provider(仅 e2b 部署形 + 旋钮 ON 时铸,一次)、时间节流锚
|
|
102
|
+
// (初始 -GRACE ⇒ 首 tick 即跑,承担 boot 半场)、重入守卫(list+逐杀可能跨多个 tick;provider RPC
|
|
103
|
+
// 带 SDK 级 requestTimeoutMs,旗不会因一次挂起永挂)。节流锚刻意用墙钟(codex R3 登记不改):判据
|
|
104
|
+
// 侧的 GRACE 比对是 E2B startedAt(墙钟)vs Date.now(墙钟)同钟系,节流锚换单调钟只护住「回拨多跑
|
|
105
|
+
// 一轮 list」这种无害面,却让本文件唯一一处异质钟——不值。锚在**发起时**推进(非成功时):失败轮也
|
|
106
|
+
// 压满 GRACE 间隔=对 provider 的天然退避;错误可见性由 e2bOrphanGuard 的连败阈值 warn 承担。
|
|
107
|
+
const e2bOrphanProvider = config.remoteExec?.provider === "e2b" && config.remoteExec.orphanReclaim !== false
|
|
108
|
+
? createE2bOrphanSandboxProvider({ apiKey: config.remoteExec.apiKey })
|
|
109
|
+
: undefined;
|
|
110
|
+
let e2bOrphanLastRunMs = Date.now() - E2B_ORPHAN_GRACE_MS;
|
|
111
|
+
let e2bOrphanInFlight = false;
|
|
100
112
|
let wfRunReapInFlight = false; // 1.108 review (lens③): serialize the all-scopes retention sweep across ticks
|
|
101
113
|
let attachmentSweepInFlight = false; // 复审 F1:附件孤儿 sweep 的重入守卫(同上)
|
|
102
114
|
let bgAgentReapInFlight = false; // core 1.364: same serialization for the durable bg-agent joint reap (per-scope serial loop)
|
|
@@ -105,6 +117,7 @@ export function startReapers(ctx) {
|
|
|
105
117
|
// per-tick, so each leg's consecutive-failure streak survives across ticks — see createThrottledReaperCatch).
|
|
106
118
|
const probeClockSkewGuard = createThrottledReaperCatch("probe_clock_skew", logger);
|
|
107
119
|
const runsReapStaleGuard = createThrottledReaperCatch("runs_reap_stale", logger);
|
|
120
|
+
const e2bOrphanGuard = createThrottledReaperCatch("e2b_orphan_reclaim", logger); // #242
|
|
108
121
|
const checkpointsReapExpiredGuard = createThrottledReaperCatch("checkpoints_reap_expired", logger);
|
|
109
122
|
const runsFailSuspendedExpiredCheckpointGuard = createThrottledReaperCatch("runs_fail_suspended_expired_checkpoint", logger);
|
|
110
123
|
const runDenySweepGuard = createThrottledReaperCatch("run_deny_sweep", logger);
|
|
@@ -483,6 +496,22 @@ export function startReapers(ctx) {
|
|
|
483
496
|
})
|
|
484
497
|
.then(() => scratchpadSweepStaleGuard.onSuccess(), scratchpadSweepStaleGuard.onError));
|
|
485
498
|
}
|
|
499
|
+
// #242(design/242 v1):E2B 孤儿沙盒回收。tick 内**时间节流**到 GRACE 周期(每 tick 打一次
|
|
500
|
+
// provider list API 是浪费——孤儿的时间尺度是分钟级);首轮在进程起后第一个 tick 就跑(boot 半场,
|
|
501
|
+
// 不单开 boot 调用点)。判据/边界全在 e2b-orphan-reclaim.ts;这里只管节流+重入+throttled catch。
|
|
502
|
+
if (e2bOrphanProvider && runStore && Date.now() - e2bOrphanLastRunMs >= E2B_ORPHAN_GRACE_MS && !e2bOrphanInFlight) {
|
|
503
|
+
e2bOrphanInFlight = true;
|
|
504
|
+
e2bOrphanLastRunMs = Date.now();
|
|
505
|
+
leg(reclaimOrphanE2bSandboxes({ provider: e2bOrphanProvider, runs: runStore, logger })
|
|
506
|
+
.then((s) => {
|
|
507
|
+
if (s.reclaimed > 0)
|
|
508
|
+
metrics.inc("e2b_orphans_reclaimed_total", {}, s.reclaimed);
|
|
509
|
+
})
|
|
510
|
+
.then(() => e2bOrphanGuard.onSuccess(), e2bOrphanGuard.onError)
|
|
511
|
+
.finally(() => {
|
|
512
|
+
e2bOrphanInFlight = false;
|
|
513
|
+
}));
|
|
514
|
+
}
|
|
486
515
|
}, config.reapIntervalSec * 1000);
|
|
487
516
|
reaper.unref?.();
|
|
488
517
|
return reaper;
|
package/dist/config-types.d.ts
CHANGED
|
@@ -359,6 +359,9 @@ export interface ServiceConfigFlat {
|
|
|
359
359
|
allowInternetAccess?: boolean;
|
|
360
360
|
sandboxEnv?: Record<string, string>;
|
|
361
361
|
mountPath?: string;
|
|
362
|
+
/** #242:崩溃遗留沙盒回收腿(E2B_ORPHAN_RECLAIM,默认 true;判据与 v1 边界见
|
|
363
|
+
* plugins/e2b-orphan-reclaim.ts 头注)。 */
|
|
364
|
+
orphanReclaim?: boolean;
|
|
362
365
|
} | {
|
|
363
366
|
provider: "k8s";
|
|
364
367
|
image: string;
|
package/dist/config.js
CHANGED
|
@@ -765,14 +765,14 @@ function parseModelDomain() {
|
|
|
765
765
|
// A non-empty value here would override every brain to one URL — breaking failover AND the Anthropic
|
|
766
766
|
// route (it would POST to the vLLM gateway). See design/15.
|
|
767
767
|
baseUrl: "",
|
|
768
|
-
reasoning:
|
|
768
|
+
reasoning: boolEnv("MODEL_REASONING", true), // #245:直读族收编词表(rc.2 登记的 7.17 件)
|
|
769
769
|
// vision precheck: the modality list drives BOTH the `model_no_vision` 422 (main.ts resolveSpec,
|
|
770
770
|
// when body.images hits a text-only model) AND the vision flag on GET /v1/models. The env-lane default model
|
|
771
771
|
// declares vision via MODEL_VISION — DEFAULT "true" keeps the historical image-capable behavior, so set
|
|
772
772
|
// MODEL_VISION=false for a text-only model (e.g. deepseek) to make the precheck actually fire instead of letting
|
|
773
773
|
// images透传 to an opaque downstream-gateway 400. (The centralized config-center lane derives input from its own
|
|
774
774
|
// per-model vision flag — see config-center/facade.ts.)
|
|
775
|
-
input:
|
|
775
|
+
input: boolEnv("MODEL_VISION", true) ? ["text", "image"] : ["text"], // #245
|
|
776
776
|
// Per-1M-token USD pricing (core `modelCostToPricing`: input→inputPer1M, …). Default 0 keeps the
|
|
777
777
|
// historical behavior (spend reads $0) — set MODEL_COST_* on a deploy to make `model_cost_micro_usd`
|
|
778
778
|
// + the Fleet card's costUsd real. Numbers are the gateway's contract price (e.g. the review gateway's
|
|
@@ -831,11 +831,8 @@ function parseModelDomain() {
|
|
|
831
831
|
// AND defeat the precheck (config.models[cheapId] is user-pickable via body.model) — the SAME dead-code-on-a-lane
|
|
832
832
|
// class clay caught on the env main model. MODEL_CHEAP_VISION declares it (strict "true"); UNSET → inherit the
|
|
833
833
|
// main model's input (prior behavior, no change — the documented "same gateway, just a different id" case).
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
: process.env.MODEL_CHEAP_VISION === "true"
|
|
837
|
-
? ["text", "image"]
|
|
838
|
-
: ["text"],
|
|
834
|
+
// #245:三态语义不变(缺席/空串 ⇒ 继承主模型),词面收编 parseBoolWord(未知词 boot 拒)。
|
|
835
|
+
input: ((v) => (v === undefined ? model.input : v ? ["text", "image"] : ["text"]))(parseBoolWord("MODEL_CHEAP_VISION", process.env.MODEL_CHEAP_VISION)),
|
|
839
836
|
// Own pricing: pricing is keyed by model id (budget.buildPricing), so without this the cheap model
|
|
840
837
|
// inherits the MAIN model's `cost` and subagent/summarize spend (council lenses, compaction) is
|
|
841
838
|
// metered at the PRO per-token price → cost metric + per-principal quota over-count. Set
|
|
@@ -857,7 +854,7 @@ function parseModelDomain() {
|
|
|
857
854
|
// wrong when the cheap slot is a different tokenizer family (e.g. CJK-dense deepseek cheap under an
|
|
858
855
|
// anthropic main). MODEL_CHEAP_CHARS_PER_TOKEN overrides; UNSET → inherit main (same knob family).
|
|
859
856
|
...(optFinitePositiveEnv("MODEL_CHEAP_CHARS_PER_TOKEN") !== undefined ? { charsPerToken: optFinitePositiveEnv("MODEL_CHEAP_CHARS_PER_TOKEN") } : {}),
|
|
860
|
-
reasoning: process.env.MODEL_CHEAP_REASONING
|
|
857
|
+
reasoning: parseBoolWord("MODEL_CHEAP_REASONING", process.env.MODEL_CHEAP_REASONING) ?? model.reasoning, // #245:三态同上
|
|
861
858
|
}
|
|
862
859
|
: undefined;
|
|
863
860
|
// [818]① 1M dual-window (core 1.289 `Model.autoCompactTokens`, additive): a 1M-window model gets
|
|
@@ -932,7 +929,7 @@ function parseModelDomain() {
|
|
|
932
929
|
authToken: firstSetEnv("ANTHROPIC_AUTH_TOKEN"),
|
|
933
930
|
baseUrl: firstSetEnv("ANTHROPIC_BASE_URL"),
|
|
934
931
|
version: process.env.ANTHROPIC_VERSION,
|
|
935
|
-
cacheBreakpoints: (
|
|
932
|
+
cacheBreakpoints: boolEnv("ANTHROPIC_CACHE_BREAKPOINTS", true), // #245
|
|
936
933
|
...(process.env.ANTHROPIC_MAX_RETRIES ? { maxRetries: numEnv("ANTHROPIC_MAX_RETRIES", "2") } : {}), // A1 同族:NaN 会把 SDK 重试面折没
|
|
937
934
|
}
|
|
938
935
|
: undefined,
|
|
@@ -1221,7 +1218,9 @@ function parseApprovalDomain(ctx) {
|
|
|
1221
1218
|
// boot 期的休眠行审计(`boot/permission-rules-audit.ts`)——对显式开过的部署说「N 条规则随默认 ON
|
|
1222
1219
|
// 激活」是假话,那些行本来就是活的。判据与 `boolEnv` 认的字面量集**同源**(只有 "true"/"false"
|
|
1223
1220
|
// 算表态;空串按未设,拼错的值走 boolEnv 的告警臂并落回默认 ⇒ 也不算表态)。
|
|
1224
|
-
|
|
1221
|
+
// #245:显式表态判别随词表放宽——八词任一在场即「显式」(旧形只认 true/false 两词,=1 的显式关/开
|
|
1222
|
+
// 会被当「未表态」滑进派生腿,恰是 operator-knob 病的判别位变体);未知词由本体 boolEnv 先拒启。
|
|
1223
|
+
permissionRulesEnabledExplicit: parseBoolWord("PERMISSION_RULES_ENABLED", process.env["PERMISSION_RULES_ENABLED"]) !== undefined,
|
|
1225
1224
|
// #151 车2(design/172 §3.3 D3):ToolApprovalCoordinator 窗长三元的安全余量。numEnv fail-loud 形,
|
|
1226
1225
|
// 照邻居旋钮(approvalTimeoutSec 等)抄——坏形(非数字)在启动期炸,不静默折成 NaN。
|
|
1227
1226
|
streamAskWindowMarginMs: numEnv("STREAM_ASK_WINDOW_MARGIN_MS", "10000"),
|
|
@@ -1664,6 +1663,10 @@ function parseOrchestrationDomain(ctx) {
|
|
|
1664
1663
|
// cwd——改此值前先 drain 挂起中的 e2b 任务,否则 resumed 任务的 exec 落新根而产物在旧根
|
|
1665
1664
|
// (根修=core resumeVM 契约带回 WorkspaceHandle.mountPath,已转 core)。
|
|
1666
1665
|
...(process.env.E2B_MOUNT_PATH ? { mountPath: process.env.E2B_MOUNT_PATH } : {}),
|
|
1666
|
+
// #242(design/242 v1):崩溃遗留沙盒回收腿(boot 一次+reaper 周期)。默认 ON——泄漏=真金
|
|
1667
|
+
// 白银,判据已保守化(活行跳过/无行 10min GRACE/拿不准恒不杀);v1 前置=每部署独立 E2B key
|
|
1668
|
+
// (跨部署异库共用一把 key 的形不支持,发车说明成文)。
|
|
1669
|
+
orphanReclaim: boolEnv("E2B_ORPHAN_RECLAIM", true),
|
|
1667
1670
|
}
|
|
1668
1671
|
: // Kata pod sandbox on our k3s: image required; auth explicit (K8S_API_URL/K8S_TOKEN) or in-cluster SA.
|
|
1669
1672
|
process.env.REMOTE_EXEC === "k8s" && process.env.K8S_SANDBOX_IMAGE
|
|
@@ -1974,7 +1977,7 @@ function parseIntegrationsDomain() {
|
|
|
1974
1977
|
? {
|
|
1975
1978
|
baseUrl: env("SEMA_REGISTRY_URL"),
|
|
1976
1979
|
token: process.env.SEMA_REGISTRY_TOKEN || "",
|
|
1977
|
-
dryRun:
|
|
1980
|
+
dryRun: boolEnv("SEMA_REGISTRY_DRY_RUN", false), // #245
|
|
1978
1981
|
worker: process.env.SEMA_REGISTRY_WORKER || undefined,
|
|
1979
1982
|
}
|
|
1980
1983
|
: undefined,
|
|
@@ -61,6 +61,10 @@ export declare const FAIL_OPEN_TAGS: {
|
|
|
61
61
|
readonly cls: "P-DEBT";
|
|
62
62
|
readonly note: "跨副本赎回 parked 审批时,祖先层在 park 时是 **auto-mode 武装**的,但那只分类器是祖先任务上的活闭包(绑着它自己的转写窗+brain),跨进程重建不出来 ⇒ 本仓交一只如实拒答的 decider,core 收到 `unavailable` 后**不产生任何自动裁决**、原样落到祖先冻结审批席那条链(本腿的席位又是无 ALS 的降级形 ⇒ 再 park 给人)。方向:分类器本会 allow 的改成问人(更严),本会 block 的也改成问人(**不是自动放行**,但比自动拒松一档)⇒ 记债不当合法兜底。计数 = 「丢了祖先分类器判决的继承 ask」次数。收口件二选一:core 把分类器判据持久进链条目,或让祖先 decider 有可跨进程重建的形。";
|
|
63
63
|
};
|
|
64
|
+
readonly "server.e2b.orphan-skip-unknown": {
|
|
65
|
+
readonly cls: "F";
|
|
66
|
+
readonly note: "#242 E2B 孤儿回收对一只沙盒**判不了属主/活性**(metadata 缺 taskId 的病态形 / run 行状态词在 LIVE 与 TERMINAL 两表之外 / getRun 抛错)⇒ 恒不杀,本轮跳过。方向纪律:回收是清理型能力,缺席判据=不动作——误杀是杀活任务的手,漏杀只是钱;但每次跳过必须计数,否则「回收在跑却总有几只杀不掉」的病灶(店故障/新状态词未收编/别家部署共用 key)永不显形。";
|
|
67
|
+
};
|
|
64
68
|
readonly "server.fleet.subscriber-callback-threw": {
|
|
65
69
|
readonly cls: "F";
|
|
66
70
|
readonly note: "fleet bus 某订阅回调抛错 ⇒ 该回调本帧作废,其余订阅方与发布方不受影响。隔离是承重的:扇出同步,修前异常会传回发布方 put/update 投影点,core 持久化 catch{} 且不推进 storeRev ⇒ durable 行冻在 running 而 notify 已 ack(#183 复审 R3 HIGH)。丢的只是一个消费方的一帧渲染,故 F 类;但必须留痕——静默吞掉等于订阅方病灶永不显形。";
|
|
@@ -84,6 +84,10 @@ export const FAIL_OPEN_TAGS = {
|
|
|
84
84
|
cls: "P-DEBT",
|
|
85
85
|
note: "跨副本赎回 parked 审批时,祖先层在 park 时是 **auto-mode 武装**的,但那只分类器是祖先任务上的活闭包(绑着它自己的转写窗+brain),跨进程重建不出来 ⇒ 本仓交一只如实拒答的 decider,core 收到 `unavailable` 后**不产生任何自动裁决**、原样落到祖先冻结审批席那条链(本腿的席位又是无 ALS 的降级形 ⇒ 再 park 给人)。方向:分类器本会 allow 的改成问人(更严),本会 block 的也改成问人(**不是自动放行**,但比自动拒松一档)⇒ 记债不当合法兜底。计数 = 「丢了祖先分类器判决的继承 ask」次数。收口件二选一:core 把分类器判据持久进链条目,或让祖先 decider 有可跨进程重建的形。",
|
|
86
86
|
},
|
|
87
|
+
"server.e2b.orphan-skip-unknown": {
|
|
88
|
+
cls: "F",
|
|
89
|
+
note: "#242 E2B 孤儿回收对一只沙盒**判不了属主/活性**(metadata 缺 taskId 的病态形 / run 行状态词在 LIVE 与 TERMINAL 两表之外 / getRun 抛错)⇒ 恒不杀,本轮跳过。方向纪律:回收是清理型能力,缺席判据=不动作——误杀是杀活任务的手,漏杀只是钱;但每次跳过必须计数,否则「回收在跑却总有几只杀不掉」的病灶(店故障/新状态词未收编/别家部署共用 key)永不显形。",
|
|
90
|
+
},
|
|
87
91
|
"server.fleet.subscriber-callback-threw": {
|
|
88
92
|
cls: "F",
|
|
89
93
|
note: "fleet bus 某订阅回调抛错 ⇒ 该回调本帧作废,其余订阅方与发布方不受影响。隔离是承重的:扇出同步,修前异常会传回发布方 put/update 投影点,core 持久化 catch{} 且不推进 storeRev ⇒ durable 行冻在 running 而 notify 已 ack(#183 复审 R3 HIGH)。丢的只是一个消费方的一帧渲染,故 F 类;但必须留痕——静默吞掉等于订阅方病灶永不显形。",
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #242([3738] / design/242 v1)— E2B 孤儿沙盒回收。
|
|
3
|
+
*
|
|
4
|
+
* 病:server 进程崩溃(SIGKILL/OOM)后 core Runner 的 `destroy()` 不跑,E2B 沙盒按自己的
|
|
5
|
+
* timeout 窗继续存活计费;崩溃循环下泄漏累积。create 侧的归属打标早已在
|
|
6
|
+
* (`remote-env-e2b.ts` factory:metadata 带 sessionId/taskId,本批补 `semaManaged:"1"`);
|
|
7
|
+
* 本模块是缺的另一半——回收腿。
|
|
8
|
+
*
|
|
9
|
+
* 判据(全部保守方向,**拿不准恒不杀**——回收是清理型能力,缺席判据=不动作,方向纪律见
|
|
10
|
+
* memory `capability-probe-fail-direction`;误杀的代价是杀死一个活任务的手,漏杀的代价只是钱):
|
|
11
|
+
* a. `metadata.semaManaged !== "1"` → 非我方铸造,provider 侧 query 过滤已排除;防御性重验。
|
|
12
|
+
* b. `metadata.taskId` 缺席 → 病态形(factory 恒打标),不杀 + failOpen 留痕。
|
|
13
|
+
* c. run 行在场:
|
|
14
|
+
* LIVE 词(running/suspended/needs_review)→ 活,跳过;
|
|
15
|
+
* TERMINAL 词 → 孤儿,kill;
|
|
16
|
+
* 两表皆不中的词 → 词表纪律(#157):不杀 + failOpen(新词先安全后收编)。
|
|
17
|
+
* d. 行不存在:
|
|
18
|
+
* startedAt 距今 < GRACE → create→行落库窗口,跳过(防误杀新沙盒);
|
|
19
|
+
* ≥ GRACE → 真孤儿,kill(单库 fleet 方向下行跨副本可见)。
|
|
20
|
+
* e. `getRun` 抛错 → 不杀 + failOpen(不知道 ≠ 孤儿)。
|
|
21
|
+
*
|
|
22
|
+
* v1 边界(design/242 §设计②,发车说明同步):跨部署**异库**共用一把 E2B key 的形不支持——
|
|
23
|
+
* 别家沙盒的 taskId 在本库查不到行,超 GRACE 会被误杀;前置=每部署独立 E2B key(本就是推荐姿势)。
|
|
24
|
+
* metadata 无跨重启稳定的部署标识可用(instanceId 每 boot 变,恰是要回收上一世代)是结构原因。
|
|
25
|
+
*
|
|
26
|
+
* kill 逐只 try/catch(一只失败不挡其余);list 整体失败由调用方(reaper throttled catch)兜底。
|
|
27
|
+
*/
|
|
28
|
+
import type { Logger } from "../observability/logger.js";
|
|
29
|
+
/** create→run 行落库窗口的误杀护栏。非行为表态面,常量不设旋钮(理由:它只需要「远大于一次
|
|
30
|
+
* create+首写延迟、远小于沙盒 timeout 窗」,10min 对两端都有量级余量;错设它的运维事故面比
|
|
31
|
+
* 它保护的窗口更大)。 */
|
|
32
|
+
export declare const E2B_ORPHAN_GRACE_MS: number;
|
|
33
|
+
/** factory 打进 metadata 的归属标(create 半场,remote-env-e2b.ts);值恒 "1"。 */
|
|
34
|
+
export declare const SEMA_MANAGED_METADATA_KEY = "semaManaged";
|
|
35
|
+
/** provider 窄面(真实装配=E2B SDK `Sandbox.list()`/`Sandbox.kill()` 包装;测试注入 fake)。 */
|
|
36
|
+
export interface OrphanSandboxProvider {
|
|
37
|
+
/** 列出**我方铸造**的沙盒(装配侧用 `query.metadata = { semaManaged: "1" }` 服务端过滤)。 */
|
|
38
|
+
listManaged(): Promise<OrphanSandboxInfo[]>;
|
|
39
|
+
kill(sandboxId: string): Promise<void>;
|
|
40
|
+
}
|
|
41
|
+
export interface OrphanSandboxInfo {
|
|
42
|
+
sandboxId: string;
|
|
43
|
+
metadata: Record<string, string>;
|
|
44
|
+
/** SDK `SandboxInfo.startedAt`。 */
|
|
45
|
+
startedAt: Date;
|
|
46
|
+
}
|
|
47
|
+
export interface OrphanReclaimDeps {
|
|
48
|
+
provider: OrphanSandboxProvider;
|
|
49
|
+
/** RunStore 三形(TiDB/Pg/File)共同的窄读口。 */
|
|
50
|
+
runs: {
|
|
51
|
+
getRun(taskId: string): Promise<{
|
|
52
|
+
status: string;
|
|
53
|
+
} | undefined>;
|
|
54
|
+
};
|
|
55
|
+
logger?: Logger;
|
|
56
|
+
now?: () => number;
|
|
57
|
+
graceMs?: number;
|
|
58
|
+
}
|
|
59
|
+
export interface OrphanReclaimSummary {
|
|
60
|
+
scanned: number;
|
|
61
|
+
reclaimed: number;
|
|
62
|
+
skippedLive: number;
|
|
63
|
+
skippedGrace: number;
|
|
64
|
+
skippedUnknown: number;
|
|
65
|
+
killFailed: number;
|
|
66
|
+
}
|
|
67
|
+
/** 一轮回收(boot 一次 + reaper interval 周期共用)。list 抛错穿透给调用方(reaper catch 族)。 */
|
|
68
|
+
export declare function reclaimOrphanE2bSandboxes(deps: OrphanReclaimDeps): Promise<OrphanReclaimSummary>;
|
|
69
|
+
/** 真实 provider(E2B SDK 包装)。SDK 静态面动态 import——回收腿只在 e2b 部署形挂载,
|
|
70
|
+
* 非 e2b 形零加载(与 execution-env 的按需装配同姿势)。 */
|
|
71
|
+
export declare function createE2bOrphanSandboxProvider(cfg: {
|
|
72
|
+
apiKey: string;
|
|
73
|
+
}): OrphanSandboxProvider;
|
|
74
|
+
//# sourceMappingURL=e2b-orphan-reclaim.d.ts.map
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #242([3738] / design/242 v1)— E2B 孤儿沙盒回收。
|
|
3
|
+
*
|
|
4
|
+
* 病:server 进程崩溃(SIGKILL/OOM)后 core Runner 的 `destroy()` 不跑,E2B 沙盒按自己的
|
|
5
|
+
* timeout 窗继续存活计费;崩溃循环下泄漏累积。create 侧的归属打标早已在
|
|
6
|
+
* (`remote-env-e2b.ts` factory:metadata 带 sessionId/taskId,本批补 `semaManaged:"1"`);
|
|
7
|
+
* 本模块是缺的另一半——回收腿。
|
|
8
|
+
*
|
|
9
|
+
* 判据(全部保守方向,**拿不准恒不杀**——回收是清理型能力,缺席判据=不动作,方向纪律见
|
|
10
|
+
* memory `capability-probe-fail-direction`;误杀的代价是杀死一个活任务的手,漏杀的代价只是钱):
|
|
11
|
+
* a. `metadata.semaManaged !== "1"` → 非我方铸造,provider 侧 query 过滤已排除;防御性重验。
|
|
12
|
+
* b. `metadata.taskId` 缺席 → 病态形(factory 恒打标),不杀 + failOpen 留痕。
|
|
13
|
+
* c. run 行在场:
|
|
14
|
+
* LIVE 词(running/suspended/needs_review)→ 活,跳过;
|
|
15
|
+
* TERMINAL 词 → 孤儿,kill;
|
|
16
|
+
* 两表皆不中的词 → 词表纪律(#157):不杀 + failOpen(新词先安全后收编)。
|
|
17
|
+
* d. 行不存在:
|
|
18
|
+
* startedAt 距今 < GRACE → create→行落库窗口,跳过(防误杀新沙盒);
|
|
19
|
+
* ≥ GRACE → 真孤儿,kill(单库 fleet 方向下行跨副本可见)。
|
|
20
|
+
* e. `getRun` 抛错 → 不杀 + failOpen(不知道 ≠ 孤儿)。
|
|
21
|
+
*
|
|
22
|
+
* v1 边界(design/242 §设计②,发车说明同步):跨部署**异库**共用一把 E2B key 的形不支持——
|
|
23
|
+
* 别家沙盒的 taskId 在本库查不到行,超 GRACE 会被误杀;前置=每部署独立 E2B key(本就是推荐姿势)。
|
|
24
|
+
* metadata 无跨重启稳定的部署标识可用(instanceId 每 boot 变,恰是要回收上一世代)是结构原因。
|
|
25
|
+
*
|
|
26
|
+
* kill 逐只 try/catch(一只失败不挡其余);list 整体失败由调用方(reaper throttled catch)兜底。
|
|
27
|
+
*/
|
|
28
|
+
import { recordFailOpen } from "../observability/fail-open.js";
|
|
29
|
+
/** create→run 行落库窗口的误杀护栏。非行为表态面,常量不设旋钮(理由:它只需要「远大于一次
|
|
30
|
+
* create+首写延迟、远小于沙盒 timeout 窗」,10min 对两端都有量级余量;错设它的运维事故面比
|
|
31
|
+
* 它保护的窗口更大)。 */
|
|
32
|
+
export const E2B_ORPHAN_GRACE_MS = 10 * 60 * 1000;
|
|
33
|
+
/** factory 打进 metadata 的归属标(create 半场,remote-env-e2b.ts);值恒 "1"。 */
|
|
34
|
+
export const SEMA_MANAGED_METADATA_KEY = "semaManaged";
|
|
35
|
+
// 构造侧 satisfies 锚类型真源;容器声明 ReadonlySet<string> 让消费侧(窄读口的 status: string)直判。
|
|
36
|
+
const LIVE_STATUSES = new Set(["running", "suspended", "needs_review"]);
|
|
37
|
+
const TERMINAL_STATUSES = new Set(["completed", "blocked", "failed"]);
|
|
38
|
+
const _statusesCovered = true;
|
|
39
|
+
void _statusesCovered;
|
|
40
|
+
/** 一轮回收(boot 一次 + reaper interval 周期共用)。list 抛错穿透给调用方(reaper catch 族)。 */
|
|
41
|
+
export async function reclaimOrphanE2bSandboxes(deps) {
|
|
42
|
+
const now = deps.now ?? Date.now;
|
|
43
|
+
const graceMs = deps.graceMs ?? E2B_ORPHAN_GRACE_MS;
|
|
44
|
+
const summary = { scanned: 0, reclaimed: 0, skippedLive: 0, skippedGrace: 0, skippedUnknown: 0, killFailed: 0 };
|
|
45
|
+
const sandboxes = await deps.provider.listManaged();
|
|
46
|
+
for (const sbx of sandboxes) {
|
|
47
|
+
summary.scanned++;
|
|
48
|
+
// a. 防御性重验(provider 装配侧已 query 过滤;fake/将来的宽 provider 不许绕过归属判别)。
|
|
49
|
+
if (sbx.metadata[SEMA_MANAGED_METADATA_KEY] !== "1") {
|
|
50
|
+
summary.skippedUnknown++;
|
|
51
|
+
recordFailOpen("server.e2b.orphan-skip-unknown");
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const taskId = sbx.metadata.taskId;
|
|
55
|
+
// b. 病态形:factory 恒打 taskId(ctx.taskId 缺席时连 sessionId 都在而 taskId 不在)——不杀,留痕。
|
|
56
|
+
if (!taskId) {
|
|
57
|
+
summary.skippedUnknown++;
|
|
58
|
+
recordFailOpen("server.e2b.orphan-skip-unknown");
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
let verdict;
|
|
62
|
+
try {
|
|
63
|
+
const run = await deps.runs.getRun(taskId);
|
|
64
|
+
if (run === undefined) {
|
|
65
|
+
// d. 无行:GRACE 内=create→落库窗口;超窗=真孤儿。
|
|
66
|
+
verdict = now() - sbx.startedAt.getTime() < graceMs ? "grace" : "kill";
|
|
67
|
+
}
|
|
68
|
+
else if (LIVE_STATUSES.has(run.status)) {
|
|
69
|
+
verdict = "live";
|
|
70
|
+
}
|
|
71
|
+
else if (TERMINAL_STATUSES.has(run.status)) {
|
|
72
|
+
verdict = "kill";
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
// c. 词表外的新词:不杀 + failOpen(先安全后收编;收编=上面两表加词,遗漏在 census 计数可见)。
|
|
76
|
+
verdict = "unknown";
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
// e. store 读错:不知道 ≠ 孤儿。
|
|
81
|
+
verdict = "unknown";
|
|
82
|
+
}
|
|
83
|
+
if (verdict === "live")
|
|
84
|
+
summary.skippedLive++;
|
|
85
|
+
else if (verdict === "grace")
|
|
86
|
+
summary.skippedGrace++;
|
|
87
|
+
else if (verdict === "unknown") {
|
|
88
|
+
summary.skippedUnknown++;
|
|
89
|
+
recordFailOpen("server.e2b.orphan-skip-unknown");
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
try {
|
|
93
|
+
await deps.provider.kill(sbx.sandboxId);
|
|
94
|
+
summary.reclaimed++;
|
|
95
|
+
deps.logger?.info("e2b_orphan_reclaimed", { sandboxId: sbx.sandboxId, taskId, ageMs: now() - sbx.startedAt.getTime() });
|
|
96
|
+
}
|
|
97
|
+
catch (e) {
|
|
98
|
+
summary.killFailed++;
|
|
99
|
+
deps.logger?.warn("e2b_orphan_kill_failed", { sandboxId: sbx.sandboxId, taskId, error: e instanceof Error ? e.message : String(e) });
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (summary.reclaimed > 0 || summary.killFailed > 0) {
|
|
104
|
+
deps.logger?.info("e2b_orphan_reclaim_round", { ...summary });
|
|
105
|
+
}
|
|
106
|
+
return summary;
|
|
107
|
+
}
|
|
108
|
+
/** provider RPC 墙钟上限。🔴 codex 对抗复审 R2(验真后修):无超时的 list/kill 一次 provider 挂起会让
|
|
109
|
+
* 调用方的重入旗永挂(finally 等的 promise 永不结算)⇒ 回收腿从此静默死掉。SDK 原生收
|
|
110
|
+
* `requestTimeoutMs`(SandboxApiOpts),用它而非外包 race——超时语义留在 SDK 的连接层。 */
|
|
111
|
+
const PROVIDER_RPC_TIMEOUT_MS = 30_000;
|
|
112
|
+
/** 真实 provider(E2B SDK 包装)。SDK 静态面动态 import——回收腿只在 e2b 部署形挂载,
|
|
113
|
+
* 非 e2b 形零加载(与 execution-env 的按需装配同姿势)。 */
|
|
114
|
+
export function createE2bOrphanSandboxProvider(cfg) {
|
|
115
|
+
return {
|
|
116
|
+
async listManaged() {
|
|
117
|
+
const { Sandbox } = await import("e2b");
|
|
118
|
+
const out = [];
|
|
119
|
+
// 服务端 metadata 过滤(SandboxListOpts.query.metadata,AND 语义);state 默认含 running+paused
|
|
120
|
+
// ——paused 也计费存储位且同样是孤儿人群,刻意不收窄。
|
|
121
|
+
const paginator = Sandbox.list({ apiKey: cfg.apiKey, requestTimeoutMs: PROVIDER_RPC_TIMEOUT_MS, query: { metadata: { [SEMA_MANAGED_METADATA_KEY]: "1" } } });
|
|
122
|
+
while (paginator.hasNext) {
|
|
123
|
+
const items = await paginator.nextItems();
|
|
124
|
+
for (const s of items)
|
|
125
|
+
out.push({ sandboxId: s.sandboxId, metadata: s.metadata ?? {}, startedAt: s.startedAt });
|
|
126
|
+
}
|
|
127
|
+
return out;
|
|
128
|
+
},
|
|
129
|
+
async kill(sandboxId) {
|
|
130
|
+
const { Sandbox } = await import("e2b");
|
|
131
|
+
await Sandbox.kill(sandboxId, { apiKey: cfg.apiKey, requestTimeoutMs: PROVIDER_RPC_TIMEOUT_MS });
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
//# sourceMappingURL=e2b-orphan-reclaim.js.map
|
|
@@ -1236,7 +1236,8 @@ export class RemoteContainerExecutionEnv {
|
|
|
1236
1236
|
export function e2bExecutionEnvFactory(config, deps) {
|
|
1237
1237
|
return (ctx) => new RemoteContainerExecutionEnv({
|
|
1238
1238
|
...config,
|
|
1239
|
-
|
|
1239
|
+
// #242:semaManaged=归属标(孤儿回收腿 e2b-orphan-reclaim.ts 按它服务端过滤;值恒 "1")。
|
|
1240
|
+
metadata: { ...config.metadata, semaManaged: "1", sessionId: ctx.sessionId, ...(ctx.taskId ? { taskId: ctx.taskId } : {}) },
|
|
1240
1241
|
}, deps);
|
|
1241
1242
|
}
|
|
1242
1243
|
// ─────────────────────────────── module helpers ───────────────────────────────
|
package/dist/tool-approval.js
CHANGED
|
@@ -1193,6 +1193,7 @@ export class ToolApprovalCoordinator {
|
|
|
1193
1193
|
// 而 `settle` 可能因为人已经答过了而整个是空操作 —— 标志式写法会在那种赛跑里给一个**人类的**终局盖上
|
|
1194
1194
|
// 「窗到期」的章。只有真正落定这次终局的那一次调用才有资格留下自报词。
|
|
1195
1195
|
let hostSettled;
|
|
1196
|
+
let denyReasonSettled; // #243:respond deny 的人写理由,shapeOutcome deny 臂消费(声明纪律见 settle 注)
|
|
1196
1197
|
/**
|
|
1197
1198
|
* 已落定的本地终局 → `AskOutcome` 的**唯一**成形口。本方法有两个出口(方法尾的正常路,与
|
|
1198
1199
|
* `earlyAbortDuringEnsure` 的早退路),它们此前各写了一份等价分派 —— 结果就是本批的 `settledBy`
|
|
@@ -1201,6 +1202,11 @@ export class ToolApprovalCoordinator {
|
|
|
1201
1202
|
* · allow + 编辑 ⇒ 对象臂带 `updatedInput`;
|
|
1202
1203
|
* · deny + 宿主自报 ⇒ 对象臂带 `settledBy`(见 {@link HostSettledBy};allow 侧恒不带 —— core 对
|
|
1203
1204
|
* `{allow:true, settledBy:"timeout"}` 是响亮拒,而自报只在 deny 分支产生);
|
|
1205
|
+
* · deny + 人写理由 ⇒ 对象臂带 `reason`(#243,core 5.29.0 收口:模型经它看到拒因——围栏
|
|
1206
|
+
* delimitUntrusted 与 REVIEWER_NOTE_MAX_BODY 截断全在 core,server 只透传不预判;空串按 core
|
|
1207
|
+
* 契约的缺席读——truthiness——不铸键;allow 侧恒不带,core 对 allow 行的 reason 恒销毁不渲染,
|
|
1208
|
+
* 审计面由店的 decisionNote 承载)。理由只从 respond 腿流入(finishRespond),窗到期/取消/
|
|
1209
|
+
* 外部清算臂无人写理由,恒缺席。
|
|
1204
1210
|
* · 其余 ⇒ 裸 boolean(falsy 纪律)。
|
|
1205
1211
|
*/
|
|
1206
1212
|
const shapeOutcome = (settled) => {
|
|
@@ -1209,13 +1215,21 @@ export class ToolApprovalCoordinator {
|
|
|
1209
1215
|
if (settled.allowed) {
|
|
1210
1216
|
return settled.updatedInput !== undefined ? { allow: true, updatedInput: settled.updatedInput } : true;
|
|
1211
1217
|
}
|
|
1212
|
-
|
|
1218
|
+
if (hostSettled !== undefined || (denyReasonSettled !== undefined && denyReasonSettled !== "")) {
|
|
1219
|
+
return {
|
|
1220
|
+
allow: false,
|
|
1221
|
+
...(hostSettled !== undefined ? { settledBy: hostSettled } : {}),
|
|
1222
|
+
...(denyReasonSettled !== undefined && denyReasonSettled !== "" ? { reason: denyReasonSettled } : {}),
|
|
1223
|
+
};
|
|
1224
|
+
}
|
|
1225
|
+
return false;
|
|
1213
1226
|
};
|
|
1214
|
-
const settle = (allowed, outcome, updatedInput, hostSettledBy) => {
|
|
1227
|
+
const settle = (allowed, outcome, updatedInput, hostSettledBy, denyReason) => {
|
|
1215
1228
|
if (done)
|
|
1216
1229
|
return;
|
|
1217
1230
|
done = true;
|
|
1218
1231
|
hostSettled = hostSettledBy;
|
|
1232
|
+
denyReasonSettled = allowed ? undefined : denyReason;
|
|
1219
1233
|
if (timer)
|
|
1220
1234
|
clearTimeout(timer);
|
|
1221
1235
|
for (const [sig, fn] of abortListeners)
|
|
@@ -1877,7 +1891,10 @@ export class ToolApprovalCoordinator {
|
|
|
1877
1891
|
this.allowAllSessions.add(sessionAllowKey(entry.owner, entry.sessionId, sessionAllowCategory(entry.toolName)));
|
|
1878
1892
|
}
|
|
1879
1893
|
const allowed = parsed.value !== "deny";
|
|
1880
|
-
|
|
1894
|
+
// #243:deny 的 note 同时是给模型的拒因(AskOutcome.reason,core 5.29.0 围栏+上限在 core);allow 的
|
|
1895
|
+
// note 只走店的审计列(core 对 allow 行 reason 恒销毁,不铸注定被丢的键)。第 4 参 hostSettledBy 恒缺席
|
|
1896
|
+
// ——respond 腿是人经 API 决,不是宿主自报臂。
|
|
1897
|
+
entry.settle(allowed, allowed ? "allowed" : "denied", parsed.updatedInput, undefined, allowed ? undefined : parsed.note);
|
|
1881
1898
|
// updatedInputForwarded=server 已透传(是否被引擎消费取决于 core OnAsk 对象臂是否在——诚实措辞,
|
|
1882
1899
|
// 不称 applied;cli 版本门+core 单落地后全链生效)。
|
|
1883
1900
|
// 🔴 codex 交叉复审 round2 R2-4(2026-08-06 真缺陷):这个回显必须以「**这次调用真的把 updatedInput
|
|
@@ -27,7 +27,7 @@ type _GuardBgNotif = AssertAllKeysHandled<Exclude<keyof BackgroundChildEvent, Bg
|
|
|
27
27
|
type RosterProjected = "name" | "agentId" | "sessionId" | "toolUseId" | "owner" | "scope" | "sessionScoped" | "rootSessionId" | "model" | "modelFallback" | "createdAt";
|
|
28
28
|
type _GuardRoster = AssertAllKeysHandled<Exclude<keyof RosterEntry, RosterProjected>>;
|
|
29
29
|
type AskProjected = "toolName" | "toolCallId" | "args" | "message" | "sourceTaskId" | "fromSubagent" | "sourceAgentName" | "delegation" | "ruleSuggestions" | "persistedRuleShadowed";
|
|
30
|
-
type AskExcluded = "preview" | "principal" | "requiresRealApproval" | "riskAxes" | "boundInputHash";
|
|
30
|
+
type AskExcluded = "preview" | "principal" | "requiresRealApproval" | "riskAxes" | "boundInputHash" | "isDelegatedChild";
|
|
31
31
|
type _GuardAsk = AssertAllKeysHandled<Exclude<keyof AskRequest, AskProjected | AskExcluded>>;
|
|
32
32
|
type TaskEventHandled = "text_delta" | "reasoning_delta" | "tool_start" | "tool_end" | "turn_end" | "compacted" | "diagnostics" | "message_committed" | "status" | "task_notification" | "task_progress" | "steering_injected" | "workspace_changed" | "done" | "context_usage" | "compaction_outcome" | "human_input" | "wiring_manifest";
|
|
33
33
|
type _GuardTaskEvent = AssertAllKeysHandled<Exclude<TaskEvent["type"], TaskEventHandled>>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sema-agent/server",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.17.0",
|
|
4
4
|
"description": "Sema Server — the server/API implementation layer for Sema, wiring core, registry, model providers, and cloud agent execution. Built on @sema-agent/core.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BUSL-1.1",
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"build:binary:run-local:darwin-arm64": "bun build --compile --target=bun-darwin-arm64 src/run-local.ts --outfile dist/run-local-darwin-arm64"
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
|
-
"@sema-agent/core": "^5.
|
|
57
|
+
"@sema-agent/core": "^5.29.0",
|
|
58
58
|
"@sema-agent/registry-core": "^0.16.0",
|
|
59
59
|
"e2b": "^2.28.0",
|
|
60
60
|
"libsodium-wrappers": "^0.8.4",
|