@sema-agent/server 7.129.0 → 7.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +37 -0
- package/USAGE.md +38 -2
- package/dist/boot/execution-env.js +1 -0
- package/dist/boot/resolve-spec.js +8 -4
- package/dist/boot/stage-10-http-server.d.ts +1 -1
- package/dist/boot/stage-10-http-server.js +2 -0
- package/dist/capabilities/mcp-probe.js +16 -3
- package/dist/capabilities/scenarios.d.ts +33 -0
- package/dist/capabilities/scenarios.js +15 -0
- package/dist/capabilities/skills.js +1 -1
- package/dist/config-catalog.js +5 -1
- package/dist/config-types.d.ts +3 -0
- package/dist/config.js +7 -1
- package/dist/exec-bounds.d.ts +69 -0
- package/dist/exec-bounds.js +37 -0
- package/dist/hooks/hook-llm.d.ts +11 -1
- package/dist/hooks/hook-llm.js +10 -13
- package/dist/hooks/hook-runner.d.ts +1 -1
- package/dist/http/routes/capabilities.js +20 -2
- package/dist/http/routes/runs.js +12 -4
- package/dist/http/server.d.ts +8 -0
- package/dist/observability/fail-open.d.ts +5 -1
- package/dist/observability/fail-open.js +5 -1
- package/dist/plugins/one-time-migrations.d.ts +12 -1
- package/dist/plugins/one-time-migrations.js +17 -0
- package/dist/plugins/remote-env-adb.js +5 -4
- package/dist/plugins/remote-env-e2b.d.ts +31 -9
- package/dist/plugins/remote-env-e2b.js +77 -26
- package/dist/plugins/remote-env-host.js +7 -5
- package/dist/plugins/remote-env-k8s.d.ts +12 -4
- package/dist/plugins/remote-env-k8s.js +22 -31
- package/dist/plugins/remote-env-local-docker.js +5 -4
- package/dist/plugins/remote-env-ssh.js +5 -4
- package/dist/runtime-caps-resolver.d.ts +7 -0
- package/dist/runtime-caps-resolver.js +1 -0
- package/dist/session-titler.d.ts +4 -1
- package/dist/session-titler.js +13 -3
- package/dist/stored-request-tool-axes-m19.d.ts +29 -0
- package/dist/stored-request-tool-axes-m19.js +27 -0
- package/dist/task-mcp.d.ts +32 -1
- package/dist/task-mcp.js +66 -33
- package/dist/tool-axis-words.d.ts +14 -17
- package/dist/web-admin-client/skills-mcp.d.ts +1 -1
- package/package.json +1 -1
package/dist/hooks/hook-llm.js
CHANGED
|
@@ -2,6 +2,7 @@ import { adjudicateModelRoute, joinRequestUrl, resolveRouteCredential, routeKeyO
|
|
|
2
2
|
import { matchCatalogModel, refusedCatalogEntry } from "../model-select.js";
|
|
3
3
|
import { modelUnavailableMessage } from "../model-entry-refusal.js";
|
|
4
4
|
import { resolveOpenaiRouteEndpoint } from "../model-route-endpoint.js";
|
|
5
|
+
import { dispatchBrainLane } from "../model-provider.js";
|
|
5
6
|
import { egressRedirectRefusal } from "../egress-redirect.js";
|
|
6
7
|
import { buildDisplaySafeRefusal, redactSecrets } from "../trace/redact.js";
|
|
7
8
|
export function buildHookFailureText(raw) {
|
|
@@ -16,21 +17,17 @@ export function buildHookFailureText(raw) {
|
|
|
16
17
|
}
|
|
17
18
|
export function createHookLlm(deps) {
|
|
18
19
|
const { config, metrics } = deps;
|
|
19
|
-
const anthropicRouteBaseUrl = () => config.anthropic
|
|
20
|
+
const anthropicRouteBaseUrl = () => config.anthropic?.baseUrl || "https://api.anthropic.com";
|
|
20
21
|
const shapeFrom = (catalogRef, m, providerModelId) => {
|
|
21
|
-
const
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
};
|
|
22
|
+
const lane = dispatchBrainLane(m?.provider, config.anthropic !== undefined);
|
|
23
|
+
switch (lane) {
|
|
24
|
+
case "anthropic":
|
|
25
|
+
return { ok: true, catalogRef, providerModelId, baseUrl: m?.baseUrl || anthropicRouteBaseUrl(), api: "anthropic-messages" };
|
|
26
|
+
case "gateway": {
|
|
27
|
+
const endpoint = resolveOpenaiRouteEndpoint(catalogRef, m?.baseUrl, config.gatewayBaseUrl);
|
|
28
|
+
return endpoint.ok ? { ok: true, catalogRef, providerModelId, baseUrl: endpoint.baseUrl, api: "openai-completions" } : { ok: false, error: endpoint.refusal };
|
|
29
29
|
}
|
|
30
|
-
return { ok: true, catalogRef, providerModelId, baseUrl, api };
|
|
31
30
|
}
|
|
32
|
-
const endpoint = resolveOpenaiRouteEndpoint(catalogRef, m?.baseUrl, config.gatewayBaseUrl);
|
|
33
|
-
return endpoint.ok ? { ok: true, catalogRef, providerModelId, baseUrl: endpoint.baseUrl, api } : { ok: false, error: endpoint.refusal };
|
|
34
31
|
};
|
|
35
32
|
const hookModelFor = (requested) => {
|
|
36
33
|
const shape = (ref) => shapeFrom(ref, config.models[ref], config.models[ref]?.id ?? ref);
|
|
@@ -135,7 +132,7 @@ export function createHookLlm(deps) {
|
|
|
135
132
|
const r = await fetchImpl(url, { method: "POST", headers, body, signal: ac.signal, redirect: "manual" });
|
|
136
133
|
const redirected = egressRedirectRefusal(r, "declare the final endpoint as this route's base URL");
|
|
137
134
|
if (redirected !== undefined)
|
|
138
|
-
return { ok: false, error: `hook llm ${redirected}
|
|
135
|
+
return { ok: false, error: `hook llm ${redirected}`, code: "redirect_refused" };
|
|
139
136
|
if (!r.ok)
|
|
140
137
|
return { ok: false, error: `hook llm HTTP ${r.status}` };
|
|
141
138
|
let text;
|
|
@@ -270,7 +270,7 @@ export type HookLlmCall = (opts: {
|
|
|
270
270
|
}>;
|
|
271
271
|
/** 失败判别码(B8:判别一律走码,禁按 error 文案分支——v3.1 批2,统检第二波 high)。error 仍是给人看的
|
|
272
272
|
* 自由文本;code 是给控制流的。缺席=未分类失败(不重试、不特判)。 */
|
|
273
|
-
export type HookLlmFailureCode = "no_content" | "hard_timeout";
|
|
273
|
+
export type HookLlmFailureCode = "no_content" | "hard_timeout" | "redirect_refused";
|
|
274
274
|
/**
|
|
275
275
|
* hook 上下文片段 → 喂给模型的单串。**所有** `additionalContext` 聚合点的唯一属主(此前是同一个
|
|
276
276
|
* `contexts.join("\n")` 表达式抄在二十余处,总量帽无处可挂 —— 那正是漂移成因)。
|
|
@@ -44,6 +44,9 @@ function retentionAdvertisement(config) {
|
|
|
44
44
|
export function createCapabilitiesLocal() {
|
|
45
45
|
return { mcpProbe: createMcpProbeLocal() };
|
|
46
46
|
}
|
|
47
|
+
function sendScenarioNotFound(res) {
|
|
48
|
+
sendError(res, 404, "scenario_not_found", "scenario not found");
|
|
49
|
+
}
|
|
47
50
|
export async function handleCapabilities(req, res, match, ctx) {
|
|
48
51
|
const { deps } = ctx;
|
|
49
52
|
switch (match.decl.id) {
|
|
@@ -51,7 +54,7 @@ export async function handleCapabilities(req, res, match, ctx) {
|
|
|
51
54
|
const name = match.params.name;
|
|
52
55
|
const detail = deps.scenarioDetails?.[name];
|
|
53
56
|
if (!detail) {
|
|
54
|
-
|
|
57
|
+
sendScenarioNotFound(res);
|
|
55
58
|
return;
|
|
56
59
|
}
|
|
57
60
|
sendJson(res, 200, detail);
|
|
@@ -217,7 +220,22 @@ export async function handleCapabilities(req, res, match, ctx) {
|
|
|
217
220
|
sendPrincipalRefusal(res, req, deps.config);
|
|
218
221
|
return;
|
|
219
222
|
}
|
|
220
|
-
|
|
223
|
+
const scenarioParams = new URLSearchParams(match.search).getAll("scenario");
|
|
224
|
+
if (scenarioParams.length > 1) {
|
|
225
|
+
sendError(res, 400, "request.field_invalid", "scenario must be given at most once (omit it to read this deployment's default scenario)");
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
const requestedScenario = scenarioParams[0];
|
|
229
|
+
let scenarioName;
|
|
230
|
+
if (requestedScenario === undefined)
|
|
231
|
+
scenarioName = deps.config.defaultScenario;
|
|
232
|
+
else if (deps.scenarioGate === undefined) {
|
|
233
|
+
sendScenarioNotFound(res);
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
else
|
|
237
|
+
scenarioName = await deps.scenarioGate(principal, requestedScenario);
|
|
238
|
+
await serveMcpProbeFace(res, ctx, principal, mcpForScenario(deps.config.mcpServers, scenarioName) ?? [], "deployment");
|
|
221
239
|
return;
|
|
222
240
|
}
|
|
223
241
|
case "capability-mcp-probe": {
|
package/dist/http/routes/runs.js
CHANGED
|
@@ -599,6 +599,14 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
599
599
|
};
|
|
600
600
|
const job = prior.then(async () => {
|
|
601
601
|
const cs = deps.checkpointStore;
|
|
602
|
+
const withdrawWakeCard = async (token, scope, arm) => {
|
|
603
|
+
try {
|
|
604
|
+
await cs.expire(token, scope);
|
|
605
|
+
}
|
|
606
|
+
catch (e) {
|
|
607
|
+
deps.logger?.warn?.("steer_wake_card_withdraw_failed", { taskId, sessionId: run.sessionId, arm, error: redactHead(e instanceof Error ? e.message : String(e), 2000) });
|
|
608
|
+
}
|
|
609
|
+
};
|
|
602
610
|
const reprobe = await tryPark();
|
|
603
611
|
if (reprobe === "parked") {
|
|
604
612
|
mintResult.parked = true;
|
|
@@ -645,18 +653,18 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
645
653
|
});
|
|
646
654
|
const stillRead = await readResumeCtx(cs.getCtx(run.sessionId));
|
|
647
655
|
if (stillRead.kind === "corrupt") {
|
|
648
|
-
await
|
|
656
|
+
await withdrawWakeCard(wakeToken, cpScope, "resume_context_corrupt");
|
|
649
657
|
mintResult.corruptCtxWhere = stillRead.where;
|
|
650
658
|
return;
|
|
651
659
|
}
|
|
652
660
|
if (stillRead.kind !== "ok") {
|
|
653
|
-
await
|
|
661
|
+
await withdrawWakeCard(wakeToken, cpScope, "resume_context_absent");
|
|
654
662
|
mintResult.noResumeCtx = true;
|
|
655
663
|
return;
|
|
656
664
|
}
|
|
657
665
|
const winner = await cs.findPendingTokenBySession(run.sessionId, undefined, { gateKinds: TERMINAL_PARK_GATE_KINDS, pick: "oldest" }).catch(() => null);
|
|
658
666
|
if (winner !== null && winner !== wakeToken) {
|
|
659
|
-
await
|
|
667
|
+
await withdrawWakeCard(wakeToken, cpScope, "converged_on_winner");
|
|
660
668
|
await parkOnce(winner, cpScope);
|
|
661
669
|
return;
|
|
662
670
|
}
|
|
@@ -680,7 +688,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
680
688
|
return;
|
|
681
689
|
}
|
|
682
690
|
if (mintResult.corruptCtxWhere !== undefined) {
|
|
683
|
-
sendError(res, 500, "corrupt", "run already ended and this session's stored resume context cannot be read —
|
|
691
|
+
sendError(res, 500, "corrupt", "run already ended and this session's stored resume context cannot be read — nothing was queued. Ask whoever operates this deployment to repair or remove the row named in `where` before steering this session again.", { where: mintResult.corruptCtxWhere });
|
|
684
692
|
return;
|
|
685
693
|
}
|
|
686
694
|
if (mintResult.unservable !== undefined) {
|
package/dist/http/server.d.ts
CHANGED
|
@@ -563,6 +563,14 @@ export interface ServiceSeamDeps {
|
|
|
563
563
|
* per-principal 查询走 caps client 的 executionRuling 同车(零额外 RTT);absent = 无 center(单用户/
|
|
564
564
|
* dry-run)⇒ 观测面不接线,零行为差。fail-open:观测失败绝不影响 sync 面本身。 */
|
|
565
565
|
sessionMirrorRuling?: (principal: string | undefined) => Promise<SessionMirrorRuling | undefined>;
|
|
566
|
+
/** 请求点名的场景名过门 —— `capabilities/scenarios.ts` `createScenarioGate` 铸的**那一只**(与 `resolveSpec` 同一个工厂、同一张场景表、同一份
|
|
567
|
+
* `principalCaps`、同一份部署缺省):`GET /v1/capabilities/mcp?scenario=<名>` 的状态面靠它回答「这个场景名在不在册、这位 principal 放不放行」,
|
|
568
|
+
* 拒体(400 `scenario_unknown` 附在册词表 / 400 `scenario_not_allowed` 附 allowlist)因此与任务提交腿逐字同。
|
|
569
|
+
* 读面上这只门走**只读腿**(`createScenarioGate(…, "peek")`):问中心裁定时 center 抖动的那一拍,不替这个主体的真提交留下 fail-closed 降级项
|
|
570
|
+
* (提交腿自己是 `"enforce"`)。只在请求**点名了**场景时才问(缺席 ⇒ 部署缺省场景,不问门、不多一次中心往返;点名空串也交门答,与提交腿 `scenario: ""` 同码)。
|
|
571
|
+
* 本接缝缺席 = 嵌入方的 HTTP 面没有场景表可问:状态面点名场景答 404 `scenario_not_found`,与场景详情读面({@link scenarioDetails} 缺席)同一个答复,
|
|
572
|
+
* 不另造空表门、不静默回缺省集。 */
|
|
573
|
+
scenarioGate?: import("../capabilities/scenarios.js").ScenarioGate;
|
|
566
574
|
/** Degenerate-output a/b instrument: on errorCode=output.degenerate, inspect the session's turns to
|
|
567
575
|
* classify where the salvageable answer was (counter + log). Fire-and-forget; no-op otherwise. */
|
|
568
576
|
instrumentDegenerate?: (result: TaskResult) => void;
|
|
@@ -117,7 +117,11 @@ export declare const FAIL_OPEN_TAGS: {
|
|
|
117
117
|
};
|
|
118
118
|
readonly "server.session-titler.gateway-unreachable": {
|
|
119
119
|
readonly cls: "F";
|
|
120
|
-
readonly note: "S-178(cli [6662] ① clay 裁定转达,`session-titler.ts logFailure`):自动会话标题那一次廉价模型调用失败(gateway 不可达 / 凭据缺失 / 超时)。标题是**纯装饰**——列表面回退 objectivePreview,零行为影响,门/裁决/账本一个字节不受它管。此前每个新会话首次同 cause 打一条 `warn`,于是一台没配模型 gateway 的单机部署在日志里被这条噪音刷屏,壳因此去关 `SESSION_AUTO_TITLE`(替上游压噪),而那是把一个部署级旋钮当消音器用。现在:逐次 `debug` + 本计数 + 一次性 warn(recorder 自己的),运维仍看得见「这台机器的 titler 一直在失败」,用户面零噪音。观测面另有 `session_title_total{outcome}` 逐次记账。";
|
|
120
|
+
readonly note: "S-178(cli [6662] ① clay 裁定转达,`session-titler.ts logFailure`):自动会话标题那一次廉价模型调用失败(gateway 不可达 / 凭据缺失 / 超时 / 非 2xx 应答;端点回 3xx 重定向被拒的那一族另记 `server.session-titler.redirect-refused`,不进本 tag)。标题是**纯装饰**——列表面回退 objectivePreview,零行为影响,门/裁决/账本一个字节不受它管。此前每个新会话首次同 cause 打一条 `warn`,于是一台没配模型 gateway 的单机部署在日志里被这条噪音刷屏,壳因此去关 `SESSION_AUTO_TITLE`(替上游压噪),而那是把一个部署级旋钮当消音器用。现在:逐次 `debug` + 本计数 + 一次性 warn(recorder 自己的),运维仍看得见「这台机器的 titler 一直在失败」,用户面零噪音。观测面另有 `session_title_total{outcome}` 逐次记账。";
|
|
121
|
+
};
|
|
122
|
+
readonly "server.session-titler.redirect-refused": {
|
|
123
|
+
readonly cls: "F";
|
|
124
|
+
readonly note: "`session-titler.ts logFailure`:自动会话标题那一次廉价模型调用被拒 —— 配置的模型端点回了 3xx 重定向。本服务发往配置端点的直连请求不跟随重定向(跟随会把请求体里的用户内容与凭据头送到 `Location` 指向的另一个 origin),所以这是**端点配置问题,不是网络问题**:出路 = 把该模型的网关 URL(`MODEL_GATEWAY_BASEURL` / 目录条目自己的 baseUrl / `ANTHROPIC_BASE_URL`)改成最终地址,查网络没有用。与 `server.session-titler.gateway-unreachable` 的分界 = 失败码(llm 面交回的 `redirect_refused`),不看文案。标题是**纯装饰**——列表面回退 objectivePreview,零行为影响,门/裁决/账本一个字节不受它管。放行的最坏后果 = 该部署的会话没有自动标题。逐次 `debug` + 本计数 + 一次性 warn(recorder 自己的);`session_title_total{outcome=llm_failed}` 照常逐次记账。";
|
|
121
125
|
};
|
|
122
126
|
readonly "server.prompts.artifact-cache-write-failed": {
|
|
123
127
|
readonly cls: "F";
|
|
@@ -98,7 +98,11 @@ export const FAIL_OPEN_TAGS = {
|
|
|
98
98
|
},
|
|
99
99
|
"server.session-titler.gateway-unreachable": {
|
|
100
100
|
cls: "F",
|
|
101
|
-
note: "S-178(cli [6662] ① clay 裁定转达,`session-titler.ts logFailure`):自动会话标题那一次廉价模型调用失败(gateway 不可达 / 凭据缺失 / 超时)。标题是**纯装饰**——列表面回退 objectivePreview,零行为影响,门/裁决/账本一个字节不受它管。此前每个新会话首次同 cause 打一条 `warn`,于是一台没配模型 gateway 的单机部署在日志里被这条噪音刷屏,壳因此去关 `SESSION_AUTO_TITLE`(替上游压噪),而那是把一个部署级旋钮当消音器用。现在:逐次 `debug` + 本计数 + 一次性 warn(recorder 自己的),运维仍看得见「这台机器的 titler 一直在失败」,用户面零噪音。观测面另有 `session_title_total{outcome}` 逐次记账。",
|
|
101
|
+
note: "S-178(cli [6662] ① clay 裁定转达,`session-titler.ts logFailure`):自动会话标题那一次廉价模型调用失败(gateway 不可达 / 凭据缺失 / 超时 / 非 2xx 应答;端点回 3xx 重定向被拒的那一族另记 `server.session-titler.redirect-refused`,不进本 tag)。标题是**纯装饰**——列表面回退 objectivePreview,零行为影响,门/裁决/账本一个字节不受它管。此前每个新会话首次同 cause 打一条 `warn`,于是一台没配模型 gateway 的单机部署在日志里被这条噪音刷屏,壳因此去关 `SESSION_AUTO_TITLE`(替上游压噪),而那是把一个部署级旋钮当消音器用。现在:逐次 `debug` + 本计数 + 一次性 warn(recorder 自己的),运维仍看得见「这台机器的 titler 一直在失败」,用户面零噪音。观测面另有 `session_title_total{outcome}` 逐次记账。",
|
|
102
|
+
},
|
|
103
|
+
"server.session-titler.redirect-refused": {
|
|
104
|
+
cls: "F",
|
|
105
|
+
note: "`session-titler.ts logFailure`:自动会话标题那一次廉价模型调用被拒 —— 配置的模型端点回了 3xx 重定向。本服务发往配置端点的直连请求不跟随重定向(跟随会把请求体里的用户内容与凭据头送到 `Location` 指向的另一个 origin),所以这是**端点配置问题,不是网络问题**:出路 = 把该模型的网关 URL(`MODEL_GATEWAY_BASEURL` / 目录条目自己的 baseUrl / `ANTHROPIC_BASE_URL`)改成最终地址,查网络没有用。与 `server.session-titler.gateway-unreachable` 的分界 = 失败码(llm 面交回的 `redirect_refused`),不看文案。标题是**纯装饰**——列表面回退 objectivePreview,零行为影响,门/裁决/账本一个字节不受它管。放行的最坏后果 = 该部署的会话没有自动标题。逐次 `debug` + 本计数 + 一次性 warn(recorder 自己的);`session_title_total{outcome=llm_failed}` 照常逐次记账。",
|
|
102
106
|
},
|
|
103
107
|
"server.prompts.artifact-cache-write-failed": {
|
|
104
108
|
cls: "F",
|
|
@@ -170,6 +170,17 @@
|
|
|
170
170
|
* **读侧半场**(同一行、同一窗,不另起登记行):由存卡唯一读者入口 `approval-card.ts` `readStoredCardEnvelope`(起服改写够不着的卡 —— 滚动升级窗里
|
|
171
171
|
* 7.128.x 副本新写的、local 车道的文件 ask 账)与停驻行入口 `currentParkArm`(**所有**停驻行 —— SQL 与 local 车道引擎 File 店的同一条路径)读时过同一只改写,
|
|
172
172
|
* **响亮**(`server.approval-card.retired-card-word-read`);读的是浅拷贝,被读的行一个字节不动。local 车道读数恒零(`NO_SQL_MIGRATIONS`)。
|
|
173
|
+
* · **M19 续跑重放腿上的存量 `toolAxes`**(7.130.0)—— 请求体 `mcpServers[].toolAxes` 的判官从本版起是引擎的 `readMcpToolAxes`(整份一坏全拒);
|
|
174
|
+
* 升级前的请求腿读者把非对象(含 `null`)整键忽略、对象逐成员剔坏留好,server 照挂。续跑重放腿重放的是升级前存下的请求体(`checkpoint_ctx`),
|
|
175
|
+
* 停驻在那台 server 某只工具审批上的任务,若这台 server 整条不进,批准即撞 `resume.tool_unavailable`、检查点重开、永远批不过去。与 M10 同形:
|
|
176
|
+
* 不改写存下的字节(体按原样留在 ctx 里),是一只**窗内才生效的行为臂** —— 窗开着时(运行期读本表 M19 行的 `removableAfter`,见 {@link migrationWindowOpen}),
|
|
177
|
+
* **只在续跑重放腿**、**只对判官判不过的那一格**按升级前的读法重读一次(`stored-request-tool-axes-m19.ts`),产物再交同一只判官(不过 ⇒ 整条不进的原臂),
|
|
178
|
+
* 真挂上的这类 server 各一行运维 warn `mcp_tool_axes_read_as_before_upgrade`;窗关之后照今天答(整条不进),删码因此是零行为变化。
|
|
179
|
+
* 窗 = 发布日 + 30 天 = 停驻行的终局兜底窗缺省(`APPROVAL_TERMINAL_BACKSTOP_MS` 缺省 30 天)—— 与 M7 / M12 同一条论证。没有起服读数(它不写库)。
|
|
180
|
+
* 窗按**发布日**算、停驻行的期限按**停驻时刻**算 ⇒ 三类部署会有升级前停驻的行活过窗、窗关后回到「整条不进」:发布后才升级的部署(发布日到它升级那天之间
|
|
181
|
+
* 停驻的行)、把 `APPROVAL_TERMINAL_BACKSTOP_MS` 调长的部署、滚动升级里仍跑 7.129.x 的副本在窗内新写的行。与 M7 / M18 一样如实登记,不为它们加长窗。
|
|
182
|
+
* **它是上游字段标准的限时例外**:引擎对 `toolAxes` 的标准是「坏键不丢、坏词不默认」(core `mcp-server-spec.d.ts`),窗内这一臂恰恰剔坏留好 ——
|
|
183
|
+
* 依据是持久数据律(存量数据不能「变红」,只许一次性迁移、窗关删码),只对升级前已经按这种读法跑过的存量体,且对运维响亮。新提交与部署腿不受它影响。
|
|
173
184
|
*
|
|
174
185
|
* ## 幂等
|
|
175
186
|
* 改写的几条都可重复跑(M4:`detail IS NOT NULL AND … = 'null'`;M5:列在场探针 + `WHERE incarnation = ''`;M6:`WHERE error_code IS NULL`;
|
|
@@ -295,7 +306,7 @@ export declare const ONE_TIME_MIGRATIONS: readonly OneTimeMigrationWindow[];
|
|
|
295
306
|
/**
|
|
296
307
|
* 一条登记行的窗**此刻**开着没有(`today` = UTC 日历日 YYYY-MM-DD):未发布形(`removableAfter: null`)⇒ 开;否则今天不晚于 `removableAfter` ⇒ 开
|
|
297
308
|
* (窗关当天仍开,与到期门「今天严格晚于才红」同一条边界)。
|
|
298
|
-
* 只给**窗内才生效的行为臂**用(M10
|
|
309
|
+
* 只给**窗内才生效的行为臂**用(M10 的认领、M19 的续跑重读):那种臂过了窗就停,不等删码车 —— 删码因此是零行为变化,而窗只写在本表这一处。
|
|
299
310
|
*/
|
|
300
311
|
export declare function migrationWindowOpen(window: Pick<OneTimeMigrationWindow, "removableAfter">, today: string): boolean;
|
|
301
312
|
/** local 车道上 SQL 独有的几条(M4 / M5 / M8 / M9 / M11 / M17 / M18)的读数(M6 两车道都有,local 腿的读数由 `LocalBackend.runOneTimeMigrations` 现算):那几张表 / 列在文件底座上从来不存在 ⇒ 如实答 absent / 0。 */
|
|
@@ -241,6 +241,23 @@ export const ONE_TIME_MIGRATIONS = [
|
|
|
241
241
|
"引擎续跑不受影响:续跑入口读行摘要、不按今日闭集重判停驻行上的这两座,行字节引擎怎么写的还是怎么在",
|
|
242
242
|
reportKeys: ["retiredCardWordAskRowsRewritten", "retiredCardWordRowsUnreadable"],
|
|
243
243
|
},
|
|
244
|
+
{
|
|
245
|
+
id: "M19",
|
|
246
|
+
introducedIn: "7.130.0",
|
|
247
|
+
releasedOn: "2026-10-16",
|
|
248
|
+
removableAfter: "2026-11-15",
|
|
249
|
+
removal: "删 `src/stored-request-tool-axes-m19.ts` 整只文件(`storedToolAxesRereadM19` / `readToolAxesAsServer7129M19` / `StoredToolAxesRereadM19`);" +
|
|
250
|
+
"`boot/resolve-spec.ts` 里递给 `resolveRequestMcp` 的那一实参(`opts?.leg === \"fresh\" ? undefined : storedToolAxesRereadM19(…)`)与它的 import;" +
|
|
251
|
+
"`task-mcp.ts` 里 `resolveRequestMcp` / `classifyRequestMcp` / `classifyHonored` / `validateRequestMcpEntries` / `normalizeServer` 的 `storedToolAxes` 形参、" +
|
|
252
|
+
"`toolAxesReread` 事实与运维行 `mcp_tool_axes_read_as_before_upgrade`,以及对本模块的型 import;`tool-axis-words.ts` 头注与两只谓词 JSDoc 里点名 M19 的那几处;" +
|
|
253
|
+
"`test/mcp-tool-axes-m19.test.ts`(窗关那一格 —— 存量坏形整条不进、warn 与通告点名 `field: toolAxes` —— 改钉到 `test/task-mcp.test.ts`);" +
|
|
254
|
+
"本文件的 `migrationWindowOpen`(若再无别的读者)",
|
|
255
|
+
afterRemoval: "没有新的臂,接住它的是今天的读法(删码零行为变化:窗关那一刻起重读臂已经不生效):升级前存下的请求体里引擎判官判不过的 `toolAxes`,续跑重放时这一条 server 整条不进," +
|
|
256
|
+
"运维 warn `mcp_injection_dropped`(`reason: malformed_entry`、`field: toolAxes`)+ 用户面通告 `mcp.injection_dropped`,兄弟照挂;停驻在这台 server 工具审批上的任务" +
|
|
257
|
+
"批准后撞 `resume.tool_unavailable`(拒绝照常可出)—— 出路是拒绝那张卡,用改好的 `toolAxes` 重新提交。会撞上它的是活过窗的升级前停驻行:发布后才升级的部署、" +
|
|
258
|
+
"调长了 `APPROVAL_TERMINAL_BACKSTOP_MS` 的部署、滚动升级里旧副本在窗内新写的行(头注 M19 段)",
|
|
259
|
+
reportKeys: [],
|
|
260
|
+
},
|
|
244
261
|
];
|
|
245
262
|
export function migrationWindowOpen(window, today) {
|
|
246
263
|
return window.removableAfter === null || today <= window.removableAfter;
|
|
@@ -4,6 +4,7 @@ import fs from "node:fs/promises";
|
|
|
4
4
|
import { spawn } from "node:child_process";
|
|
5
5
|
import { buildCwdEnvCommand, createCwdGuardedCall, shellQuote, armPipeDestroyGrace } from "./remote-shell.js";
|
|
6
6
|
import { signalNumber } from "./host-platform.js";
|
|
7
|
+
import { commandBoundsOf, livenessCutMessage } from "../exec-bounds.js";
|
|
7
8
|
import { FileError, ExecutionError, RemoteExecutionError, withRetry, RollingTailBuffer, markTruncated, } from "@sema-agent/core";
|
|
8
9
|
import { classifyFsStderr, fileErrorFromExec } from "./remote-env-file-error.js";
|
|
9
10
|
import { createPosixShellFs } from "./posix-shell-fs.js";
|
|
@@ -191,7 +192,7 @@ export class RemoteAdbExecutionEnv {
|
|
|
191
192
|
w?.();
|
|
192
193
|
};
|
|
193
194
|
let lastChunk = Date.now();
|
|
194
|
-
const
|
|
195
|
+
const liveness = commandBoundsOf({ readTimeoutMs: options?.readTimeoutMs }, undefined).liveness;
|
|
195
196
|
let paused = false;
|
|
196
197
|
let onAbort;
|
|
197
198
|
const teardown = () => {
|
|
@@ -262,7 +263,7 @@ export class RemoteAdbExecutionEnv {
|
|
|
262
263
|
throw failure;
|
|
263
264
|
if (finished)
|
|
264
265
|
break;
|
|
265
|
-
const waitMs =
|
|
266
|
+
const waitMs = liveness !== undefined ? Math.max(1, liveness.ms - (Date.now() - lastChunk)) : 30_000;
|
|
266
267
|
let waitTimer;
|
|
267
268
|
try {
|
|
268
269
|
await new Promise((r) => {
|
|
@@ -274,8 +275,8 @@ export class RemoteAdbExecutionEnv {
|
|
|
274
275
|
if (waitTimer !== undefined)
|
|
275
276
|
clearTimeout(waitTimer);
|
|
276
277
|
}
|
|
277
|
-
if (
|
|
278
|
-
throw new RemoteExecutionError("timeout",
|
|
278
|
+
if (liveness !== undefined && !finished && Date.now() - lastChunk >= liveness.ms) {
|
|
279
|
+
throw new RemoteExecutionError("timeout", livenessCutMessage(liveness));
|
|
279
280
|
}
|
|
280
281
|
}
|
|
281
282
|
while (queue.length)
|
|
@@ -71,8 +71,9 @@ export interface RemoteContainerEnvConfig extends RemoteEnvObservers {
|
|
|
71
71
|
* isRunning/files.*. A provider hang is surfaced as a typed retryable `RemoteExecutionError("timeout")` after
|
|
72
72
|
* this, instead of pinning the worker for the sandbox lifetime. Default {@link DEFAULT_RPC_TIMEOUT_MS}. */
|
|
73
73
|
rpcTimeoutMs?: number;
|
|
74
|
-
/** 🔴 (hang class-fix)
|
|
75
|
-
*
|
|
74
|
+
/** 🔴 (hang class-fix) The lane's idle/liveness bound (ms; env `E2B_LIVENESS_MS`): max time with NO output and not yet
|
|
75
|
+
* complete = suspected hang — cuts a running command ONLY when its caller stated neither a wall-clock (`timeout`) nor its own
|
|
76
|
+
* idle bound (`readTimeoutMs`); the rule: `exec-bounds.ts` (creation included). Positive only (`laneLivenessOf`). Default `LIVENESS_MS_DEFAULT`. */
|
|
76
77
|
livenessMs?: number;
|
|
77
78
|
/** 🔴 Wall-clock for data-transfer/provisioning RPCs (large file io, snapshot suspend/resume,
|
|
78
79
|
* create/connect) — wider than {@link rpcTimeoutMs} so a legit multi-minute transfer is not killed. Default
|
|
@@ -105,6 +106,8 @@ export declare class RemoteContainerExecutionEnv implements RemoteExecutionEnv,
|
|
|
105
106
|
readonly cdPath: "unknown";
|
|
106
107
|
private readonly cfg;
|
|
107
108
|
private readonly sandboxApi;
|
|
109
|
+
/** This lane's own liveness bound — what {@link commandBoundsOf} falls back to when a command's caller stated no bound. */
|
|
110
|
+
private readonly laneLiveness;
|
|
108
111
|
private sbx?;
|
|
109
112
|
/** 🔴 BL-31 keep-alive — the E2B sandbox lifetime is hard-capped (≈1h, {@link E2B_SANDBOX_MAX_MS}), but a long
|
|
110
113
|
* leader/repair run legitimately exceeds it. This interval RE-extends the sandbox (`setTimeout(timeoutMs)` resets
|
|
@@ -287,23 +290,42 @@ export declare class RemoteContainerExecutionEnv implements RemoteExecutionEnv,
|
|
|
287
290
|
* the adapter NEVER blind-retries). A caller AbortSignal still aborts independently (op surfaces it / "aborted").
|
|
288
291
|
*/
|
|
289
292
|
private rpc;
|
|
293
|
+
/**
|
|
294
|
+
* A command start that already returned to its caller (its creation deadline fired, or it failed): a handle the provider
|
|
295
|
+
* hands over LATER is killed — the caller was told the command was cut, so it must not run on. The rule for the in-flight
|
|
296
|
+
* count this start holds: it is released **once, and only when the command is confirmed not running** (council #4: never
|
|
297
|
+
* snapshot a running command). Confirmations: the provider answered `kill()` (`true` killed / `false` no such process);
|
|
298
|
+
* the handle's one `wait()` reports the process exited (a result, or `CommandExitError` for a non-zero exit). Not
|
|
299
|
+
* confirmations: a refused `kill()` (logged; the exit is still awaited), a `wait()` that ends without an exit (the
|
|
300
|
+
* connection deadline, a lost stream — the process may still run). With neither the count stays held and `suspendVM`
|
|
301
|
+
* keeps refusing `command_in_flight` — the same arm as a start that never settles. No start promise (the op never ran)
|
|
302
|
+
* ⇒ released now. A failed start is released at once too, as in 7.129.0 — and that one is NOT a confirmation: there is
|
|
303
|
+
* no handle to kill or wait on, and a start whose reply was lost (the SDK's own 60 s handshake) may have spawned the
|
|
304
|
+
* process. Closing it means classifying the start failure (provider refused ⇒ not started; lost reply ⇒ held) — open,
|
|
305
|
+
* not done here.
|
|
306
|
+
*/
|
|
307
|
+
private settleAbandonedStart;
|
|
290
308
|
/**
|
|
291
309
|
* Per-command wall-clock for the E2B SDK (seconds→ms). 🔴 This is a PHYSICAL CAP, NOT the hang
|
|
292
310
|
* detector. The old code returned `this.cfg.timeoutMs` (the sandbox lifetime) when no `timeout` was given AND
|
|
293
311
|
* had no liveness check → a hung command waited the WHOLE lifetime (~30 min, the original hang bug). The fix is
|
|
294
312
|
* NOT to shorten this cap (that would spuriously kill a long-but-progressing `npm ci`) — it's to add a LIVENESS
|
|
295
313
|
* bound ({@link cfg.livenessMs}) in exec/execStream that catches a *zero-progress* command in ~livenessMs,
|
|
296
|
-
* independent of this cap. So: explicit caller `timeout` wins
|
|
297
|
-
*
|
|
314
|
+
* independent of this cap. So: explicit caller `timeout` wins — and is then the command's bound (the liveness bound
|
|
315
|
+
* stays out, `exec-bounds.ts`); else the lifetime is the cap (a command can't outlive its VM) and the liveness bound
|
|
316
|
+
* is the hang detector.
|
|
298
317
|
*/
|
|
299
318
|
private commandTimeoutMs;
|
|
300
319
|
/**
|
|
301
|
-
* 🔴
|
|
320
|
+
* 🔴 Bounds for the buffered exec WAIT phase. `handle.wait()` is a long-poll with no read timeout —
|
|
302
321
|
* a command that hangs DURING execution (not creation) would otherwise wait the E2B command timeout (= the
|
|
303
|
-
* lifetime when none is set, the original hang bug).
|
|
304
|
-
* `
|
|
305
|
-
*
|
|
306
|
-
*
|
|
322
|
+
* lifetime when none is set, the original hang bug). With a `liveness` bound (the caller stated no `timeout` —
|
|
323
|
+
* `exec-bounds.ts`) it races wait() against an idle check: no stdout/stderr for `liveness.ms` ⇒ hung → kill + typed
|
|
324
|
+
* retryable "timeout" with the shared cut sentence. `lastProgress()` is bumped by the stdout/stderr callbacks, so a
|
|
325
|
+
* slow-but-PROGRESSING command (long `npm ci`) is never killed. Both cuts reject with an `ExecutionError` carrying
|
|
326
|
+
* its real code (exec returns it as-is) — a `RemoteExecutionError` here would reach {@link execError}'s text match
|
|
327
|
+
* and land as `unknown`. The rejection is armed BEFORE the kill: the kill rejects `wait()` too, and the race must
|
|
328
|
+
* see the bound, not the kill. (execStream has its own chunk-driven idle timer; this is the buffered-exec analogue.)
|
|
307
329
|
*/
|
|
308
330
|
private waitWithLiveness;
|
|
309
331
|
/** Lazily provision the VM on first use: a task that never touches the hand pays zero cost.
|
|
@@ -3,13 +3,13 @@ import { Sandbox, CommandExitError, FileType } from "e2b";
|
|
|
3
3
|
import { FileError, ExecutionError, RemoteExecutionError, RollingTailBuffer, markTruncated, } from "@sema-agent/core";
|
|
4
4
|
import { fileErrorFromExec } from "./remote-env-file-error.js";
|
|
5
5
|
import { numEnvOr, runSymlinkChain, shellQuote } from "./remote-shell.js";
|
|
6
|
+
import { commandBoundsOf, firstBoundOf, laneLivenessOf, livenessCutMessage, LIVENESS_MS_DEFAULT } from "../exec-bounds.js";
|
|
6
7
|
import { BackgroundShellManager } from "./background-shell-support.js";
|
|
7
8
|
const PROVIDER = "e2b";
|
|
8
9
|
const DEFAULT_MOUNT_PATH = "/home/user";
|
|
9
10
|
const DEFAULT_TIMEOUT_MS = 5 * 60_000;
|
|
10
11
|
const E2B_SANDBOX_MAX_MS = numEnvOr("E2B_SANDBOX_MAX_MS", 3_600_000, 60_000);
|
|
11
12
|
const DEFAULT_RPC_TIMEOUT_MS = 60_000;
|
|
12
|
-
const DEFAULT_LIVENESS_MS = 120_000;
|
|
13
13
|
const DEFAULT_DATA_TIMEOUT_MS = 5 * 60_000;
|
|
14
14
|
const E2B_BG_MAX_CONCURRENT = numEnvOr("E2B_BG_MAX_CONCURRENT", 8, 1);
|
|
15
15
|
const E2B_BG_DEFAULT_TIMEOUT_SEC = numEnvOr("E2B_BG_DEFAULT_TIMEOUT_SEC", 300, 1);
|
|
@@ -22,6 +22,7 @@ export class RemoteContainerExecutionEnv {
|
|
|
22
22
|
cdPath = "unknown";
|
|
23
23
|
cfg;
|
|
24
24
|
sandboxApi;
|
|
25
|
+
laneLiveness;
|
|
25
26
|
sbx;
|
|
26
27
|
keepAlive;
|
|
27
28
|
handle;
|
|
@@ -36,9 +37,10 @@ export class RemoteContainerExecutionEnv {
|
|
|
36
37
|
timeoutMs: Math.min(config.timeoutMs ?? DEFAULT_TIMEOUT_MS, E2B_SANDBOX_MAX_MS),
|
|
37
38
|
allowInternetAccess: config.allowInternetAccess ?? true,
|
|
38
39
|
rpcTimeoutMs: config.rpcTimeoutMs ?? DEFAULT_RPC_TIMEOUT_MS,
|
|
39
|
-
livenessMs: config.livenessMs ??
|
|
40
|
+
livenessMs: config.livenessMs ?? LIVENESS_MS_DEFAULT,
|
|
40
41
|
dataTimeoutMs: config.dataTimeoutMs ?? DEFAULT_DATA_TIMEOUT_MS,
|
|
41
42
|
};
|
|
43
|
+
this.laneLiveness = laneLivenessOf(this.cfg.livenessMs, "E2B_LIVENESS_MS");
|
|
42
44
|
this.cwd = this.cfg.mountPath;
|
|
43
45
|
this.homeDir = config.homeDir;
|
|
44
46
|
this.sandboxApi = deps.sandboxApi ?? Sandbox;
|
|
@@ -255,9 +257,14 @@ export class RemoteContainerExecutionEnv {
|
|
|
255
257
|
s.total += b.length;
|
|
256
258
|
};
|
|
257
259
|
let lastProgress = Date.now();
|
|
260
|
+
const startedAt = lastProgress;
|
|
261
|
+
const bounds = commandBoundsOf({ timeoutMs: options?.timeout != null ? options.timeout * 1000 : undefined }, this.laneLiveness);
|
|
262
|
+
const first = firstBoundOf(bounds);
|
|
258
263
|
let handle;
|
|
264
|
+
let starting;
|
|
265
|
+
let startCut = false;
|
|
259
266
|
try {
|
|
260
|
-
handle = await this.rpc("commands.run", () => sbx.commands.run(command, {
|
|
267
|
+
handle = await this.rpc("commands.run", () => (starting = sbx.commands.run(command, {
|
|
261
268
|
background: true,
|
|
262
269
|
cwd: this.resolve(options?.cwd ?? this.cwd),
|
|
263
270
|
...(options?.env ? { envs: options.env } : {}),
|
|
@@ -265,6 +272,8 @@ export class RemoteContainerExecutionEnv {
|
|
|
265
272
|
? Math.max(this.commandTimeoutMs(options?.timeout), (E2B_BG_MAX_TIMEOUT_SEC + E2B_BG_PROVIDER_BACKSTOP_PAD_SEC) * 1000)
|
|
266
273
|
: this.commandTimeoutMs(options?.timeout),
|
|
267
274
|
onStdout: (d) => {
|
|
275
|
+
if (startCut)
|
|
276
|
+
return;
|
|
268
277
|
if (tee)
|
|
269
278
|
teePush(tee.stdout, d);
|
|
270
279
|
if (!detached) {
|
|
@@ -273,6 +282,8 @@ export class RemoteContainerExecutionEnv {
|
|
|
273
282
|
}
|
|
274
283
|
},
|
|
275
284
|
onStderr: (d) => {
|
|
285
|
+
if (startCut)
|
|
286
|
+
return;
|
|
276
287
|
if (tee)
|
|
277
288
|
teePush(tee.stderr, d);
|
|
278
289
|
if (!detached) {
|
|
@@ -280,11 +291,13 @@ export class RemoteContainerExecutionEnv {
|
|
|
280
291
|
options?.onStderr?.(d);
|
|
281
292
|
}
|
|
282
293
|
},
|
|
283
|
-
}), options?.abortSignal,
|
|
294
|
+
})), options?.abortSignal, first.ms);
|
|
284
295
|
}
|
|
285
296
|
catch (e) {
|
|
286
|
-
|
|
287
|
-
|
|
297
|
+
startCut = true;
|
|
298
|
+
this.settleAbandonedStart(starting);
|
|
299
|
+
const cut = e instanceof RemoteExecutionError && e.code === "timeout" && !options?.abortSignal?.aborted;
|
|
300
|
+
return { ok: false, error: cut ? new ExecutionError("timeout", boundCutMessage(first)) : this.execError(e, options?.abortSignal) };
|
|
288
301
|
}
|
|
289
302
|
const onAbort = () => { if (!detached)
|
|
290
303
|
void handle.kill().catch(() => { }); };
|
|
@@ -333,7 +346,8 @@ export class RemoteContainerExecutionEnv {
|
|
|
333
346
|
const fg = this.waitWithLiveness(handle, () => lastProgress, {
|
|
334
347
|
waitP,
|
|
335
348
|
disarmed: () => detached,
|
|
336
|
-
...(tee ? {
|
|
349
|
+
...(tee ? { wall: { totalMs: this.commandTimeoutMs(options?.timeout), leftMs: Math.max(0, this.commandTimeoutMs(options?.timeout) - (Date.now() - startedAt)) } } : {}),
|
|
350
|
+
liveness: bounds.liveness,
|
|
337
351
|
});
|
|
338
352
|
const winner = await (detachedP ? Promise.race([fg, detachedP]) : fg);
|
|
339
353
|
if (winner === "detached") {
|
|
@@ -385,6 +399,9 @@ export class RemoteContainerExecutionEnv {
|
|
|
385
399
|
};
|
|
386
400
|
let bytes = 0;
|
|
387
401
|
const cap = options?.maxOutputBytes;
|
|
402
|
+
const bounds = commandBoundsOf({ timeoutMs: options?.timeout != null ? options.timeout * 1000 : undefined, readTimeoutMs: options?.readTimeoutMs }, this.laneLiveness);
|
|
403
|
+
const liveness = bounds.liveness;
|
|
404
|
+
const first = firstBoundOf(bounds);
|
|
388
405
|
const push = (type, data) => {
|
|
389
406
|
if (finished)
|
|
390
407
|
return;
|
|
@@ -394,18 +411,22 @@ export class RemoteContainerExecutionEnv {
|
|
|
394
411
|
};
|
|
395
412
|
this.activeCommands++;
|
|
396
413
|
let handle;
|
|
414
|
+
let starting;
|
|
397
415
|
try {
|
|
398
|
-
handle = await this.rpc("commands.run", () => sbx.commands.run(command, {
|
|
416
|
+
handle = await this.rpc("commands.run", () => (starting = sbx.commands.run(command, {
|
|
399
417
|
background: true,
|
|
400
418
|
cwd: this.resolve(options?.cwd ?? this.cwd),
|
|
401
419
|
...(options?.env ? { envs: options.env } : {}),
|
|
402
420
|
timeoutMs: this.commandTimeoutMs(options?.timeout),
|
|
403
421
|
onStdout: (d) => push("stdout", d),
|
|
404
422
|
onStderr: (d) => push("stderr", d),
|
|
405
|
-
}), options?.signal,
|
|
423
|
+
})), options?.signal, first.ms);
|
|
406
424
|
}
|
|
407
425
|
catch (e) {
|
|
408
|
-
|
|
426
|
+
finished = true;
|
|
427
|
+
this.settleAbandonedStart(starting);
|
|
428
|
+
if (e instanceof RemoteExecutionError && e.code === "timeout" && !options?.signal?.aborted)
|
|
429
|
+
throw new RemoteExecutionError("timeout", boundCutMessage(first), e);
|
|
409
430
|
throw this.remoteError(e, "connect_failed");
|
|
410
431
|
}
|
|
411
432
|
handle
|
|
@@ -424,7 +445,7 @@ export class RemoteContainerExecutionEnv {
|
|
|
424
445
|
exitCode = e.exitCode;
|
|
425
446
|
}
|
|
426
447
|
else {
|
|
427
|
-
failure = this.remoteError(e, "unknown");
|
|
448
|
+
failure = this.remoteError(e, providerTimedOut(e) ? "timeout" : "unknown");
|
|
428
449
|
}
|
|
429
450
|
finished = true;
|
|
430
451
|
signalReady();
|
|
@@ -465,13 +486,14 @@ export class RemoteContainerExecutionEnv {
|
|
|
465
486
|
const next = new Promise((resolve) => {
|
|
466
487
|
wake = resolve;
|
|
467
488
|
});
|
|
468
|
-
const idleMs = options?.readTimeoutMs ?? this.cfg.livenessMs;
|
|
469
489
|
let timer;
|
|
470
|
-
const idle =
|
|
471
|
-
|
|
472
|
-
|
|
490
|
+
const idle = liveness === undefined
|
|
491
|
+
? undefined
|
|
492
|
+
: new Promise((_, reject) => {
|
|
493
|
+
timer = setTimeout(() => reject(new RemoteExecutionError("timeout", livenessCutMessage(liveness))), liveness.ms);
|
|
494
|
+
});
|
|
473
495
|
try {
|
|
474
|
-
await Promise.race([next, idle]);
|
|
496
|
+
await (idle === undefined ? next : Promise.race([next, idle]));
|
|
475
497
|
}
|
|
476
498
|
catch (e) {
|
|
477
499
|
finished = true;
|
|
@@ -794,36 +816,59 @@ export class RemoteContainerExecutionEnv {
|
|
|
794
816
|
combined?.dispose();
|
|
795
817
|
}
|
|
796
818
|
}
|
|
819
|
+
settleAbandonedStart(starting) {
|
|
820
|
+
let held = true;
|
|
821
|
+
const release = () => {
|
|
822
|
+
if (!held)
|
|
823
|
+
return;
|
|
824
|
+
held = false;
|
|
825
|
+
this.activeCommands--;
|
|
826
|
+
};
|
|
827
|
+
if (starting === undefined)
|
|
828
|
+
return release();
|
|
829
|
+
void starting.then((late) => {
|
|
830
|
+
late.wait().then(release, (e) => {
|
|
831
|
+
if (e instanceof CommandExitError)
|
|
832
|
+
release();
|
|
833
|
+
});
|
|
834
|
+
late.kill().then(release, (e) => {
|
|
835
|
+
this.cfg.logger?.warn?.("e2b_abandoned_start_kill_failed", { sandboxId: this.sbx?.sandboxId, pid: late.pid, error: e instanceof Error ? e.name : typeof e });
|
|
836
|
+
});
|
|
837
|
+
}, release);
|
|
838
|
+
}
|
|
797
839
|
commandTimeoutMs(timeoutSec) {
|
|
798
840
|
return timeoutSec != null ? timeoutSec * 1000 : this.cfg.timeoutMs;
|
|
799
841
|
}
|
|
800
842
|
async waitWithLiveness(handle, lastProgress, opts) {
|
|
801
843
|
let timer;
|
|
802
844
|
let wallTimer;
|
|
845
|
+
const liveness = opts?.liveness;
|
|
803
846
|
const idle = new Promise((_, reject) => {
|
|
804
|
-
const arm = () => {
|
|
847
|
+
const arm = (bound) => {
|
|
805
848
|
timer = setTimeout(() => {
|
|
806
849
|
if (opts?.disarmed?.())
|
|
807
850
|
return;
|
|
808
851
|
const idleFor = Date.now() - lastProgress();
|
|
809
|
-
if (idleFor >=
|
|
852
|
+
if (idleFor >= bound.ms) {
|
|
853
|
+
reject(new ExecutionError("timeout", livenessCutMessage(bound)));
|
|
810
854
|
void handle.kill().catch(() => { });
|
|
811
855
|
this.cfg.metrics?.inc("remote_env_exec_idle_timeout_total", { provider: PROVIDER });
|
|
812
|
-
reject(new RemoteExecutionError("timeout", `exec idle > ${this.cfg.livenessMs}ms (provider hang / unreachable sandbox)`));
|
|
813
856
|
}
|
|
814
857
|
else {
|
|
815
|
-
arm();
|
|
858
|
+
arm(bound);
|
|
816
859
|
}
|
|
817
|
-
}, Math.max(
|
|
860
|
+
}, Math.max(bound.ms - (Date.now() - lastProgress()), 0));
|
|
818
861
|
};
|
|
819
|
-
|
|
820
|
-
|
|
862
|
+
if (liveness !== undefined)
|
|
863
|
+
arm(liveness);
|
|
864
|
+
if (opts?.wall !== undefined) {
|
|
865
|
+
const wall = opts.wall;
|
|
821
866
|
wallTimer = setTimeout(() => {
|
|
822
867
|
if (opts.disarmed?.())
|
|
823
868
|
return;
|
|
869
|
+
reject(new ExecutionError("timeout", boundCutMessage({ ms: wall.totalMs, cut: "wall" })));
|
|
824
870
|
void handle.kill().catch(() => { });
|
|
825
|
-
|
|
826
|
-
}, opts.wallMs);
|
|
871
|
+
}, wall.leftMs);
|
|
827
872
|
}
|
|
828
873
|
});
|
|
829
874
|
try {
|
|
@@ -905,7 +950,7 @@ export class RemoteContainerExecutionEnv {
|
|
|
905
950
|
const name = e?.name;
|
|
906
951
|
if (abortSignal?.aborted || name === "AbortError")
|
|
907
952
|
return new ExecutionError("aborted", msg);
|
|
908
|
-
if (
|
|
953
|
+
if (providerTimedOut(e))
|
|
909
954
|
return new ExecutionError("timeout", msg);
|
|
910
955
|
return new ExecutionError("unknown", msg, e instanceof Error ? e : undefined);
|
|
911
956
|
}
|
|
@@ -924,6 +969,12 @@ export function e2bExecutionEnvFactory(config, deps) {
|
|
|
924
969
|
function errMsg(e) {
|
|
925
970
|
return e instanceof Error ? e.message : String(e);
|
|
926
971
|
}
|
|
972
|
+
function boundCutMessage(first) {
|
|
973
|
+
return first.cut === "wall" ? `exec exceeded ${first.ms}ms` : livenessCutMessage(first.cut);
|
|
974
|
+
}
|
|
975
|
+
function providerTimedOut(e) {
|
|
976
|
+
return e?.name === "TimeoutError" || /timeout|timed out/i.test(errMsg(e));
|
|
977
|
+
}
|
|
927
978
|
function anySignal(signals) {
|
|
928
979
|
const ctor = AbortSignal;
|
|
929
980
|
if (typeof ctor.any === "function")
|