@sema-agent/server 7.101.1 → 7.103.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +84 -0
- package/README.md +4 -4
- package/README.zh-CN.md +4 -4
- package/USAGE.md +156 -9
- package/dist/approval-card.d.ts +35 -2
- package/dist/approval-card.js +19 -1
- package/dist/background-session-reap.d.ts +58 -0
- package/dist/background-session-reap.js +90 -0
- package/dist/bake-runner/main.d.ts +21 -3
- package/dist/bake-runner/main.js +18 -11
- package/dist/boot/config-center.d.ts +21 -4
- package/dist/boot/config-center.js +62 -19
- package/dist/boot/exit-boundary.js +17 -0
- package/dist/boot/runner-deps.js +1 -0
- package/dist/boot/runtime-caps.js +3 -2
- package/dist/boot/session-faces.js +3 -4
- package/dist/boot/shutdown.d.ts +2 -1
- package/dist/boot/shutdown.js +33 -12
- package/dist/boot/stage-01-config.js +2 -0
- package/dist/boot/stage-07-capability-layer.d.ts +1 -1
- package/dist/boot/stage-07-capability-layer.js +6 -4
- package/dist/boot/stage-08-reapers.d.ts +1 -1
- package/dist/boot/stage-09-leader.d.ts +1 -1
- package/dist/boot/stage-10-http-server.d.ts +1 -1
- package/dist/boot/stage-10-http-server.js +13 -4
- package/dist/brain.d.ts +28 -8
- package/dist/brain.js +20 -4
- package/dist/capabilities/builtin-tools.d.ts +14 -1
- package/dist/capabilities/builtin-tools.js +3 -0
- package/dist/capabilities/center-plugins.d.ts +9 -3
- package/dist/capabilities/center-plugins.js +5 -10
- package/dist/capabilities/scenarios.d.ts +1 -1
- package/dist/capabilities/scenarios.js +3 -3
- package/dist/center-credential-renewer.d.ts +76 -0
- package/dist/center-credential-renewer.js +359 -0
- package/dist/center-credential.d.ts +76 -22
- package/dist/center-credential.js +119 -43
- package/dist/config-catalog.js +17 -8
- package/dist/config-center/apply-effective.d.ts +16 -0
- package/dist/config-center/apply-effective.js +26 -21
- package/dist/config-center/center-request.d.ts +13 -0
- package/dist/config-center/center-request.js +23 -3
- package/dist/config-center/facade.d.ts +1 -1
- package/dist/config-center/http-client.js +2 -5
- package/dist/config-center/mcp-revocation.d.ts +2 -2
- package/dist/config-center/mcp-revocation.js +1 -1
- package/dist/config-center/read-warnings.d.ts +19 -48
- package/dist/config-center/read-warnings.js +7 -49
- package/dist/config-center/restart-signal.d.ts +8 -21
- package/dist/config-center/restart-signal.js +2 -12
- package/dist/config-center/types.d.ts +4 -2
- package/dist/config-provider.js +1 -48
- package/dist/config-types.d.ts +107 -17
- package/dist/config-types.js +7 -2
- package/dist/config.d.ts +7 -0
- package/dist/config.js +91 -29
- package/dist/device-store.d.ts +1 -1
- package/dist/env-name-allowlist-knobs.d.ts +21 -7
- package/dist/env-name-allowlist-knobs.js +7 -2
- package/dist/fleet-client.d.ts +27 -7
- package/dist/fleet-client.js +43 -12
- package/dist/fleet-lease.d.ts +1 -1
- package/dist/host-lsp-manager.d.ts +4 -3
- package/dist/host-lsp-manager.js +2 -6
- package/dist/http/approval-sweeps.js +5 -20
- package/dist/http/dispatch.js +24 -14
- package/dist/http/readiness.d.ts +87 -0
- package/dist/http/readiness.js +54 -0
- package/dist/http/resume-legs.js +5 -4
- package/dist/http/route-table.d.ts +7 -0
- package/dist/http/route-table.js +6 -0
- package/dist/http/routes/a2a-serve.js +6 -14
- package/dist/http/routes/approvals-assistant.js +1 -0
- package/dist/http/routes/background.d.ts +80 -0
- package/dist/http/routes/background.js +94 -0
- package/dist/http/routes/capabilities.js +2 -0
- package/dist/http/server.d.ts +25 -8
- package/dist/http/server.js +2 -2
- package/dist/leader/wire.d.ts +8 -1
- package/dist/leader/wire.js +4 -3
- package/dist/model-entry-refusal.d.ts +42 -19
- package/dist/model-entry-refusal.js +34 -19
- package/dist/model-route-endpoint.d.ts +10 -0
- package/dist/model-route-endpoint.js +6 -2
- package/dist/observability/fail-open.d.ts +2 -2
- package/dist/observability/fail-open.js +2 -2
- package/dist/observability/secret-env-scrub.d.ts +12 -0
- package/dist/observability/secret-env-scrub.js +12 -0
- package/dist/outbound-dispatcher.d.ts +125 -0
- package/dist/outbound-dispatcher.js +211 -0
- package/dist/parked-decide.d.ts +11 -5
- package/dist/parked-decide.js +4 -1
- package/dist/parked-revive-surface.d.ts +47 -0
- package/dist/parked-revive-surface.js +20 -0
- package/dist/plugins/background-shell-support.d.ts +20 -1
- package/dist/plugins/background-shell-support.js +26 -0
- package/dist/plugins/host-platform.d.ts +16 -1
- package/dist/plugins/host-platform.js +11 -0
- package/dist/plugins/remote-env-host.d.ts +14 -1
- package/dist/plugins/remote-env-host.js +15 -10
- package/dist/project-memory.js +3 -6
- package/dist/run-local.js +2 -0
- package/dist/server-secret-env.d.ts +97 -24
- package/dist/server-secret-env.js +37 -1
- package/dist/tool-approval.d.ts +30 -1
- package/dist/tool-approval.js +31 -8
- package/dist/trace/core-keyset-guard.d.ts +9 -4
- package/dist/trace/engine-notice-wire.d.ts +1 -1
- package/dist/trace/engine-notice-wire.js +2 -0
- package/dist/trace/project.js +13 -0
- package/dist/trace/redact.d.ts +11 -0
- package/dist/trace/redact.js +1 -0
- package/package.json +4 -3
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { type BackgroundShellHardKillReceipt } from "@sema-agent/core";
|
|
2
|
+
import type { Logger } from "./observability/logger.js";
|
|
3
|
+
/** 受据四词的计数(日志汇总行与调用方读数共用)。 */
|
|
4
|
+
export interface HardKillTally {
|
|
5
|
+
killed: number;
|
|
6
|
+
alreadyGone: number;
|
|
7
|
+
retainedSkipped: number;
|
|
8
|
+
/** 没有同步硬收落地(普通 kill 已派)—— 「未确认」,不是失败。 */
|
|
9
|
+
unconfirmed: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* 受据的**唯一读者**:逐行一条 `background_shell_hard_kill`(`{ taskId, type, outcome, reason, owner?, retained? }` —— 不带
|
|
13
|
+
* pid / shellId:core 契约「pid 不出 env」,受据上本来就没有),返回四词计数。本函数自己不抛;注入形 logger 若抛,异常**原样上抛**
|
|
14
|
+
* 给调用方(kill 已在调用前落地,受据不丢):退出边界与 E21 级联各自在外层 try 里 warn / `console.error`(生产 sink 不抛,
|
|
15
|
+
* `observability/logger.ts` 的 `emit` 已包),停止口则让它走全局 500 —— 不在这里写静默的空 catch([ref] 棘轮)。
|
|
16
|
+
* `outcome` 按 core 闭集穷举分派:集外词 = 编译红(`never` 臂),不是运行期的静默默认。
|
|
17
|
+
*/
|
|
18
|
+
export declare function reportHardKillReceipts(logger: Pick<Logger, "info"> | undefined, receipts: readonly BackgroundShellHardKillReceipt[], reason: string): HardKillTally;
|
|
19
|
+
/** {@link reapSessionBackgroundHard} 的读数。 */
|
|
20
|
+
export interface SessionHardReap {
|
|
21
|
+
/** 软面①(core 会话锚 `reapSessionBackground`)直接收到的行数(core 口径:不含受委托子代连带收掉的孙行)。 */
|
|
22
|
+
reaped: number;
|
|
23
|
+
/** 软面②(TaskStop 面)发出的行数:硬面将收、而会话锚没收的仍在跑的 shell 行(`includeRetained:true` 时的 retain 行 / 本会话 run 的
|
|
24
|
+
* 非会话驻留行)。 */
|
|
25
|
+
softStopped: number;
|
|
26
|
+
/** 硬面逐行受据(只有 shell 行:`background_bash` / `monitor`)。 */
|
|
27
|
+
receipts: BackgroundShellHardKillReceipt[];
|
|
28
|
+
tally: HardKillTally;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* **会话级收敛 —— 一只函数,「软 → 宽限 → 硬」**(车OL O1 / O2):
|
|
32
|
+
* ① 软面:core 会话锚 `reapSessionBackground(sessionId, scope)`(本会话会话驻留、非 retain 的行走 env 梯子;同时结算本会话的后台
|
|
33
|
+
* agent / workflow),再 {@link softStopRemaining}(硬面将收、会话锚没收的仍在跑的 shell 行,逐行 TaskStop 软面);
|
|
34
|
+
* ② 宽限:{@link awaitSoftStopGraces} —— 硬面不早于它将重访的任何一行的「软停时刻 + `HOST_BG_KILL_GRACE_MS`」(host 梯子的属主常量,
|
|
35
|
+
* `plugins/remote-env-host.ts` 导出,不另抄;不分车道)。本次刚软停的行 ⇒ 等满一次宽限;几十毫秒前被另一次收敛 / 模型 TaskStop 软停
|
|
36
|
+
* 的行 ⇒ 等它剩下的那段;都不在宽限内 ⇒ 不等。同一会话两次重叠收敛(壳先打停止口、别处再 DELETE)因此都不早于前者软停 + 宽限,
|
|
37
|
+
* 不需要按会话单飞;
|
|
38
|
+
* ③ 硬面:`killAllBackgroundShellsNow({ sessionId, scope, includeRetained, reason })`(会话过滤 = 会话锚自己的键:本会话的会话驻留行
|
|
39
|
+
* + run 的 task id 就是会话 id 的 run 行;同一个 id 在别的 scope 是别的会话;别的会话的行永远不碰)—— 宽限里跑完处理器的进程答
|
|
40
|
+
* `already_gone`,屏蔽 / 没跑完的在这一枪组 SIGKILL(与 host 梯子自己到点的那一枪同拍,两枪都是组 SIGKILL,幂等)。同拍是设计内的:
|
|
41
|
+
* 两枪谁先是同一毫秒的定时器次序。硬面先打 ⇒ `killed`;梯子先打、引擎还没收割组长 ⇒ 硬面那一枪落在只剩僵尸的组上:darwin EPERM ⇒
|
|
42
|
+
* `no_pgid`(未确认)、linux ⇒ `killed`;已收割 ⇒ `already_gone`。受据按观察答(S-769 车PA 删了「已硬收过」闩:组信号任一成员收到即
|
|
43
|
+
* 成功,闩在部分送达时把活着读成没了),本函数也不为「让硬面先打」挪任何一拍。
|
|
44
|
+
*
|
|
45
|
+
* `scope` = 会话行属主 `owner ?? "default"`(与 core 注册铸值 `spec.principal ?? "default"` 同源 —— 缺席 scope 在 core 的
|
|
46
|
+
* `canAccess` 上 fail-closed、一行都收不到,[ref] 钉过这一病)。
|
|
47
|
+
*
|
|
48
|
+
* 两处调用(见文件头 ①),都 `await`:E21 删除级联 `includeRetained:false`(显式活服务照留);停止口照传请求体。应答在硬面之后发,
|
|
49
|
+
* 代价 = 有 shell 行刚被软停的会话,DELETE / 停止口的应答慢到最后一行的「软停 + 宽限」(通常一次宽限;操作员旋钮 `HOST_BG_KILL_GRACE_MS`,缺省 1000;要几乎不等写 `1` ——
|
|
50
|
+
* `numEnvOr` 把 `0` 读作缺席、回落缺省,与它供的其余执行车道旋钮同一条语义)。
|
|
51
|
+
* 受委托帮手的 shell 挂在帮手自己的会话上,本函数碰不到(core KL-1107a / #1109,候 core)。
|
|
52
|
+
* 注册表是**进程内**的:只收本副本上的行(多副本部署里别的副本上的行由那个副本自己的出口收)。
|
|
53
|
+
*/
|
|
54
|
+
export declare function reapSessionBackgroundHard(sessionId: string, scope: string, opts: {
|
|
55
|
+
includeRetained: boolean;
|
|
56
|
+
reason: string;
|
|
57
|
+
}, logger: Pick<Logger, "info" | "warn"> | undefined): Promise<SessionHardReap>;
|
|
58
|
+
//# sourceMappingURL=background-session-reap.d.ts.map
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
2
|
+
import { defaultTaskRegistry } from "@sema-agent/core";
|
|
3
|
+
import { HOST_BG_KILL_GRACE_MS } from "./plugins/remote-env-host.js";
|
|
4
|
+
export function reportHardKillReceipts(logger, receipts, reason) {
|
|
5
|
+
const tally = { killed: 0, alreadyGone: 0, retainedSkipped: 0, unconfirmed: 0 };
|
|
6
|
+
for (const r of receipts) {
|
|
7
|
+
switch (r.outcome) {
|
|
8
|
+
case "killed":
|
|
9
|
+
tally.killed++;
|
|
10
|
+
break;
|
|
11
|
+
case "already_gone":
|
|
12
|
+
tally.alreadyGone++;
|
|
13
|
+
break;
|
|
14
|
+
case "retained_skipped":
|
|
15
|
+
tally.retainedSkipped++;
|
|
16
|
+
break;
|
|
17
|
+
case "no_pgid":
|
|
18
|
+
tally.unconfirmed++;
|
|
19
|
+
break;
|
|
20
|
+
default: {
|
|
21
|
+
const unreachable = r.outcome;
|
|
22
|
+
return unreachable;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
logger?.info("background_shell_hard_kill", {
|
|
26
|
+
taskId: r.taskId,
|
|
27
|
+
type: r.type,
|
|
28
|
+
outcome: r.outcome,
|
|
29
|
+
reason,
|
|
30
|
+
...(r.owner !== undefined ? { owner: r.owner } : {}),
|
|
31
|
+
...(r.retained === true ? { retained: true } : {}),
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
return tally;
|
|
35
|
+
}
|
|
36
|
+
const SOFT_STOP_LIST_LIMIT = 500;
|
|
37
|
+
const isShellRow = (type) => type === "background_bash" || type === "monitor";
|
|
38
|
+
function softStopRemaining(access, includeRetained, logger) {
|
|
39
|
+
const rows = defaultTaskRegistry
|
|
40
|
+
.list(access, { limit: SOFT_STOP_LIST_LIMIT })
|
|
41
|
+
.filter((r) => r.status === "running" && isShellRow(r.type) && (r.retained !== true || includeRetained));
|
|
42
|
+
for (const r of rows) {
|
|
43
|
+
const failed = (fields) => logger?.warn("session_background_soft_stop_failed", { taskId: r.task_id, type: r.type, ...fields, note: "the hard face right after the grace still stops this row" });
|
|
44
|
+
defaultTaskRegistry.markStopSource(r.task_id, "system");
|
|
45
|
+
void defaultTaskRegistry.stopTask(r.task_id, access).then((res) => {
|
|
46
|
+
if (res.isError === true)
|
|
47
|
+
failed({ error: res.details.code ?? res.details.error ?? "stop_failed" });
|
|
48
|
+
}, (err) => failed({ error: err instanceof Error ? err.name : "stop threw" }));
|
|
49
|
+
}
|
|
50
|
+
return rows.map((r) => r.task_id);
|
|
51
|
+
}
|
|
52
|
+
async function awaitSoftStopGraces(access, softStop) {
|
|
53
|
+
const honored = new Set();
|
|
54
|
+
let notBefore = 0;
|
|
55
|
+
const honor = (taskId, softStoppedAt) => {
|
|
56
|
+
if (honored.has(taskId))
|
|
57
|
+
return;
|
|
58
|
+
honored.add(taskId);
|
|
59
|
+
notBefore = Math.max(notBefore, softStoppedAt + HOST_BG_KILL_GRACE_MS);
|
|
60
|
+
};
|
|
61
|
+
for (const id of softStop.stoppedNow)
|
|
62
|
+
honor(id, softStop.at);
|
|
63
|
+
for (;;) {
|
|
64
|
+
const now = Date.now();
|
|
65
|
+
const rows = defaultTaskRegistry.list(access, { limit: SOFT_STOP_LIST_LIMIT });
|
|
66
|
+
for (const r of rows) {
|
|
67
|
+
if (r.status === "killed" && isShellRow(r.type))
|
|
68
|
+
honor(r.task_id, Math.min(r.updatedAt, now));
|
|
69
|
+
}
|
|
70
|
+
if (softStop.anchorReaped > 0 && rows.length >= SOFT_STOP_LIST_LIMIT)
|
|
71
|
+
notBefore = Math.max(notBefore, softStop.at + HOST_BG_KILL_GRACE_MS);
|
|
72
|
+
if (notBefore <= now)
|
|
73
|
+
return;
|
|
74
|
+
await delay(notBefore - now);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
export async function reapSessionBackgroundHard(sessionId, scope, opts, logger) {
|
|
78
|
+
const access = { owner: sessionId, sessionId, scope };
|
|
79
|
+
const reaped = defaultTaskRegistry.reapSessionBackground(sessionId, scope);
|
|
80
|
+
const stoppedNow = softStopRemaining(access, opts.includeRetained, logger);
|
|
81
|
+
const at = Date.now();
|
|
82
|
+
await awaitSoftStopGraces(access, { at, stoppedNow, anchorReaped: reaped });
|
|
83
|
+
const softStopped = stoppedNow.length;
|
|
84
|
+
const receipts = defaultTaskRegistry.killAllBackgroundShellsNow({ sessionId, scope, includeRetained: opts.includeRetained, reason: opts.reason });
|
|
85
|
+
const tally = reportHardKillReceipts(logger, receipts, opts.reason);
|
|
86
|
+
if (reaped > 0 || softStopped > 0 || receipts.length > 0)
|
|
87
|
+
logger?.info("session_background_reaped", { sessionId, reaped, softStopped, reason: opts.reason, includeRetained: opts.includeRetained, ...tally });
|
|
88
|
+
return { reaped, softStopped, receipts, tally };
|
|
89
|
+
}
|
|
90
|
+
//# sourceMappingURL=background-session-reap.js.map
|
|
@@ -11,7 +11,24 @@ interface RunnerEnv {
|
|
|
11
11
|
idlePollMs: number;
|
|
12
12
|
minFreeGb: number;
|
|
13
13
|
shutdownGraceMs: number;
|
|
14
|
+
/** `BAKE_ENV_ALLOW` 的生效名单(点到、且本进程环境里真有的名;缺省空 = 子进程 env 全剥)。 */
|
|
15
|
+
envAllow: readonly string[];
|
|
14
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* bake-runner 的 **env 契约**(本进程只读这些;成文另见 USAGE.md §9.7):
|
|
19
|
+
* · 必填:`IMAGE_API_BASE`(image-api 根,尾斜杠去掉)、`BAKE_RUNNER_TOKEN`(claim / heartbeat 的运维主体 bearer;不进日志、不进子进程)。
|
|
20
|
+
* · `BAKE_RUNNER_ID`(缺省 `bake-runner-<HOSTNAME>`)、`RECIPE_REF`(缺省 `HEAD` = 用现成 checkout)、`RECIPE_DIR`(缺省 `/opt/recipes`)、
|
|
21
|
+
* `BUILD_SH_PATH`(缺省 `e2b-template/dev-sandbox/build.sh`)、`BAKE_DATA_DIR`(缺省 `/data`)、`LOG_LEVEL`(缺省 info)。
|
|
22
|
+
* · 数值四键(`parseNumOrFailNonNegative`,非数 / 负数拒启):`BAKE_HEARTBEAT_MS`(30000)、`BAKE_IDLE_POLL_MS`(5000)、
|
|
23
|
+
* `IMAGE_BAKE_MIN_FREE_GB`(20)、`BAKE_SHUTDOWN_GRACE_MS`(5000)。
|
|
24
|
+
* · `BAKE_ENV_ALLOW=NAME,NAME`(7.103.0,S-684 残余):本进程 spawn 的每一只子进程组(build.sh 与 git fetch / docker push /
|
|
25
|
+
* docker image inspect / df)的 env 缺省 = 本进程 env 剥密(runner 自己的 `BAKE_RUNNER_TOKEN` 在内);名单点到的键从本进程 env **原样**
|
|
26
|
+
* 放回(部署方显式承担 —— 例如配方脚本真要一枚 `*_PASSWORD` 做 registry 登录)。与服务侧 `LSP_ENV_ALLOW` **同一只**解析器
|
|
27
|
+
* (`config.ts` `parseEnvNameAllowlist`)与同一只放行谓词:形错(空段 / 尾逗号 / 非变量名形 / 点名任一只名单型旋钮)⇒ **拒启**,拒句点名
|
|
28
|
+
* 旋钮与段号、不回显值;本进程环境里没有的名 ⇒ 起服 `config_env_entry_rejected` 只报序号、不放行;旋钮原文本身按凭据类,不进子进程。
|
|
29
|
+
* 没有多租户姿态(构建宿主守护进程没有主体 / 租户);生效名单见起服行 `bake_runner_start.envAllow`(只键名)。
|
|
30
|
+
* ⚠️ 这不是服务配置目录里的键(目录 `BAKE_*` 行是服务侧 bake API 的旋钮),服务进程不读它。
|
|
31
|
+
*/
|
|
15
32
|
declare function loadEnv(): RunnerEnv;
|
|
16
33
|
/** The real image-api HTTP client. Authenticates claim/heartbeat with the runner bearer; ingest additionally
|
|
17
34
|
* carries the per-bake ingest secret header (so image-api can reject a stale/rogue runner — §P2.4e). */
|
|
@@ -33,8 +50,9 @@ declare function makeApiClient(env: RunnerEnv, log: BakeRunnerLogger): ImageApiC
|
|
|
33
50
|
declare const liveGroupPgids: Set<number>;
|
|
34
51
|
declare function killLiveGroupsSync(): void;
|
|
35
52
|
/** The real child spawner: `setsid`-style detached process GROUP so a cancel/deadline kill signals the whole tree
|
|
36
|
-
* (channel-close / a plain kill of the immediate child does NOT stop a detached build — §P2.11).
|
|
37
|
-
|
|
53
|
+
* (channel-close / a plain kill of the immediate child does NOT stop a detached build — §P2.11).
|
|
54
|
+
* `envAllow` = `BAKE_ENV_ALLOW`'s effective list (the build.sh group's env = scrubbed ∪ these names). */
|
|
55
|
+
declare function makeSpawner(envAllow: readonly string[]): ChildSpawner;
|
|
38
56
|
/**
|
|
39
57
|
* 有界、整组击杀、进簿的 `sh -c` —— **本进程唯一的 host exec 面**(C5)。
|
|
40
58
|
*
|
|
@@ -42,7 +60,7 @@ declare function makeSpawner(): ChildSpawner;
|
|
|
42
60
|
* 成功 ⇒ `{ stdout }`。多出来的两件:① `timeoutMs` 到点整组 SIGKILL 并 reject;② 它的组在 {@link liveGroupPgids}
|
|
43
61
|
* 里,于是关机梯子的 ④(`process.exit`)经退出边界收得到它 —— 旧形下一只挂死的 `docker push` 是看不见也杀不死的。
|
|
44
62
|
*/
|
|
45
|
-
export declare function execGroup(cmd: string, timeoutMs: number): Promise<{
|
|
63
|
+
export declare function execGroup(cmd: string, timeoutMs: number, envAllow: readonly string[]): Promise<{
|
|
46
64
|
stdout: string;
|
|
47
65
|
}>;
|
|
48
66
|
/** The real host ops: fs manifest read, df /data, recipe checkout, and the thin docker-push big-blob retry. */
|
package/dist/bake-runner/main.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { readFile } from "node:fs/promises";
|
|
3
3
|
import { createLogger } from "../observability/logger.js";
|
|
4
|
-
import { parseNumOrFailNonNegative } from "../config.js";
|
|
4
|
+
import { logConfigDiagnostics, parseEnvNameAllowlist, parseNumOrFailNonNegative } from "../config.js";
|
|
5
|
+
import { installSecretEnvScrubRecorder } from "../observability/secret-env-scrub.js";
|
|
6
|
+
import { hostSubprocessEnv } from "../server-secret-env.js";
|
|
5
7
|
import { shellQuote as shellSafe } from "../plugins/remote-shell.js";
|
|
6
8
|
import { BakeRunner, } from "./runner.js";
|
|
7
9
|
const BAKE_CLAIM_INGEST_FETCH_TIMEOUT_MS = 30_000;
|
|
@@ -29,6 +31,7 @@ function loadEnv() {
|
|
|
29
31
|
idlePollMs: parseNumOrFailNonNegative("BAKE_IDLE_POLL_MS", process.env.BAKE_IDLE_POLL_MS || "5000"),
|
|
30
32
|
minFreeGb: parseNumOrFailNonNegative("IMAGE_BAKE_MIN_FREE_GB", process.env.IMAGE_BAKE_MIN_FREE_GB || "20"),
|
|
31
33
|
shutdownGraceMs: parseNumOrFailNonNegative("BAKE_SHUTDOWN_GRACE_MS", process.env.BAKE_SHUTDOWN_GRACE_MS || "5000"),
|
|
34
|
+
envAllow: parseEnvNameAllowlist("BAKE_ENV_ALLOW") ?? [],
|
|
32
35
|
};
|
|
33
36
|
}
|
|
34
37
|
function makeApiClient(env, log) {
|
|
@@ -119,7 +122,7 @@ function spawnGroupChild(opts) {
|
|
|
119
122
|
cwd: opts.cwd,
|
|
120
123
|
detached: true,
|
|
121
124
|
stdio: ["ignore", "pipe", "pipe"],
|
|
122
|
-
env:
|
|
125
|
+
env: hostSubprocessEnv({ site: "server.bake-runner.spawn-env", allow: opts.envAllow }),
|
|
123
126
|
});
|
|
124
127
|
const pgid = child.pid ?? -1;
|
|
125
128
|
pruneEmptyGroups();
|
|
@@ -158,15 +161,16 @@ function groupHasMembers(pgid) {
|
|
|
158
161
|
return err.code === "EPERM";
|
|
159
162
|
}
|
|
160
163
|
}
|
|
161
|
-
function makeSpawner() {
|
|
162
|
-
return { spawn: (opts) => spawnGroupChild(opts) };
|
|
164
|
+
function makeSpawner(envAllow) {
|
|
165
|
+
return { spawn: (opts) => spawnGroupChild({ ...opts, envAllow }) };
|
|
163
166
|
}
|
|
164
|
-
export async function execGroup(cmd, timeoutMs) {
|
|
167
|
+
export async function execGroup(cmd, timeoutMs, envAllow) {
|
|
165
168
|
let stdout = "";
|
|
166
169
|
let stderr = "";
|
|
167
170
|
const append = (buf, line) => (buf.length >= BAKE_HOST_EXEC_OUTPUT_CAP ? buf : `${buf}${line}\n`);
|
|
168
171
|
const child = spawnGroupChild({
|
|
169
172
|
argv: ["/bin/sh", "-c", cmd],
|
|
173
|
+
envAllow,
|
|
170
174
|
onStdoutLine: (l) => {
|
|
171
175
|
stdout = append(stdout, l);
|
|
172
176
|
},
|
|
@@ -216,7 +220,7 @@ function makeHostOps(env) {
|
|
|
216
220
|
return JSON.parse(await readFile(path, "utf8"));
|
|
217
221
|
},
|
|
218
222
|
async dfData() {
|
|
219
|
-
const { stdout } = await execGroup(`df -P -BG ${shellSafe(env.dataDir)}`, BAKE_HOST_PROBE_TIMEOUT_MS);
|
|
223
|
+
const { stdout } = await execGroup(`df -P -BG ${shellSafe(env.dataDir)}`, BAKE_HOST_PROBE_TIMEOUT_MS, env.envAllow);
|
|
220
224
|
return stdout;
|
|
221
225
|
},
|
|
222
226
|
async checkoutRecipe(ref) {
|
|
@@ -224,10 +228,10 @@ function makeHostOps(env) {
|
|
|
224
228
|
const dir = shellSafe(env.recipeDir);
|
|
225
229
|
try {
|
|
226
230
|
await execGroup(`git -C ${dir} fetch --depth 1 origin ${shellSafe(ref)} && ` +
|
|
227
|
-
`git -C ${dir} checkout --quiet --detach FETCH_HEAD`, BAKE_RECIPE_CHECKOUT_TIMEOUT_MS);
|
|
231
|
+
`git -C ${dir} checkout --quiet --detach FETCH_HEAD`, BAKE_RECIPE_CHECKOUT_TIMEOUT_MS, env.envAllow);
|
|
228
232
|
}
|
|
229
233
|
catch {
|
|
230
|
-
await execGroup(`git -C ${dir} fetch origin && git -C ${dir} checkout --quiet --detach ${shellSafe(ref)}`, BAKE_RECIPE_CHECKOUT_TIMEOUT_MS);
|
|
234
|
+
await execGroup(`git -C ${dir} fetch origin && git -C ${dir} checkout --quiet --detach ${shellSafe(ref)}`, BAKE_RECIPE_CHECKOUT_TIMEOUT_MS, env.envAllow);
|
|
231
235
|
}
|
|
232
236
|
}
|
|
233
237
|
return env.recipeDir;
|
|
@@ -235,8 +239,8 @@ function makeHostOps(env) {
|
|
|
235
239
|
async dockerPushRetry(tag, attempts) {
|
|
236
240
|
for (let i = 0; i < attempts; i++) {
|
|
237
241
|
try {
|
|
238
|
-
await execGroup(`docker push ${shellSafe(tag)}`, BAKE_DOCKER_PUSH_TIMEOUT_MS);
|
|
239
|
-
const { stdout } = await execGroup(`docker image inspect ${shellSafe(tag)} --format '{{index .RepoDigests 0}}'`, BAKE_HOST_PROBE_TIMEOUT_MS);
|
|
242
|
+
await execGroup(`docker push ${shellSafe(tag)}`, BAKE_DOCKER_PUSH_TIMEOUT_MS, env.envAllow);
|
|
243
|
+
const { stdout } = await execGroup(`docker image inspect ${shellSafe(tag)} --format '{{index .RepoDigests 0}}'`, BAKE_HOST_PROBE_TIMEOUT_MS, env.envAllow);
|
|
240
244
|
const repoDigest = stdout.trim();
|
|
241
245
|
const digest = repoDigest.includes("@") ? repoDigest.slice(repoDigest.indexOf("@") + 1) : null;
|
|
242
246
|
return { ok: true, digest };
|
|
@@ -259,16 +263,19 @@ function makeClock() {
|
|
|
259
263
|
async function main() {
|
|
260
264
|
const env = loadEnv();
|
|
261
265
|
const log = createLogger(process.env.LOG_LEVEL || "info");
|
|
266
|
+
logConfigDiagnostics(log);
|
|
267
|
+
installSecretEnvScrubRecorder({ logger: log });
|
|
262
268
|
log.info("bake_runner_start", {
|
|
263
269
|
imageApiBase: env.imageApiBase,
|
|
264
270
|
runnerId: env.runnerId,
|
|
265
271
|
recipeRef: env.recipeRef,
|
|
266
272
|
heartbeatMs: env.heartbeatMs,
|
|
267
273
|
minFreeGb: env.minFreeGb,
|
|
274
|
+
envAllow: env.envAllow,
|
|
268
275
|
});
|
|
269
276
|
const runner = new BakeRunner({
|
|
270
277
|
api: makeApiClient(env, log),
|
|
271
|
-
spawner: makeSpawner(),
|
|
278
|
+
spawner: makeSpawner(env.envAllow),
|
|
272
279
|
host: makeHostOps(env),
|
|
273
280
|
clock: makeClock(),
|
|
274
281
|
log,
|
|
@@ -1,15 +1,17 @@
|
|
|
1
|
+
import { type ModelRefusalsSignal } from "@sema-agent/settings-schema";
|
|
1
2
|
import { CenterPromptSource, type SwappableDeps } from "@sema-agent/core";
|
|
2
3
|
import { buildPricing } from "../budget.js";
|
|
3
4
|
import { type PromptsDomainFaces } from "../prompts-domain-validate.js";
|
|
4
5
|
import { type Scenario, type ScenarioDeps, type ScenarioDetail } from "../capabilities/scenarios.js";
|
|
5
6
|
import type { LoadedSkill } from "../capabilities/skills.js";
|
|
6
7
|
import { type ServiceConfig } from "../config.js";
|
|
8
|
+
import { type CenterCredentialHealthView } from "../center-credential-renewer.js";
|
|
7
9
|
import { type ModelKeyRef } from "../key-resolver.js";
|
|
8
10
|
import type { Logger } from "../observability/logger.js";
|
|
9
11
|
import type { Metrics } from "../observability/metrics.js";
|
|
10
12
|
import type { UnpricedCeilingVerdict } from "./unpriced-ceiling-verdict.js";
|
|
11
13
|
import type { ConsolidationSeatWatch } from "./memory-consolidation.js";
|
|
12
|
-
import { type
|
|
14
|
+
import { type EngineRestartSignal } from "../config-center/facade.js";
|
|
13
15
|
import { type ConfigApplyLedgerView } from "../config-center/apply-ledger.js";
|
|
14
16
|
import type { LimitSync } from "./limit-sync.js";
|
|
15
17
|
export interface ConfigCenterRuntimeCtx {
|
|
@@ -27,6 +29,8 @@ export interface CenterScenarioOverlayTargets {
|
|
|
27
29
|
/** overlay 前捕获的六内建名(centerScenarioDetails 的 shadow 判据)。 */
|
|
28
30
|
builtinScenarioNames: string[];
|
|
29
31
|
}
|
|
32
|
+
/** S-740 / S-741 / S-768:config-center 持有的三样自报状态(fleet 心跳读它们)—— 闭集:加一样 = 写口与订阅方一起表态。 */
|
|
33
|
+
export type SelfReportChange = "roster_landed" | "restart_signal" | "model_refusals";
|
|
30
34
|
/** 跨段可变状态的**唯一**对外读口 + 三个原位动作。字段语义见各方法上的注释。 */
|
|
31
35
|
export interface ConfigCenterRuntime {
|
|
32
36
|
/** 配置源形态;`undefined` = 纯 env 部署(本运行时的所有分支不存在)。 */
|
|
@@ -64,8 +68,10 @@ export interface ConfigCenterRuntime {
|
|
|
64
68
|
consolidationSeat?: ConsolidationSeatWatch;
|
|
65
69
|
checkpointStorePresent: boolean;
|
|
66
70
|
}): void;
|
|
67
|
-
/** [ref]-2:停刷新环(hardShutdown 收尾链;幂等,未起环时 no-op)。 */
|
|
71
|
+
/** [ref]-2:停刷新环(hardShutdown 收尾链;幂等,未起环时 no-op)。S-698:中心凭证续签器的定时器同停(同一条收尾链)。 */
|
|
68
72
|
stopRefreshLoop(): void;
|
|
73
|
+
/** S-698:`/health.centerCredential` 的取值口 —— 凭证来源的健康态 + 续签器状态;都健康 / 没有中心 ⇒ `undefined`(键缺席)。 */
|
|
74
|
+
centerCredentialState(): CenterCredentialHealthView | undefined;
|
|
69
75
|
/** [ref](core 5.50 [ref]):mid-turn MCP 撤销台账读口,喂 `RunnerDeps.mcpRevocations` 席。
|
|
70
76
|
* **取走即声明接线**——调用本方法后,restart 信号的 mcp 片对纯删除向豁免(台账即时承接);
|
|
71
77
|
* 不取 ⇒ 旧 restart 语义逐字保留(fail-closed)。 */
|
|
@@ -83,8 +89,19 @@ export interface ConfigCenterRuntime {
|
|
|
83
89
|
/** boot ready 门:布防(`CONFIG_REQUIRE_ROSTER=true`,远程 registry 非 dryRun)且 roster 未落地 =
|
|
84
90
|
* 计费提交 503 + /health ready:false。缺省未布防 ⇒ 恒 true。 */
|
|
85
91
|
modelReady(): boolean;
|
|
86
|
-
/** /
|
|
87
|
-
|
|
92
|
+
/** S-740 / S-741:本副本**自报状态**里由 config-center 持有的那两样 —— roster 落地(`modelReady` 翻真,one-way latch)与
|
|
93
|
+
* restart 信号(`restartState()` 出现 / 消失 / 变)—— 任一翻转的那一拍通知订阅者。装配层接 fleet 心跳的 `announceNow`,
|
|
94
|
+
* 让中心在翻转当刻就知道本副本可路由 / 要重启(不等下一拍心跳;draining 那一样由收尾链直调 `announceNow`,同一个触发)。
|
|
95
|
+
* 在状态**写定之后**调用,同步、逐个,入参 = 这一次翻的是哪一样(闭集 {@link SelfReportChange});不回放(订阅方在订阅前已读过
|
|
96
|
+
* 当刻值)。同一拍两样都翻 ⇒ 通知两次(心跳侧 `announceNow` 单飞合并,见 fleet-client)。订阅者抛错只 warn
|
|
97
|
+
* (`self_report_listener_failed`),不打断配置拍、不吞后一个订阅者。 */
|
|
98
|
+
onSelfReportChange(listener: (changed: SelfReportChange) => void): void;
|
|
99
|
+
/** /health.restart 与 fleet 心跳 `restart` 的**同一只**取值口(orchestrator 自动重启信号;settings-schema `RestartSignal` 形)。 */
|
|
100
|
+
restartState(): EngineRestartSignal | undefined;
|
|
101
|
+
/** S-768:`/health.modelRefusals` 与 fleet 心跳 `modelRefusals` 的**同一只**取值口(settings-schema 6.2.0 `ModelRefusalsSignal` 形:
|
|
102
|
+
* 名 → 码 + 组拒码)。描述**最近一次被判的那份中心候选目录**(每次候选 apply 的报告 `ApplyReport.modelRefusals` 投影;组拒时就是被拒
|
|
103
|
+
* 的那份 —— 引擎自己的拒因表不换代)。没有可报的拒 / 没有配置管道 / 还没判过候选 ⇒ `undefined`(键缺席)。值变了 ⇒ 通知自报订阅者。 */
|
|
104
|
+
modelRefusalsState(): ModelRefusalsSignal | undefined;
|
|
88
105
|
/** /health.modelPlaneDeferred:被扣着、还没有 next-boot handoff 的候选(codex R18 起的状态机)。S-668 波 0 起
|
|
89
106
|
* 模型面不再 defer,键名保留,今天承载的是 durable LKG 发布债与 skills 债(见 `planeDeferredNoHandoff`)。 */
|
|
90
107
|
planeDeferredState(): {
|
|
@@ -3,7 +3,9 @@ import { readFile as fsReadFile } from "node:fs/promises";
|
|
|
3
3
|
import { homedir } from "node:os";
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { loadRemoteExec } from "@sema-agent/settings-schema/node";
|
|
6
|
-
import { skillContentHash } from "@sema-agent/settings-schema";
|
|
6
|
+
import { CENTER_CREDENTIAL_ENV, skillContentHash } from "@sema-agent/settings-schema";
|
|
7
|
+
import { stableStringify } from "@sema-agent/settings-schema/hash";
|
|
8
|
+
import { buildModelRefusalsSignal } from "../model-entry-refusal.js";
|
|
7
9
|
import { CenterPromptSource, FilePromptArtifactStore, FilePromptSourceStateStore, MemoryPromptArtifactStore, MemoryPromptSourceStateStore, } from "@sema-agent/core";
|
|
8
10
|
import { buildPricing } from "../budget.js";
|
|
9
11
|
import { applyCenterPlugins } from "../capabilities/center-plugins.js";
|
|
@@ -12,7 +14,8 @@ import { validatePromptsDomain, CORE_ENGINE_VERSION } from "../prompts-domain-va
|
|
|
12
14
|
import { centerScenarios, centerScenarioDetails } from "../capabilities/scenarios.js";
|
|
13
15
|
import { configLkgEnabled, isRosterGateArmed } from "../config.js";
|
|
14
16
|
import { defaultLkgPath, defaultSkillCacheDir, saveLkg, loadLkg, legacyV1LkgPath, migrateV1Lkg } from "../config-lkg.js";
|
|
15
|
-
import { answerStillCurrent, centerStateBin, credentialIdentityNow } from "../center-credential.js";
|
|
17
|
+
import { answerStillCurrent, centerStateBin, credentialIdentityNow, judgeCenterCredentialAtBoot } from "../center-credential.js";
|
|
18
|
+
import { buildCenterCredentialHealthView, createCenterCredentialRenewer } from "../center-credential-renewer.js";
|
|
16
19
|
import { centerManagedConfigKeys } from "../config-center/apply-effective.js";
|
|
17
20
|
import { createConfigProvider, raceBootFetch, BOOT_FETCH_DEFERRED } from "../config-provider.js";
|
|
18
21
|
import { executionLaneCaps, executionLaneOf } from "../execution-lane-caps.js";
|
|
@@ -61,15 +64,18 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
61
64
|
let planeDeferredNoHandoff;
|
|
62
65
|
let bootSkillStale = false;
|
|
63
66
|
let pendingRestart;
|
|
67
|
+
let modelRefusals;
|
|
64
68
|
const cc = config.configCenter;
|
|
65
69
|
const bootCredential = cc?.credential.read();
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
});
|
|
70
|
+
const bootVerdict = bootCredential === undefined ? undefined : judgeCenterCredentialAtBoot(bootCredential);
|
|
71
|
+
if (bootVerdict?.verdict === "refuse")
|
|
72
|
+
throw new Error(bootVerdict.sentence);
|
|
73
|
+
if (bootVerdict?.verdict === "loud") {
|
|
74
|
+
logger.error("center_credential_unreadable", { key: CENTER_CREDENTIAL_ENV.tokenFile, failure: bootVerdict.failure, reason: bootVerdict.reason, effect: bootVerdict.effect });
|
|
72
75
|
}
|
|
76
|
+
const credentialRenewer = cc !== undefined && cc.credential.origin.from === "file"
|
|
77
|
+
? createCenterCredentialRenewer({ source: cc.credential, centerUrl: cc.baseUrl, leadMs: config.centerCredentialRenewLeadMs, logger })
|
|
78
|
+
: undefined;
|
|
73
79
|
const currentIdentity = () => (cc === undefined ? undefined : credentialIdentityNow(cc.credential));
|
|
74
80
|
const candidateIdentity = new WeakMap();
|
|
75
81
|
const candidateIdentityOf = (eff) => candidateIdentity.get(eff);
|
|
@@ -134,12 +140,41 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
134
140
|
});
|
|
135
141
|
}
|
|
136
142
|
};
|
|
143
|
+
const selfReportListeners = [];
|
|
144
|
+
const notifySelfReport = (changed) => {
|
|
145
|
+
for (const listener of selfReportListeners) {
|
|
146
|
+
try {
|
|
147
|
+
listener(changed);
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
logger.warn("self_report_listener_failed", { changed, error: err instanceof Error ? err.message : String(err), note: "a self-report subscriber threw; the state change stands regardless and the next fleet heartbeat carries it" });
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
};
|
|
137
154
|
const markRosterLanded = (eff) => {
|
|
138
155
|
if (!modelReadyState.ready && (eff.models?.models ?? []).some((m) => m.enabled !== false)) {
|
|
139
156
|
modelReadyState.ready = true;
|
|
140
157
|
logger.info("model_roster_landed", { note: "ready gate open — billable submissions accepted" });
|
|
158
|
+
notifySelfReport("roster_landed");
|
|
141
159
|
}
|
|
142
160
|
};
|
|
161
|
+
const publishRestart = (next) => {
|
|
162
|
+
const prev = pendingRestart;
|
|
163
|
+
pendingRestart = next;
|
|
164
|
+
const same = prev === undefined || next === undefined ? prev === next : prev.version === next.version && prev.since === next.since && prev.reasons.length === next.reasons.length && prev.reasons.every((r, i) => r === next.reasons[i]);
|
|
165
|
+
if (!same)
|
|
166
|
+
notifySelfReport("restart_signal");
|
|
167
|
+
};
|
|
168
|
+
const publishModelRefusals = (next) => {
|
|
169
|
+
const same = stableStringify(modelRefusals ?? null) === stableStringify(next ?? null);
|
|
170
|
+
modelRefusals = next;
|
|
171
|
+
if (!same)
|
|
172
|
+
notifySelfReport("model_refusals");
|
|
173
|
+
};
|
|
174
|
+
const recordApply = (rep) => {
|
|
175
|
+
applyLedger.recordApply(rep);
|
|
176
|
+
publishModelRefusals(buildModelRefusalsSignal(rep.modelRefusals));
|
|
177
|
+
};
|
|
143
178
|
let centerPrompts;
|
|
144
179
|
let pendingCatalogFaces;
|
|
145
180
|
const retryPendingCatalog = async (phase) => {
|
|
@@ -380,7 +415,7 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
380
415
|
generation: lkgGen,
|
|
381
416
|
report: (rep) => {
|
|
382
417
|
lkgReport = rep;
|
|
383
|
-
|
|
418
|
+
recordApply(rep);
|
|
384
419
|
},
|
|
385
420
|
});
|
|
386
421
|
applyLedger.applied(lkgGen, lkgR.effective.version);
|
|
@@ -449,7 +484,7 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
449
484
|
generation: bootGen,
|
|
450
485
|
report: (rep) => {
|
|
451
486
|
bootReport = rep;
|
|
452
|
-
|
|
487
|
+
recordApply(rep);
|
|
453
488
|
},
|
|
454
489
|
});
|
|
455
490
|
applyLedger.applied(bootGen, r.effective.version);
|
|
@@ -566,7 +601,11 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
566
601
|
getKeyResolver: () => keyResolver,
|
|
567
602
|
getCenterPrompts: () => centerPrompts,
|
|
568
603
|
modelReady: () => modelReadyState.ready,
|
|
604
|
+
onSelfReportChange: (listener) => {
|
|
605
|
+
selfReportListeners.push(listener);
|
|
606
|
+
},
|
|
569
607
|
restartState: () => pendingRestart,
|
|
608
|
+
modelRefusalsState: () => modelRefusals,
|
|
570
609
|
planeDeferredState: () => planeDeferredNoHandoff,
|
|
571
610
|
configApplyState: () => (configProvider ? applyLedger.view() : undefined),
|
|
572
611
|
centerManagedKeys: () => (configProvider ? centerManagedConfigKeys(latestEffective) : undefined),
|
|
@@ -663,7 +702,7 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
663
702
|
if (liveSkillMiss)
|
|
664
703
|
bootSkillStale = true;
|
|
665
704
|
if (liveSkillMiss && (bootPublished || (!lkgSurvivesRestart && lastPrewarmBodiesVerified))) {
|
|
666
|
-
|
|
705
|
+
publishRestart({ reasons: ["skills"], version: bootLkgCandidate.effective.version, since: Date.now() });
|
|
667
706
|
logger.warn("config_boot_skill_stale_restart", { version: bootLkgCandidate.effective.version, durableHandoff: bootPublished, note: bootPublished ? "live skill application incomplete but the durable LKG carries the full candidate — skills restart published (the next boot reads the complete LKG)" : "live skill application incomplete on a non-durable deployment but every enabled body is verified in the cache — skills restart published (transient failure proven repaired; the restarted process re-pulls live)" });
|
|
668
707
|
}
|
|
669
708
|
else if (liveSkillMiss || (!bootPublished && lkgSurvivesRestart)) {
|
|
@@ -719,14 +758,14 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
719
758
|
const promoted = lkgSurvivesRestart || skillsDebt ? await persistLkgDurable(planeDeferredNoHandoff.candidate, planeDeferredNoHandoff.candidateEtag) : false;
|
|
720
759
|
const skillsProofPromotion = !promoted && !lkgSurvivesRestart && skillsDebt && lastPrewarmBodiesVerified;
|
|
721
760
|
if (skillsProofPromotion) {
|
|
722
|
-
|
|
761
|
+
publishRestart({ reasons: ["skills"], version: planeDeferredNoHandoff.version, since: Date.now() });
|
|
723
762
|
logger.info("config_boot_skill_debt_promoted", { version: planeDeferredNoHandoff.version, note: "skill bodies verified on a source-confirmed-current candidate — the withheld skills restart is now published (non-durable lane, repair proven)" });
|
|
724
763
|
planeDeferredNoHandoff = undefined;
|
|
725
764
|
}
|
|
726
765
|
else if (promoted) {
|
|
727
766
|
const promotedReasons = [...new Set(planeDeferredNoHandoff.blocked ?? [])];
|
|
728
767
|
if (promotedReasons.length > 0) {
|
|
729
|
-
|
|
768
|
+
publishRestart({ reasons: promotedReasons, version: planeDeferredNoHandoff.version, since: Date.now() });
|
|
730
769
|
}
|
|
731
770
|
logger.info("models_tiers_deferred_promoted", { version: planeDeferredNoHandoff.version, reasons: promotedReasons, note: promotedReasons.length > 0 ? "handoff persistence recovered on a source-confirmed-current candidate — the withheld restart signal is now published" : "boot publication retry completed — LKG now durable, no restart needed (nothing was withheld)" });
|
|
732
771
|
planeDeferredNoHandoff = undefined;
|
|
@@ -775,7 +814,7 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
775
814
|
generation: tickGen,
|
|
776
815
|
report: (rep) => {
|
|
777
816
|
applyReport = rep;
|
|
778
|
-
|
|
817
|
+
recordApply(rep);
|
|
779
818
|
},
|
|
780
819
|
...(planeChanged ? { swapPlane: swapRunnerDeps } : {}),
|
|
781
820
|
});
|
|
@@ -826,13 +865,13 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
826
865
|
}
|
|
827
866
|
}
|
|
828
867
|
if (reasons.length === 0) {
|
|
829
|
-
|
|
868
|
+
publishRestart(undefined);
|
|
830
869
|
}
|
|
831
870
|
else if (!pendingRestart || pendingRestart.reasons.join(",") !== reasons.join(",")) {
|
|
832
|
-
|
|
871
|
+
publishRestart({ reasons, version: r.effective.version, since: Date.now() });
|
|
833
872
|
}
|
|
834
873
|
else {
|
|
835
|
-
|
|
874
|
+
publishRestart({ ...pendingRestart, version: r.effective.version });
|
|
836
875
|
}
|
|
837
876
|
logger.info("sema_registry_refreshed", {
|
|
838
877
|
version: r.effective.version,
|
|
@@ -907,7 +946,7 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
907
946
|
generation: lateGen,
|
|
908
947
|
report: (rep) => {
|
|
909
948
|
applyReportLate = rep;
|
|
910
|
-
|
|
949
|
+
recordApply(rep);
|
|
911
950
|
},
|
|
912
951
|
...(planeChangedLate ? { swapPlane: swapRunnerDeps } : {}),
|
|
913
952
|
});
|
|
@@ -949,7 +988,7 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
949
988
|
logger.warn("restart_candidate_publication_incomplete", { version: r.effective.version, blockedReasons: blocked, note: "late-boot candidate publication incomplete on a durable-declared deployment — restart signal withheld; retried every tick" });
|
|
950
989
|
}
|
|
951
990
|
if (reasons.length > 0)
|
|
952
|
-
|
|
991
|
+
publishRestart({ reasons, version: r.effective.version, since: Date.now() });
|
|
953
992
|
logger.info("config_loaded_deferred", { source: configProvider.kind, version: r.effective.version, models: (r.effective.models?.models ?? []).filter((m) => m.enabled !== false).length, ...(reasons.length > 0 ? { restartRequired: true, restartReasons: reasons } : {}) });
|
|
954
993
|
};
|
|
955
994
|
if (bootConfigPending) {
|
|
@@ -971,6 +1010,10 @@ export async function createConfigCenterRuntime(ctx) {
|
|
|
971
1010
|
clearInterval(ccTimer);
|
|
972
1011
|
ccTimer = undefined;
|
|
973
1012
|
}
|
|
1013
|
+
credentialRenewer?.stop();
|
|
1014
|
+
},
|
|
1015
|
+
centerCredentialState() {
|
|
1016
|
+
return cc === undefined ? undefined : buildCenterCredentialHealthView(cc.credential.health(), credentialRenewer);
|
|
974
1017
|
},
|
|
975
1018
|
};
|
|
976
1019
|
}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { defaultTaskRegistry } from "@sema-agent/core";
|
|
2
|
+
import { reportHardKillReceipts } from "../background-session-reap.js";
|
|
2
3
|
import { liveHostForegroundChildCount, reapHostForegroundChildren } from "../plugins/host-platform.js";
|
|
3
4
|
import { liveEphemeralHostWorkspaceCount, reapEphemeralHostWorkspaces } from "../plugins/host-workspace-registry.js";
|
|
5
|
+
const PROCESS_EXIT_INCLUDES_RETAINED = false;
|
|
4
6
|
export function installProcessExitBoundary({ logger }) {
|
|
5
7
|
const reapOwnChildProcesses = () => {
|
|
6
8
|
try {
|
|
@@ -22,6 +24,15 @@ export function installProcessExitBoundary({ logger }) {
|
|
|
22
24
|
logger.warn("shutdown_foreground_reap_failed", { err: String(err) });
|
|
23
25
|
}
|
|
24
26
|
};
|
|
27
|
+
const reapBackgroundHard = () => {
|
|
28
|
+
try {
|
|
29
|
+
const receipts = defaultTaskRegistry.killAllBackgroundShellsNow({ includeRetained: PROCESS_EXIT_INCLUDES_RETAINED, reason: "process_exit" });
|
|
30
|
+
reportHardKillReceipts(logger, receipts, "process_exit");
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
logger.warn("shutdown_background_hard_kill_failed", { err: String(err) });
|
|
34
|
+
}
|
|
35
|
+
};
|
|
25
36
|
const reapOwnHostWorkspaces = () => {
|
|
26
37
|
try {
|
|
27
38
|
const { removed, failed } = reapEphemeralHostWorkspaces();
|
|
@@ -42,6 +53,12 @@ export function installProcessExitBoundary({ logger }) {
|
|
|
42
53
|
catch (err) {
|
|
43
54
|
console.error("shutdown_exit_boundary_child_reap_threw", err);
|
|
44
55
|
}
|
|
56
|
+
try {
|
|
57
|
+
reapBackgroundHard();
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
console.error("shutdown_exit_boundary_background_hard_kill_threw", err);
|
|
61
|
+
}
|
|
45
62
|
try {
|
|
46
63
|
reapOwnHostWorkspaces();
|
|
47
64
|
}
|
package/dist/boot/runner-deps.js
CHANGED
|
@@ -180,6 +180,7 @@ export function createRunnerDeps(ctx) {
|
|
|
180
180
|
onAsk: createRunnerDepsOnAsk(toolApproval),
|
|
181
181
|
autoMode: {
|
|
182
182
|
...(config.autoModeClassifierFallback !== undefined ? { classifierFallback: config.autoModeClassifierFallback } : {}),
|
|
183
|
+
...(config.autoModeClassifierTimeoutMs !== undefined ? { timeoutMs: config.autoModeClassifierTimeoutMs } : {}),
|
|
183
184
|
},
|
|
184
185
|
sessionStore,
|
|
185
186
|
memoryBackend: memoryEngine ? memoryEngine.backend : undefined,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { applyGovernedVerdictAbsenceWarning, applyLocalAutoModeDeny, applyMemoryOptOutGrant, applyObserverEnvOptIn, centerEntitlementSourceWired, createPrincipalEntitlementsClient, } from "../runtime-caps-resolver.js";
|
|
2
2
|
import { scopedTokenNeedsWorker } from "../center-credential.js";
|
|
3
|
+
import { CENTER_CREDENTIAL_ENV } from "@sema-agent/settings-schema";
|
|
3
4
|
export function assertMemoryCapturePolicyWirable(input) {
|
|
4
5
|
if (input.policy !== "governed")
|
|
5
6
|
return;
|
|
@@ -13,7 +14,7 @@ export function assertMemoryCapturePolicyWirable(input) {
|
|
|
13
14
|
`refusal would read as an entitlement denial rather than the misconfiguration it is. This version has TWO verdict sources ` +
|
|
14
15
|
`— wire either one: (1) the \`memory_optout_grant\` table on a SQL store backend (DB_BACKEND=mysql|pg; the table is ` +
|
|
15
16
|
`created by the central ensureSchema and administered through /v1/admin/memory-optout), or (2) a config center that ` +
|
|
16
|
-
`EMITS \`allowMemoryOptOut\` in each principal's entitlement caps (SEMA_REGISTRY_URL +
|
|
17
|
+
`EMITS \`allowMemoryOptOut\` in each principal's entitlement caps (SEMA_REGISTRY_URL + ${CENTER_CREDENTIAL_ENV.token}, ` +
|
|
17
18
|
`SEMA_REGISTRY_DRY_RUN off — dry-run observes without applying, so it is not a source; and note the key is OPTIONAL in ` +
|
|
18
19
|
`the entitlement schema: a center that never emits it leaves every verdict absent, i.e. every opt-out still refused). ` +
|
|
19
20
|
`Or set MEMORY_CAPTURE_POLICY=open (opt-outs take effect; only an explicit per-principal false denies) or ` +
|
|
@@ -39,7 +40,7 @@ export function createRuntimeCaps(ctx) {
|
|
|
39
40
|
}
|
|
40
41
|
if (centerEntitlementSourceWired(config) && config.configCenter && scopedTokenNeedsWorker(config.configCenter.credential.read(), config.configCenter.worker)) {
|
|
41
42
|
logger.warn("runtime_caps_scoped_token_no_worker", {
|
|
42
|
-
reason:
|
|
43
|
+
reason: `${CENTER_CREDENTIAL_ENV.token} is a worker-scoped wpt_ token but SEMA_REGISTRY_WORKER is unset → center 403s per-principal caps → ALL workflow self-orchestration will be fail-closed denied. Set SEMA_REGISTRY_WORKER (the orchestrator normally injects it) or use the full SERVICE_PULL_TOKEN.`,
|
|
43
44
|
});
|
|
44
45
|
}
|
|
45
46
|
const principalCaps = centerEntitlementSourceWired(config) && config.configCenter
|