@sema-agent/server 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/budget-tracing.d.ts +48 -0
- package/dist/boot/budget-tracing.js +86 -0
- package/dist/boot/config-center.d.ts +62 -0
- package/dist/boot/config-center.js +995 -0
- package/dist/boot/coordinators.d.ts +33 -0
- package/dist/boot/coordinators.js +97 -0
- package/dist/boot/execution-env.d.ts +26 -0
- package/dist/boot/execution-env.js +370 -0
- package/dist/boot/leader.d.ts +27 -0
- package/dist/boot/leader.js +81 -0
- package/dist/boot/reapers.d.ts +53 -0
- package/dist/boot/reapers.js +252 -0
- package/dist/boot/resolve-spec.d.ts +70 -0
- package/dist/boot/resolve-spec.js +1072 -0
- package/dist/boot/runner-deps.d.ts +101 -0
- package/dist/boot/runner-deps.js +343 -0
- package/dist/boot/runtime-caps.d.ts +21 -0
- package/dist/boot/runtime-caps.js +62 -0
- package/dist/boot/session-faces.d.ts +57 -0
- package/dist/boot/session-faces.js +157 -0
- package/dist/boot/shutdown.d.ts +50 -0
- package/dist/boot/shutdown.js +129 -0
- package/dist/boot/stores.d.ts +32 -0
- package/dist/boot/stores.js +361 -0
- package/dist/boot/workflow-orchestration.d.ts +46 -0
- package/dist/boot/workflow-orchestration.js +150 -0
- package/dist/config-types.d.ts +40 -2
- package/dist/config.d.ts +14 -2
- package/dist/config.js +539 -361
- package/dist/main.js +163 -3798
- package/package.json +1 -1
|
@@ -0,0 +1,995 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* design/158 A10 尾刀:composition root 分段 —— **config-center 运行时**(boot 半场 + 60s refresh 循环 +
|
|
3
|
+
* 两者之间的 center 能力消费点)。
|
|
4
|
+
*
|
|
5
|
+
* 上一刀把 12 个 boot 段搬进 `src/boot/*.ts`(纯搬运),独独留下这两段并写明理由:它们共享一组
|
|
6
|
+
* **跨段可变状态**(`ccEtag / effective / latestEffective / lastRejectedCandidate / appliedPlaneEff /
|
|
7
|
+
* planeDeferredNoHandoff / bootSkillStale / pendingRestart / bootLkgCandidate / lastPrewarmBodiesVerified /
|
|
8
|
+
* lkgBooted / bootConfigPending / centerPrompts / pendingCatalogFaces / keyResolver / modelReadyState`),
|
|
9
|
+
* 而 codex R10–R38 的重启风暴判据密集地建立在这些状态的**读写次序**上(哪一行先落盘、哪一行先发信号、
|
|
10
|
+
* 哪个分支 splice 掉整组 reasons)。
|
|
11
|
+
*
|
|
12
|
+
* 本刀的做法:把这些 `let` 从 `main()` 的函数作用域搬进 **本工厂的闭包**——它们仍然是 `let`,**每一处
|
|
13
|
+
* 读写的字面文本、次序、分支结构一行未改**(把 `x` 改写成 `state.x` 会把 800 行里的每一处读写都变成一次
|
|
14
|
+
* 手工重写,那正是上一刀判定「不是搬运」的原因)。闭包 = 那个「持有状态的对象」;`main()` 只通过下面这
|
|
15
|
+
* 组窄口读它们,与本目录既有的晚绑 getter 惯例(`getRunStore()` / `getKeyResolver()` / `getRunDenySweep()`)
|
|
16
|
+
* 同款。
|
|
17
|
+
*
|
|
18
|
+
* ⚠️ **位置即契约**(调用点在 `main()` 里的相对次序不可动):
|
|
19
|
+
* 1. `createConfigCenterRuntime()` 必须在 logger/metrics 之后、`openStores()` 之前 —— boot 拉取要在
|
|
20
|
+
* Runner/持久层建起来之前把 models/roles/teams 应用进 `config`(env 是兜底,center 覆盖其上)。
|
|
21
|
+
* 2. `applyLocalRemoteExec()` 紧随其后、在 `createExecutionEnv()` 之前 —— 它就地改写 `config.remoteExec`,
|
|
22
|
+
* 消费点(执行环境工厂)在下游。
|
|
23
|
+
* 3. `applyCenterCapabilities()` 必须在 `loadSkills()` 之后、`buildScenarios()` 之前,且 **boot LKG 落盘点
|
|
24
|
+
* 就在它里面**(F7/codex R26:先装载 skill 正文再落盘,否则「manifest 可见、正文没进缓存」的窗口会被
|
|
25
|
+
* 发布成 boot 可见 LKG);plugins 让位判据要求 plugins 消费在 center 直发 skills **之后**。
|
|
26
|
+
* 4. `applyCenterScenarios()` 必须在 `builtinScenarioDetails()` 捕获六内建名之后(探针要打纯内建工厂)。
|
|
27
|
+
* 5. `initKeyResolver()` 留在原位(Runner 构造之后、ServiceDeps 装配之前)—— 首建要读的是 boot apply
|
|
28
|
+
* **之后**的 `config.modelApiKeyEnv/modelApiKeys`。
|
|
29
|
+
* 6. `startRefreshLoop()` 必须在 `new Runner()` 之后 —— 它要 `runnerTierFrozen`(与构造函数读的是同一
|
|
30
|
+
* config 代际),且 boot-deferred 到货的续接就挂在这里。
|
|
31
|
+
*/
|
|
32
|
+
import { createHash } from "node:crypto";
|
|
33
|
+
import { readFile as fsReadFile } from "node:fs/promises";
|
|
34
|
+
import { homedir } from "node:os";
|
|
35
|
+
import { join } from "node:path";
|
|
36
|
+
import { loadRemoteExec } from "@sema-agent/registry-core/node";
|
|
37
|
+
import { skillContentHash } from "@sema-agent/registry-core";
|
|
38
|
+
import { CenterPromptSource, FilePromptArtifactStore, FilePromptSourceStateStore, MemoryPromptArtifactStore, MemoryPromptSourceStateStore, } from "@sema-agent/core";
|
|
39
|
+
import { buildPricing } from "../budget.js";
|
|
40
|
+
import { applyCenterPlugins } from "../capabilities/center-plugins.js";
|
|
41
|
+
import { validatePromptsDomain, applyCatalogToSource, CORE_ENGINE_VERSION, withPromptArtifactBackfill } from "../capabilities/center-prompts.js";
|
|
42
|
+
import { centerScenarios, centerScenarioDetails } from "../capabilities/scenarios.js";
|
|
43
|
+
import { configLkgEnabled } from "../config.js";
|
|
44
|
+
import { defaultLkgPath, defaultSkillCacheDir, saveLkg, loadLkg } from "../config-lkg.js";
|
|
45
|
+
import { createConfigProvider, raceBootFetch, BOOT_FETCH_DEFERRED } from "../config-provider.js";
|
|
46
|
+
import { createKeyResolver } from "../key-resolver.js";
|
|
47
|
+
import { ensureSealedKeyStore, reportExecutionPublicKey } from "../sealed-key.js";
|
|
48
|
+
import { applyEffective, mutateInPlace, logEffectiveDiff, applyCenterSkills, resolveMcpServers, restartReasons, modelPlaneChanged, planeHasActiveTiers, fetchPromptArtifact } from "../sema-registry.js";
|
|
49
|
+
export async function createConfigCenterRuntime(ctx) {
|
|
50
|
+
const { config, logger, metrics, localRoot } = ctx;
|
|
51
|
+
// Sema registry (sema-registry): pull the effective config on boot and apply it OVER the env
|
|
52
|
+
// defaults — env is the fallback. Unset or unreachable → keep the env-derived config. Applied BEFORE
|
|
53
|
+
// the Runner is built so models/roles/teams take effect. (Logical config only; secrets stay in env.)
|
|
54
|
+
let ccEtag;
|
|
55
|
+
// Kept past the fetch so the deferred skills/mcp apply (which needs the live `skills` array + async
|
|
56
|
+
// content lazy-pull) can run right before buildScenarios, after applyEffective handled models/roles/teams.
|
|
57
|
+
// `effective` is the BOOT snapshot — the refresh loop never reassigns it, so restartReasons() compares each
|
|
58
|
+
// pull against what THIS process baked in (change-detection, not presence → no auto-restart loop).
|
|
59
|
+
let effective;
|
|
60
|
+
// 提示词装配协议 §9.5-2 three-store split: `effective` above = the boot snapshot (diff baseline);
|
|
61
|
+
// `latestEffective` = the last candidate that passed validation AND was applied — the explicit
|
|
62
|
+
// last-known-good (LKG). A refresh candidate failing validation is REJECTED whole (§9.5-5): never
|
|
63
|
+
// applied, so the live config keeps serving this LKG instead of a domain falling to schema-default/
|
|
64
|
+
// empty. The third store (promptCandidate — a staged prompts-domain artifact awaiting activation)
|
|
65
|
+
// lands with the prompts domain itself; `lastRejectedCandidate` is its observability forerunner and
|
|
66
|
+
// the prompts-domain validation (core S1 schema) hooks into the same gate.
|
|
67
|
+
let latestEffective;
|
|
68
|
+
let lastRejectedCandidate;
|
|
69
|
+
// codex R10 (models-tiers 窗收口): the last EffectiveConfig whose MODEL PLANE (models/roles/tiers/default) was
|
|
70
|
+
// actually applied to the live config — undefined = env plane (no center plane applied yet). When the Runner is
|
|
71
|
+
// tier-frozen, a refresh whose plane differs from THIS is deferred to restart (applyEffective deferModelPlane),
|
|
72
|
+
// keeping admission and the Runner on one catalog generation through the restart window.
|
|
73
|
+
let appliedPlaneEff;
|
|
74
|
+
// codex R18: the persistent "deferred WITHOUT next-boot handoff" state — the one-shot warn is not enough
|
|
75
|
+
// (the etag advances → later ticks 304 and never re-enter the defer branch). Set when a plane defers with no
|
|
76
|
+
// restart-surviving LKG; cleared when a plane finally applies or a forced restart reason ships. Surfaced on
|
|
77
|
+
// /health (modelPlaneDeferred) + re-warned every refresh tick (incl. 304s).
|
|
78
|
+
let planeDeferredNoHandoff;
|
|
79
|
+
// codex R35: STICKY live-skill application debt — the boot-baked skill table never rebuilds inside this
|
|
80
|
+
// process, so once the live application is known incomplete the debt survives every refresh (a
|
|
81
|
+
// same-generation 200 recomputes reasons vs boot as [] and would otherwise silently clear the signal).
|
|
82
|
+
// Set at the boot writer (R34 arms); folded into every tick's reasons; only a restart resolves it.
|
|
83
|
+
let bootSkillStale = false;
|
|
84
|
+
// Sema-registry restart signal: set by the refresh loop when a pull's restart-to-apply slices
|
|
85
|
+
// (skills/mcp/scenarios/runtime-gates) differ from boot; read by GET /health for orchestrator auto-restart.
|
|
86
|
+
let pendingRestart;
|
|
87
|
+
// DUAL-MODE-DESIGN §4: the config SOURCE is a backend-selectable seam — remote (sema-registry HTTP, the fleet
|
|
88
|
+
// default) | local (.env + config.d via @sema-agent/registry-core FileConfigStore, the single-machine/TOC source).
|
|
89
|
+
// The provider exists when SEMA_REGISTRY_URL (legacy CONFIG_CENTER_URL) is set OR CONFIG_PROVIDER=local is explicit; neither = pure-env
|
|
90
|
+
// (byte-identical to before). The remote provider DELEGATES verbatim to fetchEffective, so the fleet path is
|
|
91
|
+
// unchanged; applyEffective consumes either source's EffectiveConfig identically (one resolver → no drift).
|
|
92
|
+
const cc = config.configCenter;
|
|
93
|
+
// Sealed-box key custody: ensure this host's X25519 keypair(s) under
|
|
94
|
+
// <localDataRoot>/sealed-key/ (dir 0700, files 0600; SEVERAL may coexist — rotation keeps old private
|
|
95
|
+
// keys decrypting old ciphertexts) BEFORE the first effective pull, so a roster carrying
|
|
96
|
+
// `sealedApiKey` unseals on the boot apply. The public-key report is best-effort fire-and-forget:
|
|
97
|
+
// the center's execution-keys face is admin-gated and a worker only holds the pull token, so without
|
|
98
|
+
// an operator-provided SEMA_REGISTRY_ADMIN_TOKEN it prints the full public key for manual
|
|
99
|
+
// registration instead (honest posture — see reportExecutionPublicKey). Store failure is non-fatal:
|
|
100
|
+
// sealed models then warn per-model in applyEffective (fail-loud, never a silent apiKeyEnv fallback).
|
|
101
|
+
let sealedKeys;
|
|
102
|
+
try {
|
|
103
|
+
sealedKeys = await ensureSealedKeyStore(join(config.localDataRoot ?? localRoot, "sealed-key"), logger);
|
|
104
|
+
void reportExecutionPublicKey({
|
|
105
|
+
baseUrl: cc?.baseUrl,
|
|
106
|
+
adminToken: process.env.SEMA_REGISTRY_ADMIN_TOKEN || process.env.REGISTRY_ADMIN_TOKEN,
|
|
107
|
+
key: sealedKeys.latest,
|
|
108
|
+
...(cc?.worker ? { notes: `worker ${cc.worker}` } : {}),
|
|
109
|
+
logger,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
catch (err) {
|
|
113
|
+
logger.warn("sealed_key_store_unavailable", { err: String(err), note: "sealed api keys cannot be unsealed on this host — models carrying sealedApiKey will fail loud per-model" });
|
|
114
|
+
}
|
|
115
|
+
const configProvider = cc || config.configProvider === "local"
|
|
116
|
+
? createConfigProvider({
|
|
117
|
+
...(config.configProvider ? { provider: config.configProvider } : {}),
|
|
118
|
+
...(cc ? { configCenter: { baseUrl: cc.baseUrl, token: cc.token, worker: cc.worker, dryRun: cc.dryRun } } : {}),
|
|
119
|
+
...(config.configProvider === "local" ? { localDir: localRoot } : {}),
|
|
120
|
+
})
|
|
121
|
+
: undefined;
|
|
122
|
+
// boot ready 门 latch:仅「远程 registry 部署(非 dryRun)+ 无显式 env MODEL_ID」时等 roster——
|
|
123
|
+
// 首次 effective 落 ≥1 enabled 模型即翻 true(one-way)。其余姿势(env 模型在/local provider/dryRun/纯 env)
|
|
124
|
+
// 恒 ready=现有部署零影响。E3 废除 workers.model 后,fleet worker 的占位模型窗口由此门 fail-closed。
|
|
125
|
+
const modelReadyState = { ready: !(cc && !cc.dryRun) || Boolean(process.env.MODEL_ID) };
|
|
126
|
+
const markRosterLanded = (eff) => {
|
|
127
|
+
if (!modelReadyState.ready && (eff.models?.models ?? []).some((m) => m.enabled !== false)) {
|
|
128
|
+
modelReadyState.ready = true;
|
|
129
|
+
logger.info("model_roster_landed", { note: "ready gate open — billable submissions accepted" });
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
// center prompts ref([1057]/[1062] 投影腿):**热采用,新任务边界生效**——core 1.316([1068] 裁 a/
|
|
133
|
+
// [1070]d)把 provider 声明纳入 epoch artifactDigest:声明变=digest 变=下任务边界自动重 pin,进行中
|
|
134
|
+
// 会话由 session epoch pin 保稳,resume 记账诚实(1.222 的 restart-to-apply 是 core 件落地前的正确
|
|
135
|
+
// 过渡姿势,[1068]① 官方确认后随 1.316 采用降级为热生效——与 center「发布=新任务生效」产品语义对齐)。
|
|
136
|
+
// 判形通过才采用(形校验=防 core-assemble-THROW 底线,非语义复检 [1057]①f);坏形=warn+沿用旧包。
|
|
137
|
+
let centerPrompts;
|
|
138
|
+
/** codex-3 F1:一次失败的 catalog 过渡的目标面(undefined 目标=欠 disable)。etag 已推进时 provider
|
|
139
|
+
* 304 短路 adopt——本 marker 让每个 refresh 拍(含 304 拍)独立重试,直到过渡落地。 */
|
|
140
|
+
let pendingCatalogFaces;
|
|
141
|
+
const retryPendingCatalog = async (phase) => {
|
|
142
|
+
if (!pendingCatalogFaces)
|
|
143
|
+
return;
|
|
144
|
+
const { target } = pendingCatalogFaces;
|
|
145
|
+
if (await applyCatalogToSource(promptSource, target, centerPrompts, logger, phase)) {
|
|
146
|
+
centerPrompts = target;
|
|
147
|
+
pendingCatalogFaces = undefined;
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
// e/b(catalog 轴消费面,[1236]① seam):core 的 CenterPromptSource——server 只给根路径,core 文件形
|
|
151
|
+
// 自管 `<root>/prompt-epoch/{artifacts,source-state.json}`;boot restore=磁盘 LKG(离线重启 candidate
|
|
152
|
+
// 存续/disabled 不复活,状态机全在 core)。RunnerDeps.promptSource 挂它;adopt 时经 applyCatalogToSource
|
|
153
|
+
// 推进两态过渡。restore 失败=warn(candidate 缺席,catalog 轴等下一次 adopt;declaration 轴/内置照跑)。
|
|
154
|
+
const promptEpochDir = join(config.localDataRoot ?? localRoot, "prompt-epoch");
|
|
155
|
+
// codex-3 F2:effective 拉取是 worker 轴的(config LKG 同理 worker-keyed)——同根多 worker 进程共享
|
|
156
|
+
// content-addressed artifacts/(by-digest,天然安全),但 **candidate 状态必须按 worker 分箱**:共写一份
|
|
157
|
+
// source-state.json 会让 A 的撤下压掉 B 的 candidate、restart 恢复到别人 worker 的 catalog(错 worker
|
|
158
|
+
// 的系统提示词跑任务)。encodeURIComponent 消毒(worker 名进路径)。
|
|
159
|
+
// codex-5:无撞形分箱——缺席=字面 "global",配置了 worker= "worker-<sha256 前 16>"(哈希后天然单
|
|
160
|
+
// 段/无 dot-segment/与 "global" 不可能同名;worker 名任意字节也安全)。
|
|
161
|
+
const promptStateDir = join(promptEpochDir, "state", cc?.worker === undefined ? "global" : `worker-${createHash("sha256").update(cc.worker).digest("hex").slice(0, 16)}`);
|
|
162
|
+
// codex-6 F1:File 店构造同步 mkdir——只读/不可写数据根会把「可选的 catalog 特性」变成整服务 boot
|
|
163
|
+
// 失败。守护工厂:文件形失败 ⇒ warn + Memory 店兜底(candidate 不跨重启持久,catalog 采用仍工作;
|
|
164
|
+
// 内置/declaration 轴完全不受影响)。
|
|
165
|
+
// [985]b / [1285] — §9.4 by-digest 补拉组合店:local miss(本副本从未采用过该 candidate / 盘失)⇒
|
|
166
|
+
// 从 center 按 digest 补拉 envelope ⇒ core verifyPromptArtifact 全管线(canonical 复算;**且回验
|
|
167
|
+
// 「拿到的=要的」digest 逐位**,center 答非所问=丢弃)⇒ best-effort 回写盘(下次直命中)。pinned
|
|
168
|
+
// resume 的 fail-loud miss 自此只剩「center 也没有」的真缺席。仅 remote 治理形(cc.baseUrl)挂补拉。
|
|
169
|
+
const withByDigestBackfill = (inner) => cc?.baseUrl ? withPromptArtifactBackfill(inner, (d) => fetchPromptArtifact(cc.baseUrl, cc.token, d), logger) : inner;
|
|
170
|
+
const promptSource = (() => {
|
|
171
|
+
try {
|
|
172
|
+
return new CenterPromptSource({
|
|
173
|
+
store: withByDigestBackfill(new FilePromptArtifactStore(promptEpochDir)),
|
|
174
|
+
stateStore: new FilePromptSourceStateStore(promptStateDir),
|
|
175
|
+
engineVersion: CORE_ENGINE_VERSION,
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
catch (e) {
|
|
179
|
+
logger.warn("center_prompts_store_unwritable", { dir: promptEpochDir, err: String(e), note: "file-backed prompt-epoch store unavailable — memory fallback (catalog candidate not restart-durable); built-ins/declaration axis unaffected" });
|
|
180
|
+
return new CenterPromptSource({ store: withByDigestBackfill(new MemoryPromptArtifactStore()), stateStore: new MemoryPromptSourceStateStore(), engineVersion: CORE_ENGINE_VERSION });
|
|
181
|
+
}
|
|
182
|
+
})();
|
|
183
|
+
// codex-2 F4:只有「有配置源在治理且非 dry-run」才 restore 磁盘 candidate——provider-off/dry-run 姿势下
|
|
184
|
+
// 旧部署遗留的 active candidate 不得继续影响新任务(publish-gate-off/compare-only 语义)。不 restore 只
|
|
185
|
+
// 影响 candidate 面;pinned resume 的 by-digest 解析走 FilePromptArtifactStore.get(不依赖 restore),
|
|
186
|
+
// 旧会话恢复不受损。
|
|
187
|
+
if (configProvider && !cc?.dryRun) {
|
|
188
|
+
try {
|
|
189
|
+
await promptSource.restore();
|
|
190
|
+
}
|
|
191
|
+
catch (e) {
|
|
192
|
+
logger.warn("center_prompts_catalog_restore_failed", { err: String(e), note: "disk LKG unavailable — catalog candidate absent until the next adopt; declaration axis and built-ins unaffected" });
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
const adoptCenterPrompts = async (eff, phase) => {
|
|
196
|
+
const raw = eff?.prompts;
|
|
197
|
+
if (raw === undefined) {
|
|
198
|
+
// fail-closed 语义([1057]①d):publish 闸关=key 整体不出现 ⇒ 收敛回内置提示词(不是「保持旧包」——
|
|
199
|
+
// 闸关是显式撤下,沿用旧包会让「关闸」失效)。
|
|
200
|
+
if (centerPrompts)
|
|
201
|
+
logger.info("center_prompts_cleared", { phase, note: "effective carries no prompts key (publish gate off) — built-in providers resume" });
|
|
202
|
+
const cleared = await applyCatalogToSource(promptSource, undefined, centerPrompts, logger, phase);
|
|
203
|
+
if (cleared) {
|
|
204
|
+
centerPrompts = undefined;
|
|
205
|
+
pendingCatalogFaces = undefined;
|
|
206
|
+
}
|
|
207
|
+
else {
|
|
208
|
+
pendingCatalogFaces = { target: undefined }; // failed disable — retried by the refresh hook (codex-3 F1)
|
|
209
|
+
}
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
const v = validatePromptsDomain(raw);
|
|
213
|
+
if (!v.ok) {
|
|
214
|
+
logger.warn("center_prompts_invalid", { phase, error: v.error, kept: centerPrompts?.declarations?.packId ?? (centerPrompts?.catalog ? "(catalog)" : "(builtin)"), note: "malformed prompts face rejected (core assemble would THROW per task) — keeping the previous pack/built-ins; fix the center payload" });
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
if (centerPrompts?.identity !== v.value.identity) {
|
|
218
|
+
// [985]a §3:axes 观测两轴共存窗;declaration 轴字段照旧,catalog 轴报 artifactDigest(内容身份)。
|
|
219
|
+
logger.info("center_prompts_adopted", {
|
|
220
|
+
phase,
|
|
221
|
+
axes: v.value.axes,
|
|
222
|
+
...(v.value.declarations
|
|
223
|
+
? { packId: v.value.declarations.packId, contentDigest: v.value.declarations.contentDigest, sections: v.value.declarations.sections.length, scenarioOverrides: Object.keys(v.value.declarations.scenarioOverrides ?? {}).length }
|
|
224
|
+
: {}),
|
|
225
|
+
...(v.value.catalog ? { artifactDigest: v.value.catalog.artifact.artifactDigest, catalogDigest: v.value.catalog.artifact.payload.catalogDigest, sourceRevision: v.value.catalog.sourceRevision } : {}),
|
|
226
|
+
note: "new tasks assemble the new pack (epoch re-pins at the next task boundary); in-flight sessions stay pinned",
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
const transitioned = await applyCatalogToSource(promptSource, v.value, centerPrompts, logger, phase);
|
|
230
|
+
// codex-2 F2 + codex-3 F1:catalog 过渡失败 ⇒ 身份不推进(centerPrompts 保持 prev),且目标面记入
|
|
231
|
+
// pendingCatalogFaces——etag 会照常推进(其他域的采用不为 prompts 的盘错扣住),此后 provider 304
|
|
232
|
+
// 不再进本函数,重试由 refresh 拍上的 retryPendingCatalog 钩独立驱动(见 refresh 循环)。
|
|
233
|
+
if (transitioned) {
|
|
234
|
+
centerPrompts = v.value;
|
|
235
|
+
pendingCatalogFaces = undefined;
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
pendingCatalogFaces = { target: v.value };
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
// Boot-deferred pull (clay 2026-07-17): when the boot fetch outlives its budget, the SAME promise is
|
|
242
|
+
// stashed here and, once the refresh tick exists (defined further down), chained to trigger an immediate
|
|
243
|
+
// hot-apply on arrival — the server meanwhile starts on the env fallback in <1s instead of hanging up to
|
|
244
|
+
// ~8s on a black-holed center (真机实测 5.7s = the "local server takes 5s" report).
|
|
245
|
+
let bootConfigPending;
|
|
246
|
+
// LKG 落盘(clay 拍 2026-07-18;config-lkg.ts 顶注=全案由):remote lane 的干净候选持久化——boot 的
|
|
247
|
+
// 第三级来源(中心 > 盘上 LKG > env)。`lkgBooted` 标志决定 deferred continuation 的形:LKG 起服=有
|
|
248
|
+
// 完整 boot 基线 → 到货走普通 refreshTick(候选门+hot+restartReasons 对 LKG 比较,通常零差异=无环);
|
|
249
|
+
// 无 LKG=env 起服 → 到货走「迟到 boot」路径(deferredBootApply)。
|
|
250
|
+
const lkgEnabled = configProvider?.kind === "remote" && configLkgEnabled();
|
|
251
|
+
const lkgPath = process.env.CONFIG_LKG_PATH ?? defaultLkgPath(cc?.worker);
|
|
252
|
+
// codex R17/R20: a successful LOCAL write proves nothing about the NEXT instance seeing it — a stateless pod
|
|
253
|
+
// replaces its filesystem on restart, so treating a pod-local rename as "handoff confirmed" re-opens the R16
|
|
254
|
+
// rolling-restart storm. Next-instance visibility is a DEPLOYMENT contract only the operator can assert, and
|
|
255
|
+
// ONLY via the explicit knob: CONFIG_LKG_DURABLE=true declares the LKG path rides restart-surviving storage
|
|
256
|
+
// (host disk / PVC / shared volume). CONFIG_LKG_PATH is strictly a LOCATION override (R20: it can point at an
|
|
257
|
+
// ephemeral container path — a location says nothing about durability). Unset ⇒ the models-tiers restart
|
|
258
|
+
// reason stays suppressed on a deferred plane (old generation keeps serving; /health modelPlaneDeferred +
|
|
259
|
+
// per-tick warn carry the state) while normal LKG writes continue (they still help same-instance restarts).
|
|
260
|
+
const lkgSurvivesRestart = lkgEnabled && process.env.CONFIG_LKG_DURABLE === "true";
|
|
261
|
+
let lkgBooted = false;
|
|
262
|
+
let bootLkgCandidate; // F7:skills 装载后才落盘
|
|
263
|
+
// F1+F2(LKG 复审):awaited 落盘 + skill 正文预热——refresh 只落 manifest 的话,重启后 LKG-boot 在
|
|
264
|
+
// 中心不可达时新 skill 正文缓存缺失=软跳过(静默缺 skill)。预热=枚举 enabled manifest,把缺盘的正文
|
|
265
|
+
// 拉齐进 content-addressed 缓存(失败=warn 软降级,与 boot 的 skill_failed 同语义等级——skill 拉取
|
|
266
|
+
// 失败从不升级为配置拒绝);await 保证 restart 信号可见前 LKG+正文已落盘。
|
|
267
|
+
/** Returns TRUE iff the candidate is durably persisted for the NEXT boot (codex R16: the FORCED models-tiers
|
|
268
|
+
* restart reason must never fire without a confirmed handoff — LKG disabled/unwritable + over-budget boot
|
|
269
|
+
* pulls would otherwise turn the deferral into a rolling-restart storm: every restarted process boots the env
|
|
270
|
+
* plane again, defers again, re-signals). Change-DETECTED reasons still ride regardless (their loop is the
|
|
271
|
+
* pre-existing, documented no-LKG residual). */
|
|
272
|
+
// codex R37 loop-breaker proof: TRUE iff the LAST persistLkgDurable call hash-verified EVERY enabled skill
|
|
273
|
+
// body on disk. This is the "a restart can actually repair the live skill table" proof the NON-durable lane
|
|
274
|
+
// requires before publishing a skills restart — a successful manifest pull only proves the CONFIG endpoint
|
|
275
|
+
// works, not the SEPARATE body fetches; signaling on manifest-success alone lets a persistent body failure
|
|
276
|
+
// (404 / hash mismatch / timeout) restart-storm a stateless deployment (every replacement process fails the
|
|
277
|
+
// same body, signals again, is restarted again). With the proof, a storm cannot form: a boot that fails the
|
|
278
|
+
// body lacks the proof and only retains visible debt (no signal) until a later verify succeeds.
|
|
279
|
+
let lastPrewarmBodiesVerified = false;
|
|
280
|
+
const persistLkgDurable = async (effectiveObj, etag) => {
|
|
281
|
+
lastPrewarmBodiesVerified = false;
|
|
282
|
+
// codex R21: the LOCAL provider's candidate IS the next-boot source (config.d on the same disk boot reads
|
|
283
|
+
// BEFORE Runner construction) — the handoff is inherent, no LKG needed. Without this arm a polled config.d
|
|
284
|
+
// edit that defers the plane (tier activation / tier-frozen change) could never authorize its own restart
|
|
285
|
+
// (lkgEnabled is remote-only, so even CONFIG_LKG_DURABLE couldn't help) — stuck in modelPlaneDeferred forever.
|
|
286
|
+
// codex R28 requalified: this TRUE covers the MODEL PLANE (what the deferral gate adjudicates). Skills are
|
|
287
|
+
// NOT part of the local handoff because the local lane does not consume a center-style skills domain AT ALL
|
|
288
|
+
// yet — the only boot consumer (applyCenterSkills) requires config.configCenter (remote HTTP), absent on
|
|
289
|
+
// CONFIG_PROVIDER=local; bodies live inline in config.d (LocalConfigProvider.fetchSkillContent) but no boot
|
|
290
|
+
// path reads them. Pre-existing local-lane gap (predates this train, nothing lost across restart that was
|
|
291
|
+
// ever applied); wiring boot skills through the ConfigProvider seam is the tracked follow-up (board).
|
|
292
|
+
if (configProvider?.kind === "local")
|
|
293
|
+
return true;
|
|
294
|
+
if (!lkgEnabled)
|
|
295
|
+
return false;
|
|
296
|
+
try {
|
|
297
|
+
if (effectiveObj.skills && config.configCenter) {
|
|
298
|
+
await applyCenterSkills([], effectiveObj.skills, config.configCenter.baseUrl, config.configCenter.token, logger, undefined, defaultSkillCacheDir());
|
|
299
|
+
}
|
|
300
|
+
// codex R25/R26: skill BODIES are part of the handoff contract, and completeness gates PUBLICATION —
|
|
301
|
+
// applyCenterSkills swallows fetch/cache-write failures, so awaiting it proves nothing; verify every
|
|
302
|
+
// ENABLED manifest hash on disk (content-true) BEFORE the atomic LKG replace (R26: verifying after left
|
|
303
|
+
// an incomplete candidate as the boot-visible LKG — any crash/manual restart with the center down then
|
|
304
|
+
// boots that manifest, silently skips the uncached skills, and never re-signals because the manifest is
|
|
305
|
+
// the new boot baseline). On any miss/corruption the PREVIOUS good LKG stays in place; the per-tick
|
|
306
|
+
// retained-candidate retry re-attempts prewarm + verify + publish.
|
|
307
|
+
// Residual seam (honest, partial): the skill cache roots at SEMA_CONFIG_DIR while lkgPath can point
|
|
308
|
+
// elsewhere — CONFIG_LKG_DURABLE's contract requires BOTH on restart-surviving storage.
|
|
309
|
+
const enabledSkills = (effectiveObj.skills?.skills ?? []).filter((m) => m.enabled !== false);
|
|
310
|
+
const cacheDir = defaultSkillCacheDir();
|
|
311
|
+
for (const m of enabledSkills) {
|
|
312
|
+
const hex = m.contentHash.replace(/^sha256:/, "").toLowerCase();
|
|
313
|
+
if (!/^[0-9a-f]{64}$/.test(hex))
|
|
314
|
+
return false;
|
|
315
|
+
try {
|
|
316
|
+
const text = await fsReadFile(join(cacheDir, hex), "utf8");
|
|
317
|
+
if (`sha256:${createHash("sha256").update(text, "utf8").digest("hex")}` !== `sha256:${hex}`) {
|
|
318
|
+
logger.warn("config_lkg_skill_cache_incomplete", { skill: m.name, hash: m.contentHash, note: "cached body fails its hash — LKG NOT replaced, handoff NOT confirmed (the previous good LKG keeps serving offline boots)" });
|
|
319
|
+
return false;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
catch {
|
|
323
|
+
logger.warn("config_lkg_skill_cache_incomplete", { skill: m.name, hash: m.contentHash, note: "enabled skill body not in the cache — LKG NOT replaced, handoff NOT confirmed" });
|
|
324
|
+
return false;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
lastPrewarmBodiesVerified = true; // every enabled body verified on disk — the repair proof stands even if the LKG write below fails
|
|
328
|
+
await saveLkg(lkgPath, cc?.worker, effectiveObj, etag);
|
|
329
|
+
// codex R17: TRUE = next-BOOT visibility, not merely a successful local rename — gated on the operator's
|
|
330
|
+
// restart-surviving-storage declaration (lkgSurvivesRestart above). A stateless pod's local write still
|
|
331
|
+
// helps same-instance restarts but must not authorize a forced restart.
|
|
332
|
+
return lkgSurvivesRestart;
|
|
333
|
+
}
|
|
334
|
+
catch (err) {
|
|
335
|
+
logger.warn("config_lkg_save_failed", { path: lkgPath, err: String(err), note: "durable persist failed — restart signal proceeds anyway (a dead disk + dead center double-fault can re-loop; fix the disk)" });
|
|
336
|
+
return false;
|
|
337
|
+
}
|
|
338
|
+
};
|
|
339
|
+
if (configProvider) {
|
|
340
|
+
const dryRun = cc?.dryRun ?? false; // dry-run posture is remote-only (a local source has nothing to compare against)
|
|
341
|
+
// LKG 读腿(boot 超窗与硬失败两个入口共用):盘上有验证过的干净候选 → 当 boot 配置整套应用
|
|
342
|
+
// (skills/mcp 进 buildScenarios——这正是 env 兜底给不了的),etag 落位 → refresh 首拍 304 或正常增量。
|
|
343
|
+
const bootFromLkg = async (why) => {
|
|
344
|
+
if (!lkgEnabled || dryRun)
|
|
345
|
+
return null;
|
|
346
|
+
const read = await loadLkg(lkgPath, cc?.worker);
|
|
347
|
+
if (!read.lkg) {
|
|
348
|
+
if (read.reason !== "absent")
|
|
349
|
+
logger.warn("config_lkg_unusable", { path: lkgPath, reason: read.reason, note: "ignoring the on-disk LKG — env fallback serves; the cache rewrites on the next clean candidate" });
|
|
350
|
+
return null;
|
|
351
|
+
}
|
|
352
|
+
logger.info("config_loaded_lkg", { path: lkgPath, savedAt: read.lkg.savedAt, why, note: "booting from the persisted last-known-good (all restart-to-apply faces included) — the live pull continues in the background and reconciles on arrival" });
|
|
353
|
+
return { effective: read.lkg.effective, ...(read.lkg.etag !== undefined ? { etag: read.lkg.etag } : {}) };
|
|
354
|
+
};
|
|
355
|
+
const bootApplyLkgInline = async (lkgR) => {
|
|
356
|
+
applyEffective(config, lkgR.effective, logger, { sealedKeys });
|
|
357
|
+
await adoptCenterPrompts(lkgR.effective, "boot-lkg");
|
|
358
|
+
markRosterLanded(lkgR.effective);
|
|
359
|
+
effective = lkgR.effective;
|
|
360
|
+
appliedPlaneEff = lkgR.effective; // model plane applied pre-Runner — this IS the Runner's generation
|
|
361
|
+
latestEffective = lkgR.effective;
|
|
362
|
+
if (lkgR.etag !== undefined)
|
|
363
|
+
ccEtag = lkgR.etag;
|
|
364
|
+
const cfgModels = (lkgR.effective.models?.models ?? []).filter((m) => m.enabled !== false).length;
|
|
365
|
+
logger.info("config_loaded", { source: "lkg", version: lkgR.effective.version, worker: cc?.worker ?? "(global)", models: cfgModels });
|
|
366
|
+
};
|
|
367
|
+
try {
|
|
368
|
+
const bootFetch = configProvider.fetchEffective(undefined);
|
|
369
|
+
const raced = await raceBootFetch(bootFetch, config.configBootFetchBudgetMs);
|
|
370
|
+
let lkgFallback = null;
|
|
371
|
+
if (raced === BOOT_FETCH_DEFERRED) {
|
|
372
|
+
bootConfigPending = bootFetch;
|
|
373
|
+
bootFetch.catch(() => { }); // settled in the background — failure handling lives on the refresh cadence
|
|
374
|
+
lkgFallback = await bootFromLkg("boot fetch deferred (budget exceeded)");
|
|
375
|
+
lkgBooted = lkgFallback !== null;
|
|
376
|
+
logger.warn("config_boot_fetch_deferred", {
|
|
377
|
+
source: configProvider.kind,
|
|
378
|
+
budgetMs: config.configBootFetchBudgetMs,
|
|
379
|
+
lkg: lkgBooted,
|
|
380
|
+
note: lkgBooted
|
|
381
|
+
? "center slow — booting from the persisted LKG; the pull continues in the background and reconciles via the refresh path on arrival"
|
|
382
|
+
: "center slow/unreachable and no usable LKG — serving on the env fallback; the pull continues in the background (late-boot adoption on arrival). Raise CONFIG_BOOT_FETCH_BUDGET_MS to block boot instead.",
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
// F4(LKG 复审):LKG 载荷的 apply 用专属 catch——走主分支会让 applyEffective 半程炸落到外层
|
|
386
|
+
// catch 的「config_source_unreachable_using_env」误导日志(中心明明只是慢)。坏 LKG=点名+env 兜底
|
|
387
|
+
// (in-place mutate 的混态风险与 live 坏载荷同级,浅域判形在 loadLkg 已挡结构性坏形)。
|
|
388
|
+
if (raced === BOOT_FETCH_DEFERRED && lkgFallback) {
|
|
389
|
+
try {
|
|
390
|
+
await bootApplyLkgInline(lkgFallback);
|
|
391
|
+
}
|
|
392
|
+
catch (applyErr) {
|
|
393
|
+
lkgBooted = false;
|
|
394
|
+
logger.warn("config_lkg_apply_failed", { err: String(applyErr), note: "persisted LKG failed to apply — env fallback serves; the cache rewrites on the next clean candidate" });
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
const r = raced === BOOT_FETCH_DEFERRED ? null : raced;
|
|
398
|
+
if (r) {
|
|
399
|
+
// [898] tolerant read(registry-core 0.10.12,local lane):catalog 域坏文件不再连坐整包回落 env——
|
|
400
|
+
// 坏域按 default 落+per-domain warn 点名(文件路径在 error 文案里),好域照常生效。gate 域坏文件
|
|
401
|
+
// tolerant 仍 throw(READ 容错不放宽 gate fail-loud)⇒ 走下面既有整包 catch。
|
|
402
|
+
// §9.5-5 boot 与 refresh 的分工(codex F1:重启不得洗白坏候选):boot 照旧 tolerant APPLY(无 LKG
|
|
403
|
+
// 可保,带着好域起服务),但一个带错误的候选 ①不推进 ccEtag——refresh 首 poll 拿到同一棵树,候选门
|
|
404
|
+
// 当场整拒并持续 warn 到修复;②不写入 latestEffective——LKG 永远只指向完全干净的候选,拒绝日志的
|
|
405
|
+
// lkgVersion 也诚实(boot 带错=「(env)」)。markRosterLanded 保留:models 域自身干净才可能开门
|
|
406
|
+
// (坏 models 域 tolerant 落空=enabled 0=门不动),其他域的错误不构成扣住就绪门的理由。
|
|
407
|
+
const bootPromptsRaw = r.effective.prompts;
|
|
408
|
+
const bootPromptsOk = bootPromptsRaw === undefined || validatePromptsDomain(bootPromptsRaw).ok;
|
|
409
|
+
// 坏 prompts 面同 domainErrors 计入 not-clean(F6 boot 半场):etag 不进 → refresh 首拍同树整拒持续
|
|
410
|
+
// warn;LKG 不指向带坏 prompts 的候选。adoptCenterPrompts 自身会拒采用+点名。
|
|
411
|
+
const bootClean = (r.domainErrors ?? []).length === 0 && bootPromptsOk;
|
|
412
|
+
if (bootClean)
|
|
413
|
+
ccEtag = r.etag;
|
|
414
|
+
for (const de of r.domainErrors ?? [])
|
|
415
|
+
logger.warn("config_domain_invalid", { domain: de.domain, error: de.error, note: "tolerated at BOOT (corrupt file → schema defaults; dangling ref → reported as-is); the refresh path rejects such a candidate whole and keeps the LKG (§9.5-5)" });
|
|
416
|
+
if (dryRun) {
|
|
417
|
+
logEffectiveDiff(config, r.effective, logger); // compare-only: log the would-be changes, keep env
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
420
|
+
applyEffective(config, r.effective, logger, { sealedKeys }); // sealedKeys: unseal sealedApiKey → in-memory key pipeline
|
|
421
|
+
await adoptCenterPrompts(r.effective, "boot"); // [1057] additive prompts key(判形采用;缺席=内置)
|
|
422
|
+
markRosterLanded(r.effective); // boot pull 即落 roster 的常态:门在启动窗内就打开
|
|
423
|
+
effective = r.effective; // skills/mcp applied below, before buildScenarios (restart-to-apply)
|
|
424
|
+
appliedPlaneEff = r.effective; // model plane applied pre-Runner — this IS the Runner's generation
|
|
425
|
+
if (bootClean)
|
|
426
|
+
latestEffective = r.effective; // LKG seed (§9.5-2): only a fully clean candidate may become the LKG
|
|
427
|
+
// F7(LKG 复审三轮):boot 写点**延后**到 center skills 装载(applyCenterSkills+盘缓存)之后——
|
|
428
|
+
// 先落 LKG 会让「新 skill 的 manifest 可见但正文没进缓存」窗口存在(此窗内崩+中心断=离线 boot
|
|
429
|
+
// 静默缺 skill)。这里只记 candidate,真正落盘在 skills 装载后(main 后段)。
|
|
430
|
+
if (bootClean && !lkgBooted)
|
|
431
|
+
bootLkgCandidate = { effective: r.effective, etag: r.etag };
|
|
432
|
+
// [875]b:带上模型数与(local lane)真实目录来源——复验实录:CONFIG_LOCAL_DIR 误配(只设
|
|
433
|
+
// LOCAL_DATA_ROOT)时 config.d 静默读不到、0 模型且无告警,排障只能靠猜。
|
|
434
|
+
const cfgModels = (r.effective.models?.models ?? []).filter((m) => m.enabled !== false).length;
|
|
435
|
+
const cfgDir = configProvider.kind === "local" ? { dir: configProvider.root } : {};
|
|
436
|
+
logger.info("config_loaded", { source: configProvider.kind, version: r.effective.version, worker: cc?.worker ?? "(global)", models: cfgModels, ...cfgDir });
|
|
437
|
+
if (configProvider.kind === "local" && cfgModels === 0)
|
|
438
|
+
logger.warn("config_local_no_models", { ...cfgDir, hint: "config.d/models.json 空或 CONFIG_LOCAL_DIR 指错目录(选目录旋钮是 CONFIG_LOCAL_DIR,不是 LOCAL_DATA_ROOT)— 模型继续走 env 兜底" });
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
catch (err) {
|
|
443
|
+
logger.warn("config_source_unreachable_using_env", { source: configProvider.kind, err: String(err) });
|
|
444
|
+
// 硬失败(fast-fail RST 等,非超窗)同样值得 LKG:上一进程的干净候选好过纯 env。
|
|
445
|
+
const lkgFallback = await bootFromLkg(`boot fetch failed: ${String(err).slice(0, 120)}`);
|
|
446
|
+
if (lkgFallback) {
|
|
447
|
+
lkgBooted = true;
|
|
448
|
+
try {
|
|
449
|
+
await bootApplyLkgInline(lkgFallback);
|
|
450
|
+
}
|
|
451
|
+
catch (applyErr) {
|
|
452
|
+
lkgBooted = false;
|
|
453
|
+
logger.warn("config_lkg_apply_failed", { err: String(applyErr), note: "persisted LKG failed to apply — env fallback serves" });
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
if (!modelReadyState.ready)
|
|
459
|
+
logger.warn("model_roster_pending", { note: "registry deployment without an env model and no roster yet — billable submissions 503 until the first effective pull lands models" });
|
|
460
|
+
// Per-model API key (sema-registry `apiKeyEnv`): resolve each model's own upstream key per brain call /
|
|
461
|
+
// cascade rung. undefined when no per-model keys are configured → spec field stays unset (core default).
|
|
462
|
+
// `let` (not const): rebuilt on a sema-registry refresh so per-model key ADDITIONS hot-apply too (the
|
|
463
|
+
// resolver is undefined when no per-model keys exist, so in-place mutation alone wouldn't cover empty→non-empty).
|
|
464
|
+
let keyResolver;
|
|
465
|
+
return {
|
|
466
|
+
providerKind: configProvider?.kind,
|
|
467
|
+
promptSource,
|
|
468
|
+
getKeyResolver: () => keyResolver,
|
|
469
|
+
getCenterPrompts: () => centerPrompts,
|
|
470
|
+
modelReady: () => modelReadyState.ready,
|
|
471
|
+
restartState: () => pendingRestart,
|
|
472
|
+
planeDeferredState: () => planeDeferredNoHandoff,
|
|
473
|
+
async applyLocalRemoteExec() {
|
|
474
|
+
// DUAL-MODE §4/§5 (center #1): in LOCAL config mode, source remoteExec from the SHARED
|
|
475
|
+
// `config.d/remote-exec.json` via the registry `loadRemoteExec` — the SAME file + zod validation that the
|
|
476
|
+
// client's `agent-cli config` writes, so client doctor and the engine read one source of truth (zero drift).
|
|
477
|
+
// Semantics (center, deliberate): file MISSING → undefined → keep the env/host default (config.ts already
|
|
478
|
+
// defaults host in local mode); file PRESENT → it wins; file empty/corrupt → the loader throws → fail-closed
|
|
479
|
+
// (an empty file silently falling back to host would be an isolation downgrade if the user meant local-docker).
|
|
480
|
+
if (configProvider?.kind === "local") {
|
|
481
|
+
try {
|
|
482
|
+
const fileRx = await loadRemoteExec(localRoot); // SAME root as the config provider above (no split-brain)
|
|
483
|
+
if (fileRx) {
|
|
484
|
+
// 🔴 center contract: the package RemoteExecSpec is the CONTRACT form (env-NAME refs: apiKeyEnv/tokenEnv/…);
|
|
485
|
+
// our config.remoteExec is the RESOLVED form (values). `host` carries NO secrets so it maps 1:1 (contract
|
|
486
|
+
// `workdir` ≡ our `workspaceBase`). The ISOLATED lanes (e2b/k8s/ssh/adb) need a NAME→value resolution
|
|
487
|
+
// before use — a follow-on; until then they are sourced from REMOTE_EXEC env, so a file-source isolated
|
|
488
|
+
// arm is ignored (NOT cast verbatim — that would feed an env-NAME where a value is expected).
|
|
489
|
+
if (fileRx.provider === "host") {
|
|
490
|
+
config.remoteExec = {
|
|
491
|
+
provider: "host",
|
|
492
|
+
...(fileRx.workdir ? { workspaceBase: fileRx.workdir } : {}),
|
|
493
|
+
...(fileRx.commandTimeoutMs != null ? { commandTimeoutMs: fileRx.commandTimeoutMs } : {}),
|
|
494
|
+
};
|
|
495
|
+
logger.info("remote_exec_from_file", { provider: "host", root: localRoot });
|
|
496
|
+
}
|
|
497
|
+
else if (fileRx.provider === "local-docker") {
|
|
498
|
+
// local-docker carries NO secret env-NAMEs (image/memory/cpus/network/workdir/mountPath only), so it
|
|
499
|
+
// maps 1:1 from the contract form — no NAME→value resolution needed (unlike e2b/k8s/ssh). The contract
|
|
500
|
+
// `workdir` is the in-container workspace ROOT ≡ our `mountPath`. DOCKER_HOST/secret env stay in the
|
|
501
|
+
// worker's .env (the adapter reads its own process.env via the env-forward, not the file).
|
|
502
|
+
config.remoteExec = {
|
|
503
|
+
provider: "local-docker",
|
|
504
|
+
image: fileRx.image,
|
|
505
|
+
...(fileRx.mountPath ? { mountPath: fileRx.mountPath } : fileRx.workdir ? { mountPath: fileRx.workdir } : {}),
|
|
506
|
+
...(fileRx.memory ? { memory: fileRx.memory } : {}),
|
|
507
|
+
...(fileRx.cpus != null ? { cpus: fileRx.cpus } : {}),
|
|
508
|
+
...(fileRx.network ? { network: fileRx.network } : {}),
|
|
509
|
+
};
|
|
510
|
+
logger.info("remote_exec_from_file", { provider: "local-docker", root: localRoot, image: fileRx.image });
|
|
511
|
+
}
|
|
512
|
+
else {
|
|
513
|
+
logger.warn("remote_exec_file_isolated_lane_unwired", { provider: fileRx.provider, note: "e2b/k8s/ssh/adb are sourced from REMOTE_EXEC env until file-source NAME→value resolution lands (center field map)" });
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
catch (err) {
|
|
518
|
+
logger.error("remote_exec_file_invalid", { root: localRoot, err: String(err) });
|
|
519
|
+
throw err; // fail-closed (center semantics): a corrupt remote-exec.json must not boot a wrong/weaker lane
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
},
|
|
523
|
+
async applyCenterCapabilities(skills) {
|
|
524
|
+
// Sema-registry B1 (restart-to-apply, baked into scenarios below): overlay center skills (lazy-fetched by
|
|
525
|
+
// hash + sha256-verified) over the image baseline, center-wins-by-name; resolve center MCP servers' env-NAME
|
|
526
|
+
// refs to real values from this service's env. Both empty/unset → unchanged (image SKILLS_DIR, no MCP).
|
|
527
|
+
if (effective?.skills && config.configCenter) {
|
|
528
|
+
skills = await applyCenterSkills(skills, effective.skills, config.configCenter.baseUrl, config.configCenter.token, logger, undefined, configLkgEnabled() ? defaultSkillCacheDir() : undefined);
|
|
529
|
+
}
|
|
530
|
+
// plugins 域消费半场(契约 [1361]②/[1362]/[1366]①;2026-07-28 开工车):把 center 声明的插件引用
|
|
531
|
+
// 物化(https-only+ALLOW_HOSTS 纵深复验/sha 钉扎/隔离 clone/LKG)并装载其 skills —— **在 center 直发
|
|
532
|
+
// skills 之后**:撞名让位判据(收紧④,plugin 让位)以此刻的 skills 集为准。失败单条不连坐,永不拒启。
|
|
533
|
+
if (effective?.plugins && config.configCenter) {
|
|
534
|
+
const pluginOut = await applyCenterPlugins(skills, effective, {
|
|
535
|
+
cacheRoot: join(config.localDataRoot ?? join(homedir(), ".ai-agent"), "plugin-cache"),
|
|
536
|
+
allowHosts: config.pluginsAllowHosts,
|
|
537
|
+
logger,
|
|
538
|
+
}).catch((err) => {
|
|
539
|
+
logger.warn("center_plugins_apply_failed", { error: err instanceof Error ? err.message : String(err) });
|
|
540
|
+
return undefined;
|
|
541
|
+
});
|
|
542
|
+
if (pluginOut) {
|
|
543
|
+
skills = pluginOut.skills;
|
|
544
|
+
if (pluginOut.failures.length + pluginOut.letGo.length + pluginOut.lkgUsed.length > 0) {
|
|
545
|
+
logger.info("center_plugins_applied", { failures: pluginOut.failures, letGo: pluginOut.letGo, lkgUsed: pluginOut.lkgUsed });
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
// F7:boot 的 LKG 落盘点。codex R26:走同一只 persistLkgDurable(完整性门在前)——此前裸 saveLkg 会把
|
|
550
|
+
// applyCenterSkills 静默吞掉失败的「缺正文 manifest」发布成 boot 可见 LKG(离线重启=静默缺 skill 且不再
|
|
551
|
+
// 有信号收敛)。校验失败=不落盘(warn 已在函数内),下一个干净候选/refresh 重试重写。
|
|
552
|
+
if (bootLkgCandidate && lkgEnabled) {
|
|
553
|
+
const bootPublished = await persistLkgDurable(bootLkgCandidate.effective, bootLkgCandidate.etag).catch((err) => {
|
|
554
|
+
logger.warn("config_lkg_save_failed", { path: lkgPath, err: String(err), note: "boot LKG persist failed — retained for the per-tick retry" });
|
|
555
|
+
return false;
|
|
556
|
+
});
|
|
557
|
+
// codex R30: a failed/incomplete boot publication must enter the SAME retained-candidate retry lane as the
|
|
558
|
+
// refresh path — ccEtag has already advanced, so the next ticks 304 and nothing else would ever retry; a
|
|
559
|
+
// restart during a center outage would then boot stale/env config instead of the generation this process
|
|
560
|
+
// already applied. blocked=[] — everything WAS applied at boot, this is a publication-only retry (promotion
|
|
561
|
+
// with no withheld reasons publishes NO restart signal, it just completes the LKG and clears the state).
|
|
562
|
+
{
|
|
563
|
+
// codex R32/R34: verify the boot's LIVE application UNCONDITIONALLY — applyCenterSkills skips an
|
|
564
|
+
// unfetchable body (this process may run skill-stale / on an older same-name built-in), and the LKG
|
|
565
|
+
// prewarm inside persistLkgDurable can succeed where the live fetch failed (bootPublished true, LKG
|
|
566
|
+
// complete, process still stale). Any live miss ⇒ a skills restart: published immediately when the
|
|
567
|
+
// handoff already stands, or carried in blocked for the 304-gated promotion when it doesn't.
|
|
568
|
+
const enabledManifest = (bootLkgCandidate.effective.skills?.skills ?? []).filter((m) => m.enabled !== false);
|
|
569
|
+
// codex R33: the FULL runtime projection must match, not just name+body — a same-body built-in retained
|
|
570
|
+
// after a failed center fetch can carry different scenarios/description (e.g. global exposure where the
|
|
571
|
+
// manifest restricts to one scenario), and the scenario bundles baked right after are then stale.
|
|
572
|
+
const canonScenarios = (a) => [...(a ?? [])].sort().join("\u0000");
|
|
573
|
+
const liveSkillMiss = enabledManifest.some((m) => !skills.some((sk) => sk.spec.name === m.name &&
|
|
574
|
+
skillContentHash(sk.spec.content) === m.contentHash &&
|
|
575
|
+
sk.spec.description === m.description &&
|
|
576
|
+
canonScenarios(sk.scenarios) === canonScenarios(m.scenarios)));
|
|
577
|
+
if (liveSkillMiss)
|
|
578
|
+
bootSkillStale = true; // codex R35: sticky — survives same-generation 200 recomputes
|
|
579
|
+
if (liveSkillMiss && (bootPublished || (!lkgSurvivesRestart && lastPrewarmBodiesVerified))) {
|
|
580
|
+
// Signal NOW only on the arms where a restart PROVABLY repairs (codex R34/R36/R37):
|
|
581
|
+
// · durable + published — the complete LKG (manifest + hash-verified bodies) is the handoff;
|
|
582
|
+
// · NON-durable + body proof — the prewarm inside persistLkgDurable just hash-verified every enabled
|
|
583
|
+
// body on disk, so the miss was a TRANSIENT live-application failure and a restarted process
|
|
584
|
+
// re-pulls successfully. WITHOUT the proof (persistent 404/hash mismatch) publishing here would
|
|
585
|
+
// restart-storm a stateless deployment (codex R37) — that case falls to the retained-debt arm
|
|
586
|
+
// below: visible on /health, retried per confirming 304, promoted only once the proof holds.
|
|
587
|
+
pendingRestart = { restartRequired: true, reasons: ["skills"], version: bootLkgCandidate.effective.version, since: Date.now() };
|
|
588
|
+
logger.warn("config_boot_skill_stale_restart", { version: bootLkgCandidate.effective.version, durableHandoff: bootPublished, note: bootPublished ? "live skill application incomplete but the durable LKG carries the full candidate — skills restart published (the next boot reads the complete LKG)" : "live skill application incomplete on a non-durable deployment but every enabled body is verified in the cache — skills restart published (transient failure proven repaired; the restarted process re-pulls live)" });
|
|
589
|
+
}
|
|
590
|
+
else if (liveSkillMiss || (!bootPublished && lkgSurvivesRestart)) {
|
|
591
|
+
planeDeferredNoHandoff = { version: bootLkgCandidate.effective.version, since: Date.now(), blocked: liveSkillMiss ? ["skills"] : [], candidate: bootLkgCandidate.effective, ...(bootLkgCandidate.etag !== undefined ? { candidateEtag: bootLkgCandidate.etag } : {}) };
|
|
592
|
+
logger.warn("config_lkg_boot_publication_retained", { version: bootLkgCandidate.effective.version, liveSkillMiss, durableDeclared: lkgSurvivesRestart, note: (lkgSurvivesRestart ? "boot LKG publication incomplete on a durable-declared deployment — candidate retained; the refresh loop retries after each confirming 304" : "live skill application incomplete on a non-durable deployment and the body cache could not be verified — skills debt retained WITHOUT a restart signal (codex R37: signaling now could restart-storm; each confirming 304 re-verifies and publishes once the bodies prove fetchable)") + (liveSkillMiss && lkgSurvivesRestart ? "; this process runs skill-stale, promotion will publish a skills restart" : "") });
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
if (effective?.mcp)
|
|
597
|
+
config.mcpServers = resolveMcpServers(effective.mcp, logger);
|
|
598
|
+
return skills;
|
|
599
|
+
},
|
|
600
|
+
applyCenterScenarios(t) {
|
|
601
|
+
const { scenarioDeps, scenarios, scenarioDetails, builtinScenarioNames } = t;
|
|
602
|
+
// Center-declared scenarios (composition-as-config): overlay over the built-ins, center wins — it MAY
|
|
603
|
+
// shadow a built-in name (usability first), so shadows are logged loudly here for audit.
|
|
604
|
+
// Restart-to-apply like skills/mcp (this runs once at boot; a refresh carrying scenarios only signals).
|
|
605
|
+
if (effective?.scenarios) {
|
|
606
|
+
const { overlay, shadows } = centerScenarios(scenarioDeps, effective.scenarios.scenarios, Object.keys(scenarios), logger);
|
|
607
|
+
Object.assign(scenarios, overlay);
|
|
608
|
+
Object.assign(scenarioDetails, centerScenarioDetails(effective.scenarios.scenarios, builtinScenarioNames));
|
|
609
|
+
if (Object.keys(overlay).length > 0)
|
|
610
|
+
logger.info("sema_registry_scenarios", { scenarios: Object.keys(overlay), ...(shadows.length > 0 ? { shadowsBuiltin: shadows } : {}) });
|
|
611
|
+
}
|
|
612
|
+
},
|
|
613
|
+
initKeyResolver() {
|
|
614
|
+
keyResolver = createKeyResolver(config.modelApiKeyEnv, process.env, config.modelApiKeys);
|
|
615
|
+
// PAIR-REVIEW F-6:Bearer-only 网关 × per-model key 组合死角的 boot 响亮告警——core 的 per-call auth 是
|
|
616
|
+
// **整体替换**语义(1.345:per-call apiKey 在场即剥构造期 Authorization,x-api-key 成唯一凭据),而
|
|
617
|
+
// key-resolver 只会产 { apiKey } 形;部署若以 ANTHROPIC_AUTH_TOKEN(Bearer)直连此类网关,则每个配了
|
|
618
|
+
// per-model key 的模型必 401 且无从自诊。headers 形支持候 core/registry additive 席位(跨仓件);此前
|
|
619
|
+
// 先把组合不可用讲清楚。只 warn 不拒:混合网关(两种头都收)是存在的,fail-open+可诊断。
|
|
620
|
+
if (process.env.ANTHROPIC_AUTH_TOKEN && !process.env.ANTHROPIC_API_KEY && (Object.keys(config.modelApiKeys).length > 0 || Object.keys(config.modelApiKeyEnv).length > 0)) {
|
|
621
|
+
logger.warn("per_model_key_bearer_gateway_combo", {
|
|
622
|
+
perModelKeys: Object.keys(config.modelApiKeys).length + Object.keys(config.modelApiKeyEnv).length,
|
|
623
|
+
note: "ANTHROPIC_AUTH_TOKEN (Bearer) deployment with per-model keys: core's per-call auth REPLACES the Authorization header with x-api-key for those models — a Bearer-only gateway will 401 them. Remove the per-model key for such models, or use a gateway accepting x-api-key.",
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
},
|
|
627
|
+
startRefreshLoop(a) {
|
|
628
|
+
const { runnerTierFrozen, pricing } = a;
|
|
629
|
+
// Config refresh: re-pull with the ETag; teams hot-reload via the registry. A models/roles
|
|
630
|
+
// change updates config but the live Runner keeps its startup set — logged so ops restart to apply.
|
|
631
|
+
// 提示词装配协议 §9.5-1/-3: the loop goes through the ConfigProvider seam (never the raw HTTP fetch —
|
|
632
|
+
// one resolver, no remote/local drift) and runs for the LOCAL provider too: config.d edits are picked
|
|
633
|
+
// up by the same 60s poll (the local provider's etag = the store's raws hash, so an unchanged tree is
|
|
634
|
+
// a cheap null and a corrupt↔fixed flip bumps the version). Local mode previously read config.d
|
|
635
|
+
// exactly once at boot.
|
|
636
|
+
if (configProvider) {
|
|
637
|
+
const ccRef = config.configCenter;
|
|
638
|
+
// Single-flight (codex F2): an async tick body under setInterval has no serialization — a slow read
|
|
639
|
+
// (degraded disk/远端) outliving the 60s interval would let a LATER tick apply a newer candidate and
|
|
640
|
+
// the stalled OLDER read then complete and roll the config (and ccEtag) back. Skip the tick while one
|
|
641
|
+
// is in flight; the flag flips in `finally`, so a throw never wedges the loop shut.
|
|
642
|
+
let refreshInFlight = false;
|
|
643
|
+
const refreshTick = async (prefetched) => {
|
|
644
|
+
if (refreshInFlight)
|
|
645
|
+
return;
|
|
646
|
+
refreshInFlight = true;
|
|
647
|
+
try {
|
|
648
|
+
// codex-3/4 F1:欠账的 catalog 过渡在**每个真拍**开头无条件重试——与 fetch 成败/候选好坏解耦
|
|
649
|
+
// (304、fetch 抛错、候选被整拒的拍此前全都跳过重试=盘错恢复后过渡仍卡死)。次序安全:本拍
|
|
650
|
+
// 若随后采用了更新候选,adopt 自会覆盖/清空 pending(先重试旧账、后采新目标,永不倒退)。
|
|
651
|
+
await retryPendingCatalog("refresh-retry");
|
|
652
|
+
// codex R18: the stuck-deferred state must not vanish behind the advanced etag — re-warn + count on
|
|
653
|
+
// EVERY tick (incl. 304s and fetch-error ticks) while it persists; /health carries the same state.
|
|
654
|
+
// The RECOVERY (persist retry + promotion) moved BELOW the fetch (codex R29): promoting before
|
|
655
|
+
// adjudicating the current source generation could publish a SUPERSEDED candidate N as the
|
|
656
|
+
// boot-visible LKG while N+1 exists — a restart in that window resurrects withdrawn config.
|
|
657
|
+
if (planeDeferredNoHandoff) {
|
|
658
|
+
metrics.inc("models_tiers_plane_deferred_stuck_total");
|
|
659
|
+
logger.warn("models_tiers_deferred_no_handoff", { version: planeDeferredNoHandoff.version, since: planeDeferredNoHandoff.since, note: "candidate still retained without a next-boot handoff — mount restart-surviving LKG storage + CONFIG_LKG_DURABLE=true, or restart manually after ensuring the candidate is re-pulled at boot" });
|
|
660
|
+
}
|
|
661
|
+
// F3(LKG 复审):LKG-boot 的 deferred 到货把 unconditional 结果直接喂进来——丢弃它再按 LKG 的
|
|
662
|
+
// etag 条件拉,会在「center 重建且 etag 值撞旧」的场景永远 304=陈旧配置钉死;消费原始结果则
|
|
663
|
+
// 应用后 ccEtag=真实新值,陈旧面消除(顺带省一次拉取)。
|
|
664
|
+
const r = prefetched !== undefined ? prefetched : await configProvider.fetchEffective(ccEtag);
|
|
665
|
+
// codex R23/R29 recovery lane: a 304 (r === null) CONFIRMS the retained candidate's generation is
|
|
666
|
+
// still the source's current one (its etag advanced when it was retained) — only then is it safe to
|
|
667
|
+
// retry persistence and promote the withheld restart signal. A fetch that returned a NEWER candidate
|
|
668
|
+
// takes the normal branches below (which replace/supersede the retained state); a fetch that THREW
|
|
669
|
+
// never reaches here (freshness unknown ⇒ no promotion this tick). Retry only where promotion is
|
|
670
|
+
// possible (durable-declared lane; R37 adds the non-durable skills-debt lane, whose retry doubles as
|
|
671
|
+
// the repair proof) — elsewhere a per-tick write could never promote and just burns IO.
|
|
672
|
+
if (r === null && planeDeferredNoHandoff) {
|
|
673
|
+
// Retry where promotion is possible: the durable-declared lane (full handoff), and — codex R37 —
|
|
674
|
+
// the NON-durable skills-debt lane, where the retry's body prewarm+verify is the repair PROOF that
|
|
675
|
+
// gates the withheld skills restart (see lastPrewarmBodiesVerified).
|
|
676
|
+
const skillsDebt = (planeDeferredNoHandoff.blocked ?? []).includes("skills");
|
|
677
|
+
const promoted = lkgSurvivesRestart || skillsDebt ? await persistLkgDurable(planeDeferredNoHandoff.candidate, planeDeferredNoHandoff.candidateEtag) : false;
|
|
678
|
+
// codex R37 proof-gated promotion (non-durable lane): every enabled body now hash-verifies on disk
|
|
679
|
+
// ⇒ the center + body endpoint are BOTH live, a restarted process re-pulls successfully — publish
|
|
680
|
+
// the skills restart exactly now, never on manifest-success alone (restart-storm breaker: a boot
|
|
681
|
+
// that fails the body again lacks the proof and returns to visible-debt, not to signaling).
|
|
682
|
+
// models-tiers never rides this arm (R16: a FORCED plane restart requires a durable handoff), so a
|
|
683
|
+
// retained candidate that ALSO defers the plane keeps everything withheld until durability exists.
|
|
684
|
+
const skillsProofPromotion = !promoted && !lkgSurvivesRestart && skillsDebt && lastPrewarmBodiesVerified && planeDeferredNoHandoff.planeDeferred !== true;
|
|
685
|
+
if (skillsProofPromotion) {
|
|
686
|
+
pendingRestart = { restartRequired: true, reasons: ["skills"], version: planeDeferredNoHandoff.version, since: Date.now() };
|
|
687
|
+
logger.info("config_boot_skill_debt_promoted", { version: planeDeferredNoHandoff.version, note: "skill bodies verified on a source-confirmed-current candidate — the withheld skills restart is now published (non-durable lane, repair proven)" });
|
|
688
|
+
planeDeferredNoHandoff = undefined;
|
|
689
|
+
}
|
|
690
|
+
else if (promoted) {
|
|
691
|
+
// models-tiers joins only when the PLANE was deferred (codex R27: a skill-only retained candidate
|
|
692
|
+
// must not manufacture a plane reason on promotion). codex R30: an EMPTY promotion (boot
|
|
693
|
+
// publication retry — nothing was withheld, everything already applied) publishes NO restart
|
|
694
|
+
// signal; it just completes the LKG and clears the state.
|
|
695
|
+
const promotedReasons = [...new Set([...(planeDeferredNoHandoff.blocked ?? []), ...(planeDeferredNoHandoff.planeDeferred ? ["models-tiers"] : [])])];
|
|
696
|
+
if (promotedReasons.length > 0) {
|
|
697
|
+
pendingRestart = { restartRequired: true, reasons: promotedReasons, version: planeDeferredNoHandoff.version, since: Date.now() };
|
|
698
|
+
}
|
|
699
|
+
logger.info("models_tiers_deferred_promoted", { version: planeDeferredNoHandoff.version, reasons: promotedReasons, note: promotedReasons.length > 0 ? "handoff persistence recovered on a source-confirmed-current candidate — the withheld restart signal is now published" : "boot publication retry completed — LKG now durable, no restart needed (nothing was withheld)" });
|
|
700
|
+
planeDeferredNoHandoff = undefined;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
if (r) {
|
|
704
|
+
// §9.5-5 candidate gate: ANY invalid domain rejects the WHOLE candidate — the live config
|
|
705
|
+
// keeps serving the LKG (`latestEffective`) instead of that domain falling to schema-default/
|
|
706
|
+
// empty. ccEtag is deliberately NOT advanced: the bad tree stays visible (re-judged each poll),
|
|
707
|
+
// and fixing the file bumps the version so the next poll lands. The dedupe key is the SORTED
|
|
708
|
+
// de-duplicated domain set (codex F3: the collector's ordering isn't guaranteed, a raw join
|
|
709
|
+
// would re-warn every poll), and the first warn carries the full per-domain error detail —
|
|
710
|
+
// that's the actionable diagnosis, the key only mutes repeats. The prompts-domain artifact
|
|
711
|
+
// validation (core S1) hooks into this same gate when it ships.
|
|
712
|
+
// F6(二轮复审):prompts 面判形进候选门——坏 pack(重复 id/空数组/坏信封)若只在 boot 验,
|
|
713
|
+
// refresh 会带着它推进 etag+发 restart 信号,重启后 boot 拒之回内置=好包 A 被坏包 B 顶掉。
|
|
714
|
+
// 这里与 domainErrors 同门:坏形=整拒候选(etag 不进,LKG/现用包 A 继续服务,持续 warn 到修复)。
|
|
715
|
+
const promptsRaw = r.effective.prompts;
|
|
716
|
+
const promptsGate = promptsRaw !== undefined ? validatePromptsDomain(promptsRaw) : { ok: true };
|
|
717
|
+
const badDomains = [
|
|
718
|
+
...new Set([...(r.domainErrors ?? []).map((de) => de.domain), ...(promptsGate.ok ? [] : ["prompts"])]),
|
|
719
|
+
]
|
|
720
|
+
.sort()
|
|
721
|
+
.join(",");
|
|
722
|
+
if (badDomains) {
|
|
723
|
+
if (lastRejectedCandidate?.version !== r.effective.version || lastRejectedCandidate.domains !== badDomains) {
|
|
724
|
+
lastRejectedCandidate = { version: r.effective.version, domains: badDomains, at: Date.now() };
|
|
725
|
+
logger.warn("config_candidate_rejected", {
|
|
726
|
+
version: r.effective.version,
|
|
727
|
+
domains: badDomains,
|
|
728
|
+
errors: [
|
|
729
|
+
...(r.domainErrors ?? []).map((de) => ({ domain: de.domain, error: de.error })),
|
|
730
|
+
...(promptsGate.ok ? [] : [{ domain: "prompts", error: promptsGate.error }]),
|
|
731
|
+
],
|
|
732
|
+
lkgVersion: latestEffective?.version ?? "(env)",
|
|
733
|
+
note: "candidate rejected whole (§9.5-5) — live config keeps the last-known-good; fix the named files to unblock",
|
|
734
|
+
});
|
|
735
|
+
}
|
|
736
|
+
return;
|
|
737
|
+
}
|
|
738
|
+
lastRejectedCandidate = undefined;
|
|
739
|
+
// ccEtag advances at the END of each branch (codex lens-1 HIGH): advancing before apply meant
|
|
740
|
+
// an applyEffective throw mid-way left a MIXED-generation live config AND a cached etag — the
|
|
741
|
+
// next poll 304'd, so the failed candidate was never retried. Now a throw keeps the old etag,
|
|
742
|
+
// the next poll re-fetches the same candidate, and the (idempotent) re-apply self-heals.
|
|
743
|
+
if (ccRef?.dryRun) {
|
|
744
|
+
logEffectiveDiff(config, r.effective, logger); // compare-only: surface ongoing center changes, apply nothing
|
|
745
|
+
ccEtag = r.etag;
|
|
746
|
+
}
|
|
747
|
+
else {
|
|
748
|
+
// refresh: teams + runtime governance + MODELS/ROLES/ROSTER now hot. teamsOnly skips
|
|
749
|
+
// only the boot-only runtime GATES (rate/cost/approval). applyEffective mutates config.models/roles
|
|
750
|
+
// IN PLACE (the Runner's live ref) — but `pricing` (separate object the Runner also reads live) and
|
|
751
|
+
// `keyResolver` (closes over modelApiKeyEnv) are ours to refresh.
|
|
752
|
+
// codex R10: hot-apply of the MODEL PLANE holds ONLY when the Runner shares our map (tier-less at
|
|
753
|
+
// construction). A tier-frozen Runner keeps a private expanded copy — hot-applying a changed plane
|
|
754
|
+
// would split admission from execution (same-key retarget silently routes/bills the stale Model
|
|
755
|
+
// object through the restart window). Defer the plane; the models-tiers slice below signals the
|
|
756
|
+
// orchestrator; the restarted process applies it at boot BEFORE Runner construction.
|
|
757
|
+
// codex R12: defer also when the CANDIDATE activates tiers on a tier-less-booted Runner — it never
|
|
758
|
+
// re-runs expandTiers, so hot-applying the arriving tier table would let tier words pass the
|
|
759
|
+
// expanded admission gate while core throws "Unknown model ref" until restart. Hot-apply is safe
|
|
760
|
+
// only when BOTH generations are tier-less.
|
|
761
|
+
const planeDeferred = (runnerTierFrozen || planeHasActiveTiers(r.effective)) && modelPlaneChanged(appliedPlaneEff, r.effective);
|
|
762
|
+
applyEffective(config, r.effective, logger, { teamsOnly: true, sealedKeys, ...(planeDeferred ? { deferModelPlane: true } : {}) });
|
|
763
|
+
if (planeDeferred) {
|
|
764
|
+
logger.warn("models_tiers_plane_deferred", { version: r.effective.version, note: "tier-frozen Runner: the changed model plane (models/roles/tiers/default) is NOT hot-applied — admission stays on the Runner's generation; restart applies the new plane (models-tiers restart signal rides /health)" });
|
|
765
|
+
}
|
|
766
|
+
else {
|
|
767
|
+
appliedPlaneEff = r.effective;
|
|
768
|
+
planeDeferredNoHandoff = undefined; // plane landed — the stuck state (if any) is resolved
|
|
769
|
+
}
|
|
770
|
+
await adoptCenterPrompts(r.effective, "refresh"); // 热采用(core 1.316 声明轴:新任务边界自动重 pin,见 ref 注释)
|
|
771
|
+
// codex R13: readiness may open ONLY off an APPLIED plane — a deferred candidate's roster is not
|
|
772
|
+
// installed (config.models still the old/placeholder generation), so opening the billable gate on
|
|
773
|
+
// it would route/bill the wrong model until restart. Deferred ⇒ gate stays as-is; the restarted
|
|
774
|
+
// process applies the candidate pre-Runner and opens it at boot.
|
|
775
|
+
if (!planeDeferred)
|
|
776
|
+
markRosterLanded(r.effective); // boot pull 失败/未发布时,refresh 落 roster 同样开门
|
|
777
|
+
mutateInPlace(pricing, buildPricing(config.models)); // hot: cost/model changes; Runner reads this.deps.pricing live
|
|
778
|
+
keyResolver = createKeyResolver(config.modelApiKeyEnv, process.env, config.modelApiKeys); // hot: per-model key add/remove/change (env-ref + sealed)
|
|
779
|
+
// skills/mcp/scenarios/runtime-gates are baked into the live process at boot (buildScenarios / boot
|
|
780
|
+
// middleware) — a refresh carrying a DIFFERENT value can't hot-apply, only a restart re-reads them.
|
|
781
|
+
// We compare against the BOOT snapshot (`effective`), NOT presence: an orchestrator auto-restarts on
|
|
782
|
+
// a non-empty `reasons` WITHOUT a restart loop (same diff every 60s is idempotent, not a fresh trigger).
|
|
783
|
+
// Sticky `since`: keep the first-detected timestamp while `reasons` is unchanged; reset it when the set
|
|
784
|
+
// changes; CLEAR the signal when a later pull reverts to boot (reasons empty) — restart no longer needed.
|
|
785
|
+
// F2(LKG 复审):落盘 **await 且先于** restart 信号可见——orchestrator 见信号即杀进程,
|
|
786
|
+
// fire-and-forget 的 rename 可能没完成 → 下一进程读旧 LKG → 同差异重现 = 本功能要消的环。
|
|
787
|
+
// 顺序=先持久化(含 skill 正文预热,F1)再发信号;持久化失败=warn+照发(不发=配置永不生效;
|
|
788
|
+
// 「盘坏+中心挂」双故障下环重现,接受并点名)。
|
|
789
|
+
const lkgPersisted = await persistLkgDurable(r.effective, r.etag);
|
|
790
|
+
const reasons = restartReasons(effective, r.effective);
|
|
791
|
+
// codex R35: the sticky boot skill debt rides EVERY tick's reasons — change-detection vs boot is
|
|
792
|
+
// blind to it (the stale table IS the boot baseline). Under a no-handoff deferral it folds into
|
|
793
|
+
// blocked (promoted later); otherwise it keeps pendingRestart alive until the restart happens.
|
|
794
|
+
// codex R38: change-DETECTED skills (manifest actually changed) is distinguished from the sticky
|
|
795
|
+
// rider — the former keeps the pre-existing documented posture (rides regardless; the restart
|
|
796
|
+
// pulls the NEW generation live), the latter is proof-gated below on the non-durable lane.
|
|
797
|
+
const skillsChangeDetected = reasons.includes("skills");
|
|
798
|
+
if (bootSkillStale && !skillsChangeDetected)
|
|
799
|
+
reasons.push("skills");
|
|
800
|
+
// codex R15/R16/R19: a DEFERRED plane is adjudicated by HANDOFF STATUS alone — independent of
|
|
801
|
+
// whether change-detection already produced the reason (R19: an already-present models-tiers with
|
|
802
|
+
// no handoff sailed through = the exact storm R16 closed). With a confirmed next-boot handoff the
|
|
803
|
+
// reason is ensured present (R15: env-baseline blind spot) and the stuck state clears; without one
|
|
804
|
+
// the reason is SUPPRESSED (a restart the next process cannot satisfy — over-budget boot pull + no
|
|
805
|
+
// LKG — is a rolling-restart storm) and the persistent stuck state + per-tick warn carry the
|
|
806
|
+
// observability instead. Non-plane reasons (skills/mcp/…) keep riding either way.
|
|
807
|
+
if (planeDeferred) {
|
|
808
|
+
if (lkgPersisted) {
|
|
809
|
+
if (!reasons.includes("models-tiers"))
|
|
810
|
+
reasons.push("models-tiers");
|
|
811
|
+
planeDeferredNoHandoff = undefined; // handoff exists — the restart signal carries observability
|
|
812
|
+
}
|
|
813
|
+
else {
|
|
814
|
+
// codex R22: suppress the candidate's ENTIRE restart signal, not just models-tiers — a
|
|
815
|
+
// co-occurring skills/mcp reason would still restart the worker, and with no handoff the
|
|
816
|
+
// replacement can't land ANY of this candidate (same over-budget pull) = the same storm.
|
|
817
|
+
// The blocked reasons stay observable (state + warn) and promote once a handoff exists.
|
|
818
|
+
const blocked = reasons.splice(0, reasons.length);
|
|
819
|
+
planeDeferredNoHandoff = { version: r.effective.version, since: planeDeferredNoHandoff?.since ?? Date.now(), ...(blocked.length > 0 ? { blocked } : {}), planeDeferred: true, candidate: r.effective, ...(r.etag !== undefined ? { candidateEtag: r.etag } : {}) };
|
|
820
|
+
logger.warn("models_tiers_deferred_no_handoff", { version: r.effective.version, blockedReasons: blocked, note: "plane deferred but no NEXT-BOOT handoff (LKG disabled/unwritable, or storage not declared restart-surviving — set CONFIG_LKG_DURABLE=true) — the WHOLE restart signal is withheld (it would loop); the old generation keeps serving" });
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
else if (lkgSurvivesRestart && !lkgPersisted) {
|
|
824
|
+
// codex R27/R31: retention is NOT conditional on restart reasons — a hot-only candidate
|
|
825
|
+
// (autonomy/prompts/teams/tier-less models) with a failed publication would otherwise advance
|
|
826
|
+
// the etag and never retry (304s), and a restart during a center outage would roll back to the
|
|
827
|
+
// older LKG (weaker governance). blocked may be [] = publication-only recovery (R30 promotion
|
|
828
|
+
// publishes no signal for an empty set).
|
|
829
|
+
// codex R27: a DURABLE-DECLARED deployment whose candidate publication is incomplete (skill-body
|
|
830
|
+
// verify failed → LKG not replaced) must not signal restart off that candidate either — the next
|
|
831
|
+
// boot would land the OLD LKG and re-signal while the center stays unavailable. Retain + withhold
|
|
832
|
+
// + per-tick retry-promote, same machinery as the deferred plane (no models-tiers added on
|
|
833
|
+
// promotion — the plane was NOT deferred here). Non-declared deployments keep the pre-existing
|
|
834
|
+
// documented posture (restart converges off a live boot pull when the center is healthy).
|
|
835
|
+
const blocked = reasons.splice(0, reasons.length);
|
|
836
|
+
planeDeferredNoHandoff = { version: r.effective.version, since: planeDeferredNoHandoff?.since ?? Date.now(), blocked, candidate: r.effective, ...(r.etag !== undefined ? { candidateEtag: r.etag } : {}) };
|
|
837
|
+
logger.warn("restart_candidate_publication_incomplete", { version: r.effective.version, blockedReasons: blocked, note: "durable LKG declared but the candidate's publication is incomplete (skill bodies unverified) — restart signal withheld; retried every tick until the cache completes" });
|
|
838
|
+
}
|
|
839
|
+
else if (!planeDeferred && !lkgSurvivesRestart && bootSkillStale && !skillsChangeDetected && !lastPrewarmBodiesVerified) {
|
|
840
|
+
// codex R38: the NON-durable lane's sticky rider must clear the SAME proof gate as the boot/304
|
|
841
|
+
// arms (R37) — a same-generation 200 (center omits/rotates ETags or ignores conditionals) lands
|
|
842
|
+
// here with reasons=[skills] purely from the sticky debt, and publishing it unproven re-opens
|
|
843
|
+
// the restart storm R37 closed (every replacement fails the same body, 200s again, signals
|
|
844
|
+
// again). persistLkgDurable just ran for THIS candidate, so the proof flag is current: unproven
|
|
845
|
+
// ⇒ strip the rider from the publishable set and retain it as visible debt (the same
|
|
846
|
+
// planeDeferredNoHandoff machinery the 304 recovery lane re-verifies and proof-promotes).
|
|
847
|
+
// Proven or durable-handoff ⇒ the rider rides as before. Change-DETECTED skills is exempt
|
|
848
|
+
// (pre-existing posture — the restart pulls the NEW generation, a different repair claim).
|
|
849
|
+
const i = reasons.indexOf("skills");
|
|
850
|
+
if (i >= 0) {
|
|
851
|
+
reasons.splice(i, 1);
|
|
852
|
+
planeDeferredNoHandoff = { version: r.effective.version, since: planeDeferredNoHandoff?.since ?? Date.now(), blocked: ["skills"], candidate: r.effective, ...(r.etag !== undefined ? { candidateEtag: r.etag } : {}) };
|
|
853
|
+
logger.warn("config_boot_skill_debt_withheld", { version: r.effective.version, note: "boot skill debt still unproven on a non-durable deployment (body cache does not verify) — skills restart withheld this tick; re-verified each tick and published once the bodies prove fetchable" });
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
if (reasons.length === 0) {
|
|
857
|
+
pendingRestart = undefined;
|
|
858
|
+
}
|
|
859
|
+
else if (!pendingRestart || pendingRestart.reasons.join(",") !== reasons.join(",")) {
|
|
860
|
+
pendingRestart = { restartRequired: true, reasons, version: r.effective.version, since: Date.now() };
|
|
861
|
+
}
|
|
862
|
+
else {
|
|
863
|
+
pendingRestart = { ...pendingRestart, version: r.effective.version };
|
|
864
|
+
}
|
|
865
|
+
logger.info("sema_registry_refreshed", {
|
|
866
|
+
version: r.effective.version,
|
|
867
|
+
// hot-applied: teams + runtime governance (autonomy/commandPolicy) + MODELS/ROLES/ROSTER (+pricing
|
|
868
|
+
// +per-model keys). Still restart-to-apply: skills/mcp/scenarios + runtime GATES (rate/cost/approval).
|
|
869
|
+
note: "teams + runtime governance + models/roles/roster hot-applied; skills/mcp/runtime-gates/scenarios are restart-to-apply",
|
|
870
|
+
// Structured signal: orchestrator consumes the SAME object via GET /health.restart.
|
|
871
|
+
...(pendingRestart ? { restartRequired: true, restartReasons: pendingRestart.reasons } : {}),
|
|
872
|
+
});
|
|
873
|
+
// BOTH the LKG and the etag advance ONLY after the whole apply/pricing/keyResolver/restart
|
|
874
|
+
// chain succeeded (codex lens-3): if any step above throws, latestEffective keeps pointing at
|
|
875
|
+
// the last FULLY-applied candidate and the etag stays put, so the next poll re-fetches and
|
|
876
|
+
// re-applies the same candidate (idempotent self-heal) — the LKG never names a half-applied one.
|
|
877
|
+
latestEffective = r.effective; // LKG advance (§9.5-2): candidate validated + fully applied
|
|
878
|
+
ccEtag = r.etag;
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
catch (err) {
|
|
883
|
+
logger.warn("sema_registry_refresh_failed", { err: String(err), note: "etag NOT advanced — the same candidate is re-fetched and re-applied next poll" });
|
|
884
|
+
}
|
|
885
|
+
finally {
|
|
886
|
+
refreshInFlight = false;
|
|
887
|
+
}
|
|
888
|
+
};
|
|
889
|
+
const ccTimer = setInterval(() => void refreshTick(), 60_000);
|
|
890
|
+
ccTimer.unref?.();
|
|
891
|
+
// Boot-deferred continuation(二轮复审 F5 改形):到货结果按「迟到的 boot」处理,而不是转普通 tick——
|
|
892
|
+
// 普通 tick 的 restartReasons(undefined, r) 会把 prompts/skills 面全判为差异 → restart → 中心持续慢时
|
|
893
|
+
// 下一进程又 deferred → 无限重启环。迟到 boot:①候选门同 F6(坏形=拒,cadence 重判);②热面
|
|
894
|
+
// teamsOnly apply(boot-only gates/skills/mcp 已按 env 建,迟到不可应用——诚实);③prompts adopt
|
|
895
|
+
// (进程刚起、会话面≈空,失真面⊆重启路径,且 adopt 后 prompts 不再贡献 restart 理由=prompts 无环);
|
|
896
|
+
// ④一次性诚实 restart 信号=剩余 restart-to-apply 面(skills/mcp/scenarios/gates)非空才发;⑤基线
|
|
897
|
+
// `effective`/LKG/etag 落位 → 后续 cadence 与到货值比较,不重复触发。⚠️ 残余环面(诚实记账):center
|
|
898
|
+
// 发布了 skills/mcp 且中心**持续**慢于 boot 预算时,每个新进程仍会为真缺失的 skills 面发一次 restart
|
|
899
|
+
// ——完整解=LKG 落盘(boot 直读上个进程的干净候选),已报 clay 裁量。
|
|
900
|
+
const deferredBootApply = async (r) => {
|
|
901
|
+
if (!r)
|
|
902
|
+
return;
|
|
903
|
+
// F6(LKG 复审三轮):与 cadence tick 同一 single-flight 车道——deferred 的 skill 预热可能拖过 60s,
|
|
904
|
+
// 期间 cadence 落了更新的候选 B,A 迟到完成再覆盖盘/信号=回滚。inFlight 时直接让行(丢弃 deferred
|
|
905
|
+
// 结果无损:cadence 已经/即将拉到更新的),持有 flag 至信号发布后。
|
|
906
|
+
if (refreshInFlight)
|
|
907
|
+
return;
|
|
908
|
+
refreshInFlight = true;
|
|
909
|
+
try {
|
|
910
|
+
await deferredBootApplyInner(r);
|
|
911
|
+
}
|
|
912
|
+
finally {
|
|
913
|
+
refreshInFlight = false;
|
|
914
|
+
}
|
|
915
|
+
};
|
|
916
|
+
const deferredBootApplyInner = async (r) => {
|
|
917
|
+
const promptsRaw = r.effective.prompts;
|
|
918
|
+
const promptsGate = promptsRaw !== undefined ? validatePromptsDomain(promptsRaw) : { ok: true };
|
|
919
|
+
if ((r.domainErrors ?? []).length > 0 || !promptsGate.ok) {
|
|
920
|
+
logger.warn("config_boot_deferred_candidate_rejected", {
|
|
921
|
+
version: r.effective.version,
|
|
922
|
+
errors: [...(r.domainErrors ?? []).map((de) => ({ domain: de.domain, error: de.error })), ...(promptsGate.ok ? [] : [{ domain: "prompts", error: promptsGate.error }])],
|
|
923
|
+
note: "late boot pull carried an invalid candidate — env fallback keeps serving; the refresh cadence re-judges the same tree (etag not advanced)",
|
|
924
|
+
});
|
|
925
|
+
return;
|
|
926
|
+
}
|
|
927
|
+
if (ccRef?.dryRun) {
|
|
928
|
+
logEffectiveDiff(config, r.effective, logger);
|
|
929
|
+
ccEtag = r.etag;
|
|
930
|
+
return;
|
|
931
|
+
}
|
|
932
|
+
// codex R10 (deferred-boot twin of the refresh-lane guard): the Runner was built from the ENV plane before
|
|
933
|
+
// this late arrival — if it froze a tier-expanded copy, the arriving center plane must not hot-apply
|
|
934
|
+
// (admission/Runner split). Tier-less env boot (the common deferred-boot shape) keeps true hot-apply.
|
|
935
|
+
const planeDeferredLate = (runnerTierFrozen || planeHasActiveTiers(r.effective)) && modelPlaneChanged(appliedPlaneEff, r.effective);
|
|
936
|
+
applyEffective(config, r.effective, logger, { teamsOnly: true, sealedKeys, ...(planeDeferredLate ? { deferModelPlane: true } : {}) });
|
|
937
|
+
if (planeDeferredLate)
|
|
938
|
+
logger.warn("models_tiers_plane_deferred", { version: r.effective.version, note: "tier-frozen Runner (env tiers): the late-boot center model plane is NOT hot-applied — restart applies it" });
|
|
939
|
+
else {
|
|
940
|
+
appliedPlaneEff = r.effective;
|
|
941
|
+
planeDeferredNoHandoff = undefined;
|
|
942
|
+
}
|
|
943
|
+
await adoptCenterPrompts(r.effective, "boot-deferred");
|
|
944
|
+
if (!planeDeferredLate)
|
|
945
|
+
markRosterLanded(r.effective); // codex R13: same guard as the refresh lane — never open readiness off an unapplied plane
|
|
946
|
+
mutateInPlace(pricing, buildPricing(config.models));
|
|
947
|
+
keyResolver = createKeyResolver(config.modelApiKeyEnv, process.env, config.modelApiKeys);
|
|
948
|
+
effective = r.effective; // restart 比较基线=到货值(cadence 不再重复触发)
|
|
949
|
+
latestEffective = r.effective;
|
|
950
|
+
ccEtag = r.etag;
|
|
951
|
+
const lkgPersistedLate = await persistLkgDurable(r.effective, r.etag); // F2 parity:信号可见前落盘+skill 正文预热
|
|
952
|
+
const reasons = restartReasons(undefined, r.effective); // (prompts 不在 restart slices——热采用已在上方 adopt)
|
|
953
|
+
// codex R15 (late-boot twin): tiered env boot + tier-less late candidate ⇒ deferral fires but BOTH slice
|
|
954
|
+
// fingerprints reduce to null (undefined baseline, tier-less candidate) — no models-tiers reason, plane
|
|
955
|
+
// deferred forever. Deferral is the ground truth; force the reason so the orchestrator restarts and the
|
|
956
|
+
// next process lands the candidate at boot. codex R16: forced ONLY with a confirmed LKG handoff — without
|
|
957
|
+
// it the restarted process re-enters this exact lane (env plane, late candidate) = restart storm; keep
|
|
958
|
+
// serving the old generation and warn instead.
|
|
959
|
+
// codex R19 (late-boot twin): handoff status decides, regardless of whether change-detection already
|
|
960
|
+
// produced the reason — an active-tier candidate over an env boot deterministically carries models-tiers
|
|
961
|
+
// from restartReasons(undefined, candidate), and letting it through with no handoff is the R16 storm.
|
|
962
|
+
if (planeDeferredLate) {
|
|
963
|
+
if (lkgPersistedLate) {
|
|
964
|
+
if (!reasons.includes("models-tiers"))
|
|
965
|
+
reasons.push("models-tiers");
|
|
966
|
+
planeDeferredNoHandoff = undefined;
|
|
967
|
+
}
|
|
968
|
+
else {
|
|
969
|
+
// codex R22 (late-boot twin): withhold the WHOLE signal — co-occurring skills/mcp reasons would
|
|
970
|
+
// restart into the same over-budget/no-LKG lane and loop.
|
|
971
|
+
const blocked = reasons.splice(0, reasons.length);
|
|
972
|
+
planeDeferredNoHandoff = { version: r.effective.version, since: planeDeferredNoHandoff?.since ?? Date.now(), ...(blocked.length > 0 ? { blocked } : {}), planeDeferred: true, candidate: r.effective, ...(r.etag !== undefined ? { candidateEtag: r.etag } : {}) };
|
|
973
|
+
logger.warn("models_tiers_deferred_no_handoff", { version: r.effective.version, blockedReasons: blocked, note: "late-boot plane deferred but no NEXT-BOOT handoff (see CONFIG_LKG_DURABLE) — the WHOLE restart signal is withheld (it would loop); old generation keeps serving" });
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
else if (planeDeferredLate === false && lkgSurvivesRestart && !lkgPersistedLate) {
|
|
977
|
+
// codex R31 (late-boot twin): retain failed publications even with zero restart reasons.
|
|
978
|
+
// codex R27 (late-boot twin): durable-declared + incomplete publication ⇒ retain + withhold + retry.
|
|
979
|
+
const blocked = reasons.splice(0, reasons.length);
|
|
980
|
+
planeDeferredNoHandoff = { version: r.effective.version, since: planeDeferredNoHandoff?.since ?? Date.now(), blocked, candidate: r.effective, ...(r.etag !== undefined ? { candidateEtag: r.etag } : {}) };
|
|
981
|
+
logger.warn("restart_candidate_publication_incomplete", { version: r.effective.version, blockedReasons: blocked, note: "late-boot candidate publication incomplete on a durable-declared deployment — restart signal withheld; retried every tick" });
|
|
982
|
+
}
|
|
983
|
+
if (reasons.length > 0)
|
|
984
|
+
pendingRestart = { restartRequired: true, reasons, version: r.effective.version, since: Date.now() };
|
|
985
|
+
logger.info("config_loaded_deferred", { source: configProvider.kind, version: r.effective.version, models: (r.effective.models?.models ?? []).filter((m) => m.enabled !== false).length, ...(reasons.length > 0 ? { restartRequired: true, restartReasons: reasons } : {}) });
|
|
986
|
+
};
|
|
987
|
+
// LKG 起服的进程有完整 boot 基线 → 到货走普通 tick(候选门+hot+对 LKG 的 restartReasons,通常
|
|
988
|
+
// 零差异=无信号无环);env 起服(无 LKG)才走「迟到 boot」路径。
|
|
989
|
+
if (bootConfigPending)
|
|
990
|
+
void bootConfigPending.then((r) => (lkgBooted ? refreshTick(r) : deferredBootApply(r)), () => { });
|
|
991
|
+
}
|
|
992
|
+
},
|
|
993
|
+
};
|
|
994
|
+
}
|
|
995
|
+
//# sourceMappingURL=config-center.js.map
|