@sema-agent/server 7.120.0 → 7.121.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/MIGRATION.md +75 -0
  2. package/USAGE.md +67 -20
  3. package/deploy/sema-up/chart/templates/server-v4pro.yaml +2 -2
  4. package/deploy/sema-up/chart/templates/server.yaml +2 -2
  5. package/deploy/sema-up/stack/sema-stack-cluster.yaml +1 -1
  6. package/dist/boot/config-center.d.ts +5 -7
  7. package/dist/boot/config-center.js +33 -9
  8. package/dist/boot/engine-lease.d.ts +1 -1
  9. package/dist/boot/resolve-spec.js +1 -0
  10. package/dist/boot/shutdown.d.ts +9 -10
  11. package/dist/boot/shutdown.js +54 -16
  12. package/dist/boot/stage-09-leader.d.ts +2 -9
  13. package/dist/boot/stage-10-http-server.d.ts +1 -9
  14. package/dist/boot/stage-10-http-server.js +31 -8
  15. package/dist/capabilities/composite-members.d.ts +1 -1
  16. package/dist/capabilities/composite-members.js +1 -1
  17. package/dist/config-catalog.js +11 -5
  18. package/dist/config-center/center-request.d.ts +16 -1
  19. package/dist/config-center/center-request.js +36 -5
  20. package/dist/config-types.d.ts +39 -3
  21. package/dist/config-types.js +6 -0
  22. package/dist/config.d.ts +33 -4
  23. package/dist/config.js +99 -7
  24. package/dist/fleet/drain-entry.d.ts +87 -0
  25. package/dist/fleet/drain-entry.js +35 -0
  26. package/dist/fleet/engine-signal.d.ts +144 -0
  27. package/dist/fleet/engine-signal.js +60 -0
  28. package/dist/fleet/serving-lease.d.ts +47 -0
  29. package/dist/fleet/serving-lease.js +235 -0
  30. package/dist/fleet-client.d.ts +35 -19
  31. package/dist/fleet-client.js +49 -25
  32. package/dist/http/approval-sweeps.js +3 -5
  33. package/dist/http/dispatch.js +18 -13
  34. package/dist/http/health-self-report.d.ts +60 -0
  35. package/dist/http/health-self-report.js +53 -0
  36. package/dist/http/principal-gate.d.ts +8 -1
  37. package/dist/http/principal-gate.js +1 -0
  38. package/dist/http/readiness.d.ts +25 -5
  39. package/dist/http/readiness.js +31 -6
  40. package/dist/http/resume-legs.js +1 -1
  41. package/dist/http/route-ctx.d.ts +6 -4
  42. package/dist/http/route-table.d.ts +3 -0
  43. package/dist/http/route-table.js +4 -1
  44. package/dist/http/routes/a2a-serve.d.ts +1 -1
  45. package/dist/http/routes/a2a-serve.js +5 -5
  46. package/dist/http/routes/admin-drain.d.ts +12 -31
  47. package/dist/http/routes/admin-drain.js +15 -9
  48. package/dist/http/routes/approvals-assistant.d.ts +3 -3
  49. package/dist/http/routes/approvals-assistant.js +3 -3
  50. package/dist/http/routes/capabilities.js +2 -1
  51. package/dist/http/routes/devices.js +3 -2
  52. package/dist/http/routes/leader.d.ts +1 -1
  53. package/dist/http/routes/leader.js +1 -1
  54. package/dist/http/routes/memory-consolidation.d.ts +1 -1
  55. package/dist/http/routes/memory-consolidation.js +1 -1
  56. package/dist/http/routes/notify-wake.d.ts +1 -1
  57. package/dist/http/routes/notify-wake.js +1 -1
  58. package/dist/http/routes/rules.js +4 -3
  59. package/dist/http/routes/runs.d.ts +2 -2
  60. package/dist/http/routes/runs.js +2 -2
  61. package/dist/http/routes/side-query.d.ts +1 -1
  62. package/dist/http/routes/side-query.js +1 -1
  63. package/dist/http/routes/tasks.d.ts +2 -2
  64. package/dist/http/routes/tasks.js +3 -3
  65. package/dist/http/server.d.ts +7 -21
  66. package/dist/http/server.js +20 -1
  67. package/dist/http/wire-types.d.ts +7 -1
  68. package/dist/model-reach-scope.d.ts +15 -3
  69. package/dist/model-reach-scope.js +3 -2
  70. package/dist/model-unreachable-notice.d.ts +2 -1
  71. package/dist/model-unreachable-notice.js +1 -0
  72. package/dist/observability/fail-open.d.ts +16 -0
  73. package/dist/observability/fail-open.js +16 -0
  74. package/dist/plugins/one-time-migrations.js +1 -1
  75. package/dist/plugins/permission-rule-store-sql.d.ts +2 -1
  76. package/dist/plugins/remote-env-device.js +0 -1
  77. package/dist/request-key-closure.js +1 -1
  78. package/dist/rules-consent.d.ts +6 -2
  79. package/dist/rules-consent.js +3 -0
  80. package/dist/runs.js +1 -1
  81. package/dist/tool-approval.d.ts +7 -0
  82. package/dist/tool-approval.js +4 -1
  83. package/dist/trace/core-keyset-guard.d.ts +2 -2
  84. package/dist/trace/self-minted-wire.d.ts +64 -0
  85. package/dist/trace/self-minted-wire.js +30 -0
  86. package/package.json +2 -2
@@ -0,0 +1,235 @@
1
+ import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { SERVING_LEASE_TTL_DEFAULT_MS, readServingLeaseResponse } from "@sema-agent/settings-schema/fleet";
4
+ import { centerFetch } from "../config-center/center-request.js";
5
+ import { recordFailOpen } from "../observability/fail-open.js";
6
+ import { leaseDrainActionId } from "./drain-entry.js";
7
+ export const SERVING_LEASE_FETCH_TIMEOUT_MS = 5_000;
8
+ export const SERVING_LEASE_DENIED_FILE = "fleet-serving-lease-denied.json";
9
+ const RENEW_MIN_INTERVAL_MS = 1_000;
10
+ function denialReasonOf(d) {
11
+ switch (d.from) {
12
+ case "center":
13
+ case "data-root":
14
+ return d.reason;
15
+ case "data-root-unreadable":
16
+ return "unreadable_record";
17
+ default: {
18
+ const unhandled = d;
19
+ throw new Error(`serving lease: unknown denial source ${JSON.stringify(unhandled)}`);
20
+ }
21
+ }
22
+ }
23
+ function denialLeadOf(d) {
24
+ switch (d.from) {
25
+ case "center":
26
+ return `the fleet center denied this instance's serving lease (${d.reason})`;
27
+ case "data-root":
28
+ return `the fleet center denied this generation's serving lease earlier (${d.reason}) and the data root remembers it`;
29
+ case "data-root-unreadable":
30
+ return "the persisted lease record could not be read; treated as denied (fail-closed) — delete the record only if this generation is still meant to serve";
31
+ default: {
32
+ const unhandled = d;
33
+ throw new Error(`serving lease: unknown denial source ${JSON.stringify(unhandled)}`);
34
+ }
35
+ }
36
+ }
37
+ export function createServingLeaseClient(o) {
38
+ const mono = o.clock?.mono ?? (() => performance.now());
39
+ const epoch = o.clock?.epoch ?? Date.now;
40
+ const f = o.fetchImpl ?? fetch;
41
+ const stickyPath = o.dataRoot === undefined ? undefined : join(o.dataRoot, SERVING_LEASE_DENIED_FILE);
42
+ let enabled = false;
43
+ let enabledAtMono = 0;
44
+ let denied = false;
45
+ let stopped = false;
46
+ let lease;
47
+ let nextInMs = SERVING_LEASE_TTL_DEFAULT_MS / 3;
48
+ let timer;
49
+ let inFlight;
50
+ let failStreak = 0;
51
+ const clear = () => {
52
+ if (timer !== undefined)
53
+ clearTimeout(timer);
54
+ timer = undefined;
55
+ };
56
+ const schedule = (ms) => {
57
+ clear();
58
+ if (stopped || denied || !enabled)
59
+ return;
60
+ timer = setTimeout(() => void renewNow(), Math.max(RENEW_MIN_INTERVAL_MS, ms));
61
+ timer.unref?.();
62
+ };
63
+ const persistDenial = (reason) => {
64
+ if (stickyPath === undefined) {
65
+ o.logger?.error("fleet_serving_lease_denial_not_persisted", { note: "no data root to remember the denial in — it holds in this process only; after a restart this instance serves until its first renewal is answered" });
66
+ return false;
67
+ }
68
+ const record = { worker: o.center.worker, generation: o.generation, reason, at: epoch() };
69
+ const tmp = `${stickyPath}.tmp-${process.pid}`;
70
+ try {
71
+ writeFileSync(tmp, `${JSON.stringify(record)}\n`, { mode: 0o600 });
72
+ renameSync(tmp, stickyPath);
73
+ return true;
74
+ }
75
+ catch (err) {
76
+ o.logger?.error("fleet_serving_lease_denial_not_persisted", { path: stickyPath, error: err instanceof Error ? err.message : String(err), note: "the denial holds in this process only; after a restart this instance serves until its first renewal is answered (the center denies again then)" });
77
+ return false;
78
+ }
79
+ };
80
+ const enterDenied = (denial) => {
81
+ if (denied)
82
+ return;
83
+ denied = true;
84
+ lease = undefined;
85
+ clear();
86
+ const reason = denialReasonOf(denial);
87
+ const sticky = denial.from !== "center" ? "from-data-root" : persistDenial(reason) ? "new" : "process-only";
88
+ if (sticky === "process-only")
89
+ recordFailOpen("server.fleet.denied-marker-unpersisted", `worker=${o.center.worker} generation=${o.generation}`);
90
+ o.logger?.warn("fleet_serving_lease_denied", {
91
+ generation: o.generation,
92
+ worker: o.center.worker,
93
+ reason,
94
+ sticky,
95
+ note: `${denialLeadOf(denial)} — draining now and stopping at the drain deadline; ${sticky === "process-only" ? "the denial could NOT be recorded in the data root, so a restart of this generation does not remember it" : "the denial is remembered across restarts of this generation"}`,
96
+ });
97
+ o.drain({ actionId: leaseDrainActionId(o.generation), deadlineAt: epoch() + o.drainWindowMs() });
98
+ };
99
+ const renewOnce = async () => {
100
+ if (stopped || denied || !enabled)
101
+ return;
102
+ const sendMono = mono();
103
+ const sendEpoch = epoch();
104
+ const request = { worker: o.center.worker, generation: o.generation, instanceId: o.instanceId };
105
+ let reading;
106
+ let why = "";
107
+ try {
108
+ const { res } = await centerFetch(o.center, { kind: "fleet-serving-lease" }, { method: "POST", json: request, timeoutMs: SERVING_LEASE_FETCH_TIMEOUT_MS, fetchImpl: f });
109
+ if (res.ok) {
110
+ reading = readServingLeaseResponse(await res.json().catch(() => undefined));
111
+ if (reading === undefined)
112
+ why = "reply is not a serving-lease verdict";
113
+ }
114
+ else {
115
+ why = `HTTP ${res.status}`;
116
+ }
117
+ }
118
+ catch (err) {
119
+ why = err instanceof Error ? err.message : String(err);
120
+ }
121
+ if (reading === undefined || reading.verdict === "unknown") {
122
+ const unconfirmed = reading === undefined ? why : "verdict=unknown (the center holds no lifecycle object for this instance right now)";
123
+ recordFailOpen("server.fleet.serving-lease-unconfirmed", `worker=${o.center.worker} generation=${o.generation} why=${unconfirmed}`);
124
+ failStreak += 1;
125
+ if (failStreak === 1)
126
+ o.logger?.warn("fleet_serving_lease_unconfirmed", { error: unconfirmed, note: "the serving lease could not be renewed (the center is unreachable or did not confirm this instance) — this instance keeps serving until the lease expires and then for FLEET_LEASE_OUTAGE_MAX_MS more before it refuses new work (unconfirmed is never a denial)" });
127
+ schedule(nextInMs);
128
+ return;
129
+ }
130
+ if (failStreak > 0)
131
+ o.logger?.info("fleet_serving_lease_recovered", { after: failStreak });
132
+ failStreak = 0;
133
+ switch (reading.verdict) {
134
+ case "granted":
135
+ if (denied || !enabled)
136
+ return;
137
+ lease = { expiresAtMono: sendMono + reading.ttlMs, leaseUntil: sendEpoch + reading.ttlMs };
138
+ nextInMs = reading.ttlMs / 3;
139
+ schedule(nextInMs);
140
+ return;
141
+ case "denied":
142
+ enterDenied({ from: "center", reason: reading.reason });
143
+ return;
144
+ default: {
145
+ const unhandled = reading;
146
+ throw new Error(`serving lease: unknown verdict ${JSON.stringify(unhandled)}`);
147
+ }
148
+ }
149
+ };
150
+ const renewNow = () => {
151
+ if (inFlight !== undefined)
152
+ return inFlight;
153
+ inFlight = renewOnce().finally(() => {
154
+ inFlight = undefined;
155
+ });
156
+ return inFlight;
157
+ };
158
+ return {
159
+ noteCenterCapabilities(caps) {
160
+ if (stopped || denied)
161
+ return;
162
+ const on = caps?.servingLease === true;
163
+ if (!on)
164
+ recordFailOpen("server.fleet.serving-lease-capability-absent", `worker=${o.center.worker} generation=${o.generation} was=${enabled ? "enabled" : "none"}`);
165
+ if (on && !enabled) {
166
+ enabled = true;
167
+ enabledAtMono = mono();
168
+ o.logger?.info("fleet_serving_lease_enabled", { generation: o.generation, worker: o.center.worker });
169
+ void renewNow();
170
+ }
171
+ else if (!on && enabled) {
172
+ enabled = false;
173
+ lease = undefined;
174
+ clear();
175
+ o.logger?.warn("fleet_serving_lease_disabled", { note: "the fleet center no longer announces the servingLease capability — the lease is out of effect (leaseState none) until it does again" });
176
+ }
177
+ },
178
+ view() {
179
+ if (denied)
180
+ return { state: "denied" };
181
+ if (!enabled)
182
+ return { state: "none" };
183
+ const now = mono();
184
+ if (lease !== undefined && now < lease.expiresAtMono)
185
+ return { state: "granted", leaseUntil: lease.leaseUntil };
186
+ const expiredAt = lease?.expiresAtMono ?? enabledAtMono;
187
+ return { state: "expired", pastOutageMax: now - expiredAt >= o.outageMaxMs };
188
+ },
189
+ resumeStickyDenial() {
190
+ if (stickyPath === undefined || denied)
191
+ return;
192
+ let text;
193
+ try {
194
+ text = readFileSync(stickyPath, "utf8");
195
+ }
196
+ catch (err) {
197
+ if (err.code === "ENOENT")
198
+ return;
199
+ o.logger?.error("fleet_serving_lease_denial_unreadable", { path: stickyPath, error: err instanceof Error ? err.message : String(err), note: "a remembered lease denial could not be read — treating this instance as denied (fail-closed); delete the file only if this generation is still meant to serve" });
200
+ enterDenied({ from: "data-root-unreadable" });
201
+ return;
202
+ }
203
+ let parsed;
204
+ try {
205
+ parsed = JSON.parse(text);
206
+ }
207
+ catch {
208
+ parsed = undefined;
209
+ }
210
+ const record = typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : undefined;
211
+ if (record === undefined || typeof record.generation !== "string" || typeof record.worker !== "string") {
212
+ o.logger?.error("fleet_serving_lease_denial_unreadable", { path: stickyPath, note: "a remembered lease denial is malformed — treating this instance as denied (fail-closed); delete the file only if this generation is still meant to serve" });
213
+ enterDenied({ from: "data-root-unreadable" });
214
+ return;
215
+ }
216
+ if (record.worker !== o.center.worker || record.generation !== o.generation) {
217
+ o.logger?.info("fleet_serving_lease_denial_stale", { path: stickyPath, recorded: { worker: record.worker, generation: record.generation }, current: { worker: o.center.worker, generation: o.generation } });
218
+ try {
219
+ rmSync(stickyPath, { force: true });
220
+ }
221
+ catch (err) {
222
+ o.logger?.error("fleet_serving_lease_denial_stale_unremovable", { path: stickyPath, error: err instanceof Error ? err.message : String(err), note: "a stale lease denial of another generation could not be removed — it does not bind this generation; remove it by hand" });
223
+ }
224
+ return;
225
+ }
226
+ enterDenied({ from: "data-root", reason: typeof record.reason === "string" ? record.reason : "unknown_reason" });
227
+ },
228
+ renewNow,
229
+ stop() {
230
+ stopped = true;
231
+ clear();
232
+ },
233
+ };
234
+ }
235
+ //# sourceMappingURL=serving-lease.js.map
@@ -3,8 +3,8 @@
3
3
  * `/fleet`,center 端点 = sema-registry `/api/fleet/*`)。设计要点:
4
4
  * - **可选分层(D3),两条环各自有门(S-866 ④)**:客户端在中心车道配齐时启动(`centerLaneConfigured` —— baseUrl + 声明了
5
5
  * 凭证 + worker,与租约管理器、用量累加器**同一只**谓词);**用量环**只要这三件;**公告环**(announce / 心跳 / 注销)另要
6
- * `FLEET_ADVERTISE_ADDRESS` 显式声明(可达性是运维的话,不自猜:k8s pod IP / docker alias / 127.0.0.1:port 只有部署环境自己
7
- * 知道)。不配中心的部署零行为变化;配了中心而不公告的引擎只报用量、不注册(7.105.0 及以前它连用量都不报)。
6
+ * 配置席 `fleetAdvertiseAddress`(env `FLEET_ADVERTISE_ADDRESS`,`loadConfig` 解析一次;可达性是运维的话,不自猜:k8s pod IP /
7
+ * docker alias / 127.0.0.1:port 只有部署环境自己知道)。不配中心的部署零行为变化;配了中心而不公告的引擎只报用量、不注册(7.105.0 及以前它连用量都不报)。
8
8
  * - **announce=注册=心跳同形状幂等 upsert**:boot 后(listen 之后)POST 一次 + 每 heartbeatMs 续。
9
9
  * `ready` / `draining` 两字段由**同一只**就绪谓词的同一次读铸(S-740:`http/readiness.ts` 的 `readinessOf`,
10
10
  * 与 `/health.ready`、`GET /readyz`、计费提交门同源):`ready = verdict.ready`(= roster 已落 ∧ ¬draining),
@@ -25,8 +25,8 @@
25
25
  * 向前合并不会双计 —— 只延迟,不丢量)。
26
26
  * - 全部 fire-and-forget + swallow-guarded:fleet 面是运维旁路,绝不影响任务面;网络错误只 warn 一次/连败类。
27
27
  */
28
- import { UsageReport, type ModelRefusalsSignal, type RestartSignal } from "@sema-agent/settings-schema/fleet";
29
- import type { ReadinessVerdict } from "./http/readiness.js";
28
+ import { UsageReport, type CapabilityReading, type CenterCapabilities, type ModelRefusalsSignal } from "@sema-agent/settings-schema/fleet";
29
+ import { type EngineSignalFacts } from "./fleet/engine-signal.js";
30
30
  import { type CenterEndpoint } from "./config-center/center-request.js";
31
31
  /** Bound on distinct principal×model keys per window — a runaway/multi-tenant burst collapses into
32
32
  * `__other__` instead of growing unbounded (same posture as budget.ts cardinalityGuard). */
@@ -79,15 +79,17 @@ export interface FleetClientOpts {
79
79
  * S-866 ④:**缺席 = 公告环不装**(不注册、不心跳、不注销;`announceNow` 是空操作),用量环照装。 */
80
80
  advertiseAddress?: string | undefined;
81
81
  version: string;
82
- /** S-740:就绪判(装配层接 `readinessOf(deployment)` —— HTTP 面读的同一个 deps 组)。每条公告现读**一次**,
83
- * `ready` 与 `draining` 两字段都由这一次读铸(不会撕裂成 `ready:true` + `draining:true`)。 */
84
- readiness: () => ReadinessVerdict;
85
- /** S-741:restart 信号(装配层接 `/health` 同一只取值口 `deployment.restartState`)。每条公告现读一次;`undefined` ⇒ 公告不带
86
- * `restart` 键。必填:心跳面没有「不知道要不要重启」这一形。 */
87
- restart: () => RestartSignal | undefined;
82
+ /** 编排器片 2 件 1:引擎信号的**一只铸点**(装配层接 `mintEngineSignal(deployment)` —— `/health` 与 `/readyz` 读的同一只、同一个 deps 组)。
83
+ * 每条公告现铸**一次**,`ready` / `draining` / `restart` / 代际与租约三键 / 世代账都取自这一次(不会撕裂成 `ready:true` + `draining:true`,
84
+ * 也不会出现心跳与 `/health` 各算一份重启或租约)。修前是 `readiness` + `restart` 两只取值口(S-740 / S-741),合进这一只。 */
85
+ signal: () => EngineSignalFacts;
88
86
  /** S-768:模型拒因信号(装配层接 `/health` 同一只取值口 `deployment.modelRefusalsState`)。每条公告现读一次;`undefined` ⇒ 公告不带
89
87
  * `modelRefusals` 键。必填:与 `restart` 同 —— 心跳面没有「不知道有没有拒」这一形(老引擎不发这个键,中心按 `version` 判「不可知」)。 */
90
88
  modelRefusals: () => ModelRefusalsSignal | undefined;
89
+ /** 编排器片 2:每一次**能力读数**交出去一次(装配层接服务租约客户端的 `noteCenterCapabilities`)。规则见 {@link capabilityReadingOf}:
90
+ * 只有 `declared`(交出宣告的位)与 `undeclared`(交 `undefined` = 老中心没宣告)是读数;`unreadable`(体读不出 / 不是对象 /
91
+ * `capabilities` 键在场而形坏)不是读数 ⇒ 不调,上一次读数与它的钟原样。失败的公告同样不调(没听到 ≠ 没有)。 */
92
+ onCenterCapabilities?: (caps: CenterCapabilities | undefined) => void;
91
93
  usage?: FleetUsageAccumulator;
92
94
  heartbeatMs?: number;
93
95
  usageWindowMs?: number;
@@ -114,25 +116,39 @@ export interface FleetClient {
114
116
  flushFinalUsage: () => Promise<void>;
115
117
  }
116
118
  /**
117
- * Start the worker-side fleet loops. Caller has already gated on the center lane (startFleetClientFromEnv below).
119
+ * 心跳 2xx 答复读不成能力读数的原因 —— 一张闭集:`body` = 答复体本身读不出(空体 / 截断 / 非 JSON),其余两词 = 契约读器
120
+ * `readCapabilityReading` 的 `unreadable` 两因(`not-an-object` / `invalid-capabilities`)。
121
+ */
122
+ export type AnnounceReplyFlaw = "body" | Extract<CapabilityReading, {
123
+ kind: "unreadable";
124
+ }>["why"];
125
+ /** 本仓对一次心跳答复的读法:契约三态,`unreadable` 的原因扩成 {@link AnnounceReplyFlaw}(`detail` = 读不出体时的那句错误,当数据交出)。 */
126
+ export type AnnounceReplyReading = Exclude<CapabilityReading, {
127
+ kind: "unreadable";
128
+ }> | {
129
+ readonly kind: "unreadable";
130
+ readonly why: AnnounceReplyFlaw;
131
+ readonly detail?: string;
132
+ };
133
+ /**
134
+ * Start the worker-side fleet loops. Caller has already gated on the center lane (startFleetClientFromConfig below).
118
135
  * The announce loop runs iff `advertiseAddress` is present; the usage loop iff `usage` is present (S-866 ④: the two
119
136
  * loops no longer share one gate).
120
137
  */
121
138
  export declare function startFleetClient(opts: FleetClientOpts): FleetClient;
122
139
  /**
123
- * Env/config gate (D3 可选分层;S-866 ④ 两条环各自有门): start when the center lane is configured —
140
+ * Config gate (D3 可选分层;S-866 ④ 两条环各自有门): start when the center lane is configured —
124
141
  * {@link centerLaneConfigured}(baseUrl + a DECLARED credential + worker — S-651: declared, not "currently readable": a token
125
142
  * file that is unreadable right now still starts the client, every request re-reads it), the SAME predicate the lease
126
143
  * manager and the usage accumulator use. The usage loop needs nothing more; the announce loop additionally needs the
127
- * operator's explicit reachable address (`FLEET_ADVERTISE_ADDRESS`; a malformed one refuses boot). Lane not configured →
128
- * undefined (zero behavior change for non-fleet deployments); an address set while only the worker is missing is warned
129
- * (half-configured announce).
144
+ * operator's explicit reachable address — the config seat `fleetAdvertiseAddress` (env `FLEET_ADVERTISE_ADDRESS`, parsed once by
145
+ * `loadConfig`: a malformed one refuses boot there). Lane not configured → undefined (zero behavior change for non-fleet
146
+ * deployments); an address set while only the worker is missing is warned (half-configured announce).
130
147
  */
131
- export declare function startFleetClientFromEnv(cfg: {
148
+ export declare function startFleetClientFromConfig(cfg: {
132
149
  configCenter?: (CenterEndpoint & {
133
150
  worker?: string;
134
151
  }) | undefined;
135
- }, opts: Omit<FleetClientOpts, "center" | "advertiseAddress"> & {
136
- advertiseAddress?: string | undefined;
137
- }): FleetClient | undefined;
152
+ fleetAdvertiseAddress?: string | undefined;
153
+ }, opts: Omit<FleetClientOpts, "center" | "advertiseAddress">): FleetClient | undefined;
138
154
  //# sourceMappingURL=fleet-client.d.ts.map
@@ -1,6 +1,7 @@
1
- import { InstanceAnnouncement, UsageReport, DEFAULT_HEARTBEAT_MS } from "@sema-agent/settings-schema/fleet";
2
- import { buildDisplaySafeUrl } from "./trace/redact.js";
1
+ import { InstanceAnnouncement, UsageReport, DEFAULT_HEARTBEAT_MS, readCapabilityReading } from "@sema-agent/settings-schema/fleet";
2
+ import { drainingFlag, fleetMembershipKeys } from "./fleet/engine-signal.js";
3
3
  import { centerFetch, centerLaneConfigured } from "./config-center/center-request.js";
4
+ import { recordFailOpen } from "./observability/fail-open.js";
4
5
  export const FLEET_USAGE_KEY_CAP = 1024;
5
6
  export const FLEET_CLIENT_FETCH_TIMEOUT_MS = 5000;
6
7
  export class FleetUsageAccumulator {
@@ -74,6 +75,16 @@ async function rejectionSnippet(res) {
74
75
  return "";
75
76
  }
76
77
  }
78
+ async function capabilityReadingOf(res) {
79
+ let body;
80
+ try {
81
+ body = await res.json();
82
+ }
83
+ catch (err) {
84
+ return { kind: "unreadable", why: "body", detail: err instanceof Error ? err.message : String(err) };
85
+ }
86
+ return readCapabilityReading(body);
87
+ }
77
88
  export function startFleetClient(opts) {
78
89
  const f = opts.fetchImpl ?? fetch;
79
90
  const heartbeatMs = opts.heartbeatMs ?? DEFAULT_HEARTBEAT_MS;
@@ -81,28 +92,55 @@ export function startFleetClient(opts) {
81
92
  const send = (route, method, json) => centerFetch(opts.center, { kind: route }, { method, json, timeoutMs: FLEET_CLIENT_FETCH_TIMEOUT_MS, fetchImpl: f });
82
93
  let stopped = false;
83
94
  let failStreak = 0;
95
+ let unreadableStreak = 0;
84
96
  const address = opts.advertiseAddress;
85
97
  const announceOnce = async () => {
86
98
  if (stopped || address === undefined)
87
99
  return;
88
100
  try {
89
- const verdict = opts.readiness();
90
- const restart = opts.restart();
101
+ const signal = opts.signal();
91
102
  const modelRefusals = opts.modelRefusals();
92
103
  const body = {
93
104
  instanceId: opts.instanceId,
94
105
  worker: opts.center.worker,
95
106
  address,
96
107
  version: opts.version,
97
- ready: verdict.ready,
98
- draining: !verdict.ready && verdict.reasons.includes("draining"),
108
+ ready: signal.verdict.ready,
109
+ draining: drainingFlag(signal),
99
110
  heartbeatMs,
100
- ...(restart !== undefined ? { restart } : {}),
111
+ ...(signal.restart !== undefined ? { restart: signal.restart } : {}),
101
112
  ...(modelRefusals !== undefined ? { modelRefusals } : {}),
113
+ ...fleetMembershipKeys(signal),
102
114
  };
103
115
  const { res } = await send("fleet-instances", "POST", body);
104
116
  if (!res.ok)
105
117
  throw new Error(`announce ${res.status}${await rejectionSnippet(res)}`);
118
+ if (opts.onCenterCapabilities !== undefined) {
119
+ const reading = await capabilityReadingOf(res);
120
+ switch (reading.kind) {
121
+ case "declared":
122
+ case "undeclared":
123
+ unreadableStreak = 0;
124
+ opts.onCenterCapabilities(reading.kind === "declared" ? reading.capabilities : undefined);
125
+ break;
126
+ case "unreadable":
127
+ recordFailOpen("server.fleet.announce-reply-unreadable", `why=${reading.why} leaseState=${signal.lease.state}`);
128
+ unreadableStreak += 1;
129
+ if (unreadableStreak === 1) {
130
+ opts.logger?.warn("fleet_announce_reply_unreadable", {
131
+ status: res.status,
132
+ why: reading.why,
133
+ ...(reading.detail !== undefined ? { error: reading.detail } : {}),
134
+ note: "the fleet center's heartbeat reply is not a capability reading (unreadable body, not an object, or a malformed capabilities value) — no reading taken; the last one stands (the serving lease keeps its state and outage clock)",
135
+ });
136
+ }
137
+ break;
138
+ default: {
139
+ const unhandled = reading;
140
+ throw new Error(`heartbeat reply: unknown capability reading ${JSON.stringify(unhandled)}`);
141
+ }
142
+ }
143
+ }
106
144
  if (failStreak > 0)
107
145
  opts.logger?.info("fleet_announce_recovered", { after: failStreak });
108
146
  failStreak = 0;
@@ -156,7 +194,7 @@ export function startFleetClient(opts) {
156
194
  };
157
195
  const announcing = address !== undefined;
158
196
  if (announcing)
159
- void announceNow();
197
+ queueMicrotask(() => void announceNow());
160
198
  const hb = announcing ? setInterval(() => void announceNow(), heartbeatMs) : undefined;
161
199
  hb?.unref?.();
162
200
  const uw = opts.usage ? setInterval(() => void flushUsage(), usageWindowMs) : undefined;
@@ -190,29 +228,15 @@ export function startFleetClient(opts) {
190
228
  };
191
229
  return { announceNow, stopRouting, flushFinalUsage };
192
230
  }
193
- export function startFleetClientFromEnv(cfg, opts) {
231
+ export function startFleetClientFromConfig(cfg, opts) {
194
232
  const cc = cfg.configCenter;
195
- const raw = opts.advertiseAddress ?? process.env.FLEET_ADVERTISE_ADDRESS;
196
- const address = raw === undefined || raw === "" ? undefined : raw;
197
- if (address !== undefined && cc?.baseUrl && cc.credential.declared) {
198
- let parsed;
199
- try {
200
- parsed = new URL(address);
201
- }
202
- catch {
203
- parsed = undefined;
204
- }
205
- if (parsed === undefined || (parsed.protocol !== "http:" && parsed.protocol !== "https:")) {
206
- throw new Error(`FLEET_ADVERTISE_ADDRESS must be a parseable http(s) base URL (e.g. "http://10.0.0.5:8090"); got "${buildDisplaySafeUrl(address)}" — the fleet center rejects any other shape on every announce`);
207
- }
208
- }
233
+ const address = cfg.fleetAdvertiseAddress;
209
234
  if (!centerLaneConfigured(cfg)) {
210
235
  if (address !== undefined && cc?.baseUrl && cc.credential.declared) {
211
236
  opts.logger?.warn("fleet_client_no_worker_scope", { note: "FLEET_ADVERTISE_ADDRESS set but SEMA_REGISTRY_WORKER missing — announce needs the worker (dispatch key); fleet client not started" });
212
237
  }
213
238
  return undefined;
214
239
  }
215
- const lane = cfg.configCenter;
216
- return startFleetClient({ ...opts, center: { baseUrl: lane.baseUrl, credential: lane.credential, worker: lane.worker }, advertiseAddress: address });
240
+ return startFleetClient({ ...opts, center: cfg.configCenter, advertiseAddress: address });
217
241
  }
218
242
  //# sourceMappingURL=fleet-client.js.map
@@ -6,7 +6,7 @@ import { APPROVAL_GATE_KINDS } from "../tool-approval.js";
6
6
  import { createWallClockJumpGuard } from "../wall-clock-jump-guard.js";
7
7
  import { hasServiceAuth } from "./dispatch.js";
8
8
  import { gatedPrincipal } from "./principal-gate.js";
9
- import { readinessOf, billableRefusalOf } from "./readiness.js";
9
+ import { admissionRefusalOf, readinessOf } from "./readiness.js";
10
10
  import { readResumeCtx } from "./resume-ctx-read.js";
11
11
  import { storedBodyReplayRefusal } from "./checkpoint-ctx.js";
12
12
  const BLOCKED_SCALAR_KEYS = ["errorCode", "blockedBy", "field", "reason", "count", "cap", "index"];
@@ -107,11 +107,9 @@ export function createApprovalSweeps(ctx) {
107
107
  if (!hasServiceAuth(deps.config) && !deps.config.allowUnauthedWrites) {
108
108
  return { status: 503, body: { error: "this worker requires a service auth token (set SERVICE_AUTH_TOKEN) before accepting task submissions", errorCode: "auth.service_token_required" } };
109
109
  }
110
- const readiness = readinessOf(deps);
111
- if (!readiness.ready) {
112
- const refusal = billableRefusalOf(readiness, deps.drainState?.reason);
110
+ const refusal = admissionRefusalOf(readinessOf(deps), deps.servingLease?.(), "hitl-answer");
111
+ if (refusal !== undefined)
113
112
  return { status: refusal.status, body: refusal.body };
114
- }
115
113
  if (deps.costQuota && req.httpReq) {
116
114
  const quotaPrincipal = gatedPrincipal(req.httpReq, deps.config);
117
115
  if (quotaPrincipal !== undefined) {
@@ -3,9 +3,11 @@ import { setSsoPrincipal, ssoVerifiedPrincipal, setSsoScope, isDestructiveSessio
3
3
  import { publicStoreProbeError } from "../store-live-probe.js";
4
4
  import { looksLikeJwt } from "@sema-agent/settings-schema/api/auth-bridge";
5
5
  import { ROUTE_METHOD_ORDER } from "./route-ctx.js";
6
- import { READ_CREDENTIAL_ROUTES, PUBLIC_ROUTES, POST_CREDENTIAL_ROUTES, routeLabel, isBillableSubmitPath, isCredentialGatedRewrite, isMethodDispatchedSubmitPath, lookupRoute, isProbePath } from "./route-table.js";
6
+ import { READ_CREDENTIAL_ROUTES, PUBLIC_ROUTES, POST_CREDENTIAL_ROUTES, routeLabel, isBillableSubmitPath, billableKindOf, isCredentialGatedRewrite, isMethodDispatchedSubmitPath, lookupRoute, isProbePath } from "./route-table.js";
7
7
  import { sendJson, sendError, isSseResponse } from "./send.js";
8
- import { readinessOf, healthReadinessKeys, sendReadyz, billableRefusalOf, sendBillableRefusal } from "./readiness.js";
8
+ import { admissionRefusalOf, readinessOf, healthReadinessKeys, sendReadyz, sendBillableRefusal } from "./readiness.js";
9
+ import { fleetMembershipKeys, mintEngineSignal, restartActionable } from "../fleet/engine-signal.js";
10
+ import { buildHealthSelfReport } from "./health-self-report.js";
9
11
  import { authorized, systemFor, gatedPrincipal, sendPrincipalRefusal, sendUnauthorized, verifiedIdentitySource } from "./principal-gate.js";
10
12
  const PROCESS_STARTED_AT_MS = Date.now() - Math.round(process.uptime() * 1000);
11
13
  export function hasServiceAuth(config) {
@@ -113,7 +115,7 @@ export function createDispatch(ctx) {
113
115
  return;
114
116
  }
115
117
  if (req.method === "GET" && url === "/health") {
116
- const restart = deps.restartState?.();
118
+ const signal = mintEngineSignal(deps);
117
119
  const configHash = createHash("sha256")
118
120
  .update(JSON.stringify({ model: deps.config.model.id, sessionBackend: deps.config.sessionBackend, dbBackend: deps.config.dbBackend, dataRoot: deps.config.localDataRoot, configProvider: deps.config.configProvider, remoteExec: deps.config.remoteExec?.provider, port: deps.config.port }))
119
121
  .digest("hex")
@@ -121,24 +123,26 @@ export function createDispatch(ctx) {
121
123
  sendJson(res, 200, {
122
124
  status: "ok",
123
125
  ...(typeof deps.capabilities?.version === "string" ? { version: deps.capabilities.version } : {}),
124
- ...(deps.drainState?.draining ? { draining: true, ...(deps.drainState.since ? { drainingSince: deps.drainState.since } : {}), ...(deps.drainState.reason ? { drainReason: deps.drainState.reason } : {}) } : {}),
126
+ ...(deps.drainState?.draining ? { draining: true, ...(deps.drainState.since ? { drainingSince: deps.drainState.since } : {}) } : {}),
125
127
  ...(deps.drainState?.inflight ? { inflight: deps.drainState.inflight() } : {}),
126
128
  ...(deps.storePosture?.degradedReason !== undefined ? { durable: false, storeDegraded: true } : {}),
127
- ...healthReadinessKeys(readinessOf(deps)),
129
+ ...healthReadinessKeys(signal.verdict),
128
130
  model: deps.config.model.id,
129
131
  sessionBackend: deps.sessionStoreLabel ?? deps.config.sessionBackend,
130
132
  sessions: deps.runner.sessions.size,
131
133
  ...(deps.instanceId ? { instanceId: deps.instanceId } : {}),
134
+ ...fleetMembershipKeys(signal),
132
135
  ...(deps.config.localDataRoot ? { dataRoot: deps.config.localDataRoot } : {}),
133
136
  configHash,
134
137
  pid: process.pid,
135
138
  startedAt: PROCESS_STARTED_AT_MS,
136
139
  port: deps.config.port,
137
140
  ...(deps.config.directDoorActive ? { directDoorActive: true } : {}),
138
- ...(restart ? { restartRequired: true, restart } : {}),
141
+ ...buildHealthSelfReport(deps, hasServiceAuth(deps.config)),
142
+ ...(signal.restart ? { ...(restartActionable(signal) ? { restartRequired: true } : {}), restart: signal.restart } : {}),
139
143
  ...(() => {
140
- const stuck = deps.planeDeferredState?.();
141
- return stuck ? { modelPlaneDeferred: { version: stuck.version, since: stuck.since, noHandoff: true, ...(stuck.blocked ? { blockedReasons: stuck.blocked } : {}) } } : {};
144
+ const configHandoff = deps.configHandoffState?.();
145
+ return configHandoff ? { configHandoff } : {};
142
146
  })(),
143
147
  ...(() => {
144
148
  const gen = deps.configApplyState?.();
@@ -173,7 +177,7 @@ export function createDispatch(ctx) {
173
177
  return;
174
178
  }
175
179
  if (req.method === "GET" && url === "/readyz") {
176
- sendReadyz(res, readinessOf(deps));
180
+ sendReadyz(res, mintEngineSignal(deps).verdict);
177
181
  return;
178
182
  }
179
183
  if (req.method === "GET" && (url === "/metrics" || url === "/metrics/summary" || url === "/metrics/plan-cache")) {
@@ -244,10 +248,11 @@ export function createDispatch(ctx) {
244
248
  sendError(res, 503, "auth.service_token_required", "this worker requires a service auth token (set SERVICE_AUTH_TOKEN) before accepting task submissions");
245
249
  return;
246
250
  }
247
- if (req.method === "POST" && isBillableSubmitPath(url) && !isMethodDispatchedSubmitPath(url)) {
248
- const verdict = readinessOf(deps);
249
- if (!verdict.ready) {
250
- sendBillableRefusal(res, billableRefusalOf(verdict, deps.drainState?.reason));
251
+ const billableKind = req.method === "POST" && !isMethodDispatchedSubmitPath(url) ? billableKindOf(url) : undefined;
252
+ if (billableKind !== undefined) {
253
+ const refusal = admissionRefusalOf(readinessOf(deps), deps.servingLease?.(), billableKind);
254
+ if (refusal !== undefined) {
255
+ sendBillableRefusal(res, refusal);
251
256
  return;
252
257
  }
253
258
  }
@@ -0,0 +1,60 @@
1
+ /**
2
+ * `GET /health` 的**自报位**(7.121.0)—— `auth` / `operator` / `center` 三键:壳在**带凭据之前**要知道的三件部署事实
3
+ * (这台引擎要不要凭据、认哪几类、能不能附着身份;有没有运维方可找;接着哪个中心、会话会不会出现在那个中心的 web 面)。
4
+ *
5
+ * ## 规则(一条):自报位是**投影**,不是第二份规则
6
+ * 每一位读的是**门本身读的那只谓词 / 那只取值口**,本模块不判任何东西:
7
+ * · `auth.required` = 全局凭据门的武装谓词(`hasServiceAuth`,由 `dispatch.ts` 交进来 —— 门与本位读同一次求值的同一只函数);
8
+ * · `auth.kinds` = {@link IDENTITY_SOURCES}(`verifiedIdentitySource` 的回值闭集)里,这台部署的装配**能**回的那几个词:
9
+ * `service_token` ⇐ 全局门武装(`systemFor` 只在配了 service token 时回得出值)· `sso` ⇐ 鉴权桥装了(入口预认证只在
10
+ * `registryJwtVerifier` 在场时挂身份)· `direct_door_jwt` ⇐ 直连门开着(`verifiedIdentitySource` 那一臂的第一个合取项);
11
+ * · `auth.identityAttach` = kinds 里有没有**带调用方身份**的一类({@link carriesCallerIdentity},穷举):`service_token` 认的是系统
12
+ * (`systemFor` 回系统名或 null),不是人;
13
+ * · `operator` = 能力事实 `operatorLane`(operator 面族的 501 门读的同一只;名单本身不上 wire);
14
+ * · `center.issuer` = 鉴权桥 issuer 的唯一取值 `authBridgeIssuerOf`(装配 `boot/stage-09-leader.ts` 与 caps `centerWiring.issuer` 同读);
15
+ * · `center.webBaseUrl` = 运维旋钮 `SEMA_REGISTRY_WEB_URL`(`config.configCenter.webBaseUrl`,接了中心才挂;起服时已判成固定基址);
16
+ * · `center.sessionsVisible` = 中心的 web 面能不能列出本引擎的会话 —— 四件接线事实全在才可能:中心车道配齐(`centerLaneConfigured`,
17
+ * 租约 / 用量 / 公告同一只谓词)∧ 公告地址在(配置席 `fleetAdvertiseAddress`:web 面只拨中心登记过的实例地址)∧ 鉴权桥在(web 面带的是
18
+ * 中心铸的中继令牌,不带 service token)∧ 会话枚举面在(事实 `sessionList`,web 面中继 `GET /v1/sessions`);缺任一 ⇒ `false`。
19
+ * 四件齐时:多租户(`REQUIRE_PRINCIPAL`)每条会话都落着验过的属主 ⇒ `true`;单用户形下一条会话归谁取决于提交它的那次请求带没带
20
+ * 身份(web 面只列调用人自己的会话)—— 部署级答不了 ⇒ `"unknown"`。
21
+ *
22
+ * ## 无鉴权面
23
+ * `/health` 排在凭据门之前、缺省全网卡监听:本模块只出闭集词、布尔与运维要发布的 URL。两只 URL 值都过 display-safe 律
24
+ * (`buildDisplaySafeUrl`,所有展示面 URL 的同一个铸点):userinfo 落地、query 值成记号、尾斜杠折掉 —— issuer 的来源
25
+ * (`AUTH_BRIDGE_ISSUER` / `SEMA_REGISTRY_URL`)起服时不判凭据形,这一道是它上无鉴权面的唯一闸。令牌、名单、worker 名、凭证一概不上。
26
+ *
27
+ * ## 在场律
28
+ * 三键**恒在场**;缺席 = 老 worker(≤7.120.x)。`center.issuer` / `center.webBaseUrl` 缺席即缺席(没有鉴权桥 / 没配旋钮),不拼不猜。
29
+ */
30
+ import type { FlatServiceDeps } from "./server.js";
31
+ import { type IdentitySource } from "./principal-gate.js";
32
+ /** `auth.identityAttach` 的词表。 */
33
+ export declare const IDENTITY_ATTACH_WORDS: readonly ["available", "unavailable"];
34
+ export type IdentityAttach = (typeof IDENTITY_ATTACH_WORDS)[number];
35
+ /** `operator` 的词表。 */
36
+ export declare const OPERATOR_PRESENCE_WORDS: readonly ["present", "absent"];
37
+ export type OperatorPresence = (typeof OPERATOR_PRESENCE_WORDS)[number];
38
+ /** `center.sessionsVisible` 的值域(两个布尔 + 一个词;`"unknown"` = 接线齐了,但会话归谁要看提交它的那次请求)。 */
39
+ export declare const SESSIONS_VISIBLE_VALUES: readonly [true, false, "unknown"];
40
+ export type SessionsVisible = (typeof SESSIONS_VISIBLE_VALUES)[number];
41
+ /** 三键的 wire 形(纯数据)。 */
42
+ export interface HealthSelfReport {
43
+ readonly auth: {
44
+ readonly required: boolean;
45
+ readonly kinds: readonly IdentitySource[];
46
+ readonly identityAttach: IdentityAttach;
47
+ };
48
+ readonly operator: OperatorPresence;
49
+ readonly center: {
50
+ readonly issuer?: string;
51
+ readonly webBaseUrl?: string;
52
+ readonly sessionsVisible: SessionsVisible;
53
+ };
54
+ }
55
+ /**
56
+ * 三键的**唯一铸点**(`build*`:纯数据)。`credentialGateArmed` 由 `dispatch.ts` 交进来 —— 它是全局凭据门自己的那一句
57
+ * (`hasServiceAuth`),本模块不另求一遍(另求就是第二个写者)。
58
+ */
59
+ export declare function buildHealthSelfReport(deps: Pick<FlatServiceDeps, "config" | "registryJwtVerifier" | "sessionStorage" | "runStore">, credentialGateArmed: boolean): HealthSelfReport;
60
+ //# sourceMappingURL=health-self-report.d.ts.map