@sema-agent/server 7.149.0 → 7.150.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +122 -9
- package/README.md +1 -1
- package/README.zh-CN.md +1 -1
- package/USAGE.md +39 -15
- package/dist/approval-card.d.ts +1 -1
- package/dist/approval-gate-currency.d.ts +1 -1
- package/dist/audit.d.ts +1 -1
- package/dist/audit.js +2 -2
- package/dist/background-session-reap.d.ts +1 -1
- package/dist/boot/deferred-sandbox-path-env.d.ts +5 -3
- package/dist/boot/deferred-sandbox-path-env.js +1 -2
- package/dist/boot/parked-revive-gate.d.ts +5 -5
- package/dist/boot/parked-revive-gate.js +2 -3
- package/dist/boot/resolve-spec.d.ts +4 -5
- package/dist/boot/resolve-spec.js +61 -25
- package/dist/boot/retention-lane.d.ts +1 -1
- package/dist/boot/session-faces.js +21 -21
- package/dist/boot/stage-03-store-backend.js +2 -2
- package/dist/boot/stage-06-runners.js +1 -2
- package/dist/boot/stage-07-capability-layer.js +1 -1
- package/dist/boot/stage-10-http-server.d.ts +33 -1
- package/dist/boot/stage-10-http-server.js +9 -4
- package/dist/boot/web-admin.d.ts +1 -1
- package/dist/capability-facts.d.ts +65 -56
- package/dist/capability-facts.js +42 -35
- package/dist/config-catalog.js +1 -1
- package/dist/config-types.d.ts +12 -11
- package/dist/deployment-governance.d.ts +7 -28
- package/dist/deployment-governance.js +1 -14
- package/dist/device-ws-hub.d.ts +1 -0
- package/dist/device-ws-hub.js +64 -28
- package/dist/device-ws-protocol.d.ts +20 -6
- package/dist/device-ws-protocol.js +9 -2
- package/dist/engine-identity.d.ts +7 -1
- package/dist/engine-identity.js +5 -0
- package/dist/field-conflict.d.ts +2 -2
- package/dist/fleet/engine-signal.d.ts +14 -1
- package/dist/fleet/engine-signal.js +5 -1
- package/dist/fleet/fleet-reconciler.d.ts +1 -1
- package/dist/fleet/serving-lease.d.ts +18 -3
- package/dist/fleet/serving-lease.js +81 -35
- package/dist/fleet-client.d.ts +13 -11
- package/dist/fleet-client.js +53 -33
- package/dist/history-anchor.js +1 -1
- package/dist/hitl-delivery.d.ts +1 -3
- package/dist/hitl-delivery.js +1 -1
- package/dist/http/active-run-conflict.d.ts +3 -7
- package/dist/http/active-run-conflict.js +2 -2
- package/dist/http/admission.js +2 -2
- package/dist/http/approval-sweeps.js +2 -1
- package/dist/http/caller-trust.d.ts +2 -2
- package/dist/http/caller-trust.js +3 -3
- package/dist/http/checkpoint-ctx.d.ts +18 -63
- package/dist/http/checkpoint-ctx.js +1 -24
- package/dist/http/claim-heartbeat.d.ts +4 -4
- package/dist/http/claim-heartbeat.js +3 -3
- package/dist/http/dispatch.js +2 -2
- package/dist/http/durable-run-leg.js +4 -4
- package/dist/http/owner-access.d.ts +243 -19
- package/dist/http/owner-access.js +184 -5
- package/dist/http/readiness.d.ts +4 -2
- package/dist/http/resume-ctx-read.js +16 -1
- package/dist/http/resume-legs.js +18 -14
- package/dist/http/route-ctx.d.ts +23 -11
- package/dist/http/routes/a2a-serve.js +15 -14
- package/dist/http/routes/approvals-assistant.d.ts +3 -2
- package/dist/http/routes/approvals-assistant.js +8 -9
- package/dist/http/routes/capabilities.js +4 -3
- package/dist/http/routes/config-catalog.d.ts +1 -1
- package/dist/http/routes/mcp-reconnect.js +2 -2
- package/dist/http/routes/memory-origin.d.ts +1 -1
- package/dist/http/routes/notify-wake.js +2 -3
- package/dist/http/routes/projects.d.ts +1 -0
- package/dist/http/routes/projects.js +5 -3
- package/dist/http/routes/retention-ops.d.ts +1 -1
- package/dist/http/routes/run-permission-mode.d.ts +1 -1
- package/dist/http/routes/run-permission-mode.js +2 -2
- package/dist/http/routes/runs.d.ts +1 -1
- package/dist/http/routes/runs.js +23 -30
- package/dist/http/routes/session-memory-sync.js +4 -5
- package/dist/http/routes/session-sync.js +3 -7
- package/dist/http/routes/sessions.d.ts +1 -1
- package/dist/http/routes/sessions.js +18 -44
- package/dist/http/routes/tasks.js +9 -8
- package/dist/http/routes/trace-usage.js +14 -14
- package/dist/http/routes/workflows.d.ts +1 -0
- package/dist/http/routes/workflows.js +20 -38
- package/dist/http/server.d.ts +8 -4
- package/dist/http/server.js +5 -1
- package/dist/http/stored-body-replay.d.ts +3 -24
- package/dist/http/stored-body-replay.js +2 -12
- package/dist/http/submit-ctx.d.ts +1 -2
- package/dist/http/submit-ctx.js +3 -2
- package/dist/http/submit-replay.d.ts +15 -2
- package/dist/http/submit-replay.js +10 -4
- package/dist/http/wire-types.d.ts +6 -3
- package/dist/leader/endpoint.js +9 -8
- package/dist/leg-facts.d.ts +8 -2
- package/dist/leg-facts.js +1 -0
- package/dist/mcp-authorization-face.d.ts +18 -14
- package/dist/mcp-authorization-face.js +3 -2
- package/dist/memory-store-refusal.d.ts +1 -1
- package/dist/observability/corrupt-read-seat.d.ts +1 -1
- package/dist/observability/fail-open.d.ts +5 -5
- package/dist/observability/fail-open.js +5 -5
- package/dist/observability/logger.d.ts +7 -0
- package/dist/observability/logger.js +3 -1
- package/dist/observability/run-terminal.d.ts +1 -1
- package/dist/owner-faces.d.ts +27 -3
- package/dist/owner-faces.js +2 -0
- package/dist/peer-seat.d.ts +1 -1
- package/dist/peer-seat.js +52 -59
- package/dist/plugins/exec-path-law.d.ts +34 -0
- package/dist/plugins/exec-path-law.js +13 -0
- package/dist/plugins/file-run-store.js +1 -1
- package/dist/plugins/local-checkpoint-store.d.ts +4 -1
- package/dist/plugins/local-checkpoint-store.js +3 -0
- package/dist/plugins/memory-run-store.js +1 -1
- package/dist/plugins/one-time-migrations.d.ts +74 -49
- package/dist/plugins/one-time-migrations.js +134 -69
- package/dist/plugins/pg-pool.js +4 -0
- package/dist/plugins/posix-shell-fs.js +4 -2
- package/dist/plugins/remote-env-adb.js +3 -2
- package/dist/plugins/remote-env-device.js +8 -6
- package/dist/plugins/remote-env-e2b.d.ts +3 -1
- package/dist/plugins/remote-env-e2b.js +11 -8
- package/dist/plugins/remote-env-host.d.ts +1 -1
- package/dist/plugins/remote-env-host.js +20 -11
- package/dist/plugins/remote-env-k8s.js +3 -2
- package/dist/plugins/remote-env-local-docker.js +3 -2
- package/dist/plugins/remote-env-ssh.js +4 -3
- package/dist/plugins/remote-shell.d.ts +3 -1
- package/dist/plugins/remote-shell.js +3 -3
- package/dist/plugins/replica-registration.d.ts +35 -0
- package/dist/plugins/replica-registration.js +48 -0
- package/dist/plugins/retention-lane-store-sql.d.ts +1 -1
- package/dist/plugins/run-store-sql.d.ts +1 -1
- package/dist/plugins/run-store-sql.js +2 -2
- package/dist/plugins/scenario-prefix-switch-m42.d.ts +6 -12
- package/dist/plugins/scenario-prefix-switch-m42.js +7 -5
- package/dist/plugins/schema-consistency.d.ts +7 -5
- package/dist/plugins/schema-consistency.js +29 -16
- package/dist/plugins/schema-drift-sentence.js +1 -1
- package/dist/plugins/serving-requirement.d.ts +49 -0
- package/dist/plugins/serving-requirement.js +53 -0
- package/dist/plugins/stop-then-start-sentence.d.ts +42 -3
- package/dist/plugins/stop-then-start-sentence.js +44 -0
- package/dist/plugins/store-backend.d.ts +1 -1
- package/dist/plugins/store-contracts.d.ts +12 -2
- package/dist/plugins/store-contracts.js +3 -0
- package/dist/plugins/tidb-pool.js +4 -0
- package/dist/process-instance.d.ts +3 -0
- package/dist/process-instance.js +7 -0
- package/dist/project-context.d.ts +66 -0
- package/dist/project-context.js +70 -0
- package/dist/question.d.ts +9 -21
- package/dist/question.js +28 -19
- package/dist/run-cancel-card.d.ts +1 -1
- package/dist/run-cancel-card.js +2 -1
- package/dist/security.d.ts +1 -1
- package/dist/session-agent.d.ts +1 -10
- package/dist/session-agent.js +0 -6
- package/dist/session-incarnation.d.ts +18 -4
- package/dist/session-incarnation.js +4 -1
- package/dist/session-notify-door.d.ts +4 -4
- package/dist/session-notify-door.js +5 -6
- package/dist/session-rules-seat.d.ts +4 -3
- package/dist/session-rules-seat.js +3 -11
- package/dist/session-sync.js +3 -3
- package/dist/stores-sql/checkpoint-store-sql.d.ts +21 -2
- package/dist/stores-sql/checkpoint-store-sql.js +106 -18
- package/dist/stores-sql/retention-store-sql.d.ts +1 -1
- package/dist/task-cwd.d.ts +4 -3
- package/dist/task-settings.d.ts +13 -0
- package/dist/task-settings.js +13 -7
- package/dist/tool-approval.d.ts +7 -6
- package/dist/tool-approval.js +41 -37
- package/dist/trace/engine-notice-wire.d.ts +14 -0
- package/dist/trace/engine-notice-wire.js +18 -1
- package/dist/trace/project.d.ts +4 -3
- package/dist/trace/project.js +3 -2
- package/dist/trace/projection-table.d.ts +33 -2
- package/dist/trace/self-minted-wire.d.ts +150 -17
- package/dist/trace/self-minted-wire.js +54 -16
- package/dist/trace/tool-card.d.ts +3 -0
- package/dist/trace/tool-card.js +24 -0
- package/dist/trace/tool-output-media.d.ts +2 -1
- package/dist/unattended-permission-mode.d.ts +1 -1
- package/dist/web-admin-client/apply-effective.d.ts +1 -1
- package/dist/web-admin-client/effective-keys.d.ts +1 -1
- package/dist/web-admin-client/skills-mcp.d.ts +1 -1
- package/docs/DEVICE-EXECUTOR-PROTOCOL.md +57 -29
- package/package.json +2 -2
- package/wire-changes/server@7.150.0.json +335 -0
|
@@ -1,19 +1,6 @@
|
|
|
1
1
|
import { join, posix } from "node:path";
|
|
2
2
|
import { FileError, StubExecutionEnv, combinePolicies, createAllowDenyPolicy, createDurableQuestionPolicy, createSensitivePathPolicy, createTranscriptIntegrityPolicy, err, isAbsoluteForFamily, isProtectedWrite, ok, pathTargetValue, protectivePathTargetOf, } from "@sema-agent/core";
|
|
3
3
|
import { createDurableAskPolicy, hasApprovalListIntent } from "./approval.js";
|
|
4
|
-
export function createDurableQuestionGate(live) {
|
|
5
|
-
const durable = createDurableQuestionPolicy();
|
|
6
|
-
if (live === undefined)
|
|
7
|
-
return durable;
|
|
8
|
-
return {
|
|
9
|
-
async check(req) {
|
|
10
|
-
const answer = await durable.check(req);
|
|
11
|
-
if (answer.action !== "ask")
|
|
12
|
-
return answer;
|
|
13
|
-
return live.hasLiveContext() ? { action: "allow" } : answer;
|
|
14
|
-
},
|
|
15
|
-
};
|
|
16
|
-
}
|
|
17
4
|
export const SANDBOX_TREE_FAMILY = "posix";
|
|
18
5
|
function writeTargetFormOf(req) {
|
|
19
6
|
if (!isProtectedWrite(req))
|
|
@@ -125,7 +112,7 @@ export function buildApprovalGrantNotice(config) {
|
|
|
125
112
|
export function createApprovalBaselinePolicy(config, seat) {
|
|
126
113
|
switch (seat.kind) {
|
|
127
114
|
case "durable":
|
|
128
|
-
return combinePolicies(
|
|
115
|
+
return combinePolicies(createDurableQuestionPolicy(), createApprovalListPolicy(config, seat.exemption));
|
|
129
116
|
case "live":
|
|
130
117
|
return createApprovalListPolicy(config, undefined);
|
|
131
118
|
case "absent":
|
package/dist/device-ws-hub.d.ts
CHANGED
|
@@ -209,6 +209,7 @@ export interface DeviceWsHub {
|
|
|
209
209
|
declaredSupports(deviceId: string): ReadonlySet<DeviceInstructionKind> | undefined;
|
|
210
210
|
close(): Promise<void>;
|
|
211
211
|
}
|
|
212
|
+
export declare const DEVICE_MAX_PAYLOAD_PART_BYTES: number;
|
|
212
213
|
/** per-remote 帽键的来源(闭集;指标 `device_preauth_key_total{source}` 与 cap-hit 行 `keySource` 同一张表)。 */
|
|
213
214
|
export declare const DEVICE_PREAUTH_KEY_SOURCES: readonly ["socket", "xff", "fallback"];
|
|
214
215
|
export type DevicePreAuthKeySource = (typeof DEVICE_PREAUTH_KEY_SOURCES)[number];
|
package/dist/device-ws-hub.js
CHANGED
|
@@ -5,7 +5,7 @@ import { WebSocketServer } from "ws";
|
|
|
5
5
|
import { DEVICE_DRAINING_CODE, DEVICE_ERROR_CODES, DEVICE_ERROR_STATUS, assertDeviceIdShape, } from "./device-store.js";
|
|
6
6
|
import { MAX_TIMER_DELAY_MS } from "./config-types.js";
|
|
7
7
|
import { canonicalAddress, createTrustedProxySet } from "./net-address.js";
|
|
8
|
-
import { DEVICE_HELLO_ACK_BOUNDS, DEVICE_MIN_PROTOCOL_VERSION, DEVICE_WS_LIMITS, DEVICE_WS_PATH, DEVICE_PROTOCOL_VERSION, asInstructionKind, buildHelloSignaturePayload, deviceHelloAckNumberFlaw, formatGeneration, mintChallengeNonce, mintInstructionId, parseDeviceFrameText, verifyHelloSignature, } from "./device-ws-protocol.js";
|
|
8
|
+
import { DEVICE_HELLO_ACK_BOUNDS, DEVICE_MIN_PROTOCOL_VERSION, DEVICE_WS_LIMITS, DEVICE_WS_PATH, DEVICE_PROTOCOL_VERSION, asInstructionKind, buildHelloSignaturePayload, deviceHelloAckNumberFlaw, deviceProtocolRevisionRefusal, formatGeneration, mintChallengeNonce, mintInstructionId, parseDeviceFrameText, verifyHelloSignature, } from "./device-ws-protocol.js";
|
|
9
9
|
const OUTCOME_UNKNOWN_SENTENCE = "outcome unknown — the command may have already executed on the device; verify its effect before re-running";
|
|
10
10
|
const SETTLED_MEMO_MAX = 512;
|
|
11
11
|
const DRAIN_RETRY_AFTER_SEC = 15;
|
|
@@ -27,6 +27,25 @@ function outcomeUnknown(why) {
|
|
|
27
27
|
return { ok: false, errorCode: "outcome_unknown", message: `${why} — ${OUTCOME_UNKNOWN_SENTENCE}` };
|
|
28
28
|
}
|
|
29
29
|
const WRITE_PAYLOAD_KINDS = new Set(["writeFile", "writeFileExclusive", "writeFileGuarded", "appendFile"]);
|
|
30
|
+
const PAYLOAD_CHUNK_ENVELOPE_BYTES = Buffer.byteLength(JSON.stringify({ t: "payload", gen: String(Number.MAX_SAFE_INTEGER), instructionId: mintInstructionId(), part: "chunk", offset: Number.MAX_SAFE_INTEGER, dataB64: "" }), "utf8");
|
|
31
|
+
export const DEVICE_MAX_PAYLOAD_PART_BYTES = Math.floor((DEVICE_WS_LIMITS.maxFrameBytes - PAYLOAD_CHUNK_ENVELOPE_BYTES) / 4) * 3;
|
|
32
|
+
const FRAME_TOO_LARGE = "frame_too_large";
|
|
33
|
+
function encodeServerFrame(frame) {
|
|
34
|
+
const text = JSON.stringify(frame);
|
|
35
|
+
const bytes = Buffer.byteLength(text, "utf8");
|
|
36
|
+
return bytes <= DEVICE_WS_LIMITS.maxFrameBytes ? { ok: true, text } : { ok: false, bytes };
|
|
37
|
+
}
|
|
38
|
+
function frameTooLargeResult(kind, bytes) {
|
|
39
|
+
return {
|
|
40
|
+
kind: "never_started",
|
|
41
|
+
outcome: {
|
|
42
|
+
ok: false,
|
|
43
|
+
errorCode: "not_supported",
|
|
44
|
+
message: `[${FRAME_TOO_LARGE}] the ${kind} instruction frame would be ${bytes} bytes, over the device protocol's ${DEVICE_WS_LIMITS.maxFrameBytes}-byte frame cap — ` +
|
|
45
|
+
"it was never sent (the device transport drops a connection that carries a frame that size); keep a command and its arguments under the cap, and move large content into a file write (it travels in slices)",
|
|
46
|
+
},
|
|
47
|
+
};
|
|
48
|
+
}
|
|
30
49
|
const ALLOWED_SUCCESS_OUTCOME_KINDS = {
|
|
31
50
|
exec: ["exec"],
|
|
32
51
|
execStream: ["exec"],
|
|
@@ -145,14 +164,22 @@ export function createDeviceWsHub(options) {
|
|
|
145
164
|
.catch((err) => logger?.error("device_audit_write_failed", { event, err: err instanceof Error ? err.message : String(err) }));
|
|
146
165
|
}
|
|
147
166
|
function sendServerFrame(conn, frame) {
|
|
167
|
+
const encoded = encodeServerFrame(frame);
|
|
168
|
+
if (!encoded.ok) {
|
|
169
|
+
logger?.error("device_ws_frame_over_cap", { deviceId: conn.deviceId, t: frame.t, bytes: encoded.bytes, cap: DEVICE_WS_LIMITS.maxFrameBytes });
|
|
170
|
+
return false;
|
|
171
|
+
}
|
|
172
|
+
return writeServerText(conn, frame.t, encoded.text);
|
|
173
|
+
}
|
|
174
|
+
function writeServerText(conn, t, text) {
|
|
148
175
|
if (conn.ws.readyState !== conn.ws.OPEN)
|
|
149
176
|
return false;
|
|
150
177
|
try {
|
|
151
|
-
conn.ws.send(
|
|
178
|
+
conn.ws.send(text);
|
|
152
179
|
return true;
|
|
153
180
|
}
|
|
154
181
|
catch (err) {
|
|
155
|
-
logger?.warn("device_ws_send_failed", { deviceId: conn.deviceId, t
|
|
182
|
+
logger?.warn("device_ws_send_failed", { deviceId: conn.deviceId, t, err: err instanceof Error ? err.message : String(err) });
|
|
156
183
|
return false;
|
|
157
184
|
}
|
|
158
185
|
}
|
|
@@ -301,11 +328,11 @@ export function createDeviceWsHub(options) {
|
|
|
301
328
|
}, reconnectGraceMs);
|
|
302
329
|
reconnectTimers.set(deviceId, timer);
|
|
303
330
|
}
|
|
304
|
-
function rejectHello(conn, code,
|
|
331
|
+
function rejectHello(conn, code, version) {
|
|
305
332
|
if (conn.handshakeTimer)
|
|
306
333
|
clearTimeout(conn.handshakeTimer);
|
|
307
334
|
conn.handshakeTimer = null;
|
|
308
|
-
sendServerFrame(conn, { t: "helloReject", code, ...
|
|
335
|
+
sendServerFrame(conn, { t: "helloReject", code, ...version });
|
|
309
336
|
closeConn(conn, { releaseLease: false });
|
|
310
337
|
}
|
|
311
338
|
async function fenceConnection(conn) {
|
|
@@ -367,7 +394,10 @@ export function createDeviceWsHub(options) {
|
|
|
367
394
|
if (draining || closed)
|
|
368
395
|
return rejectHello(conn, "draining");
|
|
369
396
|
if (frame.protocolVersion < DEVICE_MIN_PROTOCOL_VERSION || frame.protocolVersion > DEVICE_PROTOCOL_VERSION) {
|
|
370
|
-
|
|
397
|
+
const message = deviceProtocolRevisionRefusal(frame.protocolVersion);
|
|
398
|
+
logger?.warn("device_hello_version_rejected", { declared: frame.protocolVersion, minProtocolVersion: DEVICE_MIN_PROTOCOL_VERSION, protocolVersion: DEVICE_PROTOCOL_VERSION, message });
|
|
399
|
+
metrics?.inc("device_hello_rejected_total", { code: "protocol_version_unsupported" });
|
|
400
|
+
return rejectHello(conn, "protocol_version_unsupported", { minProtocolVersion: DEVICE_MIN_PROTOCOL_VERSION, message });
|
|
371
401
|
}
|
|
372
402
|
let idShapeOk = true;
|
|
373
403
|
try {
|
|
@@ -860,8 +890,8 @@ export function createDeviceWsHub(options) {
|
|
|
860
890
|
async function dispatch(input) {
|
|
861
891
|
if (input.payloadPartBytes !== undefined) {
|
|
862
892
|
const n = input.payloadPartBytes;
|
|
863
|
-
if (!Number.isSafeInteger(n) || n <= 0 || n >
|
|
864
|
-
throw new Error(`device-ws: payloadPartBytes must be a positive safe integer ≤ ${DEVICE_WS_LIMITS.maxFrameBytes}
|
|
893
|
+
if (!Number.isSafeInteger(n) || n <= 0 || n > DEVICE_MAX_PAYLOAD_PART_BYTES) {
|
|
894
|
+
throw new Error(`device-ws: payloadPartBytes must be a positive safe integer ≤ ${DEVICE_MAX_PAYLOAD_PART_BYTES} (the ${DEVICE_WS_LIMITS.maxFrameBytes}-byte frame cap after base64 and the chunk envelope; got ${String(n)})`);
|
|
865
895
|
}
|
|
866
896
|
}
|
|
867
897
|
if (!Number.isSafeInteger(input.timeoutMs) || input.timeoutMs <= 0) {
|
|
@@ -963,7 +993,32 @@ export function createDeviceWsHub(options) {
|
|
|
963
993
|
releaseSlot(deviceId);
|
|
964
994
|
return abortedBeforeCommit();
|
|
965
995
|
}
|
|
996
|
+
let args = input.args;
|
|
997
|
+
let body = null;
|
|
998
|
+
let sha = "";
|
|
999
|
+
if (input.payload !== undefined && WRITE_PAYLOAD_KINDS.has(input.kind)) {
|
|
1000
|
+
body = Buffer.from(input.payload);
|
|
1001
|
+
sha = createHash("sha256").update(body).digest("hex");
|
|
1002
|
+
args = { ...input.args, contentLen: body.length, sha256: sha };
|
|
1003
|
+
}
|
|
966
1004
|
const instructionId = mintInstructionId();
|
|
1005
|
+
const instructionFrame = encodeServerFrame({
|
|
1006
|
+
t: "instruction",
|
|
1007
|
+
gen: conn.genStr,
|
|
1008
|
+
instructionId,
|
|
1009
|
+
rootSessionId: input.rootSessionId,
|
|
1010
|
+
issuedSeq: conn.issuedSeq + 1,
|
|
1011
|
+
kind: input.kind,
|
|
1012
|
+
args,
|
|
1013
|
+
...(input.cwd === undefined ? {} : { cwd: input.cwd }),
|
|
1014
|
+
...(input.shellEnv === undefined ? {} : { shellEnv: input.shellEnv }),
|
|
1015
|
+
timeoutMs: input.timeoutMs,
|
|
1016
|
+
});
|
|
1017
|
+
if (!instructionFrame.ok) {
|
|
1018
|
+
releaseSlot(deviceId);
|
|
1019
|
+
logger?.warn("device_instruction_frame_over_cap", { deviceId, kind: input.kind, bytes: instructionFrame.bytes, cap: DEVICE_WS_LIMITS.maxFrameBytes });
|
|
1020
|
+
return frameTooLargeResult(input.kind, instructionFrame.bytes);
|
|
1021
|
+
}
|
|
967
1022
|
conn.issuedSeq += 1;
|
|
968
1023
|
let resolve;
|
|
969
1024
|
const settled = new Promise((r) => {
|
|
@@ -996,26 +1051,7 @@ export function createDeviceWsHub(options) {
|
|
|
996
1051
|
resolve,
|
|
997
1052
|
};
|
|
998
1053
|
pending.set(instructionId, p);
|
|
999
|
-
|
|
1000
|
-
let body = null;
|
|
1001
|
-
let sha = "";
|
|
1002
|
-
if (input.payload !== undefined && WRITE_PAYLOAD_KINDS.has(input.kind)) {
|
|
1003
|
-
body = Buffer.from(input.payload);
|
|
1004
|
-
sha = createHash("sha256").update(body).digest("hex");
|
|
1005
|
-
args = { ...input.args, contentLen: body.length, sha256: sha };
|
|
1006
|
-
}
|
|
1007
|
-
const wrote = sendServerFrame(conn, {
|
|
1008
|
-
t: "instruction",
|
|
1009
|
-
gen: conn.genStr,
|
|
1010
|
-
instructionId,
|
|
1011
|
-
rootSessionId: input.rootSessionId,
|
|
1012
|
-
issuedSeq: conn.issuedSeq,
|
|
1013
|
-
kind: input.kind,
|
|
1014
|
-
args,
|
|
1015
|
-
...(input.cwd === undefined ? {} : { cwd: input.cwd }),
|
|
1016
|
-
...(input.shellEnv === undefined ? {} : { shellEnv: input.shellEnv }),
|
|
1017
|
-
timeoutMs: input.timeoutMs,
|
|
1018
|
-
});
|
|
1054
|
+
const wrote = writeServerText(conn, "instruction", instructionFrame.text);
|
|
1019
1055
|
if (body !== null)
|
|
1020
1056
|
sendPayloadSequence(conn, instructionId, body, input.payloadPartBytes ?? DEVICE_WS_LIMITS.defaultPayloadPartBytes, sha);
|
|
1021
1057
|
metrics?.inc("device_instruction_dispatched_total", { kind: input.kind });
|
|
@@ -4,14 +4,25 @@ export declare const DEVICE_WS_PATH = "/v1/device/ws";
|
|
|
4
4
|
/**
|
|
5
5
|
* 本 server 铸/认的协议**修订号**(wire 上的 `protocolVersion`)。
|
|
6
6
|
*
|
|
7
|
-
* 规则(§
|
|
8
|
-
*
|
|
9
|
-
* (
|
|
10
|
-
* (
|
|
7
|
+
* 规则(真源 = 协议文档 `docs/DEVICE-EXECUTOR-PROTOCOL.md` §2.1「Protocol revisions」段,此处只转述):加帧 / 加字段 = additive 不升号;
|
|
8
|
+
* 其余变更一律升号并在增量表加一行,已发布的行不改 —— **server 受理的闭集(executor → server 方向,今天 = 结局 `errorCode` 词集)加词 = 升号**,
|
|
9
|
+
* **语义变更(server 下发的帧意思变了)= 升号**。修订号是 executor 在发 hello 之前唯一能读到的「这台 server 受理什么、发来的帧是什么意思」的事实
|
|
10
|
+
* (challenge 帧带它),所以 server 受理的词集必须是修订号的函数({@link DEVICE_OUTCOME_ERROR_CODES_BY_REVISION},协议文档同名表),不另发第二份列表;
|
|
11
|
+
* 闭集变了而修订号没变 = 同一个修订号两种受理集(7.132.0 给结局码加 `liveness_no_output` 时没抬号,7.136.0 补到 2)。类型锁:抬了号而表没有这一行 ⇒ 编译红。
|
|
11
12
|
*/
|
|
12
13
|
export declare const DEVICE_PROTOCOL_VERSION: DeviceProtocolRevision;
|
|
13
|
-
/**
|
|
14
|
-
|
|
14
|
+
/**
|
|
15
|
+
* 受理下沿:低于它 ⇒ `helloReject: protocol_version_unsupported`(带 `minProtocolVersion` 与一句 {@link deviceProtocolRevisionRefusal},响亮拒,不静默降级,[ref])。
|
|
16
|
+
* 兼容窗 = [MIN, 当前]。律(协议文档 §2.1 规则句):升号的变更旧修订的 executor 守不住 ⇒ 同一版把下沿抬到那一修订号 —— 旧 executor 在 hello 被拒一次,
|
|
17
|
+
* 而不是逐调用失败。今天 = 5:修订号 5 起路径只锚不折(下沿理由见 {@link DEVICE_FLOOR_CHANGE}),写给修订号 ≤ 4 的 executor 认的是折过的路径。
|
|
18
|
+
* 这是**本 server** 的受理下沿,不是 executor 协商时的下界(executor 一侧修订号是累积的:实现了 5 的照样能按 4 服务老 server)。
|
|
19
|
+
*/
|
|
20
|
+
export declare const DEVICE_MIN_PROTOCOL_VERSION: DeviceProtocolRevision;
|
|
21
|
+
/**
|
|
22
|
+
* `protocol_version_unsupported` 的那一句 `message`(帧上与服务端 warn 行同一句;executor 可原样示人)。声明值在窗外只有两种:低于下沿 / 高于上沿。
|
|
23
|
+
* 握手之前的帧,任何能开 socket 的对端都读得到 ⇒ 句里只有修订号与下沿引入的版本,不带本进程的服务版本号。
|
|
24
|
+
*/
|
|
25
|
+
export declare function deviceProtocolRevisionRefusal(declared: number): string;
|
|
15
26
|
/** 签名的 domain tag —— 换域即换 tag,防止别处的 Ed25519 签名被拿来当握手用。 */
|
|
16
27
|
export declare const DEVICE_HELLO_DOMAIN_TAG = "sema.device.hello.v1";
|
|
17
28
|
/**
|
|
@@ -76,6 +87,7 @@ export declare const DEVICE_OUTCOME_ERROR_CODES_BY_REVISION: {
|
|
|
76
87
|
readonly 2: readonly ["liveness_no_output"];
|
|
77
88
|
readonly 3: readonly ["name_too_long"];
|
|
78
89
|
readonly 4: readonly ["symlink_loop"];
|
|
90
|
+
readonly 5: readonly [];
|
|
79
91
|
};
|
|
80
92
|
/** 本包认得的协议修订号(= 增量表的行键)。 */
|
|
81
93
|
export type DeviceProtocolRevision = keyof typeof DEVICE_OUTCOME_ERROR_CODES_BY_REVISION;
|
|
@@ -275,10 +287,12 @@ export type DeviceHelloAckFrame = {
|
|
|
275
287
|
chunkGapTimeoutMs?: number;
|
|
276
288
|
executionCeilingGraceMs?: number;
|
|
277
289
|
};
|
|
290
|
+
/** `minProtocolVersion` 与 `message` 只出现在 `protocol_version_unsupported` 上(加字段 = additive,不升号;协议文档 §1.4 / §2.3)。 */
|
|
278
291
|
export type DeviceHelloRejectFrame = {
|
|
279
292
|
t: "helloReject";
|
|
280
293
|
code: DeviceHelloRejectCode;
|
|
281
294
|
minProtocolVersion?: number;
|
|
295
|
+
message?: string;
|
|
282
296
|
};
|
|
283
297
|
/**
|
|
284
298
|
* helloAck 六个数值位的**合法域**(闭区间、整数)—— 一处成文:汇聚端创建时按它校验(越界 ⇒ 响亮拒启,不夹取、不换成缺省),
|
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
import { createPublicKey, randomBytes, randomInt, verify as edVerify } from "node:crypto";
|
|
2
2
|
import { MAX_TIMER_DELAY_MS, inWholeNumberBounds, wholeNumberBoundsRule } from "./config-types.js";
|
|
3
3
|
export const DEVICE_WS_PATH = "/v1/device/ws";
|
|
4
|
-
export const DEVICE_PROTOCOL_VERSION =
|
|
5
|
-
export const DEVICE_MIN_PROTOCOL_VERSION =
|
|
4
|
+
export const DEVICE_PROTOCOL_VERSION = 5;
|
|
5
|
+
export const DEVICE_MIN_PROTOCOL_VERSION = 5;
|
|
6
|
+
const DEVICE_FLOOR_CHANGE = 'since server 7.150.0 every path an instruction carries (args.path, args.paths, cwd) arrives unfolded, with "." / ".." / empty segments kept for the device\'s filesystem to resolve';
|
|
7
|
+
export function deviceProtocolRevisionRefusal(declared) {
|
|
8
|
+
return declared < DEVICE_MIN_PROTOCOL_VERSION
|
|
9
|
+
? `device protocol revision ${declared} is below the lowest revision this server accepts (${DEVICE_MIN_PROTOCOL_VERSION}): ${DEVICE_FLOOR_CHANGE}; upgrade the executor to one that declares revision ${DEVICE_MIN_PROTOCOL_VERSION} or above`
|
|
10
|
+
: `device protocol revision ${declared} is above the highest revision this server speaks (${DEVICE_PROTOCOL_VERSION}): declare the highest revision you implement that is not above challenge.protocolVersion`;
|
|
11
|
+
}
|
|
6
12
|
export const DEVICE_HELLO_DOMAIN_TAG = "sema.device.hello.v1";
|
|
7
13
|
export const DEVICE_WS_LIMITS = {
|
|
8
14
|
maxFrameBytes: 1024 * 1024,
|
|
@@ -68,6 +74,7 @@ export const DEVICE_OUTCOME_ERROR_CODES_BY_REVISION = {
|
|
|
68
74
|
2: ["liveness_no_output"],
|
|
69
75
|
3: ["name_too_long"],
|
|
70
76
|
4: ["symlink_loop"],
|
|
77
|
+
5: [],
|
|
71
78
|
};
|
|
72
79
|
export function deviceOutcomeErrorCodesAt(revision) {
|
|
73
80
|
if (!Number.isInteger(revision) || revision < 1 || revision > DEVICE_PROTOCOL_VERSION) {
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* 落点之前引擎自己按 `spec.principal` 分的那几面按人分(同一个人的各场景在那几面互见)—— 本仓镜像这几面的读口(工作流读面、后台登记簿的停止标记、
|
|
10
10
|
* leader 子任务的 spec)同读 {@link taskSpecIdentityOf},读到的就是引擎写下的那一个值,不各写一份。
|
|
11
11
|
*/
|
|
12
|
-
import type
|
|
12
|
+
import { type Person, type Subject, type SubjectParts } from "./owner-brands.js";
|
|
13
13
|
/** 引擎交界的身份(本仓的类型;二元)。 */
|
|
14
14
|
export interface EngineIdentity {
|
|
15
15
|
readonly principal: Person | undefined;
|
|
@@ -25,4 +25,10 @@ export declare function specIdentityOf(parts: SubjectParts | undefined): EngineI
|
|
|
25
25
|
export declare function taskSpecIdentityOf(parts: SubjectParts | undefined): {
|
|
26
26
|
readonly principal?: string;
|
|
27
27
|
};
|
|
28
|
+
/**
|
|
29
|
+
* 引擎按身份分的那几面落下的 **scope**(core 写侧 `spec.principal ?? "default"` 的同一条读法;工作流行的 `scope` 列就是它):一只主体 ⇒ 落点 {@link taskSpecIdentityOf}
|
|
30
|
+
* 写下的人;匿名 ⇒ 单机形的租户键 `"default"`(`owner-faces.ts` 的 `LEGACY_UNOWNED_AMNESTY_TENANT`)。读者只在属主存取器(`http/owner-access.ts`):工作流按 id 的门
|
|
31
|
+
* (调用方一侧按它换成引擎身份再与行的 scope 比)与工作流列表的属主过滤同读这一只;承认之后 journal / 流按承认的那一行的 scope 读。
|
|
32
|
+
*/
|
|
33
|
+
export declare function engineScopeOf(subject: Subject | undefined): string;
|
|
28
34
|
//# sourceMappingURL=engine-identity.d.ts.map
|
package/dist/engine-identity.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { partsOf } from "./owner-brands.js";
|
|
2
|
+
import { LEGACY_UNOWNED_AMNESTY_TENANT } from "./owner-faces.js";
|
|
1
3
|
export function specIdentityOf(parts) {
|
|
2
4
|
if (parts === undefined)
|
|
3
5
|
return { principal: undefined };
|
|
@@ -7,4 +9,7 @@ export function taskSpecIdentityOf(parts) {
|
|
|
7
9
|
const { principal } = specIdentityOf(parts);
|
|
8
10
|
return principal === undefined ? {} : { principal };
|
|
9
11
|
}
|
|
12
|
+
export function engineScopeOf(subject) {
|
|
13
|
+
return taskSpecIdentityOf(subject === undefined ? undefined : partsOf(subject)).principal ?? LEGACY_UNOWNED_AMNESTY_TENANT;
|
|
14
|
+
}
|
|
10
15
|
//# sourceMappingURL=engine-identity.js.map
|
package/dist/field-conflict.d.ts
CHANGED
|
@@ -10,12 +10,12 @@ export interface FieldConflictCoordinates {
|
|
|
10
10
|
readonly cascade_verify: "verify" | "cascade" | "sandboxImageProfile" | "resumeAt" | "suggestNextPrompts";
|
|
11
11
|
readonly image_profile: "capabilitiesNeeded" | "sandboxImageProfile";
|
|
12
12
|
readonly resume_anchor: "resumeAtMode" | "resumeAt";
|
|
13
|
-
readonly system_prompt_fold: "appendSystemPrompt" | "settings.outputStyle";
|
|
13
|
+
readonly system_prompt_fold: "appendSystemPrompt" | "projectId" | "settings.outputStyle";
|
|
14
14
|
readonly file_history_restore: "fileHistory" | "restoreFiles" | "rewindFilesTo";
|
|
15
15
|
readonly skills_disabled: "disableSkills" | "skills" | `agents[${number}].skills`;
|
|
16
16
|
readonly existing_session: "requireExistingSession" | "sessionId";
|
|
17
17
|
readonly fork_target: "forkId" | "forkNonce";
|
|
18
|
-
readonly scratchpad_root: "cwd" | "scratchpadDir";
|
|
18
|
+
readonly scratchpad_root: "cwd" | "projectId" | "scratchpadDir";
|
|
19
19
|
readonly ask_answer: "decision" | "answer";
|
|
20
20
|
readonly policy_person_ask: "decision" | "decidedBy";
|
|
21
21
|
readonly plan_edit: "decision" | "editedPlan";
|
|
@@ -16,6 +16,8 @@
|
|
|
16
16
|
* 那一形的面只有每拍一行运维 warn + 计数)。「扣着」在两出口上只有这一种读法(7.120.x 的 `/health.modelPlaneDeferred` 本版删除)。
|
|
17
17
|
* 合成只在 {@link buildRestartSignal} 一处,`restartState()` 是它唯一的调用方 —— 两个出口读的是同一只取值口。
|
|
18
18
|
* 4. **代际**:`generation` = env `FLEET_GENERATION` 的回声(规范十进制串;非规范在 boot 拒启,见 `config.ts`);缺席 ⇔ 非 fleet 成员。
|
|
19
|
+
* **围栏只有成员身份一个开关**:租约读数只有一只读者 {@link memberLeaseOf}(铸点与准入三处同读)—— 非成员恒 {@link LEASE_NONE},
|
|
20
|
+
* 注入了读数也不读;所以「`/health` 不报 `leaseState` 而准入照按租约拒」这一形构造不出来。
|
|
19
21
|
* **身份不是引擎态**:`instanceId`(本进程起步自铸的那一只,`boot/stage-03` 的 uuidv7)是体的身份,只在两出口体的**顶层**说一次,
|
|
20
22
|
* 铸点不读它、契约形里不重复它。
|
|
21
23
|
* 5. **两形一读(展开 / 收缩)**:两出口今天各发两形 —— 嵌套键 {@link ENGINE_SIGNAL_KEY}(契约 `EngineSignal` 整只,{@link buildEngineSignal}),
|
|
@@ -52,8 +54,19 @@ export type ServingLeaseView = {
|
|
|
52
54
|
};
|
|
53
55
|
/** 心跳体上的 fleet 成员三键 —— 类型取自契约心跳形(拼法与值域只有契约一个写者)。 */
|
|
54
56
|
export type FleetMembershipKeys = Pick<InstanceAnnouncement, "generation" | "leaseState" | "leaseUntil">;
|
|
55
|
-
/** 非 fleet 成员 / 没装租约客户端时的读数(唯一拼法)
|
|
57
|
+
/** 非 fleet 成员 / 没装租约客户端时的读数(唯一拼法)。成员的推定围栏只住在租约客户端里,不进这只缺省值。 */
|
|
56
58
|
export declare const LEASE_NONE: ServingLeaseView;
|
|
59
|
+
/**
|
|
60
|
+
* 租约读数的**唯一读者**(规则 4;铸点 {@link mintEngineSignal} 与准入三处 —— 计费门 / a2a `message/send` / PARKED 赎回席 —— 都经它,不直读
|
|
61
|
+
* `deps.servingLease`):成员身份 = `config.fleetGeneration` 在场 ⇒ 租约客户端此刻的读数(没装 ⇒ {@link LEASE_NONE});非成员 ⇒ {@link LEASE_NONE}。
|
|
62
|
+
*/
|
|
63
|
+
export declare function memberLeaseOf(deps: Pick<EngineSignalInputs, "config" | "servingLease">): ServingLeaseView;
|
|
64
|
+
/**
|
|
65
|
+
* 装配判的拒句(`createHttpServer` 构造期,fail-closed):租约读数只属于 fleet 成员,而成员身份只有 `config.fleetGeneration` 一个开关 ——
|
|
66
|
+
* 注入了 `servingLease` 却没设代际,读数永远不会被 {@link memberLeaseOf} 读到、准入永远不按租约拒。两键都是可选的,所以这是一份能编译的
|
|
67
|
+
* 自相矛盾的装配:响亮拒,不静默丢掉读数(本服务自己的组合根只给成员装租约客户端,走不到这里)。
|
|
68
|
+
*/
|
|
69
|
+
export declare const SERVING_LEASE_MEMBERSHIP_REQUIRED = "servingLease is wired but config.fleetGeneration is not: a serving-lease reading belongs to a fleet member, and membership is config.fleetGeneration (FLEET_GENERATION) alone \u2014 without it the reading is never read and admission never refuses on the lease; set config.fleetGeneration to this instance's generation, or do not pass servingLease; refusing to start rather than ignore the lease reading";
|
|
57
70
|
/** 扣着的候选(web-admin 的 `planeDeferredNoHandoff` 投影):版本、起扣时刻、被扣的重启片。 */
|
|
58
71
|
export interface RetainedCandidate {
|
|
59
72
|
readonly version: number;
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { readinessOf } from "../http/readiness.js";
|
|
2
2
|
import { drainingFactOf } from "./drain-entry.js";
|
|
3
3
|
export const LEASE_NONE = Object.freeze({ state: "none" });
|
|
4
|
+
export function memberLeaseOf(deps) {
|
|
5
|
+
return deps.config?.fleetGeneration === undefined ? LEASE_NONE : (deps.servingLease?.() ?? LEASE_NONE);
|
|
6
|
+
}
|
|
7
|
+
export const SERVING_LEASE_MEMBERSHIP_REQUIRED = "servingLease is wired but config.fleetGeneration is not: a serving-lease reading belongs to a fleet member, and membership is config.fleetGeneration (FLEET_GENERATION) alone — without it the reading is never read and admission never refuses on the lease; set config.fleetGeneration to this instance's generation, or do not pass servingLease; refusing to start rather than ignore the lease reading";
|
|
4
8
|
export function buildRestartSignal(published, retained) {
|
|
5
9
|
if (published !== undefined)
|
|
6
10
|
return published;
|
|
@@ -29,7 +33,7 @@ export function mintEngineSignal(deps) {
|
|
|
29
33
|
...(draining !== undefined ? { draining } : {}),
|
|
30
34
|
...(restart !== undefined ? { restart } : {}),
|
|
31
35
|
...(deps.config?.fleetGeneration !== undefined ? { generation: deps.config.fleetGeneration } : {}),
|
|
32
|
-
lease: deps
|
|
36
|
+
lease: memberLeaseOf(deps),
|
|
33
37
|
...(configApply !== undefined ? { configApply } : {}),
|
|
34
38
|
};
|
|
35
39
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* [ref](docs/
|
|
2
|
+
* [ref](docs/[ref]-fleet-liveness-reconcile.md v1.2,四方零驳定稿)—— fleet 总线**存活对账腿**。
|
|
3
3
|
*
|
|
4
4
|
* ## 它修什么(§0 病灶机制链)
|
|
5
5
|
* fleet 总线是进程内纯内存聚合(`fleet-bus.ts` 头注自陈 NOT durable / NOT cross-replica),行的**唯一**
|
|
@@ -32,11 +32,26 @@ export interface ServingLeaseClientOpts {
|
|
|
32
32
|
readonly epoch: () => number;
|
|
33
33
|
};
|
|
34
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* 一拍心跳听到了什么(`fleet-client.ts` 每一次公告尝试恰好交一只;读者在本模块)。
|
|
37
|
+
* · `reading` —— 中心的显式能力声明(契约 `declared`;位缺席 = 中心说不执行围栏);
|
|
38
|
+
* · `silent` —— 没有读数。`why` 的形是闭集:`HTTP <状态>`(非 2xx,不带中心体片段)/ 请求抛出的那句(不可达 / 超时 / 凭证读不出)/
|
|
39
|
+
* `reply-unreadable:<因>`(2xx 而不是声明,因 = `fleet-client.ts` `AnnounceReplyFlaw`)/ `not-sent:<那句>`(这一拍的引擎信号铸不出,没发)。
|
|
40
|
+
*/
|
|
41
|
+
export type HeartbeatHearing = {
|
|
42
|
+
readonly kind: "reading";
|
|
43
|
+
readonly capabilities: WebAdminCapabilities;
|
|
44
|
+
} | {
|
|
45
|
+
readonly kind: "silent";
|
|
46
|
+
readonly why: string;
|
|
47
|
+
};
|
|
35
48
|
export interface ServingLeaseClient {
|
|
36
|
-
/**
|
|
37
|
-
|
|
38
|
-
/** 此刻的租约读数(铸点与准入判读它)。 */
|
|
49
|
+
/** 一拍心跳听到的东西(规则 1;每一次公告尝试恰好一次)。 */
|
|
50
|
+
noteHeartbeat(hearing: HeartbeatHearing): void;
|
|
51
|
+
/** 此刻的租约读数(铸点与准入判读它 —— 经 `engine-signal.ts` 的唯一读者 `memberLeaseOf`)。 */
|
|
39
52
|
view(): ServingLeaseView;
|
|
53
|
+
/** 订阅「租约态由事件翻转」(规则 2′;一只订阅者 = 装配层的 `announceNow`,晚绑:fleet 客户端在本客户端之后造)。 */
|
|
54
|
+
onViewChange(listener: () => void): void;
|
|
40
55
|
/** 起步读黏性拒(装配层在唯一排空入口装好之后调一次)。 */
|
|
41
56
|
resumeStickyDenial(): void;
|
|
42
57
|
/** 立刻续一次(测试 / 能力位刚出现时用;单飞)。 */
|
|
@@ -40,8 +40,9 @@ export function createServingLeaseClient(o) {
|
|
|
40
40
|
const mono = o.clock?.mono ?? (() => performance.now());
|
|
41
41
|
const epoch = o.clock?.epoch ?? Date.now;
|
|
42
42
|
const stickyPath = o.dataRoot === undefined ? undefined : join(o.dataRoot, SERVING_LEASE_DENIED_FILE);
|
|
43
|
-
let
|
|
44
|
-
const enabled = () =>
|
|
43
|
+
let fence = { kind: "presumed", since: mono() };
|
|
44
|
+
const enabled = () => fence.kind === "on";
|
|
45
|
+
let viewListener;
|
|
45
46
|
let denied = false;
|
|
46
47
|
let stopped = false;
|
|
47
48
|
let lease;
|
|
@@ -49,6 +50,37 @@ export function createServingLeaseClient(o) {
|
|
|
49
50
|
let timer;
|
|
50
51
|
let inFlight;
|
|
51
52
|
let failStreak = 0;
|
|
53
|
+
const view = () => {
|
|
54
|
+
if (denied)
|
|
55
|
+
return { state: "denied" };
|
|
56
|
+
if (fence.kind === "off")
|
|
57
|
+
return { state: "none" };
|
|
58
|
+
const now = mono();
|
|
59
|
+
if (lease !== undefined && now < lease.expiresAtMono)
|
|
60
|
+
return { state: "granted", leaseUntil: lease.leaseUntil };
|
|
61
|
+
return { state: "expired", pastOutageMax: now - (lease?.expiresAtMono ?? fence.since) >= o.outageMaxMs };
|
|
62
|
+
};
|
|
63
|
+
const changing = (change) => {
|
|
64
|
+
const before = view().state;
|
|
65
|
+
change();
|
|
66
|
+
if (view().state !== before)
|
|
67
|
+
viewListener?.();
|
|
68
|
+
};
|
|
69
|
+
const unconfirmed = (why) => {
|
|
70
|
+
recordFailOpen("server.fleet.serving-lease-unconfirmed", `worker=${o.center.worker} generation=${o.generation} why=${why}`);
|
|
71
|
+
failStreak += 1;
|
|
72
|
+
if (failStreak === 1) {
|
|
73
|
+
o.logger?.warn("fleet_serving_lease_unconfirmed", {
|
|
74
|
+
error: why,
|
|
75
|
+
note: "the fleet center has not confirmed this instance's serving lease (heartbeat or renewal unanswered, refused or unreadable, or the verdict was unknown) — this instance keeps serving until its last grant, or with no grant yet the moment the fence took effect (this process's start, for a member that has not heard the center), is FLEET_LEASE_OUTAGE_MAX_MS behind, then refuses new work (unconfirmed is never a denial)",
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
const confirmed = (by) => {
|
|
80
|
+
if (failStreak > 0)
|
|
81
|
+
o.logger?.info("fleet_serving_lease_recovered", { after: failStreak, by });
|
|
82
|
+
failStreak = 0;
|
|
83
|
+
};
|
|
52
84
|
const clear = () => {
|
|
53
85
|
if (timer !== undefined)
|
|
54
86
|
clearTimeout(timer);
|
|
@@ -79,6 +111,9 @@ export function createServingLeaseClient(o) {
|
|
|
79
111
|
const enterDenied = (denial) => {
|
|
80
112
|
if (denied)
|
|
81
113
|
return;
|
|
114
|
+
changing(() => enterDeniedNow(denial));
|
|
115
|
+
};
|
|
116
|
+
const enterDeniedNow = (denial) => {
|
|
82
117
|
denied = true;
|
|
83
118
|
lease = undefined;
|
|
84
119
|
clear();
|
|
@@ -118,25 +153,22 @@ export function createServingLeaseClient(o) {
|
|
|
118
153
|
why = err instanceof Error ? err.message : String(err);
|
|
119
154
|
}
|
|
120
155
|
if (reading === undefined || reading.verdict === "unknown") {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
failStreak += 1;
|
|
124
|
-
if (failStreak === 1)
|
|
125
|
-
o.logger?.warn("fleet_serving_lease_unconfirmed", { error: unconfirmed, note: "the serving lease could not be renewed (the center is unreachable or did not confirm this instance) — this instance keeps serving until the lease expires and then for FLEET_LEASE_OUTAGE_MAX_MS more before it refuses new work (unconfirmed is never a denial)" });
|
|
156
|
+
if (fence.kind === "on")
|
|
157
|
+
unconfirmed(reading === undefined ? why : "verdict=unknown (the center holds no lifecycle object for this instance right now)");
|
|
126
158
|
schedule(nextInMs);
|
|
127
159
|
return;
|
|
128
160
|
}
|
|
129
|
-
|
|
130
|
-
o.logger?.info("fleet_serving_lease_recovered", { after: failStreak });
|
|
131
|
-
failStreak = 0;
|
|
161
|
+
confirmed("verdict");
|
|
132
162
|
switch (reading.verdict) {
|
|
133
|
-
case "granted":
|
|
163
|
+
case "granted": {
|
|
134
164
|
if (denied)
|
|
135
165
|
return;
|
|
136
|
-
|
|
166
|
+
const granted = { expiresAtMono: sendMono + reading.ttlMs, leaseUntil: sendEpoch + reading.ttlMs };
|
|
167
|
+
changing(() => (lease = granted));
|
|
137
168
|
nextInMs = reading.ttlMs / 3;
|
|
138
169
|
schedule(nextInMs);
|
|
139
170
|
return;
|
|
171
|
+
}
|
|
140
172
|
case "denied":
|
|
141
173
|
enterDenied({ from: "center", reason: reading.reason });
|
|
142
174
|
return;
|
|
@@ -154,34 +186,48 @@ export function createServingLeaseClient(o) {
|
|
|
154
186
|
});
|
|
155
187
|
return inFlight;
|
|
156
188
|
};
|
|
189
|
+
const noteDeclaration = (caps) => {
|
|
190
|
+
const on = caps.servingLease === true;
|
|
191
|
+
if (!on)
|
|
192
|
+
recordFailOpen("server.fleet.serving-lease-capability-absent", `worker=${o.center.worker} generation=${o.generation} was=${fence.kind}`);
|
|
193
|
+
if (on && fence.kind !== "on") {
|
|
194
|
+
fence = { kind: "on", since: fence.since ?? mono() };
|
|
195
|
+
o.logger?.info("fleet_serving_lease_enabled", { generation: o.generation, worker: o.center.worker });
|
|
196
|
+
void renewNow();
|
|
197
|
+
}
|
|
198
|
+
else if (!on && fence.kind !== "off") {
|
|
199
|
+
const was = fence.kind;
|
|
200
|
+
fence = { kind: "off", since: was === "on" ? fence.since : undefined };
|
|
201
|
+
clear();
|
|
202
|
+
confirmed("not-enforced");
|
|
203
|
+
if (was === "on")
|
|
204
|
+
o.logger?.warn("fleet_serving_lease_disabled", { note: "the fleet center no longer announces the servingLease capability — the lease is out of effect (leaseState none) until it does again; its outage clock is not reset (when the capability returns, the last grant and the moment the fence first took effect in this process still count)" });
|
|
205
|
+
}
|
|
206
|
+
};
|
|
157
207
|
return {
|
|
158
|
-
|
|
208
|
+
noteHeartbeat(hearing) {
|
|
159
209
|
if (stopped || denied)
|
|
160
210
|
return;
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
211
|
+
switch (hearing.kind) {
|
|
212
|
+
case "silent":
|
|
213
|
+
if (fence.kind === "presumed")
|
|
214
|
+
unconfirmed(`heartbeat:${hearing.why}`);
|
|
215
|
+
return;
|
|
216
|
+
case "reading":
|
|
217
|
+
if (fence.kind === "presumed")
|
|
218
|
+
changing(() => noteDeclaration(hearing.capabilities));
|
|
219
|
+
else
|
|
220
|
+
noteDeclaration(hearing.capabilities);
|
|
221
|
+
return;
|
|
222
|
+
default: {
|
|
223
|
+
const unhandled = hearing;
|
|
224
|
+
throw new Error(`serving lease: unknown heartbeat hearing ${JSON.stringify(unhandled)}`);
|
|
225
|
+
}
|
|
173
226
|
}
|
|
174
227
|
},
|
|
175
|
-
view
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
if (capability === undefined || !capability.announced)
|
|
179
|
-
return { state: "none" };
|
|
180
|
-
const now = mono();
|
|
181
|
-
if (lease !== undefined && now < lease.expiresAtMono)
|
|
182
|
-
return { state: "granted", leaseUntil: lease.leaseUntil };
|
|
183
|
-
const expiredAt = lease?.expiresAtMono ?? capability.firstSeenAtMono;
|
|
184
|
-
return { state: "expired", pastOutageMax: now - expiredAt >= o.outageMaxMs };
|
|
228
|
+
view,
|
|
229
|
+
onViewChange(listener) {
|
|
230
|
+
viewListener = listener;
|
|
185
231
|
},
|
|
186
232
|
resumeStickyDenial() {
|
|
187
233
|
if (stickyPath === undefined || denied)
|