@twinsearth/w2m-dsh-plugin 0.6.2 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/CHANGELOG.md +1057 -0
  2. package/README.md +9 -2
  3. package/bin/w2m-localside.mjs +0 -0
  4. package/bin/w2m-rabbit.mjs +6 -0
  5. package/bin/w2m-stun.mjs +0 -0
  6. package/deploy/server/README.md +109 -0
  7. package/deploy/server/deploy-w2m.sh +161 -0
  8. package/deploy/server/w2m-localside.service +31 -0
  9. package/deploy/server/w2m-rabbit.service +26 -0
  10. package/deploy/server/w2m-stun.service +26 -0
  11. package/docs/ACCOUNT-FLEET-DESIGN.md +369 -0
  12. package/package.json +33 -22
  13. package/src/chatops/account.mjs +361 -0
  14. package/src/chatops/actions.mjs +227 -0
  15. package/src/chatops/approval-broker.mjs +320 -0
  16. package/src/chatops/auth.mjs +376 -0
  17. package/src/chatops/bridge.mjs +344 -0
  18. package/src/chatops/channel.mjs +192 -0
  19. package/src/chatops/commands.mjs +409 -0
  20. package/src/chatops/config-route.mjs +332 -0
  21. package/src/chatops/device-signing.mjs +218 -0
  22. package/src/chatops/durability.mjs +202 -0
  23. package/src/chatops/file-safety.mjs +282 -0
  24. package/src/chatops/fleet-scope.mjs +176 -0
  25. package/src/chatops/fleet-wiring.mjs +199 -0
  26. package/src/chatops/fleet.mjs +272 -0
  27. package/src/chatops/handoff.mjs +258 -0
  28. package/src/chatops/ilink-protocol.mjs +165 -0
  29. package/src/chatops/ilink-transport.mjs +168 -0
  30. package/src/chatops/ilink.mjs +423 -0
  31. package/src/chatops/index.mjs +429 -0
  32. package/src/chatops/outbound-queue.mjs +282 -0
  33. package/src/chatops/policy.mjs +356 -0
  34. package/src/chatops/run-command.mjs +308 -0
  35. package/src/plugin/tools.mjs +1 -1
  36. package/src/relay/server.mjs +96 -0
  37. package/src/relay/state.mjs +2233 -2011
@@ -1,2011 +1,2233 @@
1
- /**
2
- * W2M Rabbit relay — in-memory state machine.
3
- *
4
- * Contract: E:\DS\_work\w2m\PROTOCOL.md (protocol_version = 1, frozen).
5
- * This module is intentionally PURE LOGIC:
6
- * - no network, no filesystem, no timers
7
- * - time is injectable via `now()` (required for lease-expiry unit tests)
8
- * - randomness is injectable for deterministic tests
9
- *
10
- * Persistence: none. All state is in-memory and is lost on restart (v0.0.1 scope);
11
- * there is intentionally no snapshot/serialization hook yet.
12
- */
13
-
14
- import { createHash, randomBytes } from 'node:crypto';
15
-
16
- export const PROTOCOL_VERSION = 1;
17
-
18
- /**
19
- * PROTOCOL §1 table + §7 code table, plus the two v0.1.2 additions:
20
- * `OPERATOR_REQUIRED` (§5.2 → 401) and `RATE_LIMITED` (§6 → 429).
21
- * A code missing from this map silently degrades to HTTP 500 via
22
- * `ProtocolError`, which keeps the right error code but lies about the status.
23
- */
24
- export const ERROR_STATUS = Object.freeze({
25
- UNAUTHORIZED: 401,
26
- BAD_REQUEST: 400,
27
- NOT_FOUND: 404,
28
- PAIRING_INVALID: 400,
29
- PAIRING_EXPIRED: 400,
30
- TASK_EXISTS: 409,
31
- FRAME_TOO_LARGE: 413,
32
- NO_ONLINE_DEVICE: 503,
33
- INTERNAL: 500,
34
- OPERATOR_REQUIRED: 401, // v0.1.2 §5.2
35
- RATE_LIMITED: 429, // v0.1.2 §6
36
-
37
- /* v0.3.0 §9 request signing. Every client-side signature failure is a 401.
38
- * `SIGNING_NOT_CONFIGURED` is deliberately NOT 401: the relay was started with
39
- * `--require-signature` but no secret, so the fault is the relay's own
40
- * configuration. Reporting that as 401 would send every operator to debug the
41
- * client -- the one side that is not broken. 500 rather than 503 because
42
- * retrying never fixes a missing configuration, so nothing should be invited to
43
- * retry. */
44
- SIGNATURE_REQUIRED: 401,
45
- SIGNATURE_INCOMPLETE: 401,
46
- SIGNATURE_MISMATCH: 401,
47
- SIGNATURE_BAD_TIMESTAMP: 401,
48
- SIGNATURE_BAD_NONCE: 401,
49
- SIGNATURE_EXPIRED: 401,
50
- SIGNATURE_REPLAY: 401,
51
- SIGNING_NOT_CONFIGURED: 500,
52
- });
53
-
54
- /** §5.1 required envelope fields — missing any one means `unverifiable`. */
55
- export const REQUIRED_ENVELOPE_FIELDS = Object.freeze([
56
- 'envelope_version', 'task_id', 'attempt', 'dedupe_key', 'machine_id', 'machine_name',
57
- 'platform', 'caps', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
58
- 'pre_tree_fingerprint', 'post_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error',
59
- 'head_commit', 'dirty_before', 'command_argv', 'command_hash', 'shell_id',
60
- 'started_at', 'ended_at', 'duration_ms', 'exit_code', 'status', 'refusal_reason',
61
- 'stdout_sha256', 'stdout_bytes', 'stderr_sha256', 'stderr_bytes', 'warnings',
62
- 'envelope_sha256',
63
- ]);
64
-
65
- /** §5.3 comparable fields — the ONLY fields that take part in consistency judgement. */
66
- export const COMPARABLE_FIELDS = Object.freeze([
67
- 'task_id', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
68
- 'pre_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error', 'head_commit',
69
- 'command_hash', 'shell_id', 'exit_code', 'status', 'refusal_reason',
70
- 'stdout_sha256', 'stdout_bytes', 'stdout_normalized_sha256',
71
- 'stderr_sha256', 'artifacts', 'diff_numstat', 'tests', 'semantic_counts', 'warnings',
72
- ]);
73
-
74
- /** §5.1 status enum. */
75
- export const RESULT_STATUSES = Object.freeze([
76
- 'ok', 'nonzero_exit', 'timeout', 'crashed', 'refused', 'unverifiable',
77
- ]);
78
-
79
- export const FAILURE_STATUSES = Object.freeze(['nonzero_exit', 'timeout', 'crashed']);
80
-
81
- /** §4.3 phase enum. */
82
- export const LEASE_PHASES = Object.freeze(['preparing', 'running', 'finalizing']);
83
-
84
- /** Lease states that can still be expired / renewed. */
85
- export const ACTIVE_LEASE_STATES = Object.freeze(['queued', 'offered', 'running']);
86
-
87
- export const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000; // §4.3: agent heartbeats every 10s
88
- export const DEFAULT_MISSED_HEARTBEATS = 2; // §4.3: 2 missed cycles = 20s
89
- export const DEFAULT_LEASE_GRACE_MS = 30_000; // §4.3: + grace 30s
90
- /**
91
- * v0.4.0: the bounded hold on the relay's own offer when a dispatcher is pushing it directly.
92
- *
93
- * Sized against what it has to cover: one punch round trip (measured 15-370 ms on the reference
94
- * networks), the frame's delivery, and the executor's first heartbeat POST — all on a host that may
95
- * be busy. 1.2 s was the first guess and it was too tight: measured on a 1-vCPU shared host, two
96
- * agent tests saw the relay's copy overtake the direct one, which for `p2pMode: direct` means a
97
- * refusal of an offer that was about to arrive. 3 s covers a slow host with room to spare.
98
- *
99
- * The cost is honest and bounded: a punch that fails delays the relay's copy by up to this much,
100
- * once per task. Nothing else waits on it.
101
- */
102
- export const DEFAULT_P2P_OFFER_GRACE_MS = 3_000;
103
- export const DEFAULT_PAIRING_TTL_MS = 24 * 60 * 60 * 1000; // §2.2: 24h
104
- export const DEFAULT_EVENT_BUFFER_SIZE = 1000; // §requirement: keep last 1000 events
105
- export const MAX_FRAME_BYTES = 64 * 1024; // 64 KiB single frame
106
- export const SHELL_ID_DIRECT = 'direct-exec'; // §5.1: v0.0.1 only direct exec
107
- /** v0.3.0: RTT sanity ceiling — anything above 24h is treated as a broken value. */
108
- export const MAX_RTT_MS = 24 * 60 * 60 * 1000;
109
- /** v0.3.0: an RTT measurement older than this is reported as stale.
110
- * Must stay comfortably ABOVE the idle-heartbeat interval (60s): at 1x the value
111
- * would flip stale just before every refresh and flap, which would make
112
- * `rtt_stale === false` useless as a "trust this number" predicate. 3x gives two
113
- * consecutive missed idle heartbeats of margin. See PROTOCOL-v0.3.0.md §7. */
114
- export const DEFAULT_RTT_STALE_MS = 180_000;
115
- /** v0.3.0: recommended cadence for an agent with no lease (reference for the agent side). */
116
- export const DEFAULT_IDLE_HEARTBEAT_MS = 60_000;
117
- export const ENVELOPE_VERSION = '1.0';
118
- export const FINGERPRINT_ALGO = 'git-temp-index-tree/v1';
119
-
120
- /* ------------------------------------------------------------------ */
121
- /* Errors */
122
- /* ------------------------------------------------------------------ */
123
-
124
- export class ProtocolError extends Error {
125
- constructor(code, message, detail = {}) {
126
- super(message);
127
- this.name = 'ProtocolError';
128
- this.code = code;
129
- this.detail = detail;
130
- this.status = ERROR_STATUS[code] ?? 500;
131
- }
132
- toBody() {
133
- return { error: { code: this.code, message: this.message, detail: this.detail } };
134
- }
135
- }
136
-
137
- /* ------------------------------------------------------------------ */
138
- /* Small pure helpers */
139
- /* ------------------------------------------------------------------ */
140
-
141
- export function sha256Hex(input) {
142
- return createHash('sha256').update(input).digest('hex');
143
- }
144
-
145
- /** RFC 8785-ish canonical JSON (JCS): sorted keys, no whitespace. */
146
- export function jcs(value) {
147
- if (value === null) return 'null';
148
- const t = typeof value;
149
- if (t === 'number') return Number.isFinite(value) ? JSON.stringify(value) : 'null';
150
- if (t === 'boolean') return value ? 'true' : 'false';
151
- if (t === 'string') return JSON.stringify(value);
152
- if (t === 'undefined' || t === 'function' || t === 'symbol') return 'null';
153
- if (Array.isArray(value)) return `[${value.map((v) => (v === undefined ? 'null' : jcs(v))).join(',')}]`;
154
- if (t === 'object') {
155
- const keys = Object.keys(value).filter((k) => value[k] !== undefined).sort();
156
- return `{${keys.map((k) => `${JSON.stringify(k)}:${jcs(value[k])}`).join(',')}}`;
157
- }
158
- return JSON.stringify(String(value));
159
- }
160
-
161
- /**
162
- * Upper bound on the stages a `pipeline` may contain.
163
- *
164
- * Each stage is another command run on someone else's machine, so an unbounded list is an unbounded
165
- * amount of work created by one request. The limit is generous for the intended use (build, test,
166
- * package) and exists so the failure is a clear refusal rather than a fleet-wide surprise.
167
- */
168
- export const MAX_PIPELINE_STAGES = 16;
169
-
170
- /**
171
- * v0.4.0: upper bound on `POST /v1/task`'s `origin_machine_id`.
172
- *
173
- * A machine id is an identifier, not a payload: 128 characters is far beyond any id this system
174
- * mints, and the ceiling is what stops the field from becoming an unbounded free-text channel into
175
- * every offer frame the relay emits.
176
- */
177
- export const MAX_ORIGIN_MACHINE_ID_LENGTH = 128;
178
-
179
- /**
180
- * v0.4.0: the P2P modes a task may be dispatched under (§1 of the frozen v0.4.0 contract).
181
- *
182
- * Mirrored here rather than imported from `src/agent/p2p-node.mjs`: the relay is a standalone
183
- * process that must not depend on agent code, and a mode list is three strings. A value outside
184
- * this list is a configuration error, never a silent fall back to the default.
185
- */
186
- export const P2P_MODES = Object.freeze(['auto', 'direct', 'relay']);
187
-
188
- /**
189
- * v0.4.0: how a result travelled — the direct P2P channel or the relay.
190
- *
191
- * A **transport is not a result**: how an envelope arrived must never take part in the consistency
192
- * judgement (§5 of the contract), or two machines that produced byte-identical output would be
193
- * reported `divergent` merely because one of them punched a hole. Hence: not required, not
194
- * comparable, and absent on every v0.3.9 envelope.
195
- */
196
- export const TRANSPORT_VALUES = Object.freeze(['p2p', 'relay']);
197
-
198
- /** §4.4 helper: command_hash = sha256(JCS(argv)+"|"+shell_id+"|"+cwd_rel). */
199
- export function computeCommandHash(commandArgv, shellId = SHELL_ID_DIRECT, cwdRel = '.') {
200
- return sha256Hex(`${jcs(commandArgv)}|${shellId}|${cwdRel}`);
201
- }
202
-
203
- /**
204
- * `command_hash` for a whole pipeline.
205
- *
206
- * The hash must bind **every** stage, not just the first. Bound to stage 0 alone, two pipelines that
207
- * differ after the first stage would carry the same hash -- so an envelope produced by the wrong
208
- * chain would still verify, and making "the machine ran what I asked" checkable is the entire point
209
- * of this field. Each stage's cwd participates for the same reason.
210
- *
211
- * The `|pipeline` suffix means this can never collide with a single-command hash: the two hash over
212
- * different shapes, and that difference is deliberate rather than incidental.
213
- */
214
- export function computePipelineCommandHash(stages, shellId = SHELL_ID_DIRECT) {
215
- const canonical = stages.map((s) => ({
216
- command_argv: s.command_argv,
217
- cwd_rel: s.cwd_rel ?? '.',
218
- continue_on_failure: s.continue_on_failure === true,
219
- }));
220
- return sha256Hex(`${jcs(canonical)}|${shellId}|pipeline`);
221
- }
222
-
223
- /** §4.4: dedupe_key = sha256(task_id + "|" + index + "|" + command_hash + "|" + base_tree). */
224
- export function computeDedupeKey(taskId, index, commandHash, baseTree) {
225
- return sha256Hex(`${taskId}|${index}|${commandHash}|${baseTree}`);
226
- }
227
-
228
- /** Epoch ms -> RFC3339 UTC, second precision (matches the PROTOCOL examples). */
229
- export function rfc3339(ms) {
230
- return new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
231
- }
232
-
233
- export function isRfc3339(value) {
234
- return typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/.test(value);
235
- }
236
-
237
- export function deepEqual(a, b) {
238
- return jcs(a === undefined ? null : a) === jcs(b === undefined ? null : b);
239
- }
240
-
241
- function normalizeNow(now) {
242
- if (typeof now !== 'function') return () => Date.now();
243
- return () => {
244
- const v = now();
245
- if (typeof v === 'number') return v;
246
- if (v instanceof Date) return v.getTime();
247
- return Date.parse(v);
248
- };
249
- }
250
-
251
- /* ------------------------------------------------------------------ */
252
- /* ULID */
253
- /* ------------------------------------------------------------------ */
254
-
255
- const CROCKFORD = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
256
-
257
- function encodeBig(value, len) {
258
- let out = '';
259
- let v = value;
260
- for (let i = 0; i < len; i += 1) {
261
- out = CROCKFORD[Number(v & 31n)] + out;
262
- v >>= 5n;
263
- }
264
- return out;
265
- }
266
-
267
- export function createIdFactory() {
268
- let lastMs = -1;
269
- let lastRand = 0n;
270
- return function nextId(nowMs = Date.now()) {
271
- const bytes = randomBytes(10);
272
- let rand = 0n;
273
- for (const b of bytes) rand = (rand << 8n) | BigInt(b);
274
- if (nowMs === lastMs) rand = (lastRand + 1n) & ((1n << 80n) - 1n);
275
- lastMs = nowMs;
276
- lastRand = rand;
277
- return encodeBig(BigInt(nowMs), 10) + encodeBig(rand, 16);
278
- };
279
- }
280
-
281
- export function ulid(nowMs = Date.now()) {
282
- return createIdFactory()(nowMs);
283
- }
284
-
285
- /* ------------------------------------------------------------------ */
286
- /* Version / capability gate (§6.1) */
287
- /* ------------------------------------------------------------------ */
288
-
289
- function parseVersion(text) {
290
- if (typeof text !== 'string') return null;
291
- const m = text.trim().match(/^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:\.(\d+))?/);
292
- if (!m) return null;
293
- return [Number(m[1]), Number(m[2] ?? 0), Number(m[3] ?? 0), Number(m[4] ?? 0)];
294
- }
295
-
296
- function cmpVersion(a, b) {
297
- for (let i = 0; i < 4; i += 1) {
298
- if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1;
299
- }
300
- return 0;
301
- }
302
-
303
- /**
304
- * Minimal semver-range check used by the §6.1 capability gate.
305
- * Supports bare versions and comma/space separated `>= > <= < = ==` conjunctions.
306
- * Unparseable ranges are treated as NOT satisfied (never silently pass).
307
- */
308
- export function satisfiesRange(version, range) {
309
- const v = parseVersion(version);
310
- if (v === null) return false;
311
- if (typeof range !== 'string') return false;
312
- const parts = range.split(/[\s,]+/).filter(Boolean);
313
- if (parts.length === 0) return false;
314
- for (const part of parts) {
315
- const m = part.match(/^(>=|<=|==|=|>|<)?\s*(.+)$/);
316
- if (!m) return false;
317
- const op = m[1] ?? '=';
318
- const target = parseVersion(m[2]);
319
- if (target === null) return false;
320
- const c = cmpVersion(v, target);
321
- if (op === '>=' && c < 0) return false;
322
- if (op === '<=' && c > 0) return false;
323
- if (op === '>' && c <= 0) return false;
324
- if (op === '<' && c >= 0) return false;
325
- if ((op === '=' || op === '==') && c !== 0) return false;
326
- }
327
- return true;
328
- }
329
-
330
- function machineWritable(caps) {
331
- if (!caps || typeof caps !== 'object') return false;
332
- if (caps.write === false || caps.writable === false || caps.write_allowed === false) return false;
333
- if (caps.read_only === true || caps.readonly === true || caps.ro_mount === true) return false;
334
- return true;
335
- }
336
-
337
- /**
338
- * §6.1 capability gate, evaluated per machine BEFORE dispatch.
339
- * @returns {null | {status:'refused', refusal_reason:string, detail:object}}
340
- */
341
- export function gateMachine(task, machine) {
342
- const caps = machine?.caps ?? {};
343
- const req = task.requirements ?? {};
344
-
345
- if (req.platform && Array.isArray(req.platform) && req.platform.length > 0) {
346
- const os = String(machine?.platform?.os ?? '').toLowerCase();
347
- if (!req.platform.map((p) => String(p).toLowerCase()).includes(os)) {
348
- return { status: 'refused', refusal_reason: 'PLATFORM_MISMATCH', detail: { os, want: req.platform } };
349
- }
350
- }
351
-
352
- const toolchain = req.toolchain ?? {};
353
- for (const [tool, range] of Object.entries(toolchain)) {
354
- const declared = caps[tool];
355
- if (declared === null || declared === undefined || declared === '') {
356
- return {
357
- status: 'refused',
358
- refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
359
- detail: { tool, want: range, have: declared ?? null },
360
- };
361
- }
362
- if (typeof range === 'string' && !satisfiesRange(declared, range)) {
363
- return {
364
- status: 'refused',
365
- refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
366
- detail: { tool, want: range, have: declared },
367
- };
368
- }
369
- }
370
-
371
- if (task.write === true && !machineWritable(caps)) {
372
- return { status: 'refused', refusal_reason: 'READ_ONLY_MACHINE', detail: { caps } };
373
- }
374
-
375
- return null;
376
- }
377
-
378
- /* ------------------------------------------------------------------ */
379
- /* RabbitState */
380
- /* ------------------------------------------------------------------ */
381
-
382
- const INSTANCE_SEP = '\u0000';
383
-
384
- export class RabbitState {
385
- constructor(options = {}) {
386
- this._now = normalizeNow(options.now);
387
- this._idFactory = options.idFactory ?? createIdFactory();
388
- this._randomToken = options.tokenFactory ?? (() => randomBytes(24).toString('hex'));
389
- this._randomChars = options.randomChars
390
- ?? ((n) => { const a = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; const b = randomBytes(n); let s = ''; for (let i = 0; i < n; i += 1) s += a[b[i] % a.length]; return s; });
391
-
392
- this.protocolVersion = PROTOCOL_VERSION;
393
- this.eventBufferSize = options.eventBufferSize ?? DEFAULT_EVENT_BUFFER_SIZE;
394
- this.pairingTtlMs = options.pairingTtlMs ?? DEFAULT_PAIRING_TTL_MS;
395
- /** When true one code stays valid for its whole TTL instead of rotating. */
396
- this.pairingCodeReusable = options.pairingCodeReusable === true;
397
- this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
398
- this.missedHeartbeats = options.missedHeartbeats ?? DEFAULT_MISSED_HEARTBEATS;
399
- this.leaseGraceMs = options.leaseGraceMs ?? DEFAULT_LEASE_GRACE_MS;
400
- /**
401
- * v0.4.0: how long the relay holds its own offer back when the dispatcher says it will push the
402
- * offer over the direct path (`origin_machine_id` + `p2p.mode`).
403
- *
404
- * Without this, the direct path could never win: the relay's offer goes out over an SSE stream
405
- * that is already open, while a punch costs at least one round trip, so the relay copy would
406
- * arrive first essentially always and `transport` would read `relay` even on a network where
407
- * the punch succeeded. Measured on loopback, and it is exactly what the first version of this
408
- * release did.
409
- *
410
- * The hold is bounded and self-healing: the executor renews its lease (a heartbeat) the moment
411
- * it starts work on the direct copy, which moves the lease out of `queued`, and the deferred
412
- * offer is then dropped. If the punch fails, nothing renews the lease and the relay offers
413
- * normally once the grace expires -- so a failed punch costs this much latency and nothing else.
414
- */
415
- this.p2pOfferGraceMs = Number.isFinite(options.p2pOfferGraceMs) && options.p2pOfferGraceMs >= 0
416
- ? options.p2pOfferGraceMs
417
- : DEFAULT_P2P_OFFER_GRACE_MS;
418
- /** Deferred offers, keyed `task_id:machine_id`, so they can be cleared on close(). */
419
- this._pendingOffers = new Map();
420
- /** v0.3.0: age beyond which a reported RTT is flagged stale. */
421
- this.rttStaleMs = Number.isFinite(options.rttStaleMs) ? options.rttStaleMs : DEFAULT_RTT_STALE_MS;
422
-
423
- this.startedAtMs = this._now();
424
- /** v0.1.2 §8.3: random per-process id; a change tells clients the relay restarted. */
425
- this.relayId = options.relayId ?? randomBytes(8).toString('hex');
426
- /** v0.1.2 §7: optional injected Persistence instance (state.mjs never touches fs itself). */
427
- this._persist = options.persistence ?? null;
428
- /** True while replaying a ledger — suppresses event emission and re-persistence. */
429
- this._restoring = false;
430
- /** Ledger entries that could not be replayed. */
431
- this.replayWarnings = [];
432
- this.seq = 0;
433
- /** @type {{seq:number,event:object,target:string|null}[]} ring buffer, last N events */
434
- this.buffer = [];
435
- this.devices = new Map(); // machine_id -> device
436
- this.tokens = new Map(); // device_token -> machine_id
437
- this.tasks = new Map(); // task_id -> task
438
- this.results = new Map(); // instanceKey -> {envelope, machine_id, received_at_ms, ...}
439
- this.pairingCodes = new Map(); // code -> {code, created_at_ms, expires_at_ms, used}
440
- this._listeners = new Set();
441
- }
442
-
443
- /* ---------------- clock / ids ---------------- */
444
-
445
- nowMs() { return this._now(); }
446
- nowIso() { return rfc3339(this.nowMs()); }
447
- nextId() { return this._idFactory(this.nowMs()); }
448
- newToken() { return this._randomToken(); }
449
-
450
- /** §4.3 lease window = 2 missed heartbeats (20s) + grace (30s) = 50s. */
451
- leaseWindowMs() { return this.heartbeatIntervalMs * this.missedHeartbeats + this.leaseGraceMs; }
452
-
453
- /* ---------------- persistence hooks (v0.1.2 §7) ---------------- */
454
-
455
- /** Inject/replace the persistence backend (also used by tests). */
456
- setPersistence(persistence) { this._persist = persistence ?? null; return this._persist; }
457
-
458
- /** Snapshot the device table; failures must never break the request path. */
459
- _saveDevices() {
460
- if (this._restoring || !this._persist) return false;
461
- try {
462
- return this._persist.saveDevices(this.devices);
463
- } catch (err) {
464
- this._persist.warn?.('devices.json', `snapshot failed: ${err.message}`);
465
- return false;
466
- }
467
- }
468
-
469
- /** Append one ledger entry; failures must never break the request path. */
470
- _ledger(entry) {
471
- if (this._restoring || !this._persist) return false;
472
- try {
473
- const appended = this._persist.append(entry);
474
- // v0.6.2: a successful append is what clears a degraded relay, so the outcome has to be
475
- // reported back rather than assumed -- otherwise the state could only ever be entered.
476
- if (appended) this._persist.noteAppendSuccess?.();
477
- return appended;
478
- } catch (err) {
479
- // Recorded as a state, not merely as a line: this is the one relay failure that keeps serving,
480
- // and so the one that is invisible from every other angle (v0.6.2).
481
- if (typeof this._persist.noteAppendFailure === 'function') this._persist.noteAppendFailure(err.message);
482
- else this._persist.warn?.('ledger.jsonl', `append failed: ${err.message}`);
483
- return false;
484
- }
485
- }
486
-
487
- _ledgerLease(task, lease) {
488
- this._ledger({ type: 'lease.changed', task_id: task.task_id, machine_id: lease.machine_id, lease: { ...lease } });
489
- }
490
-
491
- /**
492
- * v0.1.2 §7 startup recovery: rebuild the device table, token index, tasks,
493
- * leases and the result/dedupe index from a devices snapshot + ledger replay.
494
- * Never emits events and never writes back what it just read.
495
- */
496
- restore({ devices = {}, ledger = [] } = {}) {
497
- this._restoring = true;
498
- try {
499
- for (const [machineId, d] of Object.entries(devices)) {
500
- if (!d || typeof d !== 'object') continue;
501
- const pairedAt = Date.parse(d.paired_at ?? '');
502
- const device = {
503
- machine_id: machineId,
504
- machine_name: typeof d.machine_name === 'string' && d.machine_name ? d.machine_name : machineId,
505
- platform: d.platform ?? {},
506
- caps: d.caps ?? {},
507
- user_id: d.user_id ?? null,
508
- device_token: typeof d.device_token === 'string' ? d.device_token : null,
509
- paired_at_ms: Number.isFinite(pairedAt) ? pairedAt : this.nowMs(),
510
- re_paired: d.re_paired === true,
511
- last_seen_at_ms: this.nowMs(),
512
- // a snapshot proves the device exists, not that it is connected
513
- online: false,
514
- streams: 0,
515
- // v0.3.0: RTT is volatile and is NEVER restored. Even if a snapshot on
516
- // disk happens to carry `rtt_ms`, it is dropped here: after a restart the
517
- // relay genuinely does not know the latency any more, and reporting a
518
- // pre-restart number as current would be a lie that no reader can detect.
519
- rtt_ms: null,
520
- rtt_at_ms: null,
521
- last_heartbeat_at_ms: null,
522
- stream_connects: 0,
523
- };
524
- this.devices.set(machineId, device);
525
- if (device.device_token) this.tokens.set(device.device_token, machineId);
526
- }
527
-
528
- let revivedTasks = 0;
529
- for (const entry of ledger) {
530
- try {
531
- if (this._applyLedgerEntry(entry) === true) revivedTasks += 1;
532
- } catch (err) {
533
- this.replayWarnings.push({ type: entry?.type ?? '<unknown>', message: err.message });
534
- }
535
- }
536
- return { revived_devices: this.devices.size, revived_tasks: revivedTasks };
537
- } finally {
538
- this._restoring = false;
539
- }
540
- }
541
-
542
- /** @returns {boolean} true when a new task was created. */
543
- _applyLedgerEntry(entry) {
544
- if (!entry || typeof entry.type !== 'string') return false;
545
- switch (entry.type) {
546
- case 'task.created': {
547
- const t = entry.task;
548
- if (!t || typeof t.task_id !== 'string') return false;
549
- const createdAt = Number.isFinite(t.created_at_ms) ? t.created_at_ms : Date.parse(entry.ts ?? '');
550
- const task = {
551
- task_id: t.task_id,
552
- mode: t.mode ?? 'replicate',
553
- command_argv: t.command_argv ?? [],
554
- cwd_rel: t.cwd_rel ?? '.',
555
- index_total: t.index_total ?? 1,
556
- timeout_ms: t.timeout_ms ?? 300_000,
557
- write: t.write === true,
558
- write_scope: t.write_scope ?? [],
559
- require_exclusive_write: t.require_exclusive_write === true,
560
- base_commit: t.base_commit ?? null,
561
- base_tree: t.base_tree ?? null,
562
- requirements: t.requirements ?? {},
563
- compare_policy: t.compare_policy ?? {},
564
- halt: t.halt ?? 'never',
565
- created_by: t.created_by ?? null,
566
- created_at_ms: Number.isFinite(createdAt) ? createdAt : this.nowMs(),
567
- command_hash: t.command_hash ?? null,
568
- attempt: t.attempt ?? 1,
569
- leases: new Map(),
570
- cancelled: false,
571
- cancel_reason: null,
572
- degraded: t.degraded ?? null,
573
- deadline_ms: t.deadline_ms ?? 0,
574
- // v0.4.0 routing metadata. Read defensively: a ledger written by v0.3.9 has neither key,
575
- // and a replayed task with `null` for both is exactly a v0.3.9 task.
576
- origin_machine_id: t.origin_machine_id ?? null,
577
- p2p: t.p2p ?? null,
578
- result_seqs: [],
579
- };
580
- for (const lease of entry.leases ?? []) {
581
- if (lease && typeof lease.machine_id === 'string') task.leases.set(lease.machine_id, { ...lease });
582
- }
583
- this.tasks.set(task.task_id, task);
584
- return true;
585
- }
586
- case 'lease.changed': {
587
- const task = this.tasks.get(entry.task_id);
588
- const lease = task?.leases.get(entry.machine_id);
589
- if (!lease) return false;
590
- Object.assign(lease, entry.lease ?? {});
591
- return false;
592
- }
593
- case 'result.stored': {
594
- const record = entry.result;
595
- if (!record || typeof record.instance_key !== 'string') return false;
596
- this.results.set(record.instance_key, { ...record });
597
- const task = this.tasks.get(record.envelope?.task_id);
598
- if (task && Number.isInteger(record.seq)) task.result_seqs.push(record.seq);
599
- return false;
600
- }
601
- case 'task.cancelled': {
602
- const task = this.tasks.get(entry.task_id);
603
- if (task) {
604
- task.cancelled = true;
605
- task.cancel_reason = entry.reason ?? null;
606
- }
607
- return false;
608
- }
609
- default:
610
- return false; // unknown entry type: forward compatible, ignored
611
- }
612
- }
613
-
614
- /* ---------------- events (§3) ---------------- */
615
-
616
- nextSeq() { this.seq += 1; return this.seq; }
617
-
618
- /** Reserve a seq WITHOUT buffering (used for the per-connection `ready` frame). */
619
- reserveSeq() { return this.nextSeq(); }
620
-
621
- onEvent(listener) {
622
- this._listeners.add(listener);
623
- return () => this._listeners.delete(listener);
624
- }
625
-
626
- /**
627
- * Emit an event: allocates the next global seq, buffers it (ring of N) and
628
- * notifies subscribers. `target` is the machine_id the event is addressed to
629
- * (null = broadcast). The target is NOT serialized except as `machine_id`.
630
- */
631
- emit(type, fields = {}, opts = {}) {
632
- const target = opts.target ?? null;
633
- const seq = this.nextSeq();
634
- const event = {
635
- type,
636
- seq,
637
- rabbit_time: this.nowIso(),
638
- ...(target ? { machine_id: target } : {}),
639
- ...fields,
640
- };
641
- const entry = { seq, event, target };
642
- this.buffer.push(entry);
643
- if (this.buffer.length > this.eventBufferSize) {
644
- this.buffer.splice(0, this.buffer.length - this.eventBufferSize);
645
- }
646
- for (const listener of this._listeners) {
647
- try { listener(entry); } catch { /* subscriber errors must never break state */ }
648
- }
649
- return event;
650
- }
651
-
652
- oldestBufferedSeq() { return this.buffer.length ? this.buffer[0].seq : this.seq + 1; }
653
- /**
654
- * Highest seq present in the replay ring (0 when empty).
655
- *
656
- * Deliberately different from `lastSeq()`: `ready` frames allocate seq numbers
657
- * without being buffered, so `lastSeq()` can run ahead of anything a client can
658
- * actually replay. Diagnostics that describe the REPLAY WINDOW must use this one.
659
- */
660
- lastBufferedSeq() { return this.buffer.length ? this.buffer[this.buffer.length - 1].seq : 0; }
661
- lastSeq() { return this.seq; }
662
-
663
- /** Replay entries with seq >= from (used by GET /v1/stream?seq=|Last-Event-ID). */
664
- replayEntries(from) {
665
- return this.buffer.filter((e) => e.seq >= from);
666
- }
667
-
668
- eventsSince(from) { return this.replayEntries(from).map((e) => e.event); }
669
-
670
- /* ---------------- pairing (§2) ---------------- */
671
-
672
- createPairingCode({ ttlMs = this.pairingTtlMs } = {}) {
673
- let code;
674
- do { code = `PAIR-${this._randomChars(8)}`; } while (this.pairingCodes.has(code));
675
- const now = this.nowMs();
676
- this.pairingCodes.set(code, { code, created_at_ms: now, expires_at_ms: now + ttlMs, used: false });
677
- return code;
678
- }
679
-
680
- /** Revoke every outstanding code and mint a new one. */
681
- rotatePairingCode(opts) {
682
- for (const rec of this.pairingCodes.values()) rec.used = true;
683
- return this.createPairingCode(opts);
684
- }
685
-
686
- /**
687
- * Register an externally supplied code (`--pairing-code <c>`).
688
- *
689
- * v0.1.2 BUG-1: assigning the code to `relay.pairingCode` without registering it
690
- * advertised a code that `/v1/pair` then rejected with PAIRING_INVALID, so every
691
- * documented scripted-onboarding flow failed. "Which code do we use" and "is the
692
- * code registered" are two different questions; this answers the second.
693
- */
694
- registerPairingCode(code, { ttlMs = this.pairingTtlMs } = {}) {
695
- if (typeof code !== 'string' || code.trim() === '') {
696
- throw new ProtocolError('BAD_REQUEST', 'pairing code must be a non-empty string', { pairing_code: code });
697
- }
698
- const value = code.trim();
699
- const now = this.nowMs();
700
- this.pairingCodes.set(value, { code: value, created_at_ms: now, expires_at_ms: now + ttlMs, used: false });
701
- return value;
702
- }
703
-
704
- /** §2.2 step 3: validate a one-time code. */
705
- validatePairingCode(code) {
706
- if (typeof code !== 'string' || code.length === 0) {
707
- throw new ProtocolError('PAIRING_INVALID', 'pairing_code is required');
708
- }
709
- const rec = this.pairingCodes.get(code);
710
- if (!rec) throw new ProtocolError('PAIRING_INVALID', 'unknown pairing code', { pairing_code: code });
711
- if (rec.used) throw new ProtocolError('PAIRING_INVALID', 'pairing code already used', { pairing_code: code });
712
- if (this.nowMs() >= rec.expires_at_ms) {
713
- throw new ProtocolError('PAIRING_EXPIRED', 'pairing code expired', { pairing_code: code, expired_at: rfc3339(rec.expires_at_ms) });
714
- }
715
- return rec;
716
- }
717
-
718
- /** §2.2: exchange a one-time code for a device_token. §1: idempotent. */
719
- pair(request = {}) {
720
- const rec = this.validatePairingCode(request.pairing_code);
721
- const machineId = request.machine_id;
722
- if (typeof machineId !== 'string' || machineId.length === 0) {
723
- throw new ProtocolError('BAD_REQUEST', 'machine_id is required');
724
- }
725
- const now = this.nowMs();
726
- const token = this.newToken();
727
- const existing = this.devices.get(machineId);
728
- if (existing?.device_token) this.tokens.delete(existing.device_token);
729
-
730
- const device = {
731
- machine_id: machineId,
732
- machine_name: typeof request.machine_name === 'string' && request.machine_name
733
- ? request.machine_name : machineId.slice(0, 12),
734
- platform: request.platform ?? {},
735
- caps: request.caps ?? {},
736
- user_id: request.user_id ?? null,
737
- device_token: token,
738
- paired_at_ms: now,
739
- re_paired: Boolean(existing),
740
- last_seen_at_ms: now,
741
- online: true,
742
- streams: existing?.streams ?? 0,
743
- // v0.3.0 RTT: starts unknown. `null` is "never measured"; 0 would mean "instant".
744
- rtt_ms: null,
745
- rtt_at_ms: null,
746
- last_heartbeat_at_ms: null,
747
- stream_connects: existing?.stream_connects ?? 0,
748
- };
749
- this.devices.set(machineId, device);
750
- this.tokens.set(token, machineId);
751
- rec.used = true;
752
-
753
- // A group has more than one machine, and a single-use code would leave the
754
- // operator unable to pair the second one without restarting the relay. So
755
- // consuming a code immediately mints the next one and announces it: the
756
- // operator pairs machine A, sees a fresh code, and pairs machine B with it.
757
- // `pairingCodeReusable` keeps one code alive for the whole window instead,
758
- // for scripted onboarding where nobody is watching stdout.
759
- let nextCode = null;
760
- const reusable = this.pairingCodeReusable === true;
761
- if (reusable) {
762
- rec.used = false;
763
- } else {
764
- nextCode = this.rotatePairingCode();
765
- }
766
-
767
- this.emit('peer.hello', {
768
- machine_id: machineId,
769
- machine_name: device.machine_name,
770
- platform: device.platform,
771
- });
772
-
773
- if (nextCode) {
774
- this.emit('notice', {
775
- level: 'info',
776
- code: 'PAIRING_CODE_ROTATED',
777
- message: 'pairing code consumed; a new one is available',
778
- pairing_code: nextCode,
779
- });
780
- }
781
-
782
- // §7: a device change is snapshotted immediately.
783
- this._saveDevices();
784
- this._ledger({
785
- type: 'device.paired',
786
- machine_id: machineId,
787
- machine_name: device.machine_name,
788
- re_paired: device.re_paired,
789
- });
790
-
791
- return {
792
- device_token: token,
793
- protocol_version: PROTOCOL_VERSION,
794
- rabbit_time: rfc3339(now),
795
- ...(nextCode ? { next_pairing_code: nextCode } : {}),
796
- };
797
- }
798
-
799
- /** @returns {object|null} device for a bearer token. */
800
- authenticate(token) {
801
- if (typeof token !== 'string' || token.length === 0) return null;
802
- const machineId = this.tokens.get(token);
803
- if (!machineId) return null;
804
- return this.devices.get(machineId) ?? null;
805
- }
806
-
807
- touchDevice(machineId, { online = true } = {}) {
808
- const device = this.devices.get(machineId);
809
- if (!device) return null;
810
- device.last_seen_at_ms = this.nowMs();
811
- if (online) device.online = true;
812
- return device;
813
- }
814
-
815
- setOnline(machineId, online, reason = null) {
816
- const device = this.devices.get(machineId);
817
- if (!device) return null;
818
- if (device.online === online) return device;
819
- device.online = online;
820
- if (online) {
821
- this.touchDevice(machineId);
822
- this.emit('peer.hello', {
823
- machine_id: machineId,
824
- machine_name: device.machine_name,
825
- platform: device.platform,
826
- });
827
- } else {
828
- this.emit('peer.bye', { machine_id: machineId, reason: reason ?? 'stream_closed' });
829
- }
830
- return device;
831
- }
832
-
833
- /* ---------------- RTT (v0.3.0 §R) ---------------- */
834
-
835
- /**
836
- * Validate an inbound `rtt_ms`.
837
- *
838
- * The contract is deliberately liberal: a MISSING or INVALID value must never
839
- * reject the heartbeat and must never clobber a previously known good value.
840
- * RTT is a diagnostic; it must not be able to break lease renewal.
841
- *
842
- * Sanity ceiling: anything above `MAX_RTT_MS` (24h) is treated as invalid. A
843
- * single garbage value would otherwise poison the fleet aggregate.
844
- */
845
- static normalizeRtt(value) {
846
- if (value === undefined) return { provided: false, valid: false };
847
- if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
848
- return { provided: true, valid: false };
849
- }
850
- if (value > MAX_RTT_MS) return { provided: true, valid: false };
851
- return { provided: true, valid: true, value };
852
- }
853
-
854
- /**
855
- * RTT is a VOLATILE diagnostic: it is intentionally NOT written to
856
- * `devices.json` and NOT replayed from the ledger, so a restarted relay reports
857
- * `rtt_ms: null` until a machine heartbeats again. A persisted value would be
858
- * indistinguishable from a fresh one and would let the relay claim knowledge it
859
- * lost when the process died.
860
- */
861
- rttStaleAfterMs() { return this.rttStaleMs; }
862
-
863
- /**
864
- * v0.3.0: absorb the optional `rtt_ms` carried by a heartbeat.
865
- *
866
- * Deliberately liberal, because RTT is a diagnostic and must never be able to
867
- * break lease renewal:
868
- * - the field is optional; a v0.2.3 agent never sends it
869
- * - a MISSING or INVALID value is IGNORED, keeping the previous value, and the
870
- * heartbeat still succeeds
871
- * - unknown extra fields are ignored entirely (forward compatibility)
872
- *
873
- * `last_heartbeat_at` advances on EVERY accepted heartbeat, with or without a
874
- * usable RTT: it answers "is the machine alive", while `rtt_at` answers "when
875
- * was this number measured". Keeping them separate is what lets a reader tell a
876
- * genuinely slow machine from one that is simply gone.
877
- */
878
- recordHeartbeatMetrics(machineId, input = {}, nowMs = this.nowMs()) {
879
- const device = this.devices.get(machineId);
880
- if (!device) return null;
881
- device.last_heartbeat_at_ms = nowMs;
882
- const rtt = RabbitState.normalizeRtt(input.rtt_ms);
883
- if (rtt.valid) {
884
- device.rtt_ms = rtt.value;
885
- device.rtt_at_ms = nowMs;
886
- }
887
- return device;
888
- }
889
-
890
- /** True when there is no value, or the value is older than the staleness window. */
891
- isRttStale(device, nowMs = this.nowMs()) {
892
- if (!device || device.rtt_ms === null || device.rtt_ms === undefined) return true;
893
- if (device.rtt_at_ms === null || device.rtt_at_ms === undefined) return true;
894
- return nowMs - device.rtt_at_ms > this.rttStaleMs;
895
- }
896
-
897
- /** How old the RTT measurement is, in ms; null when there is none. */
898
- rttAgeMs(device, nowMs = this.nowMs()) {
899
- if (!device || device.rtt_at_ms === null || device.rtt_at_ms === undefined) return null;
900
- return Math.max(0, nowMs - device.rtt_at_ms);
901
- }
902
-
903
- /** Record that a machine attached a stream (a reconnect is any attach after the first). */
904
- noteStreamConnect(machineId) {
905
- const device = this.devices.get(machineId);
906
- if (!device) return null;
907
- device.stream_connects = (device.stream_connects ?? 0) + 1;
908
- return device.stream_connects;
909
- }
910
-
911
- /**
912
- * v0.3.0 aggregate for `/healthz`.
913
- *
914
- * Only FRESH measurements count. A machine that has been gone for a day must not
915
- * drag min/max/avg around, and with no fresh data every numeric field is null --
916
- * never 0, which would read as "instantaneous".
917
- */
918
- rttSummary(nowMs = this.nowMs()) {
919
- const fresh = [];
920
- let stale = 0;
921
- let unknown = 0;
922
- for (const device of this.devices.values()) {
923
- if (device.rtt_ms === null || device.rtt_ms === undefined) { unknown += 1; continue; }
924
- if (this.isRttStale(device, nowMs)) { stale += 1; continue; }
925
- fresh.push(device.rtt_ms);
926
- }
927
- if (fresh.length === 0) {
928
- return {
929
- machines_reporting: 0,
930
- min_ms: null,
931
- max_ms: null,
932
- avg_ms: null,
933
- machines_stale: stale,
934
- machines_unknown: unknown,
935
- };
936
- }
937
- const sum = fresh.reduce((a, b) => a + b, 0);
938
- return {
939
- machines_reporting: fresh.length,
940
- min_ms: Math.min(...fresh),
941
- max_ms: Math.max(...fresh),
942
- avg_ms: Math.round((sum / fresh.length) * 10) / 10,
943
- machines_stale: stale,
944
- machines_unknown: unknown,
945
- };
946
- }
947
-
948
- /**
949
- * v0.3.0 single-machine status view.
950
- *
951
- * `last_heartbeat_at` and `rtt_at` answer different questions, and the gap
952
- * between them is the whole point: a machine can be heartbeating (alive) while
953
- * its RTT number is old (a v0.2.3 agent that never sends `rtt_ms`), and a
954
- * machine can be gone while a plausible-looking RTT is still on file.
955
- */
956
- deviceStatus(machineId, nowMs = this.nowMs()) {
957
- const device = this.devices.get(machineId);
958
- if (!device) return null;
959
- const rttAt = device.rtt_at_ms ?? null;
960
- const lastHeartbeat = device.last_heartbeat_at_ms ?? null;
961
- return {
962
- machine_id: device.machine_id,
963
- machine_name: device.machine_name,
964
- relay_id: this.relayId,
965
- connected: (device.streams ?? 0) > 0,
966
- streams: device.streams ?? 0,
967
- online: device.online === true,
968
- rtt_ms: device.rtt_ms ?? null,
969
- rtt_at: rttAt === null ? null : rfc3339(rttAt),
970
- rtt_age_ms: this.rttAgeMs(device, nowMs),
971
- rtt_stale: this.isRttStale(device, nowMs),
972
- last_heartbeat_at: lastHeartbeat === null ? null : rfc3339(lastHeartbeat),
973
- last_seen_at: device.last_seen_at_ms === null || device.last_seen_at_ms === undefined
974
- ? null : rfc3339(device.last_seen_at_ms),
975
- reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
976
- stream_connects: device.stream_connects ?? 0,
977
- paired_at: device.paired_at_ms === null || device.paired_at_ms === undefined
978
- ? null : rfc3339(device.paired_at_ms),
979
- platform: device.platform,
980
- caps: device.caps,
981
- user_id: device.user_id ?? null,
982
- now: rfc3339(nowMs),
983
- };
984
- }
985
-
986
- /** Public device view — never leaks device_token. */
987
- publicDevice(device) {
988
- return {
989
- machine_id: device.machine_id,
990
- machine_name: device.machine_name,
991
- platform: device.platform,
992
- caps: device.caps,
993
- user_id: device.user_id,
994
- online: device.online,
995
- streams: device.streams,
996
- paired_at: rfc3339(device.paired_at_ms),
997
- last_seen_at: rfc3339(device.last_seen_at_ms),
998
- // v0.3.0: `null` means "never measured", which is NOT the same as 0ms.
999
- rtt_ms: device.rtt_ms ?? null,
1000
- rtt_at: device.rtt_at_ms === null || device.rtt_at_ms === undefined ? null : rfc3339(device.rtt_at_ms),
1001
- rtt_age_ms: this.rttAgeMs(device),
1002
- rtt_stale: this.isRttStale(device),
1003
- last_heartbeat_at: device.last_heartbeat_at_ms === null || device.last_heartbeat_at_ms === undefined
1004
- ? null : rfc3339(device.last_heartbeat_at_ms),
1005
- reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
1006
- };
1007
- }
1008
-
1009
- listDevices() { return [...this.devices.values()].map((d) => this.publicDevice(d)); }
1010
-
1011
- /* ---------------- tasks (§4) ---------------- */
1012
-
1013
- /**
1014
- * v0.4.0: validate the routing metadata a `POST /v1/task` body may carry.
1015
- *
1016
- * A machine, not a mode, is `origin_machine_id`: where the task was typed (`device.json`), so an
1017
- * offer can carry that fact to the machine that has to run it. `p2p.mode` is the dispatcher's
1018
- * resolved mode for this particular task — the plugin may be configured `direct` while a
1019
- * scheduled run is `relay`.
1020
- *
1021
- * Validation is deliberately shallow and strictly typed, in the house style of this file: a
1022
- * missing value is the v0.3.9 shape and stays valid, while a present-but-wrong value is refused
1023
- * by name. Accepting a non-string here would put a number into `origin_machine_id` and only fail
1024
- * later, on a machine that cannot tell a bad field from a bad relay.
1025
- *
1026
- * @returns {{origin_machine_id:string|null, p2p:{mode:string}|null}}
1027
- */
1028
- static validateTaskRouting(input = {}) {
1029
- let originMachineId = null;
1030
- if (input.origin_machine_id !== undefined && input.origin_machine_id !== null) {
1031
- if (typeof input.origin_machine_id !== 'string'
1032
- || input.origin_machine_id.length === 0
1033
- || input.origin_machine_id.length > MAX_ORIGIN_MACHINE_ID_LENGTH) {
1034
- throw new ProtocolError(
1035
- 'BAD_REQUEST',
1036
- `origin_machine_id must be a non-empty string of at most ${MAX_ORIGIN_MACHINE_ID_LENGTH} characters`,
1037
- { origin_machine_id: input.origin_machine_id },
1038
- );
1039
- }
1040
- originMachineId = input.origin_machine_id;
1041
- }
1042
-
1043
- let p2p = null;
1044
- if (input.p2p !== undefined && input.p2p !== null) {
1045
- if (typeof input.p2p !== 'object' || Array.isArray(input.p2p)) {
1046
- throw new ProtocolError('BAD_REQUEST', 'p2p must be an object with a `mode` field', { p2p: input.p2p });
1047
- }
1048
- if (!P2P_MODES.includes(input.p2p.mode)) {
1049
- throw new ProtocolError(
1050
- 'BAD_REQUEST',
1051
- `p2p.mode must be one of ${P2P_MODES.join('|')}`,
1052
- { 'p2p.mode': input.p2p.mode ?? null },
1053
- );
1054
- }
1055
- // Copied rather than referenced: the recorded task must not be mutable through the request
1056
- // body the caller still holds.
1057
- p2p = { ...input.p2p };
1058
- }
1059
-
1060
- return { origin_machine_id: originMachineId, p2p };
1061
- }
1062
-
1063
- /**
1064
- * v0.4.0 §5.2: validate the optional transport fields of a result envelope.
1065
- *
1066
- * Same reasoning as `validateTaskRouting`: `transport` is an enum and `p2p` is an opaque object,
1067
- * so anything else is refused by name at the door. Both fields stay OUT of
1068
- * `REQUIRED_ENVELOPE_FIELDS` (a missing one is a v0.3.9 envelope, not an incomplete one) and out
1069
- * of `COMPARABLE_FIELDS` (a path is not a result).
1070
- */
1071
- static validateTransportFields(envelope = {}) {
1072
- if (envelope.transport !== undefined && envelope.transport !== null) {
1073
- if (!TRANSPORT_VALUES.includes(envelope.transport)) {
1074
- throw new ProtocolError(
1075
- 'BAD_REQUEST',
1076
- `transport must be one of ${TRANSPORT_VALUES.join('|')}`,
1077
- { transport: envelope.transport },
1078
- );
1079
- }
1080
- }
1081
- if (envelope.p2p !== undefined && envelope.p2p !== null
1082
- && (typeof envelope.p2p !== 'object' || Array.isArray(envelope.p2p))) {
1083
- throw new ProtocolError('BAD_REQUEST', 'p2p must be an object', { p2p: envelope.p2p });
1084
- }
1085
- }
1086
-
1087
- /**
1088
- * §4.1 POST /v1/task.
1089
- * Machine selection is not specified by the PROTOCOL; we implement:
1090
- * - `target_machines` (optional explicit list), else every paired device
1091
- * - capability gate §6.1 per machine (failures become `refused` leases)
1092
- * - index assignment: replicate -> 0, split -> round-robin modulo index_total
1093
- */
1094
- createTask(input = {}) {
1095
- // v0.4.0: refused BEFORE the task literal is built and before any offer is emitted, so a bad
1096
- // value cannot leave a half-created task behind. `POST /v1/task` is all-or-nothing.
1097
- const routing = RabbitState.validateTaskRouting(input);
1098
-
1099
- const mode = input.mode ?? 'replicate';
1100
- if (mode !== 'replicate' && mode !== 'split' && mode !== 'broadcast' && mode !== 'pipeline' && mode !== 'compose') {
1101
- throw new ProtocolError(
1102
- 'BAD_REQUEST',
1103
- 'mode must be replicate|split|broadcast|pipeline|compose',
1104
- { mode },
1105
- );
1106
- }
1107
- /**
1108
- * v0.3.3 `pipeline`: the whole chain runs on each machine, stage after stage.
1109
- *
1110
- * The semantics were chosen against the alternative (fan out stage 1, converge its outputs into
1111
- * stage 2) because that one has no well-defined input: on a fleet, stage 1 produces n different
1112
- * outputs, and "feed them all to stage 2" does not say in what order, or whether stage 2 then
1113
- * runs once or n times. Worse, two machines finishing in a different order would produce
1114
- * different stage-2 inputs, so one task_id would no longer describe one reproducible execution --
1115
- * which is the property the rest of this system is built on. See docs/PIPELINE-DESIGN.md.
1116
- *
1117
- * So: a pipeline is `replicate` applied to a sequence. Every machine runs the whole chain, there
1118
- * is no cross-machine data flow, and nothing new can be silently dropped in transit.
1119
- */
1120
- const rawStages = Array.isArray(input.stages) ? input.stages : null;
1121
- const isPipeline = mode === 'pipeline';
1122
- if (isPipeline) {
1123
- if (rawStages === null || rawStages.length === 0) {
1124
- throw new ProtocolError('BAD_REQUEST', 'mode=pipeline requires a non-empty `stages` array', {
1125
- stages: rawStages === null ? null : rawStages.length,
1126
- });
1127
- }
1128
- if (rawStages.length > MAX_PIPELINE_STAGES) {
1129
- throw new ProtocolError('BAD_REQUEST', `stages must have at most ${MAX_PIPELINE_STAGES} entries`, {
1130
- stages: rawStages.length,
1131
- });
1132
- }
1133
- rawStages.forEach((stage, i) => {
1134
- const argv = stage?.command_argv;
1135
- if (!Array.isArray(argv) || argv.length === 0 || argv.some((a) => typeof a !== 'string')) {
1136
- throw new ProtocolError('BAD_REQUEST', `stages[${i}].command_argv must be a non-empty string[]`, { index: i });
1137
- }
1138
- });
1139
- // A chain is one execution on one machine; sharding it would mean each machine ran a chain of
1140
- // its own slice, which is a different feature and not what the caller asked for.
1141
- if (Number.isInteger(input.index_total) && input.index_total !== 1) {
1142
- throw new ProtocolError('BAD_REQUEST', 'mode=pipeline runs the whole chain per machine, so index_total must be 1', {
1143
- index_total: input.index_total,
1144
- });
1145
- }
1146
- } else if (rawStages !== null) {
1147
- // Refused rather than ignored: the caller believes they described a sequence, and running only
1148
- // `command_argv` would silently drop every stage after the first.
1149
- throw new ProtocolError('BAD_REQUEST', '`stages` is only valid with mode=pipeline', { mode, stages: rawStages.length });
1150
- }
1151
-
1152
- /**
1153
- * Normalized stage list. For every mode there is at least one stage, so downstream code has a
1154
- * single shape to reason about and `pipeline` is not a special case at the execution layer.
1155
- */
1156
- const stages = isPipeline
1157
- ? rawStages.map((s, i) => ({
1158
- index: i,
1159
- command_argv: [...s.command_argv],
1160
- cwd_rel: typeof s.cwd_rel === 'string' && s.cwd_rel ? s.cwd_rel : (typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.'),
1161
- timeout_ms: Number.isInteger(s.timeout_ms) ? s.timeout_ms : null,
1162
- continue_on_failure: s.continue_on_failure === true,
1163
- }))
1164
- : null;
1165
-
1166
- const commandArgv = isPipeline ? [...rawStages[0].command_argv] : input.command_argv;
1167
- if (!Array.isArray(commandArgv) || commandArgv.length === 0 || commandArgv.some((a) => typeof a !== 'string')) {
1168
- throw new ProtocolError('BAD_REQUEST', 'command_argv must be a non-empty string[]');
1169
- }
1170
- const indexTotal = Number.isInteger(input.index_total) ? input.index_total : 1;
1171
- if (indexTotal < 1) throw new ProtocolError('BAD_REQUEST', 'index_total must be >= 1');
1172
-
1173
- const halt = input.halt ?? 'never';
1174
- if (halt !== 'never' && halt !== 'now') throw new ProtocolError('BAD_REQUEST', 'halt must be never|now', { halt });
1175
-
1176
- const taskId = input.task_id ?? this.nextId();
1177
- if (this.tasks.has(taskId)) {
1178
- throw new ProtocolError('TASK_EXISTS', 'task_id already exists', { task_id: taskId });
1179
- }
1180
-
1181
- const now = this.nowMs();
1182
- const cwdRel = typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.';
1183
- const baseTree = input.base_tree ?? null;
1184
- const commandHash = input.command_hash ?? (isPipeline
1185
- ? computePipelineCommandHash(stages, SHELL_ID_DIRECT)
1186
- : computeCommandHash(commandArgv, SHELL_ID_DIRECT, cwdRel));
1187
- const timeoutMs = Number.isInteger(input.timeout_ms) ? input.timeout_ms : 300_000;
1188
-
1189
- // v0.3.3 `broadcast`: one machine runs the command, every other machine is told the outcome.
1190
- //
1191
- // The executor choice is deterministic rather than random. A random leader makes a task
1192
- // impossible to reproduce from its own record -- the same task id would run somewhere else on a
1193
- // retry -- and this project's whole value rests on being able to say exactly where something ran.
1194
- // The first target in the relay's own device order is stable for a given device table.
1195
- //
1196
- // Observers get a lease so they are tracked and notified, but their lease is `observing`, which
1197
- // every aggregation step treats as "did not run". That distinction is the point: a machine that
1198
- // never executed anything agreeing with a result would be a fabrication, and it is exactly the
1199
- // fabrication this mode could most easily produce.
1200
- //
1201
- // Resolved before the task literal because the literal records it. (`targets` is computed below,
1202
- // so the executor is re-validated against it after that.)
1203
- const targetIds = Array.isArray(input.target_machines) && input.target_machines.length > 0
1204
- ? input.target_machines
1205
- : [...this.devices.keys()];
1206
- const executorMachineId = mode === 'broadcast'
1207
- ? (typeof input.executor_machine_id === 'string' && input.executor_machine_id !== ''
1208
- ? input.executor_machine_id
1209
- : targetIds[0] ?? null)
1210
- : null;
1211
-
1212
- const task = {
1213
- task_id: taskId,
1214
- mode,
1215
- command_argv: commandArgv,
1216
- cwd_rel: cwdRel,
1217
- index_total: indexTotal,
1218
- timeout_ms: timeoutMs,
1219
- write: input.write === true,
1220
- write_scope: input.write_scope ?? [],
1221
- require_exclusive_write: input.require_exclusive_write === true,
1222
- base_commit: input.base_commit ?? null,
1223
- base_tree: baseTree,
1224
- requirements: input.requirements ?? {},
1225
- compare_policy: input.compare_policy ?? {},
1226
- halt,
1227
- created_by: input.created_by ?? null,
1228
- created_at_ms: now,
1229
- command_hash: commandHash,
1230
- attempt: 1,
1231
- leases: new Map(),
1232
- cancelled: false,
1233
- cancel_reason: null,
1234
- degraded: null,
1235
- deadline_ms: now + timeoutMs,
1236
- result_seqs: [],
1237
- /** v0.3.3: which machine actually executes under `broadcast`; null for the other modes. */
1238
- executor_machine_id: executorMachineId,
1239
- /**
1240
- * v0.3.3: the normalized stage list for `pipeline`, or null.
1241
- *
1242
- * Sent to the machine in the offer so it can run the whole chain. Kept on the task as well so
1243
- * a report can say how many stages were supposed to run even when the machine crashed before
1244
- * reporting any of them.
1245
- */
1246
- stages,
1247
- /**
1248
- * v0.4.0: where the task came from and under which P2P mode it was dispatched, or null.
1249
- *
1250
- * Kept on the task (not just passed to `emitOffer`) because an offer is emitted more than
1251
- * once: `redeliverPendingOffers` re-states unclaimed work on every reconnect and
1252
- * `_takeover` re-offers a freed index. A machine that reconnects must see the same routing
1253
- * facts as the machine that was connected at dispatch time, or the direct-push half of the
1254
- * feature would work exactly once and then quietly stop.
1255
- */
1256
- origin_machine_id: routing.origin_machine_id,
1257
- p2p: routing.p2p,
1258
- };
1259
-
1260
- const all = [...this.devices.values()];
1261
- const targets = Array.isArray(input.target_machines) && input.target_machines.length > 0
1262
- ? all.filter((d) => input.target_machines.includes(d.machine_id))
1263
- : all;
1264
-
1265
- // v0.3.3 `broadcast`: validated here, where `targets` is known. The executor id itself was
1266
- // resolved before the task literal because the literal records it.
1267
- if (mode === 'broadcast' && executorMachineId !== null && !targets.some((d) => d.machine_id === executorMachineId)) {
1268
- throw new ProtocolError('BAD_REQUEST', 'executor_machine_id is not among the target machines', {
1269
- executor_machine_id: executorMachineId,
1270
- targets: targets.map((d) => d.machine_id),
1271
- });
1272
- }
1273
-
1274
- if (targets.length === 0) {
1275
- throw new ProtocolError('NO_ONLINE_DEVICE', 'no paired device available for this task', {
1276
- known_devices: all.length,
1277
- });
1278
- }
1279
-
1280
- const dedupeKeyBase = computeDedupeKey(taskId, 0, commandHash, baseTree);
1281
- if (typeof dedupeKeyBase !== 'string' || dedupeKeyBase.length !== 64) {
1282
- throw new ProtocolError('INTERNAL', 'failed to compute dedupe_key');
1283
- }
1284
- let lastSeq = this.seq;
1285
- targets.forEach((device, i) => {
1286
- const index = mode === 'replicate' ? 0 : i % indexTotal;
1287
- const observing = mode === 'broadcast' && device.machine_id !== executorMachineId;
1288
- const lease = {
1289
- machine_id: device.machine_id,
1290
- machine_name: device.machine_name,
1291
- index,
1292
- attempt: 1,
1293
- // `observing` is a lease state, not an outcome: the machine holds a place in this task and
1294
- // will be told what happened, but it was never handed the command. `gateMachine` below is
1295
- // skipped for it because refusing to do work you were never asked to do is not a refusal.
1296
- state: observing ? 'observing' : 'queued',
1297
- refusal_reason: null,
1298
- refusal_detail: null,
1299
- dedupe_key: computeDedupeKey(taskId, index, commandHash, baseTree),
1300
- last_heartbeat_ms: now,
1301
- claimed_at_ms: now,
1302
- lease_until_ms: now + this.leaseWindowMs(),
1303
- phase: null,
1304
- progress: null,
1305
- offered_seq: null,
1306
- expired_at_ms: null,
1307
- finished_at_ms: null,
1308
- heartbeat_count: 0,
1309
- };
1310
- if (observing) {
1311
- // No capability gate and no offer: the machine is not being asked to run anything, so there
1312
- // is nothing to refuse and nothing to hand out. It is recorded so the task can tell it the
1313
- // outcome afterwards.
1314
- task.leases.set(device.machine_id, lease);
1315
- return;
1316
- }
1317
- const gate = gateMachine(task, device);
1318
- if (gate) {
1319
- lease.state = 'refused';
1320
- lease.refusal_reason = gate.refusal_reason;
1321
- lease.refusal_detail = gate.detail;
1322
- task.leases.set(device.machine_id, lease);
1323
- this.emit('notice', {
1324
- level: 'warn',
1325
- code: 'CAPABILITY_REFUSED',
1326
- message: `${device.machine_id} refused: ${gate.refusal_reason}`,
1327
- task_id: taskId,
1328
- });
1329
- return;
1330
- }
1331
- task.leases.set(device.machine_id, lease);
1332
- const grace = this._p2pOfferGraceMs(task);
1333
- if (grace > 0) {
1334
- // v0.4.0: the dispatcher is pushing this offer over the direct path right now. Hold the
1335
- // relay's copy for `grace` ms so the direct one can win; see `p2pOfferGraceMs`.
1336
- this._scheduleOffer(task, lease, grace);
1337
- } else {
1338
- const event = this.emitOffer(task, lease);
1339
- lastSeq = event.seq;
1340
- }
1341
- });
1342
-
1343
- this.tasks.set(taskId, task);
1344
-
1345
- this._ledger({
1346
- type: 'task.created',
1347
- task: {
1348
- task_id: task.task_id,
1349
- mode: task.mode,
1350
- command_argv: task.command_argv,
1351
- cwd_rel: task.cwd_rel,
1352
- index_total: task.index_total,
1353
- timeout_ms: task.timeout_ms,
1354
- write: task.write,
1355
- write_scope: task.write_scope,
1356
- require_exclusive_write: task.require_exclusive_write,
1357
- base_commit: task.base_commit,
1358
- base_tree: task.base_tree,
1359
- requirements: task.requirements,
1360
- compare_policy: task.compare_policy,
1361
- halt: task.halt,
1362
- created_by: task.created_by,
1363
- created_at_ms: task.created_at_ms,
1364
- command_hash: task.command_hash,
1365
- attempt: task.attempt,
1366
- deadline_ms: task.deadline_ms,
1367
- // v0.4.0: routing metadata, so a restarted relay re-offers with the same facts.
1368
- origin_machine_id: task.origin_machine_id ?? null,
1369
- p2p: task.p2p ?? null,
1370
- },
1371
- leases: [...task.leases.values()].map((l) => ({ ...l })),
1372
- });
1373
-
1374
- const refusedOnly = [...task.leases.values()].every((l) => l.state === 'refused');
1375
- if (refusedOnly) {
1376
- // §6.3 Step 0 territory: a task nobody may execute is still a valid, reportable task.
1377
- this.emit('notice', {
1378
- level: 'warn',
1379
- code: 'NO_ONLINE_DEVICE',
1380
- message: 'every candidate machine refused the capability gate',
1381
- task_id: taskId,
1382
- });
1383
- }
1384
-
1385
- return {
1386
- task_id: taskId,
1387
- leases: [...task.leases.values()].map((l) => ({
1388
- machine_id: l.machine_id,
1389
- index: l.index,
1390
- state: l.state,
1391
- // v0.4.0: the dispatcher pushes an offer over the direct path, and the executor dedupes it
1392
- // against the relay's copy by (task_id, attempt, dedupe_key). Without this field the
1393
- // dispatcher has to reproduce the §4.4 formula locally, which works for a single-command
1394
- // task and cannot work for a pipeline one -- the key covers the relay's normalised stage
1395
- // list. Publishing it makes the pushed copy recognisably the same work item, which is what
1396
- // keeps the direct path from turning exactly-once into at-least-twice.
1397
- dedupe_key: l.dedupe_key ?? null,
1398
- attempt: l.attempt,
1399
- ...(l.refusal_reason ? { refusal_reason: l.refusal_reason } : {}),
1400
- })),
1401
- seq: lastSeq,
1402
- };
1403
- }
1404
-
1405
- /**
1406
- * v0.4.0: should the relay hold its offer back for this task?
1407
- *
1408
- * Only when the dispatcher said two things: that it is a machine (`origin_machine_id`), and that
1409
- * the direct path is in play (`p2p.mode` of `auto` or `direct`). A `relay` mode or a v0.3.9
1410
- * dispatcher gets the old behaviour exactly, which is why this predicate is the only gate on the
1411
- * change and every pre-existing test still sees an immediate offer.
1412
- *
1413
- * @param {object} task
1414
- * @returns {number} milliseconds to hold, or 0 for "offer now"
1415
- */
1416
- _p2pOfferGraceMs(task) {
1417
- if (this.p2pOfferGraceMs <= 0) return 0;
1418
- const origin = task?.origin_machine_id;
1419
- if (typeof origin !== 'string' || origin === '') return 0;
1420
- const mode = task?.p2p?.mode;
1421
- if (mode !== 'auto' && mode !== 'direct') return 0;
1422
- return this.p2pOfferGraceMs;
1423
- }
1424
-
1425
- /**
1426
- * v0.4.0: emit `lease`'s offer after `delayMs`, unless the direct path got there first.
1427
- *
1428
- * "Got there first" is one fact and it is already tracked: the executor renews the lease
1429
- * (`POST /v1/heartbeat`) as its first act on any offer it accepts, and a renewal moves the lease
1430
- * out of `queued`. So the check is the lease's own state — no new message, no new endpoint, and
1431
- * nothing the executor has to know about.
1432
- *
1433
- * The timer is `unref()`ed: a relay with nothing else to do must still be able to exit, and a
1434
- * pending offer is not a reason to keep a process alive.
1435
- */
1436
- _scheduleOffer(task, lease, delayMs) {
1437
- const key = `${task.task_id}:${lease.machine_id}`;
1438
- const timer = setTimeout(() => {
1439
- this._pendingOffers.delete(key);
1440
- // A lease that was re-created (takeover) or swept is not this one any more.
1441
- if (task.leases.get(lease.machine_id) !== lease) return;
1442
- if (lease.state !== 'queued') return; // the direct copy won; nothing to fall back to
1443
- if (this.isLeaseExpired(lease, this.nowMs())) return; // the sweep owns it now
1444
- this.emitOffer(task, lease);
1445
- }, delayMs);
1446
- timer.unref?.();
1447
- this._pendingOffers.set(key, timer);
1448
- }
1449
-
1450
- /** Drop every deferred offer. Called by the relay's own teardown path. */
1451
- clearPendingOffers() {
1452
- for (const timer of this._pendingOffers.values()) clearTimeout(timer);
1453
- this._pendingOffers.clear();
1454
- }
1455
-
1456
- emitOffer(task, lease) {
1457
- const event = this.emit('task.offer', {
1458
- task_id: task.task_id,
1459
- attempt: lease.attempt,
1460
- mode: task.mode,
1461
- index: lease.index,
1462
- index_total: task.index_total,
1463
- command_argv: task.command_argv,
1464
- // v0.3.3: present only for `pipeline`, and only then does the machine run a chain. Omitted
1465
- // entirely otherwise, so an offer for a single-command task is byte-identical to before --
1466
- // which is what keeps a v0.2.3 agent working against this relay.
1467
- ...(task.stages ? { stages: task.stages } : {}),
1468
- /**
1469
- * v0.4.0: always present, `null` when the caller supplied nothing.
1470
- *
1471
- * Deliberately NOT conditional (unlike `stages` above, which is omitted so a single-command
1472
- * offer stays byte-identical for an old agent): the contract fixes the shape of these two as
1473
- * `string|null` and `{mode}|null`, and `null` is the honest value for "the dispatcher did not
1474
- * say" — which is exactly the v0.3.9 case. They are additive keys in an event whose consumers
1475
- * already ignore unknown fields.
1476
- */
1477
- origin_machine_id: task.origin_machine_id ?? null,
1478
- p2p: task.p2p ?? null,
1479
- cwd_rel: task.cwd_rel,
1480
- write: task.write,
1481
- timeout_ms: task.timeout_ms,
1482
- base_commit: task.base_commit,
1483
- base_tree: task.base_tree,
1484
- requirements: task.requirements,
1485
- compare_policy: task.compare_policy,
1486
- dedupe_key: lease.dedupe_key,
1487
- deadline: rfc3339(task.deadline_ms),
1488
- }, { target: lease.machine_id });
1489
- lease.state = 'offered';
1490
- lease.offered_seq = event.seq;
1491
- lease.last_heartbeat_ms = this.nowMs();
1492
- lease.lease_until_ms = lease.last_heartbeat_ms + this.leaseWindowMs();
1493
- return event;
1494
- }
1495
-
1496
- getTask(taskId) { return this.tasks.get(taskId) ?? null; }
1497
-
1498
- requireTask(taskId) {
1499
- const task = this.tasks.get(taskId);
1500
- if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
1501
- return task;
1502
- }
1503
-
1504
- /**
1505
- * Re-offer the work a machine was handed but has not yet started (v0.1.2 §8.6).
1506
- *
1507
- * WHY THIS EXISTS
1508
- *
1509
- * An offer is a point-in-time event on a stream. `createTask` publishes
1510
- * `task.offer` once, and if the target machine's stream happens to be down at
1511
- * that instant -- the ordinary situation after a relay restart, a tunnel drop,
1512
- * or any cross-network interruption -- the event is simply gone. The lease sits
1513
- * in `offered` until the sweep expires it, the task never runs, and the
1514
- * operator sees a timeout with no explanation. Measured: the machine is back
1515
- * online with a live stream within milliseconds, and still receives nothing.
1516
- *
1517
- * So the relay re-states unclaimed work when a machine connects. This is safe
1518
- * because the state it acts on is durable: the lease the offer refers to is
1519
- * still in the ledger.
1520
- *
1521
- * WHY ONLY `offered` AND `queued`
1522
- *
1523
- * `running` is deliberately excluded, and that exclusion is the whole safety
1524
- * argument. A running lease means a machine has already STARTED the command --
1525
- * it acknowledged the offer and is heartbeating. Re-offering it would make that
1526
- * machine execute the work a second time, so the fix for a lost offer would
1527
- * introduce the one failure this project is built to avoid: two machines, or
1528
- * one machine twice, running the same side-effecting command.
1529
- *
1530
- * A machine that dies mid-run therefore does not get its work back here; that
1531
- * path stays with the sweep, which expires the lease and hands the index to
1532
- * another machine with `attempt + 1`.
1533
- *
1534
- * Re-emitted offers are new events and are replayed like any other, so a client
1535
- * that reconnects repeatedly can see the same offer more than once. That is
1536
- * intentional: the agent already de-duplicates by `task_id` (it ignores an
1537
- * offer whose task is queued or running), and an at-least-once offer with a
1538
- * client-side dedupe is strictly better than an at-most-once offer that is
1539
- * silently dropped.
1540
- *
1541
- * @param {string} machineId
1542
- * @returns {{count: number, task_ids: string[]}}
1543
- */
1544
- redeliverPendingOffers(machineId) {
1545
- const taskIds = [];
1546
- const now = this.nowMs();
1547
-
1548
- for (const task of this.tasks.values()) {
1549
- if (task.cancelled === true) continue;
1550
- const lease = task.leases.get(machineId);
1551
- if (!lease) continue;
1552
- // Only work the machine has not begun. See the note above: `running` is
1553
- // excluded on purpose, not by omission.
1554
- if (lease.state !== 'offered' && lease.state !== 'queued') continue;
1555
- // Do not resurrect a lease the sweep has already written off; the takeover
1556
- // path owns that decision.
1557
- if (this.isLeaseExpired(lease, now)) continue;
1558
-
1559
- this.emitOffer(task, lease);
1560
- taskIds.push(task.task_id);
1561
- }
1562
-
1563
- return { count: taskIds.length, task_ids: taskIds };
1564
- }
1565
-
1566
- listTasks({ limit = 50 } = {}) {
1567
- const n = Number.isInteger(limit) && limit > 0 ? limit : 50;
1568
- return [...this.tasks.values()]
1569
- .sort((a, b) => b.created_at_ms - a.created_at_ms)
1570
- .slice(0, n)
1571
- .map((t) => this.taskSummary(t));
1572
- }
1573
-
1574
- taskSummary(task) {
1575
- return {
1576
- task_id: task.task_id,
1577
- mode: task.mode,
1578
- created_at: rfc3339(task.created_at_ms),
1579
- created_by: task.created_by,
1580
- index_total: task.index_total,
1581
- cancelled: task.cancelled,
1582
- degraded: task.degraded,
1583
- leases: [...task.leases.values()].map((l) => l.machine_id),
1584
- lease_states: Object.fromEntries([...task.leases.values()].map((l) => [l.machine_id, l.state])),
1585
- };
1586
- }
1587
-
1588
- cancelTask(taskId, reason = 'cancelled_by_operator') {
1589
- const task = this.requireTask(taskId);
1590
- if (task.cancelled) return task;
1591
- task.cancelled = true;
1592
- task.cancel_reason = reason;
1593
- for (const lease of task.leases.values()) {
1594
- if (ACTIVE_LEASE_STATES.includes(lease.state)) lease.state = 'cancelled';
1595
- }
1596
- this.emit('task.cancel', { task_id: taskId, reason });
1597
- this._ledger({ type: 'task.cancelled', task_id: taskId, reason });
1598
- return task;
1599
- }
1600
-
1601
- /* ---------------- leases / heartbeat (§4.3) ---------------- */
1602
-
1603
- /** Shared `phase`/`progress` validation for both heartbeat modes. */
1604
- _validatePhaseProgress(input) {
1605
- if (input.phase !== undefined && input.phase !== null && !LEASE_PHASES.includes(input.phase)) {
1606
- throw new ProtocolError('BAD_REQUEST', 'phase must be preparing|running|finalizing', { phase: input.phase });
1607
- }
1608
- if (input.progress !== undefined && input.progress !== null
1609
- && (typeof input.progress !== 'number' || input.progress < 0 || input.progress > 1)) {
1610
- throw new ProtocolError('BAD_REQUEST', 'progress must be a number in [0,1]', { progress: input.progress });
1611
- }
1612
- }
1613
-
1614
- /**
1615
- * v0.3.0 IDLE (diagnostic) heartbeat — an agent with no lease keeps its latency
1616
- * and liveness visible to other machines.
1617
- *
1618
- * This path deliberately NEVER looks at `this.tasks`. That is the whole safety
1619
- * argument, and it is structural rather than a matter of remembering a rule: an
1620
- * idle heartbeat cannot renew, resurrect or otherwise touch a lease because it
1621
- * never obtains one. Without that, an idle ping from a machine whose lease had
1622
- * expired would flip `expired` back to `running` and manufacture a live task out
1623
- * of nothing -- a fake "the work is still being done" signal, which is the most
1624
- * dangerous lie this relay can tell.
1625
- */
1626
- diagnosticHeartbeat(input = {}) {
1627
- const machineId = typeof input.machine_id === 'string' && input.machine_id !== ''
1628
- ? input.machine_id : null;
1629
- if (machineId === null) {
1630
- throw new ProtocolError('BAD_REQUEST', 'machine_id is required', {});
1631
- }
1632
- if (!this.devices.has(machineId)) {
1633
- throw new ProtocolError('NOT_FOUND', 'unknown machine_id', { machine_id: machineId });
1634
- }
1635
- this._validatePhaseProgress(input);
1636
-
1637
- const device = this.touchDevice(machineId);
1638
- this.recordHeartbeatMetrics(machineId, input);
1639
- return {
1640
- // No lease is involved, and the response says so instead of inventing one.
1641
- lease_until: null,
1642
- cancel: false,
1643
- diagnostic: true,
1644
- machine_id: machineId,
1645
- rtt_ms: device?.rtt_ms ?? null,
1646
- };
1647
- }
1648
-
1649
- /**
1650
- * §4.3 POST /v1/heartbeat — renews the lease using SERVER time only.
1651
- *
1652
- * Two modes since v0.3.0:
1653
- * - `task_id` present -> LEASE heartbeat (unchanged §4.3 behaviour)
1654
- * - `diagnostic: true`, no task -> IDLE heartbeat (diagnostics only, no lease)
1655
- * - neither -> 400. A heartbeat that silently renewed
1656
- * nothing would let an agent whose `task_id` went missing stop renewing while
1657
- * believing it was fine, ending in a falsely `expired` task. §0 forbids
1658
- * silent degradation, so this is loud.
1659
- *
1660
- * @returns {{lease_until:string|null, cancel:boolean, phase?:string, progress?:number, diagnostic?:boolean}}
1661
- */
1662
- heartbeat(input = {}) {
1663
- const hasTask = typeof input.task_id === 'string' && input.task_id !== '';
1664
- if (!hasTask) {
1665
- if (input.diagnostic !== true) {
1666
- throw new ProtocolError('BAD_REQUEST',
1667
- 'a heartbeat must carry task_id (lease renewal) or diagnostic:true (idle diagnostics); '
1668
- + 'refusing to treat a missing task_id as a no-op renewal', {
1669
- received_keys: Object.keys(input).sort(),
1670
- });
1671
- }
1672
- return this.diagnosticHeartbeat(input);
1673
- }
1674
- if (input.diagnostic === true) {
1675
- // task_id wins: a heartbeat that names a task IS a lease heartbeat.
1676
- // (documented; the flag is simply redundant here)
1677
- }
1678
- return this.leaseHeartbeat(input);
1679
- }
1680
-
1681
- leaseHeartbeat(input = {}) {
1682
- const task = this.requireTask(input.task_id);
1683
- const lease = task.leases.get(input.machine_id);
1684
- if (!lease) {
1685
- throw new ProtocolError('NOT_FOUND', 'no lease for this machine on this task', {
1686
- task_id: input.task_id, machine_id: input.machine_id,
1687
- });
1688
- }
1689
- this._validatePhaseProgress(input);
1690
-
1691
- const now = this.nowMs();
1692
- this.touchDevice(input.machine_id);
1693
- this.recordHeartbeatMetrics(input.machine_id, input, now);
1694
-
1695
- if (task.cancelled) {
1696
- return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: task.cancel_reason };
1697
- }
1698
-
1699
- // §4.3: once halt=now has handed this index to another attempt, the old owner
1700
- // must stop — a late heartbeat from the expired machine is not a valid re-claim.
1701
- if (lease.state === 'expired' && task.attempt > lease.attempt) {
1702
- return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: 'LEASE_EXPIRED_TAKEN_OVER' };
1703
- }
1704
-
1705
- const stateBefore = lease.state;
1706
- lease.last_heartbeat_ms = now;
1707
- lease.lease_until_ms = now + this.leaseWindowMs();
1708
- lease.heartbeat_count += 1;
1709
- if (input.phase !== undefined && input.phase !== null) lease.phase = input.phase;
1710
- if (input.progress !== undefined && input.progress !== null) lease.progress = input.progress;
1711
- if (Number.isInteger(input.attempt) && input.attempt >= 1) lease.attempt = input.attempt;
1712
- if (lease.state === 'queued' || lease.state === 'offered') lease.state = 'running';
1713
- // A live heartbeat is stronger evidence than a timer: with halt=never nobody took
1714
- // the work away, so the late-but-alive agent re-claims its own lease.
1715
- if (lease.state === 'expired') { lease.state = 'running'; lease.expired_at_ms = null; }
1716
-
1717
- // §7 ledger records lease STATE changes, not every 10s heartbeat.
1718
- if (lease.state !== stateBefore) this._ledgerLease(task, lease);
1719
-
1720
- return { lease_until: rfc3339(lease.lease_until_ms), cancel: false, phase: lease.phase, progress: lease.progress };
1721
- }
1722
-
1723
- /** §4.3: expiry is decided by Rabbit's clock only. */
1724
- isLeaseExpired(lease, nowMs = this.nowMs()) {
1725
- if (!ACTIVE_LEASE_STATES.includes(lease.state)) return false;
1726
- return nowMs >= lease.lease_until_ms;
1727
- }
1728
-
1729
- /**
1730
- * §4.3 expiry sweep.
1731
- * - halt=never → book-keeping only, task is marked `partial`
1732
- * - halt=now → notify remaining machines immediately that they may take over
1733
- */
1734
- sweepExpired(nowMs = this.nowMs()) {
1735
- const expired = [];
1736
- for (const task of this.tasks.values()) {
1737
- if (task.cancelled) continue;
1738
- let changed = false;
1739
- for (const lease of task.leases.values()) {
1740
- if (!this.isLeaseExpired(lease, nowMs)) continue;
1741
- lease.state = 'expired';
1742
- lease.expired_at_ms = nowMs;
1743
- changed = true;
1744
- expired.push({ task, lease });
1745
- this._ledgerLease(task, lease);
1746
- this.emit('notice', {
1747
- level: 'warn',
1748
- code: 'LEASE_EXPIRED',
1749
- message: `lease expired: machine=${lease.machine_id} task=${task.task_id} attempt=${lease.attempt}`,
1750
- task_id: task.task_id,
1751
- machine_id: lease.machine_id,
1752
- });
1753
- if (task.halt === 'now') {
1754
- task.attempt += 1;
1755
- this._takeover(task, lease, nowMs);
1756
- }
1757
- }
1758
- if (changed) task.degraded = 'partial';
1759
- }
1760
- return expired;
1761
- }
1762
-
1763
- /**
1764
- * §4.3 `halt=now`: "立即通知其他机器可接管".
1765
- *
1766
- * v0.0.1's PROTOCOL defines no per-index lease reassignment, so takeover is
1767
- * implemented as: re-offer the freed `index` (attempt+1) to every live machine
1768
- * that is idle — no lease yet, or a lease still queued/offered — and broadcast a
1769
- * notice. Machines already `running` are never re-offered, to avoid double
1770
- * execution of the same index.
1771
- */
1772
- _takeover(task, expiredLease, nowMs = this.nowMs()) {
1773
- const candidates = [];
1774
- for (const device of this.devices.values()) {
1775
- if (device.machine_id === expiredLease.machine_id) continue;
1776
- const lease = task.leases.get(device.machine_id);
1777
- if (lease && !['queued', 'offered'].includes(lease.state)) continue;
1778
- if (!lease && gateMachine(task, device)) continue;
1779
- candidates.push({ device, lease: lease ?? null });
1780
- }
1781
-
1782
- for (const { device, lease } of candidates) {
1783
- const target = lease ?? {
1784
- machine_id: device.machine_id,
1785
- machine_name: device.machine_name,
1786
- index: expiredLease.index,
1787
- attempt: task.attempt,
1788
- state: 'queued',
1789
- refusal_reason: null,
1790
- refusal_detail: null,
1791
- dedupe_key: null,
1792
- last_heartbeat_ms: nowMs,
1793
- claimed_at_ms: nowMs,
1794
- lease_until_ms: nowMs + this.leaseWindowMs(),
1795
- phase: null,
1796
- progress: null,
1797
- offered_seq: null,
1798
- expired_at_ms: null,
1799
- finished_at_ms: null,
1800
- heartbeat_count: 0,
1801
- };
1802
- target.index = expiredLease.index;
1803
- target.attempt = task.attempt;
1804
- target.dedupe_key = computeDedupeKey(task.task_id, target.index, task.command_hash, task.base_tree);
1805
- task.leases.set(device.machine_id, target);
1806
- this.emitOffer(task, target);
1807
- this._ledgerLease(task, target);
1808
- }
1809
-
1810
- this.emit('notice', {
1811
- level: 'info',
1812
- code: candidates.length > 0 ? 'LEASE_TAKEOVER' : 'LEASE_TAKEOVER_UNAVAILABLE',
1813
- message: candidates.length > 0
1814
- ? `task ${task.task_id} index ${expiredLease.index} re-offered at attempt ${task.attempt} to: `
1815
- + candidates.map((c) => c.device.machine_id).join(',')
1816
- : `task ${task.task_id} index ${expiredLease.index} has no idle machine to take over`,
1817
- task_id: task.task_id,
1818
- });
1819
- return candidates.map((c) => c.device.machine_id);
1820
- }
1821
-
1822
- /* ---------------- results (§5) ---------------- */
1823
-
1824
- static instanceKey(machineId, dedupeKey, attempt) {
1825
- return `${machineId}${INSTANCE_SEP}${dedupeKey}${INSTANCE_SEP}${attempt}`;
1826
- }
1827
-
1828
- /**
1829
- * §5 result envelope validation.
1830
- * Missing §5.1 required fields do NOT reject the frame — they make the result
1831
- * `unverifiable` (§5.1 "缺一即 unverifiable").
1832
- */
1833
- static validateEnvelope(envelope) {
1834
- const missing = [];
1835
- if (!envelope || typeof envelope !== 'object' || Array.isArray(envelope)) {
1836
- return { missing: [...REQUIRED_ENVELOPE_FIELDS], malformed: true };
1837
- }
1838
- for (const field of REQUIRED_ENVELOPE_FIELDS) {
1839
- if (!(field in envelope) || envelope[field] === undefined) missing.push(field);
1840
- }
1841
- const problems = [];
1842
- if ('status' in envelope && !RESULT_STATUSES.includes(envelope.status)) {
1843
- problems.push(`status must be one of ${RESULT_STATUSES.join('|')}`);
1844
- }
1845
- if (envelope.status === 'refused' && !envelope.refusal_reason) {
1846
- problems.push('refusal_reason is required when status=refused');
1847
- }
1848
- if ('warnings' in envelope && !Array.isArray(envelope.warnings)) problems.push('warnings must be string[]');
1849
- return { missing, problems };
1850
- }
1851
-
1852
- /**
1853
- * §1 idempotent POST /v1/result.
1854
- *
1855
- * NOTE (contract ambiguity, see report): §4.4 `dedupe_key` deliberately excludes
1856
- * `machine_id`/`attempt`, but §6.3 aggregates results from *several* machines in
1857
- * replicate mode — they all share one dedupe_key. Deduping purely on dedupe_key
1858
- * would silently drop every machine's result but the first. We therefore key the
1859
- * idempotency check on (machine_id, dedupe_key, attempt): an identical redelivery
1860
- * is `deduped:true`, distinct machines/attempts are all recorded.
1861
- */
1862
- submitResult(envelope = {}) {
1863
- const { missing, problems, malformed } = RabbitState.validateEnvelope(envelope);
1864
- if (malformed) throw new ProtocolError('BAD_REQUEST', 'result must be a JSON object');
1865
-
1866
- /**
1867
- * v0.4.0: `transport` / `p2p` are OPTIONAL envelope fields, and they are validated as values
1868
- * rather than added to the §5.1 required set. A v0.3.9 agent sends neither and must keep
1869
- * working unchanged; an agent that sends `transport: 'satellite'` is refused by name instead of
1870
- * storing a value the report would then show as a path that does not exist.
1871
- */
1872
- RabbitState.validateTransportFields(envelope);
1873
-
1874
- const taskId = envelope.task_id;
1875
- if (typeof taskId !== 'string' || taskId.length === 0) {
1876
- throw new ProtocolError('BAD_REQUEST', 'task_id is required');
1877
- }
1878
- const task = this.tasks.get(taskId);
1879
- if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
1880
-
1881
- const machineId = typeof envelope.machine_id === 'string' ? envelope.machine_id : null;
1882
- const dedupeKey = typeof envelope.dedupe_key === 'string' && envelope.dedupe_key
1883
- ? envelope.dedupe_key : null;
1884
- const attempt = Number.isInteger(envelope.attempt) && envelope.attempt >= 1 ? envelope.attempt : 1;
1885
-
1886
- if (dedupeKey === null) {
1887
- // Cannot dedupe without a key; still record it, as unverifiable.
1888
- missing.push('dedupe_key');
1889
- }
1890
- const instanceKey = RabbitState.instanceKey(machineId ?? '<unknown>', dedupeKey ?? '<none>', attempt);
1891
-
1892
- // §4.4 sanity check against the lease this machine actually holds. Booked as a
1893
- // problem instead of a hard rejection so a buggy client cannot lose its result.
1894
- const leaseForMachine = machineId ? task.leases.get(machineId) : null;
1895
- if (leaseForMachine?.dedupe_key && dedupeKey && leaseForMachine.dedupe_key !== dedupeKey) {
1896
- problems.push('dedupe_key does not match the §4.4 formula for this lease');
1897
- }
1898
-
1899
- if (this.results.has(instanceKey)) {
1900
- const prior = this.results.get(instanceKey);
1901
- return { deduped: true, task_id: taskId, dedupe_key: dedupeKey, seq: prior.seq };
1902
- }
1903
-
1904
- const now = this.nowMs();
1905
- let envelopeShaOk = null;
1906
- if (typeof envelope.envelope_sha256 === 'string') {
1907
- const { envelope_sha256: _drop, ...rest } = envelope;
1908
- envelopeShaOk = sha256Hex(jcs(rest)) === envelope.envelope_sha256;
1909
- }
1910
-
1911
- const record = {
1912
- envelope,
1913
- machine_id: machineId,
1914
- dedupe_key: dedupeKey,
1915
- attempt,
1916
- instance_key: instanceKey,
1917
- received_at_ms: now,
1918
- validation: {
1919
- missing: [...new Set(missing)],
1920
- problems,
1921
- envelope_sha256_ok: envelopeShaOk,
1922
- },
1923
- };
1924
-
1925
- const effectiveStatus = record.validation.missing.length > 0 ? 'unverifiable' : envelope.status;
1926
-
1927
- const seq = this.nextSeq();
1928
- record.seq = seq;
1929
- record.effective_status = effectiveStatus;
1930
- this.results.set(instanceKey, record);
1931
-
1932
- if (machineId) {
1933
- const lease = task.leases.get(machineId);
1934
- if (lease) {
1935
- lease.state = effectiveStatus === 'refused' ? 'refused' : 'done';
1936
- if (effectiveStatus === 'refused' && !lease.refusal_reason) {
1937
- lease.refusal_reason = envelope.refusal_reason ?? 'REFUSED';
1938
- }
1939
- lease.finished_at_ms = now;
1940
- lease.attempt = attempt;
1941
- }
1942
- this.touchDevice(machineId);
1943
- }
1944
- task.result_seqs.push(seq);
1945
-
1946
- this.emit('task.result', {
1947
- task_id: taskId,
1948
- dedupe_key: dedupeKey,
1949
- attempt,
1950
- index: envelope.index ?? null,
1951
- status: effectiveStatus,
1952
- machine_id: machineId,
1953
- deduped: false,
1954
- validation_errors: record.validation.missing,
1955
- });
1956
-
1957
- // §7: results go into the ledger so the dedupe index survives a restart.
1958
- this._ledger({ type: 'result.stored', task_id: taskId, machine_id: machineId, result: record });
1959
- if (machineId) {
1960
- const changed = task.leases.get(machineId);
1961
- if (changed) this._ledgerLease(task, changed);
1962
- }
1963
-
1964
- return { deduped: false, task_id: taskId, dedupe_key: dedupeKey, seq, _instanceKey: instanceKey };
1965
- }
1966
-
1967
- /** All recorded results for a task, keyed by machine_id (later attempts win). */
1968
- taskResults(taskId) {
1969
- const task = this.tasks.get(taskId);
1970
- if (!task) return new Map();
1971
- const out = new Map();
1972
- for (const rec of this.results.values()) {
1973
- if (rec.envelope.task_id !== taskId) continue;
1974
- const prev = out.get(rec.machine_id);
1975
- if (!prev || rec.received_at_ms >= prev.received_at_ms) out.set(rec.machine_id, rec);
1976
- }
1977
- return out;
1978
- }
1979
-
1980
- /** Global dedupe book-keeping: dedupe_key -> instances seen. */
1981
- dedupeReport() {
1982
- const out = {};
1983
- for (const rec of this.results.values()) {
1984
- if (!rec.dedupe_key) continue;
1985
- out[rec.dedupe_key] = out[rec.dedupe_key] ?? [];
1986
- out[rec.dedupe_key].push({
1987
- machine_id: rec.machine_id,
1988
- attempt: rec.attempt,
1989
- received_at: rfc3339(rec.received_at_ms),
1990
- });
1991
- }
1992
- return out;
1993
- }
1994
-
1995
- /* ---------------- serialization ---------------- */
1996
-
1997
- stats() {
1998
- return {
1999
- protocol_version: PROTOCOL_VERSION,
2000
- seq: this.seq,
2001
- buffered_events: this.buffer.length,
2002
- devices: this.devices.size,
2003
- tasks: this.tasks.size,
2004
- results: this.results.size,
2005
- pairing_codes: [...this.pairingCodes.values()].filter((c) => !c.used && this.nowMs() < c.expires_at_ms).length,
2006
- uptime_ms: this.nowMs() - this.startedAtMs,
2007
- };
2008
- }
2009
- }
2010
-
2011
- export default RabbitState;
1
+ /**
2
+ * W2M Rabbit relay — in-memory state machine.
3
+ *
4
+ * Contract: E:\DS\_work\w2m\PROTOCOL.md (protocol_version = 1, frozen).
5
+ * This module is intentionally PURE LOGIC:
6
+ * - no network, no filesystem, no timers
7
+ * - time is injectable via `now()` (required for lease-expiry unit tests)
8
+ * - randomness is injectable for deterministic tests
9
+ *
10
+ * Persistence: none. All state is in-memory and is lost on restart (v0.0.1 scope);
11
+ * there is intentionally no snapshot/serialization hook yet.
12
+ */
13
+
14
+ import { createHash, randomBytes } from 'node:crypto';
15
+
16
+ // v0.7.7: fleet scoping. Imported rather than inlined so the rule has one home with its own
17
+ // exhaustive tests — three lines inside this object would be untestable without a live relay.
18
+ import { targetsFor } from '../chatops/fleet-wiring.mjs';
19
+ import { fleetForPairingCode, deviceFleetFields } from '../chatops/fleet-wiring.mjs';
20
+
21
+ export const PROTOCOL_VERSION = 1;
22
+
23
+ /**
24
+ * PROTOCOL §1 table + §7 code table, plus the two v0.1.2 additions:
25
+ * `OPERATOR_REQUIRED` (§5.2 → 401) and `RATE_LIMITED` (§6 → 429).
26
+ * A code missing from this map silently degrades to HTTP 500 via
27
+ * `ProtocolError`, which keeps the right error code but lies about the status.
28
+ */
29
+ export const ERROR_STATUS = Object.freeze({
30
+ UNAUTHORIZED: 401,
31
+ BAD_REQUEST: 400,
32
+ NOT_FOUND: 404,
33
+ PAIRING_INVALID: 400,
34
+ PAIRING_EXPIRED: 400,
35
+ TASK_EXISTS: 409,
36
+ FRAME_TOO_LARGE: 413,
37
+ NO_ONLINE_DEVICE: 503,
38
+ INTERNAL: 500,
39
+ OPERATOR_REQUIRED: 401, // v0.1.2 §5.2
40
+ RATE_LIMITED: 429, // v0.1.2 §6
41
+
42
+ /* v0.3.0 §9 request signing. Every client-side signature failure is a 401.
43
+ * `SIGNING_NOT_CONFIGURED` is deliberately NOT 401: the relay was started with
44
+ * `--require-signature` but no secret, so the fault is the relay's own
45
+ * configuration. Reporting that as 401 would send every operator to debug the
46
+ * client -- the one side that is not broken. 500 rather than 503 because
47
+ * retrying never fixes a missing configuration, so nothing should be invited to
48
+ * retry. */
49
+ SIGNATURE_REQUIRED: 401,
50
+ SIGNATURE_INCOMPLETE: 401,
51
+ SIGNATURE_MISMATCH: 401,
52
+ SIGNATURE_BAD_TIMESTAMP: 401,
53
+ SIGNATURE_BAD_NONCE: 401,
54
+ SIGNATURE_EXPIRED: 401,
55
+ SIGNATURE_REPLAY: 401,
56
+ SIGNING_NOT_CONFIGURED: 500,
57
+ });
58
+
59
+ /** §5.1 required envelope fields — missing any one means `unverifiable`. */
60
+ export const REQUIRED_ENVELOPE_FIELDS = Object.freeze([
61
+ 'envelope_version', 'task_id', 'attempt', 'dedupe_key', 'machine_id', 'machine_name',
62
+ 'platform', 'caps', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
63
+ 'pre_tree_fingerprint', 'post_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error',
64
+ 'head_commit', 'dirty_before', 'command_argv', 'command_hash', 'shell_id',
65
+ 'started_at', 'ended_at', 'duration_ms', 'exit_code', 'status', 'refusal_reason',
66
+ 'stdout_sha256', 'stdout_bytes', 'stderr_sha256', 'stderr_bytes', 'warnings',
67
+ 'envelope_sha256',
68
+ ]);
69
+
70
+ /** §5.3 comparable fields — the ONLY fields that take part in consistency judgement. */
71
+ export const COMPARABLE_FIELDS = Object.freeze([
72
+ 'task_id', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
73
+ 'pre_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error', 'head_commit',
74
+ 'command_hash', 'shell_id', 'exit_code', 'status', 'refusal_reason',
75
+ 'stdout_sha256', 'stdout_bytes', 'stdout_normalized_sha256',
76
+ 'stderr_sha256', 'artifacts', 'diff_numstat', 'tests', 'semantic_counts', 'warnings',
77
+ ]);
78
+
79
+ /** §5.1 status enum. */
80
+ export const RESULT_STATUSES = Object.freeze([
81
+ 'ok', 'nonzero_exit', 'timeout', 'crashed', 'refused', 'unverifiable',
82
+ ]);
83
+
84
+ export const FAILURE_STATUSES = Object.freeze(['nonzero_exit', 'timeout', 'crashed']);
85
+
86
+ /** §4.3 phase enum. */
87
+ export const LEASE_PHASES = Object.freeze(['preparing', 'running', 'finalizing']);
88
+
89
+ /** Lease states that can still be expired / renewed. */
90
+ export const ACTIVE_LEASE_STATES = Object.freeze(['queued', 'offered', 'running']);
91
+
92
+ export const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000; // §4.3: agent heartbeats every 10s
93
+ export const DEFAULT_MISSED_HEARTBEATS = 2; // §4.3: 2 missed cycles = 20s
94
+ export const DEFAULT_LEASE_GRACE_MS = 30_000; // §4.3: + grace 30s
95
+ /**
96
+ * v0.4.0: the bounded hold on the relay's own offer when a dispatcher is pushing it directly.
97
+ *
98
+ * Sized against what it has to cover: one punch round trip (measured 15-370 ms on the reference
99
+ * networks), the frame's delivery, and the executor's first heartbeat POST — all on a host that may
100
+ * be busy. 1.2 s was the first guess and it was too tight: measured on a 1-vCPU shared host, two
101
+ * agent tests saw the relay's copy overtake the direct one, which for `p2pMode: direct` means a
102
+ * refusal of an offer that was about to arrive. 3 s covers a slow host with room to spare.
103
+ *
104
+ * The cost is honest and bounded: a punch that fails delays the relay's copy by up to this much,
105
+ * once per task. Nothing else waits on it.
106
+ */
107
+ export const DEFAULT_P2P_OFFER_GRACE_MS = 3_000;
108
+ export const DEFAULT_PAIRING_TTL_MS = 24 * 60 * 60 * 1000; // §2.2: 24h
109
+ export const DEFAULT_EVENT_BUFFER_SIZE = 1000; // §requirement: keep last 1000 events
110
+ export const MAX_FRAME_BYTES = 64 * 1024; // 64 KiB single frame
111
+ export const SHELL_ID_DIRECT = 'direct-exec'; // §5.1: v0.0.1 only direct exec
112
+ /** v0.3.0: RTT sanity ceiling — anything above 24h is treated as a broken value. */
113
+ export const MAX_RTT_MS = 24 * 60 * 60 * 1000;
114
+ /** v0.3.0: an RTT measurement older than this is reported as stale.
115
+ * Must stay comfortably ABOVE the idle-heartbeat interval (60s): at 1x the value
116
+ * would flip stale just before every refresh and flap, which would make
117
+ * `rtt_stale === false` useless as a "trust this number" predicate. 3x gives two
118
+ * consecutive missed idle heartbeats of margin. See PROTOCOL-v0.3.0.md §7. */
119
+ export const DEFAULT_RTT_STALE_MS = 180_000;
120
+ /** v0.3.0: recommended cadence for an agent with no lease (reference for the agent side). */
121
+ export const DEFAULT_IDLE_HEARTBEAT_MS = 60_000;
122
+ export const ENVELOPE_VERSION = '1.0';
123
+ export const FINGERPRINT_ALGO = 'git-temp-index-tree/v1';
124
+
125
+ /* ------------------------------------------------------------------ */
126
+ /* Errors */
127
+ /* ------------------------------------------------------------------ */
128
+
129
+ export class ProtocolError extends Error {
130
+ constructor(code, message, detail = {}) {
131
+ super(message);
132
+ this.name = 'ProtocolError';
133
+ this.code = code;
134
+ this.detail = detail;
135
+ this.status = ERROR_STATUS[code] ?? 500;
136
+ }
137
+ toBody() {
138
+ return { error: { code: this.code, message: this.message, detail: this.detail } };
139
+ }
140
+ }
141
+
142
+ /* ------------------------------------------------------------------ */
143
+ /* Small pure helpers */
144
+ /* ------------------------------------------------------------------ */
145
+
146
+ export function sha256Hex(input) {
147
+ return createHash('sha256').update(input).digest('hex');
148
+ }
149
+
150
+ /** RFC 8785-ish canonical JSON (JCS): sorted keys, no whitespace. */
151
+ export function jcs(value) {
152
+ if (value === null) return 'null';
153
+ const t = typeof value;
154
+ if (t === 'number') return Number.isFinite(value) ? JSON.stringify(value) : 'null';
155
+ if (t === 'boolean') return value ? 'true' : 'false';
156
+ if (t === 'string') return JSON.stringify(value);
157
+ if (t === 'undefined' || t === 'function' || t === 'symbol') return 'null';
158
+ if (Array.isArray(value)) return `[${value.map((v) => (v === undefined ? 'null' : jcs(v))).join(',')}]`;
159
+ if (t === 'object') {
160
+ const keys = Object.keys(value).filter((k) => value[k] !== undefined).sort();
161
+ return `{${keys.map((k) => `${JSON.stringify(k)}:${jcs(value[k])}`).join(',')}}`;
162
+ }
163
+ return JSON.stringify(String(value));
164
+ }
165
+
166
+ /**
167
+ * Upper bound on the stages a `pipeline` may contain.
168
+ *
169
+ * Each stage is another command run on someone else's machine, so an unbounded list is an unbounded
170
+ * amount of work created by one request. The limit is generous for the intended use (build, test,
171
+ * package) and exists so the failure is a clear refusal rather than a fleet-wide surprise.
172
+ */
173
+ export const MAX_PIPELINE_STAGES = 16;
174
+
175
+ /**
176
+ * v0.4.0: upper bound on `POST /v1/task`'s `origin_machine_id`.
177
+ *
178
+ * A machine id is an identifier, not a payload: 128 characters is far beyond any id this system
179
+ * mints, and the ceiling is what stops the field from becoming an unbounded free-text channel into
180
+ * every offer frame the relay emits.
181
+ */
182
+ export const MAX_ORIGIN_MACHINE_ID_LENGTH = 128;
183
+
184
+ /**
185
+ * v0.4.0: the P2P modes a task may be dispatched under (§1 of the frozen v0.4.0 contract).
186
+ *
187
+ * Mirrored here rather than imported from `src/agent/p2p-node.mjs`: the relay is a standalone
188
+ * process that must not depend on agent code, and a mode list is three strings. A value outside
189
+ * this list is a configuration error, never a silent fall back to the default.
190
+ */
191
+ export const P2P_MODES = Object.freeze(['auto', 'direct', 'relay']);
192
+
193
+ /**
194
+ * v0.4.0: how a result travelled — the direct P2P channel or the relay.
195
+ *
196
+ * A **transport is not a result**: how an envelope arrived must never take part in the consistency
197
+ * judgement (§5 of the contract), or two machines that produced byte-identical output would be
198
+ * reported `divergent` merely because one of them punched a hole. Hence: not required, not
199
+ * comparable, and absent on every v0.3.9 envelope.
200
+ */
201
+ export const TRANSPORT_VALUES = Object.freeze(['p2p', 'relay']);
202
+
203
+ /** §4.4 helper: command_hash = sha256(JCS(argv)+"|"+shell_id+"|"+cwd_rel). */
204
+ export function computeCommandHash(commandArgv, shellId = SHELL_ID_DIRECT, cwdRel = '.') {
205
+ return sha256Hex(`${jcs(commandArgv)}|${shellId}|${cwdRel}`);
206
+ }
207
+
208
+ /**
209
+ * `command_hash` for a whole pipeline.
210
+ *
211
+ * The hash must bind **every** stage, not just the first. Bound to stage 0 alone, two pipelines that
212
+ * differ after the first stage would carry the same hash -- so an envelope produced by the wrong
213
+ * chain would still verify, and making "the machine ran what I asked" checkable is the entire point
214
+ * of this field. Each stage's cwd participates for the same reason.
215
+ *
216
+ * The `|pipeline` suffix means this can never collide with a single-command hash: the two hash over
217
+ * different shapes, and that difference is deliberate rather than incidental.
218
+ */
219
+ export function computePipelineCommandHash(stages, shellId = SHELL_ID_DIRECT) {
220
+ const canonical = stages.map((s) => ({
221
+ command_argv: s.command_argv,
222
+ cwd_rel: s.cwd_rel ?? '.',
223
+ continue_on_failure: s.continue_on_failure === true,
224
+ }));
225
+ return sha256Hex(`${jcs(canonical)}|${shellId}|pipeline`);
226
+ }
227
+
228
+ /** §4.4: dedupe_key = sha256(task_id + "|" + index + "|" + command_hash + "|" + base_tree). */
229
+ export function computeDedupeKey(taskId, index, commandHash, baseTree) {
230
+ return sha256Hex(`${taskId}|${index}|${commandHash}|${baseTree}`);
231
+ }
232
+
233
+ /** Epoch ms -> RFC3339 UTC, second precision (matches the PROTOCOL examples). */
234
+ export function rfc3339(ms) {
235
+ return new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
236
+ }
237
+
238
+ export function isRfc3339(value) {
239
+ return typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/.test(value);
240
+ }
241
+
242
+ export function deepEqual(a, b) {
243
+ return jcs(a === undefined ? null : a) === jcs(b === undefined ? null : b);
244
+ }
245
+
246
+ function normalizeNow(now) {
247
+ if (typeof now !== 'function') return () => Date.now();
248
+ return () => {
249
+ const v = now();
250
+ if (typeof v === 'number') return v;
251
+ if (v instanceof Date) return v.getTime();
252
+ return Date.parse(v);
253
+ };
254
+ }
255
+
256
+ /* ------------------------------------------------------------------ */
257
+ /* ULID */
258
+ /* ------------------------------------------------------------------ */
259
+
260
+ const CROCKFORD = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
261
+
262
+ function encodeBig(value, len) {
263
+ let out = '';
264
+ let v = value;
265
+ for (let i = 0; i < len; i += 1) {
266
+ out = CROCKFORD[Number(v & 31n)] + out;
267
+ v >>= 5n;
268
+ }
269
+ return out;
270
+ }
271
+
272
+ export function createIdFactory() {
273
+ let lastMs = -1;
274
+ let lastRand = 0n;
275
+ return function nextId(nowMs = Date.now()) {
276
+ const bytes = randomBytes(10);
277
+ let rand = 0n;
278
+ for (const b of bytes) rand = (rand << 8n) | BigInt(b);
279
+ if (nowMs === lastMs) rand = (lastRand + 1n) & ((1n << 80n) - 1n);
280
+ lastMs = nowMs;
281
+ lastRand = rand;
282
+ return encodeBig(BigInt(nowMs), 10) + encodeBig(rand, 16);
283
+ };
284
+ }
285
+
286
+ export function ulid(nowMs = Date.now()) {
287
+ return createIdFactory()(nowMs);
288
+ }
289
+
290
+ /* ------------------------------------------------------------------ */
291
+ /* Version / capability gate (§6.1) */
292
+ /* ------------------------------------------------------------------ */
293
+
294
+ function parseVersion(text) {
295
+ if (typeof text !== 'string') return null;
296
+ const m = text.trim().match(/^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:\.(\d+))?/);
297
+ if (!m) return null;
298
+ return [Number(m[1]), Number(m[2] ?? 0), Number(m[3] ?? 0), Number(m[4] ?? 0)];
299
+ }
300
+
301
+ function cmpVersion(a, b) {
302
+ for (let i = 0; i < 4; i += 1) {
303
+ if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1;
304
+ }
305
+ return 0;
306
+ }
307
+
308
+ /**
309
+ * Minimal semver-range check used by the §6.1 capability gate.
310
+ * Supports bare versions and comma/space separated `>= > <= < = ==` conjunctions.
311
+ * Unparseable ranges are treated as NOT satisfied (never silently pass).
312
+ */
313
+ export function satisfiesRange(version, range) {
314
+ const v = parseVersion(version);
315
+ if (v === null) return false;
316
+ if (typeof range !== 'string') return false;
317
+ const parts = range.split(/[\s,]+/).filter(Boolean);
318
+ if (parts.length === 0) return false;
319
+ for (const part of parts) {
320
+ const m = part.match(/^(>=|<=|==|=|>|<)?\s*(.+)$/);
321
+ if (!m) return false;
322
+ const op = m[1] ?? '=';
323
+ const target = parseVersion(m[2]);
324
+ if (target === null) return false;
325
+ const c = cmpVersion(v, target);
326
+ if (op === '>=' && c < 0) return false;
327
+ if (op === '<=' && c > 0) return false;
328
+ if (op === '>' && c <= 0) return false;
329
+ if (op === '<' && c >= 0) return false;
330
+ if ((op === '=' || op === '==') && c !== 0) return false;
331
+ }
332
+ return true;
333
+ }
334
+
335
+ function machineWritable(caps) {
336
+ if (!caps || typeof caps !== 'object') return false;
337
+ if (caps.write === false || caps.writable === false || caps.write_allowed === false) return false;
338
+ if (caps.read_only === true || caps.readonly === true || caps.ro_mount === true) return false;
339
+ return true;
340
+ }
341
+
342
+ /**
343
+ * §6.1 capability gate, evaluated per machine BEFORE dispatch.
344
+ * @returns {null | {status:'refused', refusal_reason:string, detail:object}}
345
+ */
346
+ export function gateMachine(task, machine) {
347
+ const caps = machine?.caps ?? {};
348
+ const req = task.requirements ?? {};
349
+
350
+ if (req.platform && Array.isArray(req.platform) && req.platform.length > 0) {
351
+ const os = String(machine?.platform?.os ?? '').toLowerCase();
352
+ if (!req.platform.map((p) => String(p).toLowerCase()).includes(os)) {
353
+ return { status: 'refused', refusal_reason: 'PLATFORM_MISMATCH', detail: { os, want: req.platform } };
354
+ }
355
+ }
356
+
357
+ const toolchain = req.toolchain ?? {};
358
+ for (const [tool, range] of Object.entries(toolchain)) {
359
+ const declared = caps[tool];
360
+ if (declared === null || declared === undefined || declared === '') {
361
+ return {
362
+ status: 'refused',
363
+ refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
364
+ detail: { tool, want: range, have: declared ?? null },
365
+ };
366
+ }
367
+ if (typeof range === 'string' && !satisfiesRange(declared, range)) {
368
+ return {
369
+ status: 'refused',
370
+ refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
371
+ detail: { tool, want: range, have: declared },
372
+ };
373
+ }
374
+ }
375
+
376
+ if (task.write === true && !machineWritable(caps)) {
377
+ return { status: 'refused', refusal_reason: 'READ_ONLY_MACHINE', detail: { caps } };
378
+ }
379
+
380
+ return null;
381
+ }
382
+
383
+ /* ------------------------------------------------------------------ */
384
+ /* RabbitState */
385
+ /* ------------------------------------------------------------------ */
386
+
387
+ const INSTANCE_SEP = '\u0000';
388
+
389
+ export class RabbitState {
390
+ constructor(options = {}) {
391
+ this._now = normalizeNow(options.now);
392
+ this._idFactory = options.idFactory ?? createIdFactory();
393
+ this._randomToken = options.tokenFactory ?? (() => randomBytes(24).toString('hex'));
394
+ this._randomChars = options.randomChars
395
+ ?? ((n) => { const a = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; const b = randomBytes(n); let s = ''; for (let i = 0; i < n; i += 1) s += a[b[i] % a.length]; return s; });
396
+
397
+ this.protocolVersion = PROTOCOL_VERSION;
398
+ this.eventBufferSize = options.eventBufferSize ?? DEFAULT_EVENT_BUFFER_SIZE;
399
+ /**
400
+ * v0.7.7: the fleet this relay serves, or null for a relay that serves every device.
401
+ *
402
+ * **Null is the migration-safe default and is what the shared relay runs today.** With it,
403
+ * task targeting behaves exactly as it did before this release, so deploying cannot be an
404
+ * outage. Set it (via `--fleet <id>` or `fleetId`) and the relay becomes scoped: a device
405
+ * with a `fleet_id` reaches only its own fleet, and a device with none reaches nothing.
406
+ *
407
+ * This is the relay's *own* fleet, not a device's. A device carries its own `fleet_id`, which
408
+ * the relay wrote at pairing time from the pairing code.
409
+ */
410
+ this.fleetId = typeof options.fleetId === 'string' && options.fleetId !== '' ? options.fleetId : null;
411
+ this.pairingTtlMs = options.pairingTtlMs ?? DEFAULT_PAIRING_TTL_MS;
412
+ /** When true one code stays valid for its whole TTL instead of rotating. */
413
+ this.pairingCodeReusable = options.pairingCodeReusable === true;
414
+ this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
415
+ this.missedHeartbeats = options.missedHeartbeats ?? DEFAULT_MISSED_HEARTBEATS;
416
+ this.leaseGraceMs = options.leaseGraceMs ?? DEFAULT_LEASE_GRACE_MS;
417
+ /**
418
+ * v0.4.0: how long the relay holds its own offer back when the dispatcher says it will push the
419
+ * offer over the direct path (`origin_machine_id` + `p2p.mode`).
420
+ *
421
+ * Without this, the direct path could never win: the relay's offer goes out over an SSE stream
422
+ * that is already open, while a punch costs at least one round trip, so the relay copy would
423
+ * arrive first essentially always and `transport` would read `relay` even on a network where
424
+ * the punch succeeded. Measured on loopback, and it is exactly what the first version of this
425
+ * release did.
426
+ *
427
+ * The hold is bounded and self-healing: the executor renews its lease (a heartbeat) the moment
428
+ * it starts work on the direct copy, which moves the lease out of `queued`, and the deferred
429
+ * offer is then dropped. If the punch fails, nothing renews the lease and the relay offers
430
+ * normally once the grace expires -- so a failed punch costs this much latency and nothing else.
431
+ */
432
+ this.p2pOfferGraceMs = Number.isFinite(options.p2pOfferGraceMs) && options.p2pOfferGraceMs >= 0
433
+ ? options.p2pOfferGraceMs
434
+ : DEFAULT_P2P_OFFER_GRACE_MS;
435
+ /** Deferred offers, keyed `task_id:machine_id`, so they can be cleared on close(). */
436
+ this._pendingOffers = new Map();
437
+ /** v0.3.0: age beyond which a reported RTT is flagged stale. */
438
+ this.rttStaleMs = Number.isFinite(options.rttStaleMs) ? options.rttStaleMs : DEFAULT_RTT_STALE_MS;
439
+
440
+ this.startedAtMs = this._now();
441
+ /** v0.1.2 §8.3: random per-process id; a change tells clients the relay restarted. */
442
+ this.relayId = options.relayId ?? randomBytes(8).toString('hex');
443
+ /** v0.1.2 §7: optional injected Persistence instance (state.mjs never touches fs itself). */
444
+ this._persist = options.persistence ?? null;
445
+ /** True while replaying a ledger — suppresses event emission and re-persistence. */
446
+ this._restoring = false;
447
+ /** Ledger entries that could not be replayed. */
448
+ this.replayWarnings = [];
449
+ this.seq = 0;
450
+ /** @type {{seq:number,event:object,target:string|null}[]} ring buffer, last N events */
451
+ this.buffer = [];
452
+ this.devices = new Map(); // machine_id -> device
453
+ this.tokens = new Map(); // device_token -> machine_id
454
+ this.tasks = new Map(); // task_id -> task
455
+ this.results = new Map(); // instanceKey -> {envelope, machine_id, received_at_ms, ...}
456
+ this.pairingCodes = new Map(); // code -> {code, created_at_ms, expires_at_ms, used}
457
+ this._listeners = new Set();
458
+ }
459
+
460
+ /* ---------------- clock / ids ---------------- */
461
+
462
+ nowMs() { return this._now(); }
463
+ nowIso() { return rfc3339(this.nowMs()); }
464
+ nextId() { return this._idFactory(this.nowMs()); }
465
+ newToken() { return this._randomToken(); }
466
+
467
+ /** §4.3 lease window = 2 missed heartbeats (20s) + grace (30s) = 50s. */
468
+ leaseWindowMs() { return this.heartbeatIntervalMs * this.missedHeartbeats + this.leaseGraceMs; }
469
+
470
+ /* ---------------- persistence hooks (v0.1.2 §7) ---------------- */
471
+
472
+ /** Inject/replace the persistence backend (also used by tests). */
473
+ setPersistence(persistence) { this._persist = persistence ?? null; return this._persist; }
474
+
475
+ /** Snapshot the device table; failures must never break the request path. */
476
+ _saveDevices() {
477
+ if (this._restoring || !this._persist) return false;
478
+ try {
479
+ return this._persist.saveDevices(this.devices);
480
+ } catch (err) {
481
+ this._persist.warn?.('devices.json', `snapshot failed: ${err.message}`);
482
+ return false;
483
+ }
484
+ }
485
+
486
+ /** Append one ledger entry; failures must never break the request path. */
487
+ _ledger(entry) {
488
+ if (this._restoring || !this._persist) return false;
489
+ try {
490
+ const appended = this._persist.append(entry);
491
+ // v0.6.2: a successful append is what clears a degraded relay, so the outcome has to be
492
+ // reported back rather than assumed -- otherwise the state could only ever be entered.
493
+ if (appended) this._persist.noteAppendSuccess?.();
494
+ return appended;
495
+ } catch (err) {
496
+ // Recorded as a state, not merely as a line: this is the one relay failure that keeps serving,
497
+ // and so the one that is invisible from every other angle (v0.6.2).
498
+ if (typeof this._persist.noteAppendFailure === 'function') this._persist.noteAppendFailure(err.message);
499
+ else this._persist.warn?.('ledger.jsonl', `append failed: ${err.message}`);
500
+ return false;
501
+ }
502
+ }
503
+
504
+ _ledgerLease(task, lease) {
505
+ this._ledger({ type: 'lease.changed', task_id: task.task_id, machine_id: lease.machine_id, lease: { ...lease } });
506
+ }
507
+
508
+ /**
509
+ * v0.1.2 §7 startup recovery: rebuild the device table, token index, tasks,
510
+ * leases and the result/dedupe index from a devices snapshot + ledger replay.
511
+ * Never emits events and never writes back what it just read.
512
+ */
513
+ restore({ devices = {}, ledger = [] } = {}) {
514
+ this._restoring = true;
515
+ try {
516
+ for (const [machineId, d] of Object.entries(devices)) {
517
+ if (!d || typeof d !== 'object') continue;
518
+ const pairedAt = Date.parse(d.paired_at ?? '');
519
+ const device = {
520
+ machine_id: machineId,
521
+ machine_name: typeof d.machine_name === 'string' && d.machine_name ? d.machine_name : machineId,
522
+ platform: d.platform ?? {},
523
+ caps: d.caps ?? {},
524
+ user_id: d.user_id ?? null,
525
+ device_token: typeof d.device_token === 'string' ? d.device_token : null,
526
+ paired_at_ms: Number.isFinite(pairedAt) ? pairedAt : this.nowMs(),
527
+ re_paired: d.re_paired === true,
528
+ last_seen_at_ms: this.nowMs(),
529
+ // a snapshot proves the device exists, not that it is connected
530
+ online: false,
531
+ streams: 0,
532
+ // v0.3.0: RTT is volatile and is NEVER restored. Even if a snapshot on
533
+ // disk happens to carry `rtt_ms`, it is dropped here: after a restart the
534
+ // relay genuinely does not know the latency any more, and reporting a
535
+ // pre-restart number as current would be a lie that no reader can detect.
536
+ rtt_ms: null,
537
+ rtt_at_ms: null,
538
+ last_heartbeat_at_ms: null,
539
+ stream_connects: 0,
540
+ };
541
+ this.devices.set(machineId, device);
542
+ if (device.device_token) this.tokens.set(device.device_token, machineId);
543
+ }
544
+
545
+ let revivedTasks = 0;
546
+ for (const entry of ledger) {
547
+ try {
548
+ if (this._applyLedgerEntry(entry) === true) revivedTasks += 1;
549
+ } catch (err) {
550
+ this.replayWarnings.push({ type: entry?.type ?? '<unknown>', message: err.message });
551
+ }
552
+ }
553
+ return { revived_devices: this.devices.size, revived_tasks: revivedTasks };
554
+ } finally {
555
+ this._restoring = false;
556
+ }
557
+ }
558
+
559
+ /** @returns {boolean} true when a new task was created. */
560
+ _applyLedgerEntry(entry) {
561
+ if (!entry || typeof entry.type !== 'string') return false;
562
+ switch (entry.type) {
563
+ case 'task.created': {
564
+ const t = entry.task;
565
+ if (!t || typeof t.task_id !== 'string') return false;
566
+ const createdAt = Number.isFinite(t.created_at_ms) ? t.created_at_ms : Date.parse(entry.ts ?? '');
567
+ const task = {
568
+ task_id: t.task_id,
569
+ mode: t.mode ?? 'replicate',
570
+ command_argv: t.command_argv ?? [],
571
+ cwd_rel: t.cwd_rel ?? '.',
572
+ index_total: t.index_total ?? 1,
573
+ timeout_ms: t.timeout_ms ?? 300_000,
574
+ write: t.write === true,
575
+ write_scope: t.write_scope ?? [],
576
+ require_exclusive_write: t.require_exclusive_write === true,
577
+ base_commit: t.base_commit ?? null,
578
+ base_tree: t.base_tree ?? null,
579
+ requirements: t.requirements ?? {},
580
+ compare_policy: t.compare_policy ?? {},
581
+ halt: t.halt ?? 'never',
582
+ created_by: t.created_by ?? null,
583
+ created_at_ms: Number.isFinite(createdAt) ? createdAt : this.nowMs(),
584
+ command_hash: t.command_hash ?? null,
585
+ attempt: t.attempt ?? 1,
586
+ leases: new Map(),
587
+ cancelled: false,
588
+ cancel_reason: null,
589
+ degraded: t.degraded ?? null,
590
+ deadline_ms: t.deadline_ms ?? 0,
591
+ // v0.4.0 routing metadata. Read defensively: a ledger written by v0.3.9 has neither key,
592
+ // and a replayed task with `null` for both is exactly a v0.3.9 task.
593
+ origin_machine_id: t.origin_machine_id ?? null,
594
+ p2p: t.p2p ?? null,
595
+ result_seqs: [],
596
+ };
597
+ for (const lease of entry.leases ?? []) {
598
+ if (lease && typeof lease.machine_id === 'string') task.leases.set(lease.machine_id, { ...lease });
599
+ }
600
+ this.tasks.set(task.task_id, task);
601
+ return true;
602
+ }
603
+ case 'lease.changed': {
604
+ const task = this.tasks.get(entry.task_id);
605
+ const lease = task?.leases.get(entry.machine_id);
606
+ if (!lease) return false;
607
+ Object.assign(lease, entry.lease ?? {});
608
+ return false;
609
+ }
610
+ case 'result.stored': {
611
+ const record = entry.result;
612
+ if (!record || typeof record.instance_key !== 'string') return false;
613
+ this.results.set(record.instance_key, { ...record });
614
+ const task = this.tasks.get(record.envelope?.task_id);
615
+ if (task && Number.isInteger(record.seq)) task.result_seqs.push(record.seq);
616
+ return false;
617
+ }
618
+ case 'task.cancelled': {
619
+ const task = this.tasks.get(entry.task_id);
620
+ if (task) {
621
+ task.cancelled = true;
622
+ task.cancel_reason = entry.reason ?? null;
623
+ }
624
+ return false;
625
+ }
626
+ default:
627
+ return false; // unknown entry type: forward compatible, ignored
628
+ }
629
+ }
630
+
631
+ /* ---------------- events (§3) ---------------- */
632
+
633
+ nextSeq() { this.seq += 1; return this.seq; }
634
+
635
+ /** Reserve a seq WITHOUT buffering (used for the per-connection `ready` frame). */
636
+ reserveSeq() { return this.nextSeq(); }
637
+
638
+ onEvent(listener) {
639
+ this._listeners.add(listener);
640
+ return () => this._listeners.delete(listener);
641
+ }
642
+
643
+ /**
644
+ * Emit an event: allocates the next global seq, buffers it (ring of N) and
645
+ * notifies subscribers. `target` is the machine_id the event is addressed to
646
+ * (null = broadcast). The target is NOT serialized except as `machine_id`.
647
+ */
648
+ emit(type, fields = {}, opts = {}) {
649
+ const target = opts.target ?? null;
650
+ const seq = this.nextSeq();
651
+ const event = {
652
+ type,
653
+ seq,
654
+ rabbit_time: this.nowIso(),
655
+ ...(target ? { machine_id: target } : {}),
656
+ ...fields,
657
+ };
658
+ const entry = { seq, event, target };
659
+ this.buffer.push(entry);
660
+ if (this.buffer.length > this.eventBufferSize) {
661
+ this.buffer.splice(0, this.buffer.length - this.eventBufferSize);
662
+ }
663
+ for (const listener of this._listeners) {
664
+ try { listener(entry); } catch { /* subscriber errors must never break state */ }
665
+ }
666
+ return event;
667
+ }
668
+
669
+ oldestBufferedSeq() { return this.buffer.length ? this.buffer[0].seq : this.seq + 1; }
670
+ /**
671
+ * Highest seq present in the replay ring (0 when empty).
672
+ *
673
+ * Deliberately different from `lastSeq()`: `ready` frames allocate seq numbers
674
+ * without being buffered, so `lastSeq()` can run ahead of anything a client can
675
+ * actually replay. Diagnostics that describe the REPLAY WINDOW must use this one.
676
+ */
677
+ lastBufferedSeq() { return this.buffer.length ? this.buffer[this.buffer.length - 1].seq : 0; }
678
+ lastSeq() { return this.seq; }
679
+
680
+ /** Replay entries with seq >= from (used by GET /v1/stream?seq=|Last-Event-ID). */
681
+ replayEntries(from) {
682
+ return this.buffer.filter((e) => e.seq >= from);
683
+ }
684
+
685
+ eventsSince(from) { return this.replayEntries(from).map((e) => e.event); }
686
+
687
+ /* ---------------- pairing (§2) ---------------- */
688
+
689
+ /**
690
+ * Mint a pairing code.
691
+ *
692
+ * **v0.7.7: the code carries the relay's fleet.** This is the link that makes the scheme work,
693
+ * and it is easy to leave out: `pair()` reads the fleet *from the code*, so a code with no
694
+ * fleet pairs a device into no fleet -- and a device in no fleet reaches nothing on a scoped
695
+ * relay. `--fleet X` would then have had no effect on anything, silently.
696
+ *
697
+ * Defaulting to `this.fleetId` rather than requiring the caller to pass it means there is no
698
+ * path that mints a fleetless code on a scoped relay. An explicit `fleetId` overrides, which
699
+ * is what a relay serving several fleets would need.
700
+ */
701
+ createPairingCode({ ttlMs = this.pairingTtlMs, fleetId = this.fleetId } = {}) {
702
+ let code;
703
+ do { code = `PAIR-${this._randomChars(8)}`; } while (this.pairingCodes.has(code));
704
+ const now = this.nowMs();
705
+ this.pairingCodes.set(code, {
706
+ code,
707
+ created_at_ms: now,
708
+ expires_at_ms: now + ttlMs,
709
+ used: false,
710
+ // Present as an explicit key even when null, so a code dump shows whether a fleet was
711
+ // attached rather than leaving it to be inferred from an absent field.
712
+ fleet_id: typeof fleetId === 'string' && fleetId !== '' ? fleetId : null,
713
+ });
714
+ return code;
715
+ }
716
+
717
+ /** Revoke every outstanding code and mint a new one. */
718
+ rotatePairingCode(opts) {
719
+ for (const rec of this.pairingCodes.values()) rec.used = true;
720
+ return this.createPairingCode(opts);
721
+ }
722
+
723
+ /**
724
+ * Register an externally supplied code (`--pairing-code <c>`).
725
+ *
726
+ * v0.1.2 BUG-1: assigning the code to `relay.pairingCode` without registering it
727
+ * advertised a code that `/v1/pair` then rejected with PAIRING_INVALID, so every
728
+ * documented scripted-onboarding flow failed. "Which code do we use" and "is the
729
+ * code registered" are two different questions; this answers the second.
730
+ */
731
+ registerPairingCode(code, { ttlMs = this.pairingTtlMs } = {}) {
732
+ if (typeof code !== 'string' || code.trim() === '') {
733
+ throw new ProtocolError('BAD_REQUEST', 'pairing code must be a non-empty string', { pairing_code: code });
734
+ }
735
+ const value = code.trim();
736
+ const now = this.nowMs();
737
+ this.pairingCodes.set(value, { code: value, created_at_ms: now, expires_at_ms: now + ttlMs, used: false });
738
+ return value;
739
+ }
740
+
741
+ /** §2.2 step 3: validate a one-time code. */
742
+ validatePairingCode(code) {
743
+ if (typeof code !== 'string' || code.length === 0) {
744
+ throw new ProtocolError('PAIRING_INVALID', 'pairing_code is required');
745
+ }
746
+ const rec = this.pairingCodes.get(code);
747
+ if (!rec) throw new ProtocolError('PAIRING_INVALID', 'unknown pairing code', { pairing_code: code });
748
+ if (rec.used) throw new ProtocolError('PAIRING_INVALID', 'pairing code already used', { pairing_code: code });
749
+ if (this.nowMs() >= rec.expires_at_ms) {
750
+ throw new ProtocolError('PAIRING_EXPIRED', 'pairing code expired', { pairing_code: code, expired_at: rfc3339(rec.expires_at_ms) });
751
+ }
752
+ return rec;
753
+ }
754
+
755
+ /** §2.2: exchange a one-time code for a device_token. §1: idempotent. */
756
+ pair(request = {}) {
757
+ const rec = this.validatePairingCode(request.pairing_code);
758
+ const machineId = request.machine_id;
759
+ if (typeof machineId !== 'string' || machineId.length === 0) {
760
+ throw new ProtocolError('BAD_REQUEST', 'machine_id is required');
761
+ }
762
+ const now = this.nowMs();
763
+ const token = this.newToken();
764
+ const existing = this.devices.get(machineId);
765
+ if (existing?.device_token) this.tokens.delete(existing.device_token);
766
+
767
+ const device = {
768
+ machine_id: machineId,
769
+ machine_name: typeof request.machine_name === 'string' && request.machine_name
770
+ ? request.machine_name : machineId.slice(0, 12),
771
+ platform: request.platform ?? {},
772
+ caps: request.caps ?? {},
773
+ user_id: request.user_id ?? null,
774
+ // v0.7.7: which fleet this device belongs to, and where that fact came from.
775
+ //
776
+ // **The fleet comes from the pairing code, not from the request.** A device that could
777
+ // name its own fleet would join any fleet, and every signature would faithfully attest
778
+ // to it. `deviceFleetFields` is the single place that binding is made, and it records
779
+ // its provenance so a fleet that arrived from a request is visible as such in
780
+ // `devices.json` rather than looking identical to one that came from a code.
781
+ ...deviceFleetFields({ codeRecord: rec, requestedFleet: null }),
782
+ /**
783
+ * v0.7.10: the device's Ed25519 public key, if it offered one.
784
+ *
785
+ * A public key is not a secret, so storing it costs nothing and it is the whole basis of
786
+ * per-device verification: the relay can then check a signature only *this* machine could
787
+ * have made, rather than a shared secret every machine holds.
788
+ *
789
+ * Not validated here, deliberately. A malformed key fails at verification with a named
790
+ * reason, while rejecting a pairing over it would lock out every existing device the
791
+ * moment this field was added -- and devices that send no key keep working, with
792
+ * `public_key: null` meaning "signs with the shared secret or not at all".
793
+ */
794
+ public_key: typeof request.public_key === 'string' && request.public_key !== '' ? request.public_key : null,
795
+ device_token: token,
796
+ paired_at_ms: now,
797
+ re_paired: Boolean(existing),
798
+ last_seen_at_ms: now,
799
+ online: true,
800
+ streams: existing?.streams ?? 0,
801
+ // v0.3.0 RTT: starts unknown. `null` is "never measured"; 0 would mean "instant".
802
+ rtt_ms: null,
803
+ rtt_at_ms: null,
804
+ last_heartbeat_at_ms: null,
805
+ stream_connects: existing?.stream_connects ?? 0,
806
+ };
807
+ this.devices.set(machineId, device);
808
+ this.tokens.set(token, machineId);
809
+ rec.used = true;
810
+
811
+ // A group has more than one machine, and a single-use code would leave the
812
+ // operator unable to pair the second one without restarting the relay. So
813
+ // consuming a code immediately mints the next one and announces it: the
814
+ // operator pairs machine A, sees a fresh code, and pairs machine B with it.
815
+ // `pairingCodeReusable` keeps one code alive for the whole window instead,
816
+ // for scripted onboarding where nobody is watching stdout.
817
+ let nextCode = null;
818
+ const reusable = this.pairingCodeReusable === true;
819
+ if (reusable) {
820
+ rec.used = false;
821
+ } else {
822
+ nextCode = this.rotatePairingCode();
823
+ }
824
+
825
+ this.emit('peer.hello', {
826
+ machine_id: machineId,
827
+ machine_name: device.machine_name,
828
+ platform: device.platform,
829
+ });
830
+
831
+ if (nextCode) {
832
+ this.emit('notice', {
833
+ level: 'info',
834
+ code: 'PAIRING_CODE_ROTATED',
835
+ message: 'pairing code consumed; a new one is available',
836
+ pairing_code: nextCode,
837
+ });
838
+ }
839
+
840
+ // §7: a device change is snapshotted immediately.
841
+ this._saveDevices();
842
+ this._ledger({
843
+ type: 'device.paired',
844
+ machine_id: machineId,
845
+ machine_name: device.machine_name,
846
+ re_paired: device.re_paired,
847
+ });
848
+
849
+ return {
850
+ device_token: token,
851
+ protocol_version: PROTOCOL_VERSION,
852
+ rabbit_time: rfc3339(now),
853
+ ...(nextCode ? { next_pairing_code: nextCode } : {}),
854
+ };
855
+ }
856
+
857
+ /** @returns {object|null} device for a bearer token. */
858
+ authenticate(token) {
859
+ if (typeof token !== 'string' || token.length === 0) return null;
860
+ const machineId = this.tokens.get(token);
861
+ if (!machineId) return null;
862
+ return this.devices.get(machineId) ?? null;
863
+ }
864
+
865
+ touchDevice(machineId, { online = true } = {}) {
866
+ const device = this.devices.get(machineId);
867
+ if (!device) return null;
868
+ device.last_seen_at_ms = this.nowMs();
869
+ if (online) device.online = true;
870
+ return device;
871
+ }
872
+
873
+ setOnline(machineId, online, reason = null) {
874
+ const device = this.devices.get(machineId);
875
+ if (!device) return null;
876
+ if (device.online === online) return device;
877
+ device.online = online;
878
+ if (online) {
879
+ this.touchDevice(machineId);
880
+ this.emit('peer.hello', {
881
+ machine_id: machineId,
882
+ machine_name: device.machine_name,
883
+ platform: device.platform,
884
+ });
885
+ } else {
886
+ this.emit('peer.bye', { machine_id: machineId, reason: reason ?? 'stream_closed' });
887
+ }
888
+ return device;
889
+ }
890
+
891
+ /* ---------------- RTT (v0.3.0 §R) ---------------- */
892
+
893
+ /**
894
+ * Validate an inbound `rtt_ms`.
895
+ *
896
+ * The contract is deliberately liberal: a MISSING or INVALID value must never
897
+ * reject the heartbeat and must never clobber a previously known good value.
898
+ * RTT is a diagnostic; it must not be able to break lease renewal.
899
+ *
900
+ * Sanity ceiling: anything above `MAX_RTT_MS` (24h) is treated as invalid. A
901
+ * single garbage value would otherwise poison the fleet aggregate.
902
+ */
903
+ static normalizeRtt(value) {
904
+ if (value === undefined) return { provided: false, valid: false };
905
+ if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
906
+ return { provided: true, valid: false };
907
+ }
908
+ if (value > MAX_RTT_MS) return { provided: true, valid: false };
909
+ return { provided: true, valid: true, value };
910
+ }
911
+
912
+ /**
913
+ * RTT is a VOLATILE diagnostic: it is intentionally NOT written to
914
+ * `devices.json` and NOT replayed from the ledger, so a restarted relay reports
915
+ * `rtt_ms: null` until a machine heartbeats again. A persisted value would be
916
+ * indistinguishable from a fresh one and would let the relay claim knowledge it
917
+ * lost when the process died.
918
+ */
919
+ rttStaleAfterMs() { return this.rttStaleMs; }
920
+
921
+ /**
922
+ * v0.3.0: absorb the optional `rtt_ms` carried by a heartbeat.
923
+ *
924
+ * Deliberately liberal, because RTT is a diagnostic and must never be able to
925
+ * break lease renewal:
926
+ * - the field is optional; a v0.2.3 agent never sends it
927
+ * - a MISSING or INVALID value is IGNORED, keeping the previous value, and the
928
+ * heartbeat still succeeds
929
+ * - unknown extra fields are ignored entirely (forward compatibility)
930
+ *
931
+ * `last_heartbeat_at` advances on EVERY accepted heartbeat, with or without a
932
+ * usable RTT: it answers "is the machine alive", while `rtt_at` answers "when
933
+ * was this number measured". Keeping them separate is what lets a reader tell a
934
+ * genuinely slow machine from one that is simply gone.
935
+ */
936
+ recordHeartbeatMetrics(machineId, input = {}, nowMs = this.nowMs()) {
937
+ const device = this.devices.get(machineId);
938
+ if (!device) return null;
939
+ device.last_heartbeat_at_ms = nowMs;
940
+ const rtt = RabbitState.normalizeRtt(input.rtt_ms);
941
+ if (rtt.valid) {
942
+ device.rtt_ms = rtt.value;
943
+ device.rtt_at_ms = nowMs;
944
+ }
945
+ return device;
946
+ }
947
+
948
+ /** True when there is no value, or the value is older than the staleness window. */
949
+ isRttStale(device, nowMs = this.nowMs()) {
950
+ if (!device || device.rtt_ms === null || device.rtt_ms === undefined) return true;
951
+ if (device.rtt_at_ms === null || device.rtt_at_ms === undefined) return true;
952
+ return nowMs - device.rtt_at_ms > this.rttStaleMs;
953
+ }
954
+
955
+ /** How old the RTT measurement is, in ms; null when there is none. */
956
+ rttAgeMs(device, nowMs = this.nowMs()) {
957
+ if (!device || device.rtt_at_ms === null || device.rtt_at_ms === undefined) return null;
958
+ return Math.max(0, nowMs - device.rtt_at_ms);
959
+ }
960
+
961
+ /** Record that a machine attached a stream (a reconnect is any attach after the first). */
962
+ noteStreamConnect(machineId) {
963
+ const device = this.devices.get(machineId);
964
+ if (!device) return null;
965
+ device.stream_connects = (device.stream_connects ?? 0) + 1;
966
+ return device.stream_connects;
967
+ }
968
+
969
+ /**
970
+ * v0.3.0 aggregate for `/healthz`.
971
+ *
972
+ * Only FRESH measurements count. A machine that has been gone for a day must not
973
+ * drag min/max/avg around, and with no fresh data every numeric field is null --
974
+ * never 0, which would read as "instantaneous".
975
+ */
976
+ rttSummary(nowMs = this.nowMs()) {
977
+ const fresh = [];
978
+ let stale = 0;
979
+ let unknown = 0;
980
+ for (const device of this.devices.values()) {
981
+ if (device.rtt_ms === null || device.rtt_ms === undefined) { unknown += 1; continue; }
982
+ if (this.isRttStale(device, nowMs)) { stale += 1; continue; }
983
+ fresh.push(device.rtt_ms);
984
+ }
985
+ if (fresh.length === 0) {
986
+ return {
987
+ machines_reporting: 0,
988
+ min_ms: null,
989
+ max_ms: null,
990
+ avg_ms: null,
991
+ machines_stale: stale,
992
+ machines_unknown: unknown,
993
+ };
994
+ }
995
+ const sum = fresh.reduce((a, b) => a + b, 0);
996
+ return {
997
+ machines_reporting: fresh.length,
998
+ min_ms: Math.min(...fresh),
999
+ max_ms: Math.max(...fresh),
1000
+ avg_ms: Math.round((sum / fresh.length) * 10) / 10,
1001
+ machines_stale: stale,
1002
+ machines_unknown: unknown,
1003
+ };
1004
+ }
1005
+
1006
+ /**
1007
+ * v0.3.0 single-machine status view.
1008
+ *
1009
+ * `last_heartbeat_at` and `rtt_at` answer different questions, and the gap
1010
+ * between them is the whole point: a machine can be heartbeating (alive) while
1011
+ * its RTT number is old (a v0.2.3 agent that never sends `rtt_ms`), and a
1012
+ * machine can be gone while a plausible-looking RTT is still on file.
1013
+ */
1014
+ deviceStatus(machineId, nowMs = this.nowMs()) {
1015
+ const device = this.devices.get(machineId);
1016
+ if (!device) return null;
1017
+ const rttAt = device.rtt_at_ms ?? null;
1018
+ const lastHeartbeat = device.last_heartbeat_at_ms ?? null;
1019
+ return {
1020
+ machine_id: device.machine_id,
1021
+ machine_name: device.machine_name,
1022
+ relay_id: this.relayId,
1023
+ connected: (device.streams ?? 0) > 0,
1024
+ streams: device.streams ?? 0,
1025
+ online: device.online === true,
1026
+ rtt_ms: device.rtt_ms ?? null,
1027
+ rtt_at: rttAt === null ? null : rfc3339(rttAt),
1028
+ rtt_age_ms: this.rttAgeMs(device, nowMs),
1029
+ rtt_stale: this.isRttStale(device, nowMs),
1030
+ last_heartbeat_at: lastHeartbeat === null ? null : rfc3339(lastHeartbeat),
1031
+ last_seen_at: device.last_seen_at_ms === null || device.last_seen_at_ms === undefined
1032
+ ? null : rfc3339(device.last_seen_at_ms),
1033
+ reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
1034
+ stream_connects: device.stream_connects ?? 0,
1035
+ paired_at: device.paired_at_ms === null || device.paired_at_ms === undefined
1036
+ ? null : rfc3339(device.paired_at_ms),
1037
+ platform: device.platform,
1038
+ caps: device.caps,
1039
+ user_id: device.user_id ?? null,
1040
+ now: rfc3339(nowMs),
1041
+ };
1042
+ }
1043
+
1044
+ /** Public device view — never leaks device_token. */
1045
+ publicDevice(device) {
1046
+ return {
1047
+ machine_id: device.machine_id,
1048
+ machine_name: device.machine_name,
1049
+ platform: device.platform,
1050
+ caps: device.caps,
1051
+ user_id: device.user_id,
1052
+ /**
1053
+ * v0.7.7: the fleet this device belongs to, and where the fact came from.
1054
+ *
1055
+ * **Carried in the public shape on purpose, and it has to be**, because `devices.json`
1056
+ * is written from this shape -- a fleet that lived only on the in-memory record would
1057
+ * vanish on the next restart, which is precisely when an operator is looking at it. A
1058
+ * first version omitted these and the live test caught the fleet disappearing from
1059
+ * `listDevices()`.
1060
+ *
1061
+ * Safe to expose: `fleet_id` is a truncated hash of the account id, not the account id,
1062
+ * and devices on one relay can already enumerate each other. Narrowing `/v1/devices` to
1063
+ * a device's own fleet is a further step, recorded as such rather than implied.
1064
+ */
1065
+ fleet_id: device.fleet_id ?? null,
1066
+ fleet_source: device.fleet_source ?? null,
1067
+ // v0.7.10: a boolean rather than the key itself. The key is not a secret, but a device
1068
+ // list is read by every device, and there is no reason to hand every machine every other
1069
+ // machine's key when the verification happens relay-side.
1070
+ public_key_present: typeof device.public_key === 'string' && device.public_key !== '',
1071
+ online: device.online,
1072
+ streams: device.streams,
1073
+ paired_at: rfc3339(device.paired_at_ms),
1074
+ last_seen_at: rfc3339(device.last_seen_at_ms),
1075
+ // v0.3.0: `null` means "never measured", which is NOT the same as 0ms.
1076
+ rtt_ms: device.rtt_ms ?? null,
1077
+ rtt_at: device.rtt_at_ms === null || device.rtt_at_ms === undefined ? null : rfc3339(device.rtt_at_ms),
1078
+ rtt_age_ms: this.rttAgeMs(device),
1079
+ rtt_stale: this.isRttStale(device),
1080
+ last_heartbeat_at: device.last_heartbeat_at_ms === null || device.last_heartbeat_at_ms === undefined
1081
+ ? null : rfc3339(device.last_heartbeat_at_ms),
1082
+ reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
1083
+ };
1084
+ }
1085
+
1086
+ listDevices() { return [...this.devices.values()].map((d) => this.publicDevice(d)); }
1087
+
1088
+ /* ---------------- tasks (§4) ---------------- */
1089
+
1090
+ /**
1091
+ * v0.4.0: validate the routing metadata a `POST /v1/task` body may carry.
1092
+ *
1093
+ * A machine, not a mode, is `origin_machine_id`: where the task was typed (`device.json`), so an
1094
+ * offer can carry that fact to the machine that has to run it. `p2p.mode` is the dispatcher's
1095
+ * resolved mode for this particular task — the plugin may be configured `direct` while a
1096
+ * scheduled run is `relay`.
1097
+ *
1098
+ * Validation is deliberately shallow and strictly typed, in the house style of this file: a
1099
+ * missing value is the v0.3.9 shape and stays valid, while a present-but-wrong value is refused
1100
+ * by name. Accepting a non-string here would put a number into `origin_machine_id` and only fail
1101
+ * later, on a machine that cannot tell a bad field from a bad relay.
1102
+ *
1103
+ * @returns {{origin_machine_id:string|null, p2p:{mode:string}|null}}
1104
+ */
1105
+ static validateTaskRouting(input = {}) {
1106
+ let originMachineId = null;
1107
+ if (input.origin_machine_id !== undefined && input.origin_machine_id !== null) {
1108
+ if (typeof input.origin_machine_id !== 'string'
1109
+ || input.origin_machine_id.length === 0
1110
+ || input.origin_machine_id.length > MAX_ORIGIN_MACHINE_ID_LENGTH) {
1111
+ throw new ProtocolError(
1112
+ 'BAD_REQUEST',
1113
+ `origin_machine_id must be a non-empty string of at most ${MAX_ORIGIN_MACHINE_ID_LENGTH} characters`,
1114
+ { origin_machine_id: input.origin_machine_id },
1115
+ );
1116
+ }
1117
+ originMachineId = input.origin_machine_id;
1118
+ }
1119
+
1120
+ let p2p = null;
1121
+ if (input.p2p !== undefined && input.p2p !== null) {
1122
+ if (typeof input.p2p !== 'object' || Array.isArray(input.p2p)) {
1123
+ throw new ProtocolError('BAD_REQUEST', 'p2p must be an object with a `mode` field', { p2p: input.p2p });
1124
+ }
1125
+ if (!P2P_MODES.includes(input.p2p.mode)) {
1126
+ throw new ProtocolError(
1127
+ 'BAD_REQUEST',
1128
+ `p2p.mode must be one of ${P2P_MODES.join('|')}`,
1129
+ { 'p2p.mode': input.p2p.mode ?? null },
1130
+ );
1131
+ }
1132
+ // Copied rather than referenced: the recorded task must not be mutable through the request
1133
+ // body the caller still holds.
1134
+ p2p = { ...input.p2p };
1135
+ }
1136
+
1137
+ return { origin_machine_id: originMachineId, p2p };
1138
+ }
1139
+
1140
+ /**
1141
+ * v0.4.0 §5.2: validate the optional transport fields of a result envelope.
1142
+ *
1143
+ * Same reasoning as `validateTaskRouting`: `transport` is an enum and `p2p` is an opaque object,
1144
+ * so anything else is refused by name at the door. Both fields stay OUT of
1145
+ * `REQUIRED_ENVELOPE_FIELDS` (a missing one is a v0.3.9 envelope, not an incomplete one) and out
1146
+ * of `COMPARABLE_FIELDS` (a path is not a result).
1147
+ */
1148
+ static validateTransportFields(envelope = {}) {
1149
+ if (envelope.transport !== undefined && envelope.transport !== null) {
1150
+ if (!TRANSPORT_VALUES.includes(envelope.transport)) {
1151
+ throw new ProtocolError(
1152
+ 'BAD_REQUEST',
1153
+ `transport must be one of ${TRANSPORT_VALUES.join('|')}`,
1154
+ { transport: envelope.transport },
1155
+ );
1156
+ }
1157
+ }
1158
+ if (envelope.p2p !== undefined && envelope.p2p !== null
1159
+ && (typeof envelope.p2p !== 'object' || Array.isArray(envelope.p2p))) {
1160
+ throw new ProtocolError('BAD_REQUEST', 'p2p must be an object', { p2p: envelope.p2p });
1161
+ }
1162
+ }
1163
+
1164
+ /**
1165
+ * §4.1 POST /v1/task.
1166
+ * Machine selection is not specified by the PROTOCOL; we implement:
1167
+ * - `target_machines` (optional explicit list), else every paired device
1168
+ * - capability gate §6.1 per machine (failures become `refused` leases)
1169
+ * - index assignment: replicate -> 0, split -> round-robin modulo index_total
1170
+ */
1171
+ createTask(input = {}) {
1172
+ // v0.4.0: refused BEFORE the task literal is built and before any offer is emitted, so a bad
1173
+ // value cannot leave a half-created task behind. `POST /v1/task` is all-or-nothing.
1174
+ const routing = RabbitState.validateTaskRouting(input);
1175
+
1176
+ const mode = input.mode ?? 'replicate';
1177
+ if (mode !== 'replicate' && mode !== 'split' && mode !== 'broadcast' && mode !== 'pipeline' && mode !== 'compose') {
1178
+ throw new ProtocolError(
1179
+ 'BAD_REQUEST',
1180
+ 'mode must be replicate|split|broadcast|pipeline|compose',
1181
+ { mode },
1182
+ );
1183
+ }
1184
+ /**
1185
+ * v0.3.3 `pipeline`: the whole chain runs on each machine, stage after stage.
1186
+ *
1187
+ * The semantics were chosen against the alternative (fan out stage 1, converge its outputs into
1188
+ * stage 2) because that one has no well-defined input: on a fleet, stage 1 produces n different
1189
+ * outputs, and "feed them all to stage 2" does not say in what order, or whether stage 2 then
1190
+ * runs once or n times. Worse, two machines finishing in a different order would produce
1191
+ * different stage-2 inputs, so one task_id would no longer describe one reproducible execution --
1192
+ * which is the property the rest of this system is built on. See docs/PIPELINE-DESIGN.md.
1193
+ *
1194
+ * So: a pipeline is `replicate` applied to a sequence. Every machine runs the whole chain, there
1195
+ * is no cross-machine data flow, and nothing new can be silently dropped in transit.
1196
+ */
1197
+ const rawStages = Array.isArray(input.stages) ? input.stages : null;
1198
+ const isPipeline = mode === 'pipeline';
1199
+ if (isPipeline) {
1200
+ if (rawStages === null || rawStages.length === 0) {
1201
+ throw new ProtocolError('BAD_REQUEST', 'mode=pipeline requires a non-empty `stages` array', {
1202
+ stages: rawStages === null ? null : rawStages.length,
1203
+ });
1204
+ }
1205
+ if (rawStages.length > MAX_PIPELINE_STAGES) {
1206
+ throw new ProtocolError('BAD_REQUEST', `stages must have at most ${MAX_PIPELINE_STAGES} entries`, {
1207
+ stages: rawStages.length,
1208
+ });
1209
+ }
1210
+ rawStages.forEach((stage, i) => {
1211
+ const argv = stage?.command_argv;
1212
+ if (!Array.isArray(argv) || argv.length === 0 || argv.some((a) => typeof a !== 'string')) {
1213
+ throw new ProtocolError('BAD_REQUEST', `stages[${i}].command_argv must be a non-empty string[]`, { index: i });
1214
+ }
1215
+ });
1216
+ // A chain is one execution on one machine; sharding it would mean each machine ran a chain of
1217
+ // its own slice, which is a different feature and not what the caller asked for.
1218
+ if (Number.isInteger(input.index_total) && input.index_total !== 1) {
1219
+ throw new ProtocolError('BAD_REQUEST', 'mode=pipeline runs the whole chain per machine, so index_total must be 1', {
1220
+ index_total: input.index_total,
1221
+ });
1222
+ }
1223
+ } else if (rawStages !== null) {
1224
+ // Refused rather than ignored: the caller believes they described a sequence, and running only
1225
+ // `command_argv` would silently drop every stage after the first.
1226
+ throw new ProtocolError('BAD_REQUEST', '`stages` is only valid with mode=pipeline', { mode, stages: rawStages.length });
1227
+ }
1228
+
1229
+ /**
1230
+ * Normalized stage list. For every mode there is at least one stage, so downstream code has a
1231
+ * single shape to reason about and `pipeline` is not a special case at the execution layer.
1232
+ */
1233
+ const stages = isPipeline
1234
+ ? rawStages.map((s, i) => ({
1235
+ index: i,
1236
+ command_argv: [...s.command_argv],
1237
+ cwd_rel: typeof s.cwd_rel === 'string' && s.cwd_rel ? s.cwd_rel : (typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.'),
1238
+ timeout_ms: Number.isInteger(s.timeout_ms) ? s.timeout_ms : null,
1239
+ continue_on_failure: s.continue_on_failure === true,
1240
+ }))
1241
+ : null;
1242
+
1243
+ const commandArgv = isPipeline ? [...rawStages[0].command_argv] : input.command_argv;
1244
+ if (!Array.isArray(commandArgv) || commandArgv.length === 0 || commandArgv.some((a) => typeof a !== 'string')) {
1245
+ throw new ProtocolError('BAD_REQUEST', 'command_argv must be a non-empty string[]');
1246
+ }
1247
+ const indexTotal = Number.isInteger(input.index_total) ? input.index_total : 1;
1248
+ if (indexTotal < 1) throw new ProtocolError('BAD_REQUEST', 'index_total must be >= 1');
1249
+
1250
+ const halt = input.halt ?? 'never';
1251
+ if (halt !== 'never' && halt !== 'now') throw new ProtocolError('BAD_REQUEST', 'halt must be never|now', { halt });
1252
+
1253
+ const taskId = input.task_id ?? this.nextId();
1254
+ if (this.tasks.has(taskId)) {
1255
+ throw new ProtocolError('TASK_EXISTS', 'task_id already exists', { task_id: taskId });
1256
+ }
1257
+
1258
+ const now = this.nowMs();
1259
+ const cwdRel = typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.';
1260
+ const baseTree = input.base_tree ?? null;
1261
+ const commandHash = input.command_hash ?? (isPipeline
1262
+ ? computePipelineCommandHash(stages, SHELL_ID_DIRECT)
1263
+ : computeCommandHash(commandArgv, SHELL_ID_DIRECT, cwdRel));
1264
+ const timeoutMs = Number.isInteger(input.timeout_ms) ? input.timeout_ms : 300_000;
1265
+
1266
+ // v0.3.3 `broadcast`: one machine runs the command, every other machine is told the outcome.
1267
+ //
1268
+ // The executor choice is deterministic rather than random. A random leader makes a task
1269
+ // impossible to reproduce from its own record -- the same task id would run somewhere else on a
1270
+ // retry -- and this project's whole value rests on being able to say exactly where something ran.
1271
+ // The first target in the relay's own device order is stable for a given device table.
1272
+ //
1273
+ // Observers get a lease so they are tracked and notified, but their lease is `observing`, which
1274
+ // every aggregation step treats as "did not run". That distinction is the point: a machine that
1275
+ // never executed anything agreeing with a result would be a fabrication, and it is exactly the
1276
+ // fabrication this mode could most easily produce.
1277
+ //
1278
+ // Resolved before the task literal because the literal records it. (`targets` is computed below,
1279
+ // so the executor is re-validated against it after that.)
1280
+ const targetIds = Array.isArray(input.target_machines) && input.target_machines.length > 0
1281
+ ? input.target_machines
1282
+ : [...this.devices.keys()];
1283
+ const executorMachineId = mode === 'broadcast'
1284
+ ? (typeof input.executor_machine_id === 'string' && input.executor_machine_id !== ''
1285
+ ? input.executor_machine_id
1286
+ : targetIds[0] ?? null)
1287
+ : null;
1288
+
1289
+ const task = {
1290
+ task_id: taskId,
1291
+ mode,
1292
+ command_argv: commandArgv,
1293
+ cwd_rel: cwdRel,
1294
+ index_total: indexTotal,
1295
+ timeout_ms: timeoutMs,
1296
+ write: input.write === true,
1297
+ write_scope: input.write_scope ?? [],
1298
+ require_exclusive_write: input.require_exclusive_write === true,
1299
+ base_commit: input.base_commit ?? null,
1300
+ base_tree: baseTree,
1301
+ requirements: input.requirements ?? {},
1302
+ compare_policy: input.compare_policy ?? {},
1303
+ halt,
1304
+ created_by: input.created_by ?? null,
1305
+ created_at_ms: now,
1306
+ command_hash: commandHash,
1307
+ attempt: 1,
1308
+ leases: new Map(),
1309
+ cancelled: false,
1310
+ cancel_reason: null,
1311
+ degraded: null,
1312
+ deadline_ms: now + timeoutMs,
1313
+ result_seqs: [],
1314
+ /** v0.3.3: which machine actually executes under `broadcast`; null for the other modes. */
1315
+ executor_machine_id: executorMachineId,
1316
+ /**
1317
+ * v0.3.3: the normalized stage list for `pipeline`, or null.
1318
+ *
1319
+ * Sent to the machine in the offer so it can run the whole chain. Kept on the task as well so
1320
+ * a report can say how many stages were supposed to run even when the machine crashed before
1321
+ * reporting any of them.
1322
+ */
1323
+ stages,
1324
+ /**
1325
+ * v0.4.0: where the task came from and under which P2P mode it was dispatched, or null.
1326
+ *
1327
+ * Kept on the task (not just passed to `emitOffer`) because an offer is emitted more than
1328
+ * once: `redeliverPendingOffers` re-states unclaimed work on every reconnect and
1329
+ * `_takeover` re-offers a freed index. A machine that reconnects must see the same routing
1330
+ * facts as the machine that was connected at dispatch time, or the direct-push half of the
1331
+ * feature would work exactly once and then quietly stop.
1332
+ */
1333
+ origin_machine_id: routing.origin_machine_id,
1334
+ p2p: routing.p2p,
1335
+ };
1336
+
1337
+ /**
1338
+ * Fleet scoping (v0.7.7).
1339
+ *
1340
+ * This replaces `targets = [...this.devices.values()]`, which reached **every device on the
1341
+ * relay** with no filter — the finding that accounts were not isolated. The rule now lives in
1342
+ * `fleet-wiring.mjs` so it is one function with its own exhaustive tests rather than three
1343
+ * lines inside a two-thousand-line object.
1344
+ *
1345
+ * ## The migration is deliberate, and so is its limit
1346
+ *
1347
+ * The shared relay holds four devices that paired before fleets existed, and none of them has
1348
+ * a `fleet_id`. `targetsFor` therefore has two modes: a relay with no fleet configured keeps
1349
+ * exactly the old behaviour, and a relay that *is* scoped refuses a fleetless requester rather
1350
+ * than handing it the run of the fleet. So deploying this cannot be an outage, and scoping it
1351
+ * cannot be a silent grant.
1352
+ *
1353
+ * ## What this does NOT do, stated here rather than discovered later
1354
+ *
1355
+ * `POST /v1/task` authenticates with the **operator token**, and `authorizeTaskDispatch`
1356
+ * returns `{ role: 'operator', machine_id: null }` for it — there is no device, so there is no
1357
+ * fleet to scope by. Every device that authenticates with a **device token** gets full
1358
+ * isolation from this change; an operator-token dispatch remains relay-wide, because the
1359
+ * operator token *is* the relay-wide credential by design.
1360
+ *
1361
+ * `input.created_by` is the one device identity a task carries, and it is used when present:
1362
+ * a task submitted on behalf of a machine is scoped to that machine's fleet.
1363
+ */
1364
+ const requester =
1365
+ (typeof input.created_by === 'string' && this.devices.get(input.created_by)) ||
1366
+ (typeof input.requester_machine_id === 'string' && this.devices.get(input.requester_machine_id)) ||
1367
+ null;
1368
+
1369
+ /**
1370
+ * An operator-token dispatch, on a relay that has been scoped, acts **within that scope**.
1371
+ *
1372
+ * A first version passed `requester: null` and let it be refused -- which broke every
1373
+ * operator dispatch on a scoped relay, and broke it in the least useful way: the operator had
1374
+ * just configured the fleet, and configuring it made them unable to dispatch at all.
1375
+ *
1376
+ * The reasoning for the fix: the relay's own `fleetId` *is* an admin's statement about which
1377
+ * fleet this relay serves. An operator dispatch therefore has an unambiguous scope, and
1378
+ * refusing it would be refusing the case that was explicitly asked for.
1379
+ *
1380
+ * What stays genuinely ambiguous is an operator dispatch on a relay holding **several** fleets
1381
+ * with no fleet of its own -- there is no way to tell which was meant, so `targetsFor` refuses
1382
+ * that rather than guessing.
1383
+ */
1384
+ const scopedRequester =
1385
+ requester ??
1386
+ (this.fleetId === null ? null : { machine_id: null, fleet_id: this.fleetId, fleet_source: 'relay_scope' });
1387
+
1388
+ const scoped = targetsFor({
1389
+ devices: [...this.devices.values()],
1390
+ requester: scopedRequester,
1391
+ targetMachines: input.target_machines,
1392
+ relayFleetId: this.fleetId ?? null,
1393
+ // The existing code did not filter offline devices at selection time -- it offered to them and
1394
+ // let the lease expire. Keeping that here avoids changing two behaviours at once, and the
1395
+ // lease expiry is what `handoff.mjs` is for.
1396
+ requireOnline: false,
1397
+ });
1398
+ if (!scoped.ok) {
1399
+ /**
1400
+ * `NO_ONLINE_DEVICE`, not a new code.
1401
+ *
1402
+ * A first version invented `NO_ELIGIBLE_DEVICE`, which broke the one relay test that pins
1403
+ * this contract — and worse, it would have broken **callers**: `chatops/fleet.mjs` reads
1404
+ * `NO_ONLINE_DEVICE` to produce *"start an agent on each machine"*, and a code it has never
1405
+ * seen falls through to a generic refusal. The protocol's code table is part of the
1406
+ * protocol, and adding a member to it is a decision for a protocol revision, not for a
1407
+ * wiring change.
1408
+ *
1409
+ * What *is* new is why nothing was eligible, and that belongs in `detail`, where a caller
1410
+ * can read it without a new code entering the table.
1411
+ */
1412
+ throw new ProtocolError('NO_ONLINE_DEVICE', scoped.text, {
1413
+ reason: scoped.reason,
1414
+ legacy: scoped.legacy,
1415
+ fleet_scoped: scoped.legacy === false,
1416
+ hint:
1417
+ scoped.reason === 'requester_has_no_fleet'
1418
+ ? 'this relay is bound to a fleet and the requesting device is in none; pair it with a code issued for that fleet'
1419
+ : scoped.reason === 'target_outside_fleet'
1420
+ ? 'the named target is not in this fleet'
1421
+ : 'check that at least one device in this fleet is paired and streaming',
1422
+ });
1423
+ }
1424
+ const targets = scoped.targets;
1425
+ if (scoped.legacy) {
1426
+ // Logged at debug so an operator reading logs during a migration can see which rule applied.
1427
+ // "N targets" alone does not say whether isolation is on, and that is the question being asked.
1428
+ this.log?.('debug', 'task targets selected without a fleet (legacy path)', {
1429
+ task_candidates: targets.length,
1430
+ relay_fleet_id: this.fleetId ?? null,
1431
+ });
1432
+ }
1433
+
1434
+ // v0.3.3 `broadcast`: validated here, where `targets` is known. The executor id itself was
1435
+ // resolved before the task literal because the literal records it.
1436
+ if (mode === 'broadcast' && executorMachineId !== null && !targets.some((d) => d.machine_id === executorMachineId)) {
1437
+ throw new ProtocolError('BAD_REQUEST', 'executor_machine_id is not among the target machines', {
1438
+ executor_machine_id: executorMachineId,
1439
+ targets: targets.map((d) => d.machine_id),
1440
+ });
1441
+ }
1442
+
1443
+ if (targets.length === 0) {
1444
+ throw new ProtocolError('NO_ONLINE_DEVICE', 'no paired device available for this task', {
1445
+ known_devices: all.length,
1446
+ });
1447
+ }
1448
+
1449
+ const dedupeKeyBase = computeDedupeKey(taskId, 0, commandHash, baseTree);
1450
+ if (typeof dedupeKeyBase !== 'string' || dedupeKeyBase.length !== 64) {
1451
+ throw new ProtocolError('INTERNAL', 'failed to compute dedupe_key');
1452
+ }
1453
+ let lastSeq = this.seq;
1454
+ targets.forEach((device, i) => {
1455
+ const index = mode === 'replicate' ? 0 : i % indexTotal;
1456
+ const observing = mode === 'broadcast' && device.machine_id !== executorMachineId;
1457
+ const lease = {
1458
+ machine_id: device.machine_id,
1459
+ machine_name: device.machine_name,
1460
+ index,
1461
+ attempt: 1,
1462
+ // `observing` is a lease state, not an outcome: the machine holds a place in this task and
1463
+ // will be told what happened, but it was never handed the command. `gateMachine` below is
1464
+ // skipped for it because refusing to do work you were never asked to do is not a refusal.
1465
+ state: observing ? 'observing' : 'queued',
1466
+ refusal_reason: null,
1467
+ refusal_detail: null,
1468
+ dedupe_key: computeDedupeKey(taskId, index, commandHash, baseTree),
1469
+ last_heartbeat_ms: now,
1470
+ claimed_at_ms: now,
1471
+ lease_until_ms: now + this.leaseWindowMs(),
1472
+ phase: null,
1473
+ progress: null,
1474
+ offered_seq: null,
1475
+ expired_at_ms: null,
1476
+ finished_at_ms: null,
1477
+ heartbeat_count: 0,
1478
+ };
1479
+ if (observing) {
1480
+ // No capability gate and no offer: the machine is not being asked to run anything, so there
1481
+ // is nothing to refuse and nothing to hand out. It is recorded so the task can tell it the
1482
+ // outcome afterwards.
1483
+ task.leases.set(device.machine_id, lease);
1484
+ return;
1485
+ }
1486
+ const gate = gateMachine(task, device);
1487
+ if (gate) {
1488
+ lease.state = 'refused';
1489
+ lease.refusal_reason = gate.refusal_reason;
1490
+ lease.refusal_detail = gate.detail;
1491
+ task.leases.set(device.machine_id, lease);
1492
+ this.emit('notice', {
1493
+ level: 'warn',
1494
+ code: 'CAPABILITY_REFUSED',
1495
+ message: `${device.machine_id} refused: ${gate.refusal_reason}`,
1496
+ task_id: taskId,
1497
+ });
1498
+ return;
1499
+ }
1500
+ task.leases.set(device.machine_id, lease);
1501
+ const grace = this._p2pOfferGraceMs(task);
1502
+ if (grace > 0) {
1503
+ // v0.4.0: the dispatcher is pushing this offer over the direct path right now. Hold the
1504
+ // relay's copy for `grace` ms so the direct one can win; see `p2pOfferGraceMs`.
1505
+ this._scheduleOffer(task, lease, grace);
1506
+ } else {
1507
+ const event = this.emitOffer(task, lease);
1508
+ lastSeq = event.seq;
1509
+ }
1510
+ });
1511
+
1512
+ this.tasks.set(taskId, task);
1513
+
1514
+ this._ledger({
1515
+ type: 'task.created',
1516
+ task: {
1517
+ task_id: task.task_id,
1518
+ mode: task.mode,
1519
+ command_argv: task.command_argv,
1520
+ cwd_rel: task.cwd_rel,
1521
+ index_total: task.index_total,
1522
+ timeout_ms: task.timeout_ms,
1523
+ write: task.write,
1524
+ write_scope: task.write_scope,
1525
+ require_exclusive_write: task.require_exclusive_write,
1526
+ base_commit: task.base_commit,
1527
+ base_tree: task.base_tree,
1528
+ requirements: task.requirements,
1529
+ compare_policy: task.compare_policy,
1530
+ halt: task.halt,
1531
+ created_by: task.created_by,
1532
+ created_at_ms: task.created_at_ms,
1533
+ command_hash: task.command_hash,
1534
+ attempt: task.attempt,
1535
+ deadline_ms: task.deadline_ms,
1536
+ // v0.4.0: routing metadata, so a restarted relay re-offers with the same facts.
1537
+ origin_machine_id: task.origin_machine_id ?? null,
1538
+ p2p: task.p2p ?? null,
1539
+ },
1540
+ leases: [...task.leases.values()].map((l) => ({ ...l })),
1541
+ });
1542
+
1543
+ const refusedOnly = [...task.leases.values()].every((l) => l.state === 'refused');
1544
+ if (refusedOnly) {
1545
+ // §6.3 Step 0 territory: a task nobody may execute is still a valid, reportable task.
1546
+ this.emit('notice', {
1547
+ level: 'warn',
1548
+ code: 'NO_ONLINE_DEVICE',
1549
+ message: 'every candidate machine refused the capability gate',
1550
+ task_id: taskId,
1551
+ });
1552
+ }
1553
+
1554
+ return {
1555
+ task_id: taskId,
1556
+ leases: [...task.leases.values()].map((l) => ({
1557
+ machine_id: l.machine_id,
1558
+ index: l.index,
1559
+ state: l.state,
1560
+ // v0.4.0: the dispatcher pushes an offer over the direct path, and the executor dedupes it
1561
+ // against the relay's copy by (task_id, attempt, dedupe_key). Without this field the
1562
+ // dispatcher has to reproduce the §4.4 formula locally, which works for a single-command
1563
+ // task and cannot work for a pipeline one -- the key covers the relay's normalised stage
1564
+ // list. Publishing it makes the pushed copy recognisably the same work item, which is what
1565
+ // keeps the direct path from turning exactly-once into at-least-twice.
1566
+ dedupe_key: l.dedupe_key ?? null,
1567
+ attempt: l.attempt,
1568
+ ...(l.refusal_reason ? { refusal_reason: l.refusal_reason } : {}),
1569
+ })),
1570
+ seq: lastSeq,
1571
+ };
1572
+ }
1573
+
1574
+ /**
1575
+ * v0.4.0: should the relay hold its offer back for this task?
1576
+ *
1577
+ * Only when the dispatcher said two things: that it is a machine (`origin_machine_id`), and that
1578
+ * the direct path is in play (`p2p.mode` of `auto` or `direct`). A `relay` mode or a v0.3.9
1579
+ * dispatcher gets the old behaviour exactly, which is why this predicate is the only gate on the
1580
+ * change and every pre-existing test still sees an immediate offer.
1581
+ *
1582
+ * @param {object} task
1583
+ * @returns {number} milliseconds to hold, or 0 for "offer now"
1584
+ */
1585
+ _p2pOfferGraceMs(task) {
1586
+ if (this.p2pOfferGraceMs <= 0) return 0;
1587
+ const origin = task?.origin_machine_id;
1588
+ if (typeof origin !== 'string' || origin === '') return 0;
1589
+ const mode = task?.p2p?.mode;
1590
+ if (mode !== 'auto' && mode !== 'direct') return 0;
1591
+ return this.p2pOfferGraceMs;
1592
+ }
1593
+
1594
+ /**
1595
+ * v0.4.0: emit `lease`'s offer after `delayMs`, unless the direct path got there first.
1596
+ *
1597
+ * "Got there first" is one fact and it is already tracked: the executor renews the lease
1598
+ * (`POST /v1/heartbeat`) as its first act on any offer it accepts, and a renewal moves the lease
1599
+ * out of `queued`. So the check is the lease's own state — no new message, no new endpoint, and
1600
+ * nothing the executor has to know about.
1601
+ *
1602
+ * The timer is `unref()`ed: a relay with nothing else to do must still be able to exit, and a
1603
+ * pending offer is not a reason to keep a process alive.
1604
+ */
1605
+ _scheduleOffer(task, lease, delayMs) {
1606
+ const key = `${task.task_id}:${lease.machine_id}`;
1607
+ const timer = setTimeout(() => {
1608
+ this._pendingOffers.delete(key);
1609
+ // A lease that was re-created (takeover) or swept is not this one any more.
1610
+ if (task.leases.get(lease.machine_id) !== lease) return;
1611
+ if (lease.state !== 'queued') return; // the direct copy won; nothing to fall back to
1612
+ if (this.isLeaseExpired(lease, this.nowMs())) return; // the sweep owns it now
1613
+ this.emitOffer(task, lease);
1614
+ }, delayMs);
1615
+ timer.unref?.();
1616
+ this._pendingOffers.set(key, timer);
1617
+ }
1618
+
1619
+ /** Drop every deferred offer. Called by the relay's own teardown path. */
1620
+ clearPendingOffers() {
1621
+ for (const timer of this._pendingOffers.values()) clearTimeout(timer);
1622
+ this._pendingOffers.clear();
1623
+ }
1624
+
1625
+ emitOffer(task, lease) {
1626
+ const event = this.emit('task.offer', {
1627
+ task_id: task.task_id,
1628
+ attempt: lease.attempt,
1629
+ mode: task.mode,
1630
+ index: lease.index,
1631
+ index_total: task.index_total,
1632
+ command_argv: task.command_argv,
1633
+ // v0.3.3: present only for `pipeline`, and only then does the machine run a chain. Omitted
1634
+ // entirely otherwise, so an offer for a single-command task is byte-identical to before --
1635
+ // which is what keeps a v0.2.3 agent working against this relay.
1636
+ ...(task.stages ? { stages: task.stages } : {}),
1637
+ /**
1638
+ * v0.4.0: always present, `null` when the caller supplied nothing.
1639
+ *
1640
+ * Deliberately NOT conditional (unlike `stages` above, which is omitted so a single-command
1641
+ * offer stays byte-identical for an old agent): the contract fixes the shape of these two as
1642
+ * `string|null` and `{mode}|null`, and `null` is the honest value for "the dispatcher did not
1643
+ * say" — which is exactly the v0.3.9 case. They are additive keys in an event whose consumers
1644
+ * already ignore unknown fields.
1645
+ */
1646
+ origin_machine_id: task.origin_machine_id ?? null,
1647
+ p2p: task.p2p ?? null,
1648
+ cwd_rel: task.cwd_rel,
1649
+ write: task.write,
1650
+ timeout_ms: task.timeout_ms,
1651
+ base_commit: task.base_commit,
1652
+ base_tree: task.base_tree,
1653
+ requirements: task.requirements,
1654
+ compare_policy: task.compare_policy,
1655
+ dedupe_key: lease.dedupe_key,
1656
+ deadline: rfc3339(task.deadline_ms),
1657
+ }, { target: lease.machine_id });
1658
+ lease.state = 'offered';
1659
+ lease.offered_seq = event.seq;
1660
+ lease.last_heartbeat_ms = this.nowMs();
1661
+ lease.lease_until_ms = lease.last_heartbeat_ms + this.leaseWindowMs();
1662
+ return event;
1663
+ }
1664
+
1665
+ getTask(taskId) { return this.tasks.get(taskId) ?? null; }
1666
+
1667
+ requireTask(taskId) {
1668
+ const task = this.tasks.get(taskId);
1669
+ if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
1670
+ return task;
1671
+ }
1672
+
1673
+ /**
1674
+ * Re-offer the work a machine was handed but has not yet started (v0.1.2 §8.6).
1675
+ *
1676
+ * WHY THIS EXISTS
1677
+ *
1678
+ * An offer is a point-in-time event on a stream. `createTask` publishes
1679
+ * `task.offer` once, and if the target machine's stream happens to be down at
1680
+ * that instant -- the ordinary situation after a relay restart, a tunnel drop,
1681
+ * or any cross-network interruption -- the event is simply gone. The lease sits
1682
+ * in `offered` until the sweep expires it, the task never runs, and the
1683
+ * operator sees a timeout with no explanation. Measured: the machine is back
1684
+ * online with a live stream within milliseconds, and still receives nothing.
1685
+ *
1686
+ * So the relay re-states unclaimed work when a machine connects. This is safe
1687
+ * because the state it acts on is durable: the lease the offer refers to is
1688
+ * still in the ledger.
1689
+ *
1690
+ * WHY ONLY `offered` AND `queued`
1691
+ *
1692
+ * `running` is deliberately excluded, and that exclusion is the whole safety
1693
+ * argument. A running lease means a machine has already STARTED the command --
1694
+ * it acknowledged the offer and is heartbeating. Re-offering it would make that
1695
+ * machine execute the work a second time, so the fix for a lost offer would
1696
+ * introduce the one failure this project is built to avoid: two machines, or
1697
+ * one machine twice, running the same side-effecting command.
1698
+ *
1699
+ * A machine that dies mid-run therefore does not get its work back here; that
1700
+ * path stays with the sweep, which expires the lease and hands the index to
1701
+ * another machine with `attempt + 1`.
1702
+ *
1703
+ * Re-emitted offers are new events and are replayed like any other, so a client
1704
+ * that reconnects repeatedly can see the same offer more than once. That is
1705
+ * intentional: the agent already de-duplicates by `task_id` (it ignores an
1706
+ * offer whose task is queued or running), and an at-least-once offer with a
1707
+ * client-side dedupe is strictly better than an at-most-once offer that is
1708
+ * silently dropped.
1709
+ *
1710
+ * @param {string} machineId
1711
+ * @returns {{count: number, task_ids: string[]}}
1712
+ */
1713
+ redeliverPendingOffers(machineId) {
1714
+ const taskIds = [];
1715
+ const now = this.nowMs();
1716
+
1717
+ for (const task of this.tasks.values()) {
1718
+ if (task.cancelled === true) continue;
1719
+ const lease = task.leases.get(machineId);
1720
+ if (!lease) continue;
1721
+ // Only work the machine has not begun. See the note above: `running` is
1722
+ // excluded on purpose, not by omission.
1723
+ if (lease.state !== 'offered' && lease.state !== 'queued') continue;
1724
+ // Do not resurrect a lease the sweep has already written off; the takeover
1725
+ // path owns that decision.
1726
+ if (this.isLeaseExpired(lease, now)) continue;
1727
+
1728
+ this.emitOffer(task, lease);
1729
+ taskIds.push(task.task_id);
1730
+ }
1731
+
1732
+ return { count: taskIds.length, task_ids: taskIds };
1733
+ }
1734
+
1735
+ listTasks({ limit = 50 } = {}) {
1736
+ const n = Number.isInteger(limit) && limit > 0 ? limit : 50;
1737
+ return [...this.tasks.values()]
1738
+ .sort((a, b) => b.created_at_ms - a.created_at_ms)
1739
+ .slice(0, n)
1740
+ .map((t) => this.taskSummary(t));
1741
+ }
1742
+
1743
+ taskSummary(task) {
1744
+ return {
1745
+ task_id: task.task_id,
1746
+ mode: task.mode,
1747
+ created_at: rfc3339(task.created_at_ms),
1748
+ created_by: task.created_by,
1749
+ index_total: task.index_total,
1750
+ cancelled: task.cancelled,
1751
+ degraded: task.degraded,
1752
+ leases: [...task.leases.values()].map((l) => l.machine_id),
1753
+ lease_states: Object.fromEntries([...task.leases.values()].map((l) => [l.machine_id, l.state])),
1754
+ };
1755
+ }
1756
+
1757
+ cancelTask(taskId, reason = 'cancelled_by_operator') {
1758
+ const task = this.requireTask(taskId);
1759
+ if (task.cancelled) return task;
1760
+ task.cancelled = true;
1761
+ task.cancel_reason = reason;
1762
+ for (const lease of task.leases.values()) {
1763
+ if (ACTIVE_LEASE_STATES.includes(lease.state)) lease.state = 'cancelled';
1764
+ }
1765
+ this.emit('task.cancel', { task_id: taskId, reason });
1766
+ this._ledger({ type: 'task.cancelled', task_id: taskId, reason });
1767
+ return task;
1768
+ }
1769
+
1770
+ /* ---------------- leases / heartbeat (§4.3) ---------------- */
1771
+
1772
+ /** Shared `phase`/`progress` validation for both heartbeat modes. */
1773
+ _validatePhaseProgress(input) {
1774
+ if (input.phase !== undefined && input.phase !== null && !LEASE_PHASES.includes(input.phase)) {
1775
+ throw new ProtocolError('BAD_REQUEST', 'phase must be preparing|running|finalizing', { phase: input.phase });
1776
+ }
1777
+ if (input.progress !== undefined && input.progress !== null
1778
+ && (typeof input.progress !== 'number' || input.progress < 0 || input.progress > 1)) {
1779
+ throw new ProtocolError('BAD_REQUEST', 'progress must be a number in [0,1]', { progress: input.progress });
1780
+ }
1781
+ }
1782
+
1783
+ /**
1784
+ * v0.3.0 IDLE (diagnostic) heartbeat — an agent with no lease keeps its latency
1785
+ * and liveness visible to other machines.
1786
+ *
1787
+ * This path deliberately NEVER looks at `this.tasks`. That is the whole safety
1788
+ * argument, and it is structural rather than a matter of remembering a rule: an
1789
+ * idle heartbeat cannot renew, resurrect or otherwise touch a lease because it
1790
+ * never obtains one. Without that, an idle ping from a machine whose lease had
1791
+ * expired would flip `expired` back to `running` and manufacture a live task out
1792
+ * of nothing -- a fake "the work is still being done" signal, which is the most
1793
+ * dangerous lie this relay can tell.
1794
+ */
1795
+ diagnosticHeartbeat(input = {}) {
1796
+ const machineId = typeof input.machine_id === 'string' && input.machine_id !== ''
1797
+ ? input.machine_id : null;
1798
+ if (machineId === null) {
1799
+ throw new ProtocolError('BAD_REQUEST', 'machine_id is required', {});
1800
+ }
1801
+ if (!this.devices.has(machineId)) {
1802
+ throw new ProtocolError('NOT_FOUND', 'unknown machine_id', { machine_id: machineId });
1803
+ }
1804
+ this._validatePhaseProgress(input);
1805
+
1806
+ const device = this.touchDevice(machineId);
1807
+ this.recordHeartbeatMetrics(machineId, input);
1808
+ return {
1809
+ // No lease is involved, and the response says so instead of inventing one.
1810
+ lease_until: null,
1811
+ cancel: false,
1812
+ diagnostic: true,
1813
+ machine_id: machineId,
1814
+ rtt_ms: device?.rtt_ms ?? null,
1815
+ };
1816
+ }
1817
+
1818
+ /**
1819
+ * §4.3 POST /v1/heartbeat — renews the lease using SERVER time only.
1820
+ *
1821
+ * Two modes since v0.3.0:
1822
+ * - `task_id` present -> LEASE heartbeat (unchanged §4.3 behaviour)
1823
+ * - `diagnostic: true`, no task -> IDLE heartbeat (diagnostics only, no lease)
1824
+ * - neither -> 400. A heartbeat that silently renewed
1825
+ * nothing would let an agent whose `task_id` went missing stop renewing while
1826
+ * believing it was fine, ending in a falsely `expired` task. §0 forbids
1827
+ * silent degradation, so this is loud.
1828
+ *
1829
+ * @returns {{lease_until:string|null, cancel:boolean, phase?:string, progress?:number, diagnostic?:boolean}}
1830
+ */
1831
+ heartbeat(input = {}) {
1832
+ const hasTask = typeof input.task_id === 'string' && input.task_id !== '';
1833
+ if (!hasTask) {
1834
+ if (input.diagnostic !== true) {
1835
+ throw new ProtocolError('BAD_REQUEST',
1836
+ 'a heartbeat must carry task_id (lease renewal) or diagnostic:true (idle diagnostics); '
1837
+ + 'refusing to treat a missing task_id as a no-op renewal', {
1838
+ received_keys: Object.keys(input).sort(),
1839
+ });
1840
+ }
1841
+ return this.diagnosticHeartbeat(input);
1842
+ }
1843
+ if (input.diagnostic === true) {
1844
+ // task_id wins: a heartbeat that names a task IS a lease heartbeat.
1845
+ // (documented; the flag is simply redundant here)
1846
+ }
1847
+ return this.leaseHeartbeat(input);
1848
+ }
1849
+
1850
+ leaseHeartbeat(input = {}) {
1851
+ const task = this.requireTask(input.task_id);
1852
+ const lease = task.leases.get(input.machine_id);
1853
+ if (!lease) {
1854
+ throw new ProtocolError('NOT_FOUND', 'no lease for this machine on this task', {
1855
+ task_id: input.task_id, machine_id: input.machine_id,
1856
+ });
1857
+ }
1858
+ this._validatePhaseProgress(input);
1859
+
1860
+ const now = this.nowMs();
1861
+ this.touchDevice(input.machine_id);
1862
+ this.recordHeartbeatMetrics(input.machine_id, input, now);
1863
+
1864
+ if (task.cancelled) {
1865
+ return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: task.cancel_reason };
1866
+ }
1867
+
1868
+ // §4.3: once halt=now has handed this index to another attempt, the old owner
1869
+ // must stop — a late heartbeat from the expired machine is not a valid re-claim.
1870
+ if (lease.state === 'expired' && task.attempt > lease.attempt) {
1871
+ return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: 'LEASE_EXPIRED_TAKEN_OVER' };
1872
+ }
1873
+
1874
+ const stateBefore = lease.state;
1875
+ lease.last_heartbeat_ms = now;
1876
+ lease.lease_until_ms = now + this.leaseWindowMs();
1877
+ lease.heartbeat_count += 1;
1878
+ if (input.phase !== undefined && input.phase !== null) lease.phase = input.phase;
1879
+ if (input.progress !== undefined && input.progress !== null) lease.progress = input.progress;
1880
+ if (Number.isInteger(input.attempt) && input.attempt >= 1) lease.attempt = input.attempt;
1881
+ if (lease.state === 'queued' || lease.state === 'offered') lease.state = 'running';
1882
+ // A live heartbeat is stronger evidence than a timer: with halt=never nobody took
1883
+ // the work away, so the late-but-alive agent re-claims its own lease.
1884
+ if (lease.state === 'expired') { lease.state = 'running'; lease.expired_at_ms = null; }
1885
+
1886
+ // §7 ledger records lease STATE changes, not every 10s heartbeat.
1887
+ if (lease.state !== stateBefore) this._ledgerLease(task, lease);
1888
+
1889
+ return { lease_until: rfc3339(lease.lease_until_ms), cancel: false, phase: lease.phase, progress: lease.progress };
1890
+ }
1891
+
1892
+ /** §4.3: expiry is decided by Rabbit's clock only. */
1893
+ isLeaseExpired(lease, nowMs = this.nowMs()) {
1894
+ if (!ACTIVE_LEASE_STATES.includes(lease.state)) return false;
1895
+ return nowMs >= lease.lease_until_ms;
1896
+ }
1897
+
1898
+ /**
1899
+ * §4.3 expiry sweep.
1900
+ * - halt=never → book-keeping only, task is marked `partial`
1901
+ * - halt=now → notify remaining machines immediately that they may take over
1902
+ */
1903
+ sweepExpired(nowMs = this.nowMs()) {
1904
+ const expired = [];
1905
+ for (const task of this.tasks.values()) {
1906
+ if (task.cancelled) continue;
1907
+ let changed = false;
1908
+ for (const lease of task.leases.values()) {
1909
+ if (!this.isLeaseExpired(lease, nowMs)) continue;
1910
+ lease.state = 'expired';
1911
+ lease.expired_at_ms = nowMs;
1912
+ changed = true;
1913
+ expired.push({ task, lease });
1914
+ this._ledgerLease(task, lease);
1915
+ this.emit('notice', {
1916
+ level: 'warn',
1917
+ code: 'LEASE_EXPIRED',
1918
+ message: `lease expired: machine=${lease.machine_id} task=${task.task_id} attempt=${lease.attempt}`,
1919
+ task_id: task.task_id,
1920
+ machine_id: lease.machine_id,
1921
+ });
1922
+ if (task.halt === 'now') {
1923
+ task.attempt += 1;
1924
+ this._takeover(task, lease, nowMs);
1925
+ }
1926
+ }
1927
+ if (changed) task.degraded = 'partial';
1928
+ }
1929
+ return expired;
1930
+ }
1931
+
1932
+ /**
1933
+ * §4.3 `halt=now`: "立即通知其他机器可接管".
1934
+ *
1935
+ * v0.0.1's PROTOCOL defines no per-index lease reassignment, so takeover is
1936
+ * implemented as: re-offer the freed `index` (attempt+1) to every live machine
1937
+ * that is idle — no lease yet, or a lease still queued/offered — and broadcast a
1938
+ * notice. Machines already `running` are never re-offered, to avoid double
1939
+ * execution of the same index.
1940
+ */
1941
+ _takeover(task, expiredLease, nowMs = this.nowMs()) {
1942
+ /**
1943
+ * v0.7.8: the two safety guards this path was missing.
1944
+ *
1945
+ * `_takeover` already existed and is invoked on `halt=now`, so hand-off was not absent — but it
1946
+ * would re-offer a dead machine's index to **every idle device the relay holds**, with no fleet
1947
+ * filter and no consideration of whether the task writes. Both are reachable defects:
1948
+ *
1949
+ * 1. **A device in another account could be handed the work.** On a relay serving several
1950
+ * fleets, `this.devices` includes all of them, so a takeover crossed accounts — which is
1951
+ * the isolation this release series is about, broken in the one path that is easy to miss
1952
+ * because it only runs when something has already gone wrong.
1953
+ * 2. **A writing task could be run a second time.** Hand-off makes execution at-least-once,
1954
+ * which is harmless for `node --test` and is a duplicated write for anything else. The
1955
+ * design forbids automatic hand-off for writing tasks until a fencing token exists.
1956
+ *
1957
+ * The re-offer shape itself is unchanged: PROTOCOL v0.0.1 defines no per-index lease
1958
+ * reassignment, so takeover is expressed as a re-offer to idle machines. Narrowing that to a
1959
+ * single successor is a protocol revision, not a wiring change, and is not made here.
1960
+ */
1961
+ if (task.write === true) {
1962
+ this.emit('notice', {
1963
+ level: 'warn',
1964
+ code: 'LEASE_TAKEOVER_REFUSED',
1965
+ message:
1966
+ `task ${task.task_id} index ${expiredLease.index} is a writing task, so no machine takes it over: `
1967
+ + 'a second run could apply the same change twice, and the caller cannot tell from the verdict',
1968
+ task_id: task.task_id,
1969
+ reason: 'write_not_idempotent',
1970
+ });
1971
+ return [];
1972
+ }
1973
+
1974
+ const requesterFleet = this.fleetId ?? null;
1975
+
1976
+ const candidates = [];
1977
+ for (const device of this.devices.values()) {
1978
+ if (device.machine_id === expiredLease.machine_id) continue;
1979
+ /**
1980
+ * Fleet membership, when the relay is scoped.
1981
+ *
1982
+ * A device with its own `fleet_id` must match. A device with none cannot take over on a
1983
+ * scoped relay, because there is no fleet to match it against — the same rule `targetsFor`
1984
+ * applies to dispatch, kept identical here so the two paths cannot disagree.
1985
+ */
1986
+ if (requesterFleet !== null) {
1987
+ const deviceFleet = typeof device.fleet_id === 'string' && device.fleet_id !== '' ? device.fleet_id : null;
1988
+ if (deviceFleet !== requesterFleet) {
1989
+ this.log?.('debug', 'takeover candidate skipped: not in this fleet', {
1990
+ task_id: task.task_id,
1991
+ machine_id: device.machine_id,
1992
+ device_fleet: deviceFleet,
1993
+ relay_fleet: requesterFleet,
1994
+ });
1995
+ continue;
1996
+ }
1997
+ }
1998
+ const lease = task.leases.get(device.machine_id);
1999
+ if (lease && !['queued', 'offered'].includes(lease.state)) continue;
2000
+ if (!lease && gateMachine(task, device)) continue;
2001
+ candidates.push({ device, lease: lease ?? null });
2002
+ }
2003
+
2004
+ for (const { device, lease } of candidates) {
2005
+ const target = lease ?? {
2006
+ machine_id: device.machine_id,
2007
+ machine_name: device.machine_name,
2008
+ index: expiredLease.index,
2009
+ attempt: task.attempt,
2010
+ state: 'queued',
2011
+ refusal_reason: null,
2012
+ refusal_detail: null,
2013
+ dedupe_key: null,
2014
+ last_heartbeat_ms: nowMs,
2015
+ claimed_at_ms: nowMs,
2016
+ lease_until_ms: nowMs + this.leaseWindowMs(),
2017
+ phase: null,
2018
+ progress: null,
2019
+ offered_seq: null,
2020
+ expired_at_ms: null,
2021
+ finished_at_ms: null,
2022
+ heartbeat_count: 0,
2023
+ };
2024
+ target.index = expiredLease.index;
2025
+ target.attempt = task.attempt;
2026
+ target.dedupe_key = computeDedupeKey(task.task_id, target.index, task.command_hash, task.base_tree);
2027
+ task.leases.set(device.machine_id, target);
2028
+ this.emitOffer(task, target);
2029
+ this._ledgerLease(task, target);
2030
+ }
2031
+
2032
+ this.emit('notice', {
2033
+ level: 'info',
2034
+ code: candidates.length > 0 ? 'LEASE_TAKEOVER' : 'LEASE_TAKEOVER_UNAVAILABLE',
2035
+ message: candidates.length > 0
2036
+ ? `task ${task.task_id} index ${expiredLease.index} re-offered at attempt ${task.attempt} to: `
2037
+ + candidates.map((c) => c.device.machine_id).join(',')
2038
+ : `task ${task.task_id} index ${expiredLease.index} has no idle machine to take over`,
2039
+ task_id: task.task_id,
2040
+ });
2041
+ return candidates.map((c) => c.device.machine_id);
2042
+ }
2043
+
2044
+ /* ---------------- results (§5) ---------------- */
2045
+
2046
+ static instanceKey(machineId, dedupeKey, attempt) {
2047
+ return `${machineId}${INSTANCE_SEP}${dedupeKey}${INSTANCE_SEP}${attempt}`;
2048
+ }
2049
+
2050
+ /**
2051
+ * §5 result envelope validation.
2052
+ * Missing §5.1 required fields do NOT reject the frame — they make the result
2053
+ * `unverifiable` (§5.1 "缺一即 unverifiable").
2054
+ */
2055
+ static validateEnvelope(envelope) {
2056
+ const missing = [];
2057
+ if (!envelope || typeof envelope !== 'object' || Array.isArray(envelope)) {
2058
+ return { missing: [...REQUIRED_ENVELOPE_FIELDS], malformed: true };
2059
+ }
2060
+ for (const field of REQUIRED_ENVELOPE_FIELDS) {
2061
+ if (!(field in envelope) || envelope[field] === undefined) missing.push(field);
2062
+ }
2063
+ const problems = [];
2064
+ if ('status' in envelope && !RESULT_STATUSES.includes(envelope.status)) {
2065
+ problems.push(`status must be one of ${RESULT_STATUSES.join('|')}`);
2066
+ }
2067
+ if (envelope.status === 'refused' && !envelope.refusal_reason) {
2068
+ problems.push('refusal_reason is required when status=refused');
2069
+ }
2070
+ if ('warnings' in envelope && !Array.isArray(envelope.warnings)) problems.push('warnings must be string[]');
2071
+ return { missing, problems };
2072
+ }
2073
+
2074
+ /**
2075
+ * §1 idempotent POST /v1/result.
2076
+ *
2077
+ * NOTE (contract ambiguity, see report): §4.4 `dedupe_key` deliberately excludes
2078
+ * `machine_id`/`attempt`, but §6.3 aggregates results from *several* machines in
2079
+ * replicate mode — they all share one dedupe_key. Deduping purely on dedupe_key
2080
+ * would silently drop every machine's result but the first. We therefore key the
2081
+ * idempotency check on (machine_id, dedupe_key, attempt): an identical redelivery
2082
+ * is `deduped:true`, distinct machines/attempts are all recorded.
2083
+ */
2084
+ submitResult(envelope = {}) {
2085
+ const { missing, problems, malformed } = RabbitState.validateEnvelope(envelope);
2086
+ if (malformed) throw new ProtocolError('BAD_REQUEST', 'result must be a JSON object');
2087
+
2088
+ /**
2089
+ * v0.4.0: `transport` / `p2p` are OPTIONAL envelope fields, and they are validated as values
2090
+ * rather than added to the §5.1 required set. A v0.3.9 agent sends neither and must keep
2091
+ * working unchanged; an agent that sends `transport: 'satellite'` is refused by name instead of
2092
+ * storing a value the report would then show as a path that does not exist.
2093
+ */
2094
+ RabbitState.validateTransportFields(envelope);
2095
+
2096
+ const taskId = envelope.task_id;
2097
+ if (typeof taskId !== 'string' || taskId.length === 0) {
2098
+ throw new ProtocolError('BAD_REQUEST', 'task_id is required');
2099
+ }
2100
+ const task = this.tasks.get(taskId);
2101
+ if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
2102
+
2103
+ const machineId = typeof envelope.machine_id === 'string' ? envelope.machine_id : null;
2104
+ const dedupeKey = typeof envelope.dedupe_key === 'string' && envelope.dedupe_key
2105
+ ? envelope.dedupe_key : null;
2106
+ const attempt = Number.isInteger(envelope.attempt) && envelope.attempt >= 1 ? envelope.attempt : 1;
2107
+
2108
+ if (dedupeKey === null) {
2109
+ // Cannot dedupe without a key; still record it, as unverifiable.
2110
+ missing.push('dedupe_key');
2111
+ }
2112
+ const instanceKey = RabbitState.instanceKey(machineId ?? '<unknown>', dedupeKey ?? '<none>', attempt);
2113
+
2114
+ // §4.4 sanity check against the lease this machine actually holds. Booked as a
2115
+ // problem instead of a hard rejection so a buggy client cannot lose its result.
2116
+ const leaseForMachine = machineId ? task.leases.get(machineId) : null;
2117
+ if (leaseForMachine?.dedupe_key && dedupeKey && leaseForMachine.dedupe_key !== dedupeKey) {
2118
+ problems.push('dedupe_key does not match the §4.4 formula for this lease');
2119
+ }
2120
+
2121
+ if (this.results.has(instanceKey)) {
2122
+ const prior = this.results.get(instanceKey);
2123
+ return { deduped: true, task_id: taskId, dedupe_key: dedupeKey, seq: prior.seq };
2124
+ }
2125
+
2126
+ const now = this.nowMs();
2127
+ let envelopeShaOk = null;
2128
+ if (typeof envelope.envelope_sha256 === 'string') {
2129
+ const { envelope_sha256: _drop, ...rest } = envelope;
2130
+ envelopeShaOk = sha256Hex(jcs(rest)) === envelope.envelope_sha256;
2131
+ }
2132
+
2133
+ const record = {
2134
+ envelope,
2135
+ machine_id: machineId,
2136
+ dedupe_key: dedupeKey,
2137
+ attempt,
2138
+ instance_key: instanceKey,
2139
+ received_at_ms: now,
2140
+ validation: {
2141
+ missing: [...new Set(missing)],
2142
+ problems,
2143
+ envelope_sha256_ok: envelopeShaOk,
2144
+ },
2145
+ };
2146
+
2147
+ const effectiveStatus = record.validation.missing.length > 0 ? 'unverifiable' : envelope.status;
2148
+
2149
+ const seq = this.nextSeq();
2150
+ record.seq = seq;
2151
+ record.effective_status = effectiveStatus;
2152
+ this.results.set(instanceKey, record);
2153
+
2154
+ if (machineId) {
2155
+ const lease = task.leases.get(machineId);
2156
+ if (lease) {
2157
+ lease.state = effectiveStatus === 'refused' ? 'refused' : 'done';
2158
+ if (effectiveStatus === 'refused' && !lease.refusal_reason) {
2159
+ lease.refusal_reason = envelope.refusal_reason ?? 'REFUSED';
2160
+ }
2161
+ lease.finished_at_ms = now;
2162
+ lease.attempt = attempt;
2163
+ }
2164
+ this.touchDevice(machineId);
2165
+ }
2166
+ task.result_seqs.push(seq);
2167
+
2168
+ this.emit('task.result', {
2169
+ task_id: taskId,
2170
+ dedupe_key: dedupeKey,
2171
+ attempt,
2172
+ index: envelope.index ?? null,
2173
+ status: effectiveStatus,
2174
+ machine_id: machineId,
2175
+ deduped: false,
2176
+ validation_errors: record.validation.missing,
2177
+ });
2178
+
2179
+ // §7: results go into the ledger so the dedupe index survives a restart.
2180
+ this._ledger({ type: 'result.stored', task_id: taskId, machine_id: machineId, result: record });
2181
+ if (machineId) {
2182
+ const changed = task.leases.get(machineId);
2183
+ if (changed) this._ledgerLease(task, changed);
2184
+ }
2185
+
2186
+ return { deduped: false, task_id: taskId, dedupe_key: dedupeKey, seq, _instanceKey: instanceKey };
2187
+ }
2188
+
2189
+ /** All recorded results for a task, keyed by machine_id (later attempts win). */
2190
+ taskResults(taskId) {
2191
+ const task = this.tasks.get(taskId);
2192
+ if (!task) return new Map();
2193
+ const out = new Map();
2194
+ for (const rec of this.results.values()) {
2195
+ if (rec.envelope.task_id !== taskId) continue;
2196
+ const prev = out.get(rec.machine_id);
2197
+ if (!prev || rec.received_at_ms >= prev.received_at_ms) out.set(rec.machine_id, rec);
2198
+ }
2199
+ return out;
2200
+ }
2201
+
2202
+ /** Global dedupe book-keeping: dedupe_key -> instances seen. */
2203
+ dedupeReport() {
2204
+ const out = {};
2205
+ for (const rec of this.results.values()) {
2206
+ if (!rec.dedupe_key) continue;
2207
+ out[rec.dedupe_key] = out[rec.dedupe_key] ?? [];
2208
+ out[rec.dedupe_key].push({
2209
+ machine_id: rec.machine_id,
2210
+ attempt: rec.attempt,
2211
+ received_at: rfc3339(rec.received_at_ms),
2212
+ });
2213
+ }
2214
+ return out;
2215
+ }
2216
+
2217
+ /* ---------------- serialization ---------------- */
2218
+
2219
+ stats() {
2220
+ return {
2221
+ protocol_version: PROTOCOL_VERSION,
2222
+ seq: this.seq,
2223
+ buffered_events: this.buffer.length,
2224
+ devices: this.devices.size,
2225
+ tasks: this.tasks.size,
2226
+ results: this.results.size,
2227
+ pairing_codes: [...this.pairingCodes.values()].filter((c) => !c.used && this.nowMs() < c.expires_at_ms).length,
2228
+ uptime_ms: this.nowMs() - this.startedAtMs,
2229
+ };
2230
+ }
2231
+ }
2232
+
2233
+ export default RabbitState;