@twinsearth/w2m-dsh-plugin 0.8.10 → 0.8.99-tokenprobe.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,2279 +1,2279 @@
1
- /**
2
- * W2M Rabbit relay — in-memory state machine.
3
- *
4
- * Contract: E:\DS\_work\w2m\PROTOCOL.md (protocol_version = 1, frozen).
5
- * This module is intentionally PURE LOGIC:
6
- * - no network, no filesystem, no timers
7
- * - time is injectable via `now()` (required for lease-expiry unit tests)
8
- * - randomness is injectable for deterministic tests
9
- *
10
- * Persistence: none. All state is in-memory and is lost on restart (v0.0.1 scope);
11
- * there is intentionally no snapshot/serialization hook yet.
12
- */
13
-
14
- import { createHash, randomBytes } from 'node:crypto';
15
-
16
- // v0.7.7: fleet scoping. Imported rather than inlined so the rule has one home with its own
17
- // exhaustive tests — three lines inside this object would be untestable without a live relay.
18
- import { targetsFor } from '../chatops/fleet-wiring.mjs';
1
+ /**
2
+ * W2M Rabbit relay — in-memory state machine.
3
+ *
4
+ * Contract: E:\DS\_work\w2m\PROTOCOL.md (protocol_version = 1, frozen).
5
+ * This module is intentionally PURE LOGIC:
6
+ * - no network, no filesystem, no timers
7
+ * - time is injectable via `now()` (required for lease-expiry unit tests)
8
+ * - randomness is injectable for deterministic tests
9
+ *
10
+ * Persistence: none. All state is in-memory and is lost on restart (v0.0.1 scope);
11
+ * there is intentionally no snapshot/serialization hook yet.
12
+ */
13
+
14
+ import { createHash, randomBytes } from 'node:crypto';
15
+
16
+ // v0.7.7: fleet scoping. Imported rather than inlined so the rule has one home with its own
17
+ // exhaustive tests — three lines inside this object would be untestable without a live relay.
18
+ import { targetsFor } from '../chatops/fleet-wiring.mjs';
19
19
  import { fleetForPairingCode, deviceFleetFields } from '../chatops/fleet-wiring.mjs';
20
20
  // v0.8.1: the account-derived fleet key, so a machine can join with no pairing code.
21
- import { planJoin } from '../chatops/fleet-join.mjs';
22
-
23
- export const PROTOCOL_VERSION = 1;
24
-
25
- /**
26
- * PROTOCOL §1 table + §7 code table, plus the two v0.1.2 additions:
27
- * `OPERATOR_REQUIRED` (§5.2 → 401) and `RATE_LIMITED` (§6 → 429).
28
- * A code missing from this map silently degrades to HTTP 500 via
29
- * `ProtocolError`, which keeps the right error code but lies about the status.
30
- */
31
- export const ERROR_STATUS = Object.freeze({
32
- UNAUTHORIZED: 401,
33
- BAD_REQUEST: 400,
34
- NOT_FOUND: 404,
35
- PAIRING_INVALID: 400,
36
- PAIRING_EXPIRED: 400,
37
- TASK_EXISTS: 409,
38
- FRAME_TOO_LARGE: 413,
39
- NO_ONLINE_DEVICE: 503,
40
- INTERNAL: 500,
41
- OPERATOR_REQUIRED: 401, // v0.1.2 §5.2
42
- RATE_LIMITED: 429, // v0.1.2 §6
43
-
44
- /* v0.3.0 §9 request signing. Every client-side signature failure is a 401.
45
- * `SIGNING_NOT_CONFIGURED` is deliberately NOT 401: the relay was started with
46
- * `--require-signature` but no secret, so the fault is the relay's own
47
- * configuration. Reporting that as 401 would send every operator to debug the
48
- * client -- the one side that is not broken. 500 rather than 503 because
49
- * retrying never fixes a missing configuration, so nothing should be invited to
50
- * retry. */
51
- SIGNATURE_REQUIRED: 401,
52
- SIGNATURE_INCOMPLETE: 401,
53
- SIGNATURE_MISMATCH: 401,
54
- SIGNATURE_BAD_TIMESTAMP: 401,
55
- SIGNATURE_BAD_NONCE: 401,
56
- SIGNATURE_EXPIRED: 401,
57
- SIGNATURE_REPLAY: 401,
58
- SIGNING_NOT_CONFIGURED: 500,
59
- });
60
-
61
- /** §5.1 required envelope fields — missing any one means `unverifiable`. */
62
- export const REQUIRED_ENVELOPE_FIELDS = Object.freeze([
63
- 'envelope_version', 'task_id', 'attempt', 'dedupe_key', 'machine_id', 'machine_name',
64
- 'platform', 'caps', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
65
- 'pre_tree_fingerprint', 'post_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error',
66
- 'head_commit', 'dirty_before', 'command_argv', 'command_hash', 'shell_id',
67
- 'started_at', 'ended_at', 'duration_ms', 'exit_code', 'status', 'refusal_reason',
68
- 'stdout_sha256', 'stdout_bytes', 'stderr_sha256', 'stderr_bytes', 'warnings',
69
- 'envelope_sha256',
70
- ]);
71
-
72
- /** §5.3 comparable fields — the ONLY fields that take part in consistency judgement. */
73
- export const COMPARABLE_FIELDS = Object.freeze([
74
- 'task_id', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
75
- 'pre_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error', 'head_commit',
76
- 'command_hash', 'shell_id', 'exit_code', 'status', 'refusal_reason',
77
- 'stdout_sha256', 'stdout_bytes', 'stdout_normalized_sha256',
78
- 'stderr_sha256', 'artifacts', 'diff_numstat', 'tests', 'semantic_counts', 'warnings',
79
- ]);
80
-
81
- /** §5.1 status enum. */
82
- export const RESULT_STATUSES = Object.freeze([
83
- 'ok', 'nonzero_exit', 'timeout', 'crashed', 'refused', 'unverifiable',
84
- ]);
85
-
86
- export const FAILURE_STATUSES = Object.freeze(['nonzero_exit', 'timeout', 'crashed']);
87
-
88
- /** §4.3 phase enum. */
89
- export const LEASE_PHASES = Object.freeze(['preparing', 'running', 'finalizing']);
90
-
91
- /** Lease states that can still be expired / renewed. */
92
- export const ACTIVE_LEASE_STATES = Object.freeze(['queued', 'offered', 'running']);
93
-
94
- export const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000; // §4.3: agent heartbeats every 10s
95
- export const DEFAULT_MISSED_HEARTBEATS = 2; // §4.3: 2 missed cycles = 20s
96
- export const DEFAULT_LEASE_GRACE_MS = 30_000; // §4.3: + grace 30s
97
- /**
98
- * v0.4.0: the bounded hold on the relay's own offer when a dispatcher is pushing it directly.
99
- *
100
- * Sized against what it has to cover: one punch round trip (measured 15-370 ms on the reference
101
- * networks), the frame's delivery, and the executor's first heartbeat POST — all on a host that may
102
- * be busy. 1.2 s was the first guess and it was too tight: measured on a 1-vCPU shared host, two
103
- * agent tests saw the relay's copy overtake the direct one, which for `p2pMode: direct` means a
104
- * refusal of an offer that was about to arrive. 3 s covers a slow host with room to spare.
105
- *
106
- * The cost is honest and bounded: a punch that fails delays the relay's copy by up to this much,
107
- * once per task. Nothing else waits on it.
108
- */
109
- export const DEFAULT_P2P_OFFER_GRACE_MS = 3_000;
110
- export const DEFAULT_PAIRING_TTL_MS = 24 * 60 * 60 * 1000; // §2.2: 24h
111
- export const DEFAULT_EVENT_BUFFER_SIZE = 1000; // §requirement: keep last 1000 events
112
- export const MAX_FRAME_BYTES = 64 * 1024; // 64 KiB single frame
113
- export const SHELL_ID_DIRECT = 'direct-exec'; // §5.1: v0.0.1 only direct exec
114
- /** v0.3.0: RTT sanity ceiling — anything above 24h is treated as a broken value. */
115
- export const MAX_RTT_MS = 24 * 60 * 60 * 1000;
116
- /** v0.3.0: an RTT measurement older than this is reported as stale.
117
- * Must stay comfortably ABOVE the idle-heartbeat interval (60s): at 1x the value
118
- * would flip stale just before every refresh and flap, which would make
119
- * `rtt_stale === false` useless as a "trust this number" predicate. 3x gives two
120
- * consecutive missed idle heartbeats of margin. See PROTOCOL-v0.3.0.md §7. */
121
- export const DEFAULT_RTT_STALE_MS = 180_000;
122
- /** v0.3.0: recommended cadence for an agent with no lease (reference for the agent side). */
123
- export const DEFAULT_IDLE_HEARTBEAT_MS = 60_000;
124
- export const ENVELOPE_VERSION = '1.0';
125
- export const FINGERPRINT_ALGO = 'git-temp-index-tree/v1';
126
-
127
- /* ------------------------------------------------------------------ */
128
- /* Errors */
129
- /* ------------------------------------------------------------------ */
130
-
131
- export class ProtocolError extends Error {
132
- constructor(code, message, detail = {}) {
133
- super(message);
134
- this.name = 'ProtocolError';
135
- this.code = code;
136
- this.detail = detail;
137
- this.status = ERROR_STATUS[code] ?? 500;
138
- }
139
- toBody() {
140
- return { error: { code: this.code, message: this.message, detail: this.detail } };
141
- }
142
- }
143
-
144
- /* ------------------------------------------------------------------ */
145
- /* Small pure helpers */
146
- /* ------------------------------------------------------------------ */
147
-
148
- export function sha256Hex(input) {
149
- return createHash('sha256').update(input).digest('hex');
150
- }
151
-
152
- /** RFC 8785-ish canonical JSON (JCS): sorted keys, no whitespace. */
153
- export function jcs(value) {
154
- if (value === null) return 'null';
155
- const t = typeof value;
156
- if (t === 'number') return Number.isFinite(value) ? JSON.stringify(value) : 'null';
157
- if (t === 'boolean') return value ? 'true' : 'false';
158
- if (t === 'string') return JSON.stringify(value);
159
- if (t === 'undefined' || t === 'function' || t === 'symbol') return 'null';
160
- if (Array.isArray(value)) return `[${value.map((v) => (v === undefined ? 'null' : jcs(v))).join(',')}]`;
161
- if (t === 'object') {
162
- const keys = Object.keys(value).filter((k) => value[k] !== undefined).sort();
163
- return `{${keys.map((k) => `${JSON.stringify(k)}:${jcs(value[k])}`).join(',')}}`;
164
- }
165
- return JSON.stringify(String(value));
166
- }
167
-
168
- /**
169
- * Upper bound on the stages a `pipeline` may contain.
170
- *
171
- * Each stage is another command run on someone else's machine, so an unbounded list is an unbounded
172
- * amount of work created by one request. The limit is generous for the intended use (build, test,
173
- * package) and exists so the failure is a clear refusal rather than a fleet-wide surprise.
174
- */
175
- export const MAX_PIPELINE_STAGES = 16;
176
-
177
- /**
178
- * v0.4.0: upper bound on `POST /v1/task`'s `origin_machine_id`.
179
- *
180
- * A machine id is an identifier, not a payload: 128 characters is far beyond any id this system
181
- * mints, and the ceiling is what stops the field from becoming an unbounded free-text channel into
182
- * every offer frame the relay emits.
183
- */
184
- export const MAX_ORIGIN_MACHINE_ID_LENGTH = 128;
185
-
186
- /**
187
- * v0.4.0: the P2P modes a task may be dispatched under (§1 of the frozen v0.4.0 contract).
188
- *
189
- * Mirrored here rather than imported from `src/agent/p2p-node.mjs`: the relay is a standalone
190
- * process that must not depend on agent code, and a mode list is three strings. A value outside
191
- * this list is a configuration error, never a silent fall back to the default.
192
- */
193
- export const P2P_MODES = Object.freeze(['auto', 'direct', 'relay']);
194
-
195
- /**
196
- * v0.4.0: how a result travelled — the direct P2P channel or the relay.
197
- *
198
- * A **transport is not a result**: how an envelope arrived must never take part in the consistency
199
- * judgement (§5 of the contract), or two machines that produced byte-identical output would be
200
- * reported `divergent` merely because one of them punched a hole. Hence: not required, not
201
- * comparable, and absent on every v0.3.9 envelope.
202
- */
203
- export const TRANSPORT_VALUES = Object.freeze(['p2p', 'relay']);
204
-
205
- /** §4.4 helper: command_hash = sha256(JCS(argv)+"|"+shell_id+"|"+cwd_rel). */
206
- export function computeCommandHash(commandArgv, shellId = SHELL_ID_DIRECT, cwdRel = '.') {
207
- return sha256Hex(`${jcs(commandArgv)}|${shellId}|${cwdRel}`);
208
- }
209
-
210
- /**
211
- * `command_hash` for a whole pipeline.
212
- *
213
- * The hash must bind **every** stage, not just the first. Bound to stage 0 alone, two pipelines that
214
- * differ after the first stage would carry the same hash -- so an envelope produced by the wrong
215
- * chain would still verify, and making "the machine ran what I asked" checkable is the entire point
216
- * of this field. Each stage's cwd participates for the same reason.
217
- *
218
- * The `|pipeline` suffix means this can never collide with a single-command hash: the two hash over
219
- * different shapes, and that difference is deliberate rather than incidental.
220
- */
221
- export function computePipelineCommandHash(stages, shellId = SHELL_ID_DIRECT) {
222
- const canonical = stages.map((s) => ({
223
- command_argv: s.command_argv,
224
- cwd_rel: s.cwd_rel ?? '.',
225
- continue_on_failure: s.continue_on_failure === true,
226
- }));
227
- return sha256Hex(`${jcs(canonical)}|${shellId}|pipeline`);
228
- }
229
-
230
- /** §4.4: dedupe_key = sha256(task_id + "|" + index + "|" + command_hash + "|" + base_tree). */
231
- export function computeDedupeKey(taskId, index, commandHash, baseTree) {
232
- return sha256Hex(`${taskId}|${index}|${commandHash}|${baseTree}`);
233
- }
234
-
235
- /** Epoch ms -> RFC3339 UTC, second precision (matches the PROTOCOL examples). */
236
- export function rfc3339(ms) {
237
- return new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
238
- }
239
-
240
- export function isRfc3339(value) {
241
- return typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/.test(value);
242
- }
243
-
244
- export function deepEqual(a, b) {
245
- return jcs(a === undefined ? null : a) === jcs(b === undefined ? null : b);
246
- }
247
-
248
- function normalizeNow(now) {
249
- if (typeof now !== 'function') return () => Date.now();
250
- return () => {
251
- const v = now();
252
- if (typeof v === 'number') return v;
253
- if (v instanceof Date) return v.getTime();
254
- return Date.parse(v);
255
- };
256
- }
257
-
258
- /* ------------------------------------------------------------------ */
259
- /* ULID */
260
- /* ------------------------------------------------------------------ */
261
-
262
- const CROCKFORD = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
263
-
264
- function encodeBig(value, len) {
265
- let out = '';
266
- let v = value;
267
- for (let i = 0; i < len; i += 1) {
268
- out = CROCKFORD[Number(v & 31n)] + out;
269
- v >>= 5n;
270
- }
271
- return out;
272
- }
273
-
274
- export function createIdFactory() {
275
- let lastMs = -1;
276
- let lastRand = 0n;
277
- return function nextId(nowMs = Date.now()) {
278
- const bytes = randomBytes(10);
279
- let rand = 0n;
280
- for (const b of bytes) rand = (rand << 8n) | BigInt(b);
281
- if (nowMs === lastMs) rand = (lastRand + 1n) & ((1n << 80n) - 1n);
282
- lastMs = nowMs;
283
- lastRand = rand;
284
- return encodeBig(BigInt(nowMs), 10) + encodeBig(rand, 16);
285
- };
286
- }
287
-
288
- export function ulid(nowMs = Date.now()) {
289
- return createIdFactory()(nowMs);
290
- }
291
-
292
- /* ------------------------------------------------------------------ */
293
- /* Version / capability gate (§6.1) */
294
- /* ------------------------------------------------------------------ */
295
-
296
- function parseVersion(text) {
297
- if (typeof text !== 'string') return null;
298
- const m = text.trim().match(/^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:\.(\d+))?/);
299
- if (!m) return null;
300
- return [Number(m[1]), Number(m[2] ?? 0), Number(m[3] ?? 0), Number(m[4] ?? 0)];
301
- }
302
-
303
- function cmpVersion(a, b) {
304
- for (let i = 0; i < 4; i += 1) {
305
- if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1;
306
- }
307
- return 0;
308
- }
309
-
310
- /**
311
- * Minimal semver-range check used by the §6.1 capability gate.
312
- * Supports bare versions and comma/space separated `>= > <= < = ==` conjunctions.
313
- * Unparseable ranges are treated as NOT satisfied (never silently pass).
314
- */
315
- export function satisfiesRange(version, range) {
316
- const v = parseVersion(version);
317
- if (v === null) return false;
318
- if (typeof range !== 'string') return false;
319
- const parts = range.split(/[\s,]+/).filter(Boolean);
320
- if (parts.length === 0) return false;
321
- for (const part of parts) {
322
- const m = part.match(/^(>=|<=|==|=|>|<)?\s*(.+)$/);
323
- if (!m) return false;
324
- const op = m[1] ?? '=';
325
- const target = parseVersion(m[2]);
326
- if (target === null) return false;
327
- const c = cmpVersion(v, target);
328
- if (op === '>=' && c < 0) return false;
329
- if (op === '<=' && c > 0) return false;
330
- if (op === '>' && c <= 0) return false;
331
- if (op === '<' && c >= 0) return false;
332
- if ((op === '=' || op === '==') && c !== 0) return false;
333
- }
334
- return true;
335
- }
336
-
337
- function machineWritable(caps) {
338
- if (!caps || typeof caps !== 'object') return false;
339
- if (caps.write === false || caps.writable === false || caps.write_allowed === false) return false;
340
- if (caps.read_only === true || caps.readonly === true || caps.ro_mount === true) return false;
341
- return true;
342
- }
343
-
344
- /**
345
- * §6.1 capability gate, evaluated per machine BEFORE dispatch.
346
- * @returns {null | {status:'refused', refusal_reason:string, detail:object}}
347
- */
348
- export function gateMachine(task, machine) {
349
- const caps = machine?.caps ?? {};
350
- const req = task.requirements ?? {};
351
-
352
- if (req.platform && Array.isArray(req.platform) && req.platform.length > 0) {
353
- const os = String(machine?.platform?.os ?? '').toLowerCase();
354
- if (!req.platform.map((p) => String(p).toLowerCase()).includes(os)) {
355
- return { status: 'refused', refusal_reason: 'PLATFORM_MISMATCH', detail: { os, want: req.platform } };
356
- }
357
- }
358
-
359
- const toolchain = req.toolchain ?? {};
360
- for (const [tool, range] of Object.entries(toolchain)) {
361
- const declared = caps[tool];
362
- if (declared === null || declared === undefined || declared === '') {
363
- return {
364
- status: 'refused',
365
- refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
366
- detail: { tool, want: range, have: declared ?? null },
367
- };
368
- }
369
- if (typeof range === 'string' && !satisfiesRange(declared, range)) {
370
- return {
371
- status: 'refused',
372
- refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
373
- detail: { tool, want: range, have: declared },
374
- };
375
- }
376
- }
377
-
378
- if (task.write === true && !machineWritable(caps)) {
379
- return { status: 'refused', refusal_reason: 'READ_ONLY_MACHINE', detail: { caps } };
380
- }
381
-
382
- return null;
383
- }
384
-
385
- /* ------------------------------------------------------------------ */
386
- /* RabbitState */
387
- /* ------------------------------------------------------------------ */
388
-
389
- const INSTANCE_SEP = '\u0000';
390
-
391
- export class RabbitState {
392
- constructor(options = {}) {
393
- this._now = normalizeNow(options.now);
394
- this._idFactory = options.idFactory ?? createIdFactory();
395
- this._randomToken = options.tokenFactory ?? (() => randomBytes(24).toString('hex'));
396
- this._randomChars = options.randomChars
397
- ?? ((n) => { const a = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; const b = randomBytes(n); let s = ''; for (let i = 0; i < n; i += 1) s += a[b[i] % a.length]; return s; });
398
-
399
- this.protocolVersion = PROTOCOL_VERSION;
400
- this.eventBufferSize = options.eventBufferSize ?? DEFAULT_EVENT_BUFFER_SIZE;
401
- /**
402
- * v0.7.7: the fleet this relay serves, or null for a relay that serves every device.
403
- *
404
- * **Null is the migration-safe default and is what the shared relay runs today.** With it,
405
- * task targeting behaves exactly as it did before this release, so deploying cannot be an
406
- * outage. Set it (via `--fleet <id>` or `fleetId`) and the relay becomes scoped: a device
407
- * with a `fleet_id` reaches only its own fleet, and a device with none reaches nothing.
408
- *
409
- * This is the relay's *own* fleet, not a device's. A device carries its own `fleet_id`, which
410
- * the relay wrote at pairing time from the pairing code.
411
- */
412
- this.fleetId = typeof options.fleetId === 'string' && options.fleetId !== '' ? options.fleetId : null;
413
- this.pairingTtlMs = options.pairingTtlMs ?? DEFAULT_PAIRING_TTL_MS;
414
- /** When true one code stays valid for its whole TTL instead of rotating. */
415
- this.pairingCodeReusable = options.pairingCodeReusable === true;
416
- this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
417
- this.missedHeartbeats = options.missedHeartbeats ?? DEFAULT_MISSED_HEARTBEATS;
418
- this.leaseGraceMs = options.leaseGraceMs ?? DEFAULT_LEASE_GRACE_MS;
419
- /**
420
- * v0.4.0: how long the relay holds its own offer back when the dispatcher says it will push the
421
- * offer over the direct path (`origin_machine_id` + `p2p.mode`).
422
- *
423
- * Without this, the direct path could never win: the relay's offer goes out over an SSE stream
424
- * that is already open, while a punch costs at least one round trip, so the relay copy would
425
- * arrive first essentially always and `transport` would read `relay` even on a network where
426
- * the punch succeeded. Measured on loopback, and it is exactly what the first version of this
427
- * release did.
428
- *
429
- * The hold is bounded and self-healing: the executor renews its lease (a heartbeat) the moment
430
- * it starts work on the direct copy, which moves the lease out of `queued`, and the deferred
431
- * offer is then dropped. If the punch fails, nothing renews the lease and the relay offers
432
- * normally once the grace expires -- so a failed punch costs this much latency and nothing else.
433
- */
434
- this.p2pOfferGraceMs = Number.isFinite(options.p2pOfferGraceMs) && options.p2pOfferGraceMs >= 0
435
- ? options.p2pOfferGraceMs
436
- : DEFAULT_P2P_OFFER_GRACE_MS;
437
- /** Deferred offers, keyed `task_id:machine_id`, so they can be cleared on close(). */
438
- this._pendingOffers = new Map();
439
- /** v0.3.0: age beyond which a reported RTT is flagged stale. */
440
- this.rttStaleMs = Number.isFinite(options.rttStaleMs) ? options.rttStaleMs : DEFAULT_RTT_STALE_MS;
441
-
442
- this.startedAtMs = this._now();
443
- /** v0.1.2 §8.3: random per-process id; a change tells clients the relay restarted. */
444
- this.relayId = options.relayId ?? randomBytes(8).toString('hex');
445
- /** v0.1.2 §7: optional injected Persistence instance (state.mjs never touches fs itself). */
446
- this._persist = options.persistence ?? null;
447
- /** True while replaying a ledger — suppresses event emission and re-persistence. */
448
- this._restoring = false;
449
- /** Ledger entries that could not be replayed. */
450
- this.replayWarnings = [];
451
- this.seq = 0;
452
- /** @type {{seq:number,event:object,target:string|null}[]} ring buffer, last N events */
453
- this.buffer = [];
454
- this.devices = new Map(); // machine_id -> device
455
- this.tokens = new Map(); // device_token -> machine_id
456
- this.tasks = new Map(); // task_id -> task
457
- this.results = new Map(); // instanceKey -> {envelope, machine_id, received_at_ms, ...}
458
- this.pairingCodes = new Map(); // code -> {code, created_at_ms, expires_at_ms, used}
459
- this._listeners = new Set();
460
- }
461
-
462
- /* ---------------- clock / ids ---------------- */
463
-
464
- nowMs() { return this._now(); }
465
- nowIso() { return rfc3339(this.nowMs()); }
466
- nextId() { return this._idFactory(this.nowMs()); }
467
- newToken() { return this._randomToken(); }
468
-
469
- /** §4.3 lease window = 2 missed heartbeats (20s) + grace (30s) = 50s. */
470
- leaseWindowMs() { return this.heartbeatIntervalMs * this.missedHeartbeats + this.leaseGraceMs; }
471
-
472
- /* ---------------- persistence hooks (v0.1.2 §7) ---------------- */
473
-
474
- /** Inject/replace the persistence backend (also used by tests). */
475
- setPersistence(persistence) { this._persist = persistence ?? null; return this._persist; }
476
-
477
- /** Snapshot the device table; failures must never break the request path. */
478
- _saveDevices() {
479
- if (this._restoring || !this._persist) return false;
480
- try {
481
- return this._persist.saveDevices(this.devices);
482
- } catch (err) {
483
- this._persist.warn?.('devices.json', `snapshot failed: ${err.message}`);
484
- return false;
485
- }
486
- }
487
-
488
- /** Append one ledger entry; failures must never break the request path. */
489
- _ledger(entry) {
490
- if (this._restoring || !this._persist) return false;
491
- try {
492
- const appended = this._persist.append(entry);
493
- // v0.6.2: a successful append is what clears a degraded relay, so the outcome has to be
494
- // reported back rather than assumed -- otherwise the state could only ever be entered.
495
- if (appended) this._persist.noteAppendSuccess?.();
496
- return appended;
497
- } catch (err) {
498
- // Recorded as a state, not merely as a line: this is the one relay failure that keeps serving,
499
- // and so the one that is invisible from every other angle (v0.6.2).
500
- if (typeof this._persist.noteAppendFailure === 'function') this._persist.noteAppendFailure(err.message);
501
- else this._persist.warn?.('ledger.jsonl', `append failed: ${err.message}`);
502
- return false;
503
- }
504
- }
505
-
506
- _ledgerLease(task, lease) {
507
- this._ledger({ type: 'lease.changed', task_id: task.task_id, machine_id: lease.machine_id, lease: { ...lease } });
508
- }
509
-
510
- /**
511
- * v0.1.2 §7 startup recovery: rebuild the device table, token index, tasks,
512
- * leases and the result/dedupe index from a devices snapshot + ledger replay.
513
- * Never emits events and never writes back what it just read.
514
- */
515
- restore({ devices = {}, ledger = [] } = {}) {
516
- this._restoring = true;
517
- try {
518
- for (const [machineId, d] of Object.entries(devices)) {
519
- if (!d || typeof d !== 'object') continue;
520
- const pairedAt = Date.parse(d.paired_at ?? '');
521
- const device = {
522
- machine_id: machineId,
523
- machine_name: typeof d.machine_name === 'string' && d.machine_name ? d.machine_name : machineId,
524
- platform: d.platform ?? {},
525
- caps: d.caps ?? {},
526
- user_id: d.user_id ?? null,
527
- device_token: typeof d.device_token === 'string' ? d.device_token : null,
528
- paired_at_ms: Number.isFinite(pairedAt) ? pairedAt : this.nowMs(),
529
- re_paired: d.re_paired === true,
530
- last_seen_at_ms: this.nowMs(),
531
- // a snapshot proves the device exists, not that it is connected
532
- online: false,
533
- streams: 0,
534
- // v0.3.0: RTT is volatile and is NEVER restored. Even if a snapshot on
535
- // disk happens to carry `rtt_ms`, it is dropped here: after a restart the
536
- // relay genuinely does not know the latency any more, and reporting a
537
- // pre-restart number as current would be a lie that no reader can detect.
538
- rtt_ms: null,
539
- rtt_at_ms: null,
540
- last_heartbeat_at_ms: null,
541
- stream_connects: 0,
542
- };
543
- this.devices.set(machineId, device);
544
- if (device.device_token) this.tokens.set(device.device_token, machineId);
545
- }
546
-
547
- let revivedTasks = 0;
548
- for (const entry of ledger) {
549
- try {
550
- if (this._applyLedgerEntry(entry) === true) revivedTasks += 1;
551
- } catch (err) {
552
- this.replayWarnings.push({ type: entry?.type ?? '<unknown>', message: err.message });
553
- }
554
- }
555
- return { revived_devices: this.devices.size, revived_tasks: revivedTasks };
556
- } finally {
557
- this._restoring = false;
558
- }
559
- }
560
-
561
- /** @returns {boolean} true when a new task was created. */
562
- _applyLedgerEntry(entry) {
563
- if (!entry || typeof entry.type !== 'string') return false;
564
- switch (entry.type) {
565
- case 'task.created': {
566
- const t = entry.task;
567
- if (!t || typeof t.task_id !== 'string') return false;
568
- const createdAt = Number.isFinite(t.created_at_ms) ? t.created_at_ms : Date.parse(entry.ts ?? '');
569
- const task = {
570
- task_id: t.task_id,
571
- mode: t.mode ?? 'replicate',
572
- command_argv: t.command_argv ?? [],
573
- cwd_rel: t.cwd_rel ?? '.',
574
- index_total: t.index_total ?? 1,
575
- timeout_ms: t.timeout_ms ?? 300_000,
576
- write: t.write === true,
577
- write_scope: t.write_scope ?? [],
578
- require_exclusive_write: t.require_exclusive_write === true,
579
- base_commit: t.base_commit ?? null,
580
- base_tree: t.base_tree ?? null,
581
- requirements: t.requirements ?? {},
582
- compare_policy: t.compare_policy ?? {},
583
- halt: t.halt ?? 'never',
584
- created_by: t.created_by ?? null,
585
- created_at_ms: Number.isFinite(createdAt) ? createdAt : this.nowMs(),
586
- command_hash: t.command_hash ?? null,
587
- attempt: t.attempt ?? 1,
588
- leases: new Map(),
589
- cancelled: false,
590
- cancel_reason: null,
591
- degraded: t.degraded ?? null,
592
- deadline_ms: t.deadline_ms ?? 0,
593
- // v0.4.0 routing metadata. Read defensively: a ledger written by v0.3.9 has neither key,
594
- // and a replayed task with `null` for both is exactly a v0.3.9 task.
595
- origin_machine_id: t.origin_machine_id ?? null,
596
- p2p: t.p2p ?? null,
597
- result_seqs: [],
598
- };
599
- for (const lease of entry.leases ?? []) {
600
- if (lease && typeof lease.machine_id === 'string') task.leases.set(lease.machine_id, { ...lease });
601
- }
602
- this.tasks.set(task.task_id, task);
603
- return true;
604
- }
605
- case 'lease.changed': {
606
- const task = this.tasks.get(entry.task_id);
607
- const lease = task?.leases.get(entry.machine_id);
608
- if (!lease) return false;
609
- Object.assign(lease, entry.lease ?? {});
610
- return false;
611
- }
612
- case 'result.stored': {
613
- const record = entry.result;
614
- if (!record || typeof record.instance_key !== 'string') return false;
615
- this.results.set(record.instance_key, { ...record });
616
- const task = this.tasks.get(record.envelope?.task_id);
617
- if (task && Number.isInteger(record.seq)) task.result_seqs.push(record.seq);
618
- return false;
619
- }
620
- case 'task.cancelled': {
621
- const task = this.tasks.get(entry.task_id);
622
- if (task) {
623
- task.cancelled = true;
624
- task.cancel_reason = entry.reason ?? null;
625
- }
626
- return false;
627
- }
628
- default:
629
- return false; // unknown entry type: forward compatible, ignored
630
- }
631
- }
632
-
633
- /* ---------------- events (§3) ---------------- */
634
-
635
- nextSeq() { this.seq += 1; return this.seq; }
636
-
637
- /** Reserve a seq WITHOUT buffering (used for the per-connection `ready` frame). */
638
- reserveSeq() { return this.nextSeq(); }
639
-
640
- onEvent(listener) {
641
- this._listeners.add(listener);
642
- return () => this._listeners.delete(listener);
643
- }
644
-
645
- /**
646
- * Emit an event: allocates the next global seq, buffers it (ring of N) and
647
- * notifies subscribers. `target` is the machine_id the event is addressed to
648
- * (null = broadcast). The target is NOT serialized except as `machine_id`.
649
- */
650
- emit(type, fields = {}, opts = {}) {
651
- const target = opts.target ?? null;
652
- const seq = this.nextSeq();
653
- const event = {
654
- type,
655
- seq,
656
- rabbit_time: this.nowIso(),
657
- ...(target ? { machine_id: target } : {}),
658
- ...fields,
659
- };
660
- const entry = { seq, event, target };
661
- this.buffer.push(entry);
662
- if (this.buffer.length > this.eventBufferSize) {
663
- this.buffer.splice(0, this.buffer.length - this.eventBufferSize);
664
- }
665
- for (const listener of this._listeners) {
666
- try { listener(entry); } catch { /* subscriber errors must never break state */ }
667
- }
668
- return event;
669
- }
670
-
671
- oldestBufferedSeq() { return this.buffer.length ? this.buffer[0].seq : this.seq + 1; }
672
- /**
673
- * Highest seq present in the replay ring (0 when empty).
674
- *
675
- * Deliberately different from `lastSeq()`: `ready` frames allocate seq numbers
676
- * without being buffered, so `lastSeq()` can run ahead of anything a client can
677
- * actually replay. Diagnostics that describe the REPLAY WINDOW must use this one.
678
- */
679
- lastBufferedSeq() { return this.buffer.length ? this.buffer[this.buffer.length - 1].seq : 0; }
680
- lastSeq() { return this.seq; }
681
-
682
- /** Replay entries with seq >= from (used by GET /v1/stream?seq=|Last-Event-ID). */
683
- replayEntries(from) {
684
- return this.buffer.filter((e) => e.seq >= from);
685
- }
686
-
687
- eventsSince(from) { return this.replayEntries(from).map((e) => e.event); }
688
-
689
- /* ---------------- pairing (§2) ---------------- */
690
-
691
- /**
692
- * Mint a pairing code.
693
- *
694
- * **v0.7.7: the code carries the relay's fleet.** This is the link that makes the scheme work,
695
- * and it is easy to leave out: `pair()` reads the fleet *from the code*, so a code with no
696
- * fleet pairs a device into no fleet -- and a device in no fleet reaches nothing on a scoped
697
- * relay. `--fleet X` would then have had no effect on anything, silently.
698
- *
699
- * Defaulting to `this.fleetId` rather than requiring the caller to pass it means there is no
700
- * path that mints a fleetless code on a scoped relay. An explicit `fleetId` overrides, which
701
- * is what a relay serving several fleets would need.
702
- */
703
- createPairingCode({ ttlMs = this.pairingTtlMs, fleetId = this.fleetId } = {}) {
704
- let code;
705
- do { code = `PAIR-${this._randomChars(8)}`; } while (this.pairingCodes.has(code));
706
- const now = this.nowMs();
707
- this.pairingCodes.set(code, {
708
- code,
709
- created_at_ms: now,
710
- expires_at_ms: now + ttlMs,
711
- used: false,
712
- // Present as an explicit key even when null, so a code dump shows whether a fleet was
713
- // attached rather than leaving it to be inferred from an absent field.
714
- fleet_id: typeof fleetId === 'string' && fleetId !== '' ? fleetId : null,
715
- });
716
- return code;
717
- }
718
-
719
- /** Revoke every outstanding code and mint a new one. */
720
- rotatePairingCode(opts) {
721
- for (const rec of this.pairingCodes.values()) rec.used = true;
722
- return this.createPairingCode(opts);
723
- }
724
-
725
- /**
726
- * Register an externally supplied code (`--pairing-code <c>`).
727
- *
728
- * v0.1.2 BUG-1: assigning the code to `relay.pairingCode` without registering it
729
- * advertised a code that `/v1/pair` then rejected with PAIRING_INVALID, so every
730
- * documented scripted-onboarding flow failed. "Which code do we use" and "is the
731
- * code registered" are two different questions; this answers the second.
732
- */
733
- registerPairingCode(code, { ttlMs = this.pairingTtlMs } = {}) {
734
- if (typeof code !== 'string' || code.trim() === '') {
735
- throw new ProtocolError('BAD_REQUEST', 'pairing code must be a non-empty string', { pairing_code: code });
736
- }
737
- const value = code.trim();
738
- const now = this.nowMs();
739
- this.pairingCodes.set(value, { code: value, created_at_ms: now, expires_at_ms: now + ttlMs, used: false });
740
- return value;
741
- }
742
-
743
- /** §2.2 step 3: validate a one-time code. */
744
- validatePairingCode(code) {
745
- if (typeof code !== 'string' || code.length === 0) {
746
- throw new ProtocolError('PAIRING_INVALID', 'pairing_code is required');
747
- }
748
- const rec = this.pairingCodes.get(code);
749
- if (!rec) throw new ProtocolError('PAIRING_INVALID', 'unknown pairing code', { pairing_code: code });
750
- if (rec.used) throw new ProtocolError('PAIRING_INVALID', 'pairing code already used', { pairing_code: code });
751
- if (this.nowMs() >= rec.expires_at_ms) {
752
- throw new ProtocolError('PAIRING_EXPIRED', 'pairing code expired', { pairing_code: code, expired_at: rfc3339(rec.expires_at_ms) });
753
- }
754
- return rec;
755
- }
756
-
757
- /** §2.2: exchange a one-time code for a device_token. §1: idempotent. */
758
- pair(request = {}) {
759
- /**
760
- * v0.8.1: a pairing code **or** an account-derived fleet key.
761
- *
762
- * The code path is unchanged. The key path is what lets a second machine on one account join
763
- * with no human step and no relay configuration — the gap that left a new PC invisible beside
764
- * its own fleet.
765
- *
766
- * ## Why a key may stand in for a code
767
- *
768
- * They authorise different things. A **code** is the operator saying *"let this machine in"*,
769
- * and it is the only way to place a device in a fleet the device could not compute for itself.
770
- * A **key** is the account saying *"this machine is mine"*, and it can only ever place the
771
- * device in the fleet derived from that key — so it cannot be used to reach a fleet its holder
772
- * is not already entitled to.
773
- *
774
- * ## Why a malformed key refuses rather than falling back
775
- *
776
- * A request with a `fleet_key` is a request that *meant* to join by account. Quietly demanding a
777
- * code instead would surface at the client as an unexplained `PAIRING_INVALID`, which is the
778
- * silent-wrong-answer shape this codebase keeps producing; the reason is named instead.
779
- */
780
- const hasKey = typeof request.fleet_key === 'string' && request.fleet_key !== '';
781
- let rec = null;
782
- if (hasKey) {
783
- const join = planJoin({ fleetKeyEncoded: request.fleet_key });
784
- if (!join.ok) {
785
- throw new ProtocolError('PAIRING_INVALID', join.text, { reason: join.reason });
786
- }
787
- // A synthetic code record so the fleet binding below reads from one place. `fleet_id` is the
788
- // DERIVED value, never anything the request claimed.
789
- rec = { code: null, fleet_id: join.fleetId, from_fleet_key: true };
790
- } else {
791
- rec = this.validatePairingCode(request.pairing_code);
792
- }
793
-
794
- const machineId = request.machine_id;
795
- if (typeof machineId !== 'string' || machineId.length === 0) {
796
- throw new ProtocolError('BAD_REQUEST', 'machine_id is required');
797
- }
798
- const now = this.nowMs();
799
- const token = this.newToken();
800
- const existing = this.devices.get(machineId);
801
- if (existing?.device_token) this.tokens.delete(existing.device_token);
802
-
803
- const device = {
804
- machine_id: machineId,
805
- machine_name: typeof request.machine_name === 'string' && request.machine_name
806
- ? request.machine_name : machineId.slice(0, 12),
807
- platform: request.platform ?? {},
808
- caps: request.caps ?? {},
809
- user_id: request.user_id ?? null,
810
- // v0.7.7: which fleet this device belongs to, and where that fact came from.
811
- //
812
- // **The fleet comes from the pairing code, not from the request.** A device that could
813
- // name its own fleet would join any fleet, and every signature would faithfully attest
814
- // to it. `deviceFleetFields` is the single place that binding is made, and it records
815
- // its provenance so a fleet that arrived from a request is visible as such in
816
- // `devices.json` rather than looking identical to one that came from a code.
817
- //
818
- // v0.8.1 adds the third source: an account-derived `fleet_key`. That is what lets a second
819
- // machine on one account join with **no human step and no relay configuration** -- the gap
820
- // that left a new PC invisible beside its own fleet. The relay does not trust a claimed
821
- // fleet id; it *derives* one from the key, so a device can only land in the fleet whose key
822
- // it holds and anything else forms a separate, empty fleet.
823
- ...deviceFleetFields({
824
- codeRecord: rec,
825
- fleetKeyEncoded: typeof request.fleet_key === 'string' ? request.fleet_key : null,
826
- requestedFleet: null,
827
- }),
828
- /**
829
- * v0.7.10: the device's Ed25519 public key, if it offered one.
830
- *
831
- * A public key is not a secret, so storing it costs nothing and it is the whole basis of
832
- * per-device verification: the relay can then check a signature only *this* machine could
833
- * have made, rather than a shared secret every machine holds.
834
- *
835
- * Not validated here, deliberately. A malformed key fails at verification with a named
836
- * reason, while rejecting a pairing over it would lock out every existing device the
837
- * moment this field was added -- and devices that send no key keep working, with
838
- * `public_key: null` meaning "signs with the shared secret or not at all".
839
- */
840
- public_key: typeof request.public_key === 'string' && request.public_key !== '' ? request.public_key : null,
841
- device_token: token,
842
- paired_at_ms: now,
843
- re_paired: Boolean(existing),
844
- last_seen_at_ms: now,
845
- online: true,
846
- streams: existing?.streams ?? 0,
847
- // v0.3.0 RTT: starts unknown. `null` is "never measured"; 0 would mean "instant".
848
- rtt_ms: null,
849
- rtt_at_ms: null,
850
- last_heartbeat_at_ms: null,
851
- stream_connects: existing?.stream_connects ?? 0,
852
- };
853
- this.devices.set(machineId, device);
854
- this.tokens.set(token, machineId);
855
- rec.used = true;
856
-
857
- // A group has more than one machine, and a single-use code would leave the
858
- // operator unable to pair the second one without restarting the relay. So
859
- // consuming a code immediately mints the next one and announces it: the
860
- // operator pairs machine A, sees a fresh code, and pairs machine B with it.
861
- // `pairingCodeReusable` keeps one code alive for the whole window instead,
862
- // for scripted onboarding where nobody is watching stdout.
863
- let nextCode = null;
864
- const reusable = this.pairingCodeReusable === true;
865
- if (reusable) {
866
- rec.used = false;
867
- } else {
868
- nextCode = this.rotatePairingCode();
869
- }
870
-
871
- this.emit('peer.hello', {
872
- machine_id: machineId,
873
- machine_name: device.machine_name,
874
- platform: device.platform,
875
- });
876
-
877
- if (nextCode) {
878
- this.emit('notice', {
879
- level: 'info',
880
- code: 'PAIRING_CODE_ROTATED',
881
- message: 'pairing code consumed; a new one is available',
882
- pairing_code: nextCode,
883
- });
884
- }
885
-
886
- // §7: a device change is snapshotted immediately.
887
- this._saveDevices();
888
- this._ledger({
889
- type: 'device.paired',
890
- machine_id: machineId,
891
- machine_name: device.machine_name,
892
- re_paired: device.re_paired,
893
- });
894
-
895
- return {
896
- device_token: token,
897
- protocol_version: PROTOCOL_VERSION,
898
- rabbit_time: rfc3339(now),
899
- ...(nextCode ? { next_pairing_code: nextCode } : {}),
900
- };
901
- }
902
-
903
- /** @returns {object|null} device for a bearer token. */
904
- authenticate(token) {
905
- if (typeof token !== 'string' || token.length === 0) return null;
906
- const machineId = this.tokens.get(token);
907
- if (!machineId) return null;
908
- return this.devices.get(machineId) ?? null;
909
- }
910
-
911
- touchDevice(machineId, { online = true } = {}) {
912
- const device = this.devices.get(machineId);
913
- if (!device) return null;
914
- device.last_seen_at_ms = this.nowMs();
915
- if (online) device.online = true;
916
- return device;
917
- }
918
-
919
- setOnline(machineId, online, reason = null) {
920
- const device = this.devices.get(machineId);
921
- if (!device) return null;
922
- if (device.online === online) return device;
923
- device.online = online;
924
- if (online) {
925
- this.touchDevice(machineId);
926
- this.emit('peer.hello', {
927
- machine_id: machineId,
928
- machine_name: device.machine_name,
929
- platform: device.platform,
930
- });
931
- } else {
932
- this.emit('peer.bye', { machine_id: machineId, reason: reason ?? 'stream_closed' });
933
- }
934
- return device;
935
- }
936
-
937
- /* ---------------- RTT (v0.3.0 §R) ---------------- */
938
-
939
- /**
940
- * Validate an inbound `rtt_ms`.
941
- *
942
- * The contract is deliberately liberal: a MISSING or INVALID value must never
943
- * reject the heartbeat and must never clobber a previously known good value.
944
- * RTT is a diagnostic; it must not be able to break lease renewal.
945
- *
946
- * Sanity ceiling: anything above `MAX_RTT_MS` (24h) is treated as invalid. A
947
- * single garbage value would otherwise poison the fleet aggregate.
948
- */
949
- static normalizeRtt(value) {
950
- if (value === undefined) return { provided: false, valid: false };
951
- if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
952
- return { provided: true, valid: false };
953
- }
954
- if (value > MAX_RTT_MS) return { provided: true, valid: false };
955
- return { provided: true, valid: true, value };
956
- }
957
-
958
- /**
959
- * RTT is a VOLATILE diagnostic: it is intentionally NOT written to
960
- * `devices.json` and NOT replayed from the ledger, so a restarted relay reports
961
- * `rtt_ms: null` until a machine heartbeats again. A persisted value would be
962
- * indistinguishable from a fresh one and would let the relay claim knowledge it
963
- * lost when the process died.
964
- */
965
- rttStaleAfterMs() { return this.rttStaleMs; }
966
-
967
- /**
968
- * v0.3.0: absorb the optional `rtt_ms` carried by a heartbeat.
969
- *
970
- * Deliberately liberal, because RTT is a diagnostic and must never be able to
971
- * break lease renewal:
972
- * - the field is optional; a v0.2.3 agent never sends it
973
- * - a MISSING or INVALID value is IGNORED, keeping the previous value, and the
974
- * heartbeat still succeeds
975
- * - unknown extra fields are ignored entirely (forward compatibility)
976
- *
977
- * `last_heartbeat_at` advances on EVERY accepted heartbeat, with or without a
978
- * usable RTT: it answers "is the machine alive", while `rtt_at` answers "when
979
- * was this number measured". Keeping them separate is what lets a reader tell a
980
- * genuinely slow machine from one that is simply gone.
981
- */
982
- recordHeartbeatMetrics(machineId, input = {}, nowMs = this.nowMs()) {
983
- const device = this.devices.get(machineId);
984
- if (!device) return null;
985
- device.last_heartbeat_at_ms = nowMs;
986
- const rtt = RabbitState.normalizeRtt(input.rtt_ms);
987
- if (rtt.valid) {
988
- device.rtt_ms = rtt.value;
989
- device.rtt_at_ms = nowMs;
990
- }
991
- return device;
992
- }
993
-
994
- /** True when there is no value, or the value is older than the staleness window. */
995
- isRttStale(device, nowMs = this.nowMs()) {
996
- if (!device || device.rtt_ms === null || device.rtt_ms === undefined) return true;
997
- if (device.rtt_at_ms === null || device.rtt_at_ms === undefined) return true;
998
- return nowMs - device.rtt_at_ms > this.rttStaleMs;
999
- }
1000
-
1001
- /** How old the RTT measurement is, in ms; null when there is none. */
1002
- rttAgeMs(device, nowMs = this.nowMs()) {
1003
- if (!device || device.rtt_at_ms === null || device.rtt_at_ms === undefined) return null;
1004
- return Math.max(0, nowMs - device.rtt_at_ms);
1005
- }
1006
-
1007
- /** Record that a machine attached a stream (a reconnect is any attach after the first). */
1008
- noteStreamConnect(machineId) {
1009
- const device = this.devices.get(machineId);
1010
- if (!device) return null;
1011
- device.stream_connects = (device.stream_connects ?? 0) + 1;
1012
- return device.stream_connects;
1013
- }
1014
-
1015
- /**
1016
- * v0.3.0 aggregate for `/healthz`.
1017
- *
1018
- * Only FRESH measurements count. A machine that has been gone for a day must not
1019
- * drag min/max/avg around, and with no fresh data every numeric field is null --
1020
- * never 0, which would read as "instantaneous".
1021
- */
1022
- rttSummary(nowMs = this.nowMs()) {
1023
- const fresh = [];
1024
- let stale = 0;
1025
- let unknown = 0;
1026
- for (const device of this.devices.values()) {
1027
- if (device.rtt_ms === null || device.rtt_ms === undefined) { unknown += 1; continue; }
1028
- if (this.isRttStale(device, nowMs)) { stale += 1; continue; }
1029
- fresh.push(device.rtt_ms);
1030
- }
1031
- if (fresh.length === 0) {
1032
- return {
1033
- machines_reporting: 0,
1034
- min_ms: null,
1035
- max_ms: null,
1036
- avg_ms: null,
1037
- machines_stale: stale,
1038
- machines_unknown: unknown,
1039
- };
1040
- }
1041
- const sum = fresh.reduce((a, b) => a + b, 0);
1042
- return {
1043
- machines_reporting: fresh.length,
1044
- min_ms: Math.min(...fresh),
1045
- max_ms: Math.max(...fresh),
1046
- avg_ms: Math.round((sum / fresh.length) * 10) / 10,
1047
- machines_stale: stale,
1048
- machines_unknown: unknown,
1049
- };
1050
- }
1051
-
1052
- /**
1053
- * v0.3.0 single-machine status view.
1054
- *
1055
- * `last_heartbeat_at` and `rtt_at` answer different questions, and the gap
1056
- * between them is the whole point: a machine can be heartbeating (alive) while
1057
- * its RTT number is old (a v0.2.3 agent that never sends `rtt_ms`), and a
1058
- * machine can be gone while a plausible-looking RTT is still on file.
1059
- */
1060
- deviceStatus(machineId, nowMs = this.nowMs()) {
1061
- const device = this.devices.get(machineId);
1062
- if (!device) return null;
1063
- const rttAt = device.rtt_at_ms ?? null;
1064
- const lastHeartbeat = device.last_heartbeat_at_ms ?? null;
1065
- return {
1066
- machine_id: device.machine_id,
1067
- machine_name: device.machine_name,
1068
- relay_id: this.relayId,
1069
- connected: (device.streams ?? 0) > 0,
1070
- streams: device.streams ?? 0,
1071
- online: device.online === true,
1072
- rtt_ms: device.rtt_ms ?? null,
1073
- rtt_at: rttAt === null ? null : rfc3339(rttAt),
1074
- rtt_age_ms: this.rttAgeMs(device, nowMs),
1075
- rtt_stale: this.isRttStale(device, nowMs),
1076
- last_heartbeat_at: lastHeartbeat === null ? null : rfc3339(lastHeartbeat),
1077
- last_seen_at: device.last_seen_at_ms === null || device.last_seen_at_ms === undefined
1078
- ? null : rfc3339(device.last_seen_at_ms),
1079
- reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
1080
- stream_connects: device.stream_connects ?? 0,
1081
- paired_at: device.paired_at_ms === null || device.paired_at_ms === undefined
1082
- ? null : rfc3339(device.paired_at_ms),
1083
- platform: device.platform,
1084
- caps: device.caps,
1085
- user_id: device.user_id ?? null,
1086
- now: rfc3339(nowMs),
1087
- };
1088
- }
1089
-
1090
- /** Public device view — never leaks device_token. */
1091
- publicDevice(device) {
1092
- return {
1093
- machine_id: device.machine_id,
1094
- machine_name: device.machine_name,
1095
- platform: device.platform,
1096
- caps: device.caps,
1097
- user_id: device.user_id,
1098
- /**
1099
- * v0.7.7: the fleet this device belongs to, and where the fact came from.
1100
- *
1101
- * **Carried in the public shape on purpose, and it has to be**, because `devices.json`
1102
- * is written from this shape -- a fleet that lived only on the in-memory record would
1103
- * vanish on the next restart, which is precisely when an operator is looking at it. A
1104
- * first version omitted these and the live test caught the fleet disappearing from
1105
- * `listDevices()`.
1106
- *
1107
- * Safe to expose: `fleet_id` is a truncated hash of the account id, not the account id,
1108
- * and devices on one relay can already enumerate each other. Narrowing `/v1/devices` to
1109
- * a device's own fleet is a further step, recorded as such rather than implied.
1110
- */
1111
- fleet_id: device.fleet_id ?? null,
1112
- fleet_source: device.fleet_source ?? null,
1113
- // v0.7.10: a boolean rather than the key itself. The key is not a secret, but a device
1114
- // list is read by every device, and there is no reason to hand every machine every other
1115
- // machine's key when the verification happens relay-side.
1116
- public_key_present: typeof device.public_key === 'string' && device.public_key !== '',
1117
- online: device.online,
1118
- streams: device.streams,
1119
- paired_at: rfc3339(device.paired_at_ms),
1120
- last_seen_at: rfc3339(device.last_seen_at_ms),
1121
- // v0.3.0: `null` means "never measured", which is NOT the same as 0ms.
1122
- rtt_ms: device.rtt_ms ?? null,
1123
- rtt_at: device.rtt_at_ms === null || device.rtt_at_ms === undefined ? null : rfc3339(device.rtt_at_ms),
1124
- rtt_age_ms: this.rttAgeMs(device),
1125
- rtt_stale: this.isRttStale(device),
1126
- last_heartbeat_at: device.last_heartbeat_at_ms === null || device.last_heartbeat_at_ms === undefined
1127
- ? null : rfc3339(device.last_heartbeat_at_ms),
1128
- reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
1129
- };
1130
- }
1131
-
1132
- listDevices() { return [...this.devices.values()].map((d) => this.publicDevice(d)); }
1133
-
1134
- /* ---------------- tasks (§4) ---------------- */
1135
-
1136
- /**
1137
- * v0.4.0: validate the routing metadata a `POST /v1/task` body may carry.
1138
- *
1139
- * A machine, not a mode, is `origin_machine_id`: where the task was typed (`device.json`), so an
1140
- * offer can carry that fact to the machine that has to run it. `p2p.mode` is the dispatcher's
1141
- * resolved mode for this particular task — the plugin may be configured `direct` while a
1142
- * scheduled run is `relay`.
1143
- *
1144
- * Validation is deliberately shallow and strictly typed, in the house style of this file: a
1145
- * missing value is the v0.3.9 shape and stays valid, while a present-but-wrong value is refused
1146
- * by name. Accepting a non-string here would put a number into `origin_machine_id` and only fail
1147
- * later, on a machine that cannot tell a bad field from a bad relay.
1148
- *
1149
- * @returns {{origin_machine_id:string|null, p2p:{mode:string}|null}}
1150
- */
1151
- static validateTaskRouting(input = {}) {
1152
- let originMachineId = null;
1153
- if (input.origin_machine_id !== undefined && input.origin_machine_id !== null) {
1154
- if (typeof input.origin_machine_id !== 'string'
1155
- || input.origin_machine_id.length === 0
1156
- || input.origin_machine_id.length > MAX_ORIGIN_MACHINE_ID_LENGTH) {
1157
- throw new ProtocolError(
1158
- 'BAD_REQUEST',
1159
- `origin_machine_id must be a non-empty string of at most ${MAX_ORIGIN_MACHINE_ID_LENGTH} characters`,
1160
- { origin_machine_id: input.origin_machine_id },
1161
- );
1162
- }
1163
- originMachineId = input.origin_machine_id;
1164
- }
1165
-
1166
- let p2p = null;
1167
- if (input.p2p !== undefined && input.p2p !== null) {
1168
- if (typeof input.p2p !== 'object' || Array.isArray(input.p2p)) {
1169
- throw new ProtocolError('BAD_REQUEST', 'p2p must be an object with a `mode` field', { p2p: input.p2p });
1170
- }
1171
- if (!P2P_MODES.includes(input.p2p.mode)) {
1172
- throw new ProtocolError(
1173
- 'BAD_REQUEST',
1174
- `p2p.mode must be one of ${P2P_MODES.join('|')}`,
1175
- { 'p2p.mode': input.p2p.mode ?? null },
1176
- );
1177
- }
1178
- // Copied rather than referenced: the recorded task must not be mutable through the request
1179
- // body the caller still holds.
1180
- p2p = { ...input.p2p };
1181
- }
1182
-
1183
- return { origin_machine_id: originMachineId, p2p };
1184
- }
1185
-
1186
- /**
1187
- * v0.4.0 §5.2: validate the optional transport fields of a result envelope.
1188
- *
1189
- * Same reasoning as `validateTaskRouting`: `transport` is an enum and `p2p` is an opaque object,
1190
- * so anything else is refused by name at the door. Both fields stay OUT of
1191
- * `REQUIRED_ENVELOPE_FIELDS` (a missing one is a v0.3.9 envelope, not an incomplete one) and out
1192
- * of `COMPARABLE_FIELDS` (a path is not a result).
1193
- */
1194
- static validateTransportFields(envelope = {}) {
1195
- if (envelope.transport !== undefined && envelope.transport !== null) {
1196
- if (!TRANSPORT_VALUES.includes(envelope.transport)) {
1197
- throw new ProtocolError(
1198
- 'BAD_REQUEST',
1199
- `transport must be one of ${TRANSPORT_VALUES.join('|')}`,
1200
- { transport: envelope.transport },
1201
- );
1202
- }
1203
- }
1204
- if (envelope.p2p !== undefined && envelope.p2p !== null
1205
- && (typeof envelope.p2p !== 'object' || Array.isArray(envelope.p2p))) {
1206
- throw new ProtocolError('BAD_REQUEST', 'p2p must be an object', { p2p: envelope.p2p });
1207
- }
1208
- }
1209
-
1210
- /**
1211
- * §4.1 POST /v1/task.
1212
- * Machine selection is not specified by the PROTOCOL; we implement:
1213
- * - `target_machines` (optional explicit list), else every paired device
1214
- * - capability gate §6.1 per machine (failures become `refused` leases)
1215
- * - index assignment: replicate -> 0, split -> round-robin modulo index_total
1216
- */
1217
- createTask(input = {}) {
1218
- // v0.4.0: refused BEFORE the task literal is built and before any offer is emitted, so a bad
1219
- // value cannot leave a half-created task behind. `POST /v1/task` is all-or-nothing.
1220
- const routing = RabbitState.validateTaskRouting(input);
1221
-
1222
- const mode = input.mode ?? 'replicate';
1223
- if (mode !== 'replicate' && mode !== 'split' && mode !== 'broadcast' && mode !== 'pipeline' && mode !== 'compose') {
1224
- throw new ProtocolError(
1225
- 'BAD_REQUEST',
1226
- 'mode must be replicate|split|broadcast|pipeline|compose',
1227
- { mode },
1228
- );
1229
- }
1230
- /**
1231
- * v0.3.3 `pipeline`: the whole chain runs on each machine, stage after stage.
1232
- *
1233
- * The semantics were chosen against the alternative (fan out stage 1, converge its outputs into
1234
- * stage 2) because that one has no well-defined input: on a fleet, stage 1 produces n different
1235
- * outputs, and "feed them all to stage 2" does not say in what order, or whether stage 2 then
1236
- * runs once or n times. Worse, two machines finishing in a different order would produce
1237
- * different stage-2 inputs, so one task_id would no longer describe one reproducible execution --
1238
- * which is the property the rest of this system is built on. See docs/PIPELINE-DESIGN.md.
1239
- *
1240
- * So: a pipeline is `replicate` applied to a sequence. Every machine runs the whole chain, there
1241
- * is no cross-machine data flow, and nothing new can be silently dropped in transit.
1242
- */
1243
- const rawStages = Array.isArray(input.stages) ? input.stages : null;
1244
- const isPipeline = mode === 'pipeline';
1245
- if (isPipeline) {
1246
- if (rawStages === null || rawStages.length === 0) {
1247
- throw new ProtocolError('BAD_REQUEST', 'mode=pipeline requires a non-empty `stages` array', {
1248
- stages: rawStages === null ? null : rawStages.length,
1249
- });
1250
- }
1251
- if (rawStages.length > MAX_PIPELINE_STAGES) {
1252
- throw new ProtocolError('BAD_REQUEST', `stages must have at most ${MAX_PIPELINE_STAGES} entries`, {
1253
- stages: rawStages.length,
1254
- });
1255
- }
1256
- rawStages.forEach((stage, i) => {
1257
- const argv = stage?.command_argv;
1258
- if (!Array.isArray(argv) || argv.length === 0 || argv.some((a) => typeof a !== 'string')) {
1259
- throw new ProtocolError('BAD_REQUEST', `stages[${i}].command_argv must be a non-empty string[]`, { index: i });
1260
- }
1261
- });
1262
- // A chain is one execution on one machine; sharding it would mean each machine ran a chain of
1263
- // its own slice, which is a different feature and not what the caller asked for.
1264
- if (Number.isInteger(input.index_total) && input.index_total !== 1) {
1265
- throw new ProtocolError('BAD_REQUEST', 'mode=pipeline runs the whole chain per machine, so index_total must be 1', {
1266
- index_total: input.index_total,
1267
- });
1268
- }
1269
- } else if (rawStages !== null) {
1270
- // Refused rather than ignored: the caller believes they described a sequence, and running only
1271
- // `command_argv` would silently drop every stage after the first.
1272
- throw new ProtocolError('BAD_REQUEST', '`stages` is only valid with mode=pipeline', { mode, stages: rawStages.length });
1273
- }
1274
-
1275
- /**
1276
- * Normalized stage list. For every mode there is at least one stage, so downstream code has a
1277
- * single shape to reason about and `pipeline` is not a special case at the execution layer.
1278
- */
1279
- const stages = isPipeline
1280
- ? rawStages.map((s, i) => ({
1281
- index: i,
1282
- command_argv: [...s.command_argv],
1283
- cwd_rel: typeof s.cwd_rel === 'string' && s.cwd_rel ? s.cwd_rel : (typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.'),
1284
- timeout_ms: Number.isInteger(s.timeout_ms) ? s.timeout_ms : null,
1285
- continue_on_failure: s.continue_on_failure === true,
1286
- }))
1287
- : null;
1288
-
1289
- const commandArgv = isPipeline ? [...rawStages[0].command_argv] : input.command_argv;
1290
- if (!Array.isArray(commandArgv) || commandArgv.length === 0 || commandArgv.some((a) => typeof a !== 'string')) {
1291
- throw new ProtocolError('BAD_REQUEST', 'command_argv must be a non-empty string[]');
1292
- }
1293
- const indexTotal = Number.isInteger(input.index_total) ? input.index_total : 1;
1294
- if (indexTotal < 1) throw new ProtocolError('BAD_REQUEST', 'index_total must be >= 1');
1295
-
1296
- const halt = input.halt ?? 'never';
1297
- if (halt !== 'never' && halt !== 'now') throw new ProtocolError('BAD_REQUEST', 'halt must be never|now', { halt });
1298
-
1299
- const taskId = input.task_id ?? this.nextId();
1300
- if (this.tasks.has(taskId)) {
1301
- throw new ProtocolError('TASK_EXISTS', 'task_id already exists', { task_id: taskId });
1302
- }
1303
-
1304
- const now = this.nowMs();
1305
- const cwdRel = typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.';
1306
- const baseTree = input.base_tree ?? null;
1307
- const commandHash = input.command_hash ?? (isPipeline
1308
- ? computePipelineCommandHash(stages, SHELL_ID_DIRECT)
1309
- : computeCommandHash(commandArgv, SHELL_ID_DIRECT, cwdRel));
1310
- const timeoutMs = Number.isInteger(input.timeout_ms) ? input.timeout_ms : 300_000;
1311
-
1312
- // v0.3.3 `broadcast`: one machine runs the command, every other machine is told the outcome.
1313
- //
1314
- // The executor choice is deterministic rather than random. A random leader makes a task
1315
- // impossible to reproduce from its own record -- the same task id would run somewhere else on a
1316
- // retry -- and this project's whole value rests on being able to say exactly where something ran.
1317
- // The first target in the relay's own device order is stable for a given device table.
1318
- //
1319
- // Observers get a lease so they are tracked and notified, but their lease is `observing`, which
1320
- // every aggregation step treats as "did not run". That distinction is the point: a machine that
1321
- // never executed anything agreeing with a result would be a fabrication, and it is exactly the
1322
- // fabrication this mode could most easily produce.
1323
- //
1324
- // Resolved before the task literal because the literal records it. (`targets` is computed below,
1325
- // so the executor is re-validated against it after that.)
1326
- const targetIds = Array.isArray(input.target_machines) && input.target_machines.length > 0
1327
- ? input.target_machines
1328
- : [...this.devices.keys()];
1329
- const executorMachineId = mode === 'broadcast'
1330
- ? (typeof input.executor_machine_id === 'string' && input.executor_machine_id !== ''
1331
- ? input.executor_machine_id
1332
- : targetIds[0] ?? null)
1333
- : null;
1334
-
1335
- const task = {
1336
- task_id: taskId,
1337
- mode,
1338
- command_argv: commandArgv,
1339
- cwd_rel: cwdRel,
1340
- index_total: indexTotal,
1341
- timeout_ms: timeoutMs,
1342
- write: input.write === true,
1343
- write_scope: input.write_scope ?? [],
1344
- require_exclusive_write: input.require_exclusive_write === true,
1345
- base_commit: input.base_commit ?? null,
1346
- base_tree: baseTree,
1347
- requirements: input.requirements ?? {},
1348
- compare_policy: input.compare_policy ?? {},
1349
- halt,
1350
- created_by: input.created_by ?? null,
1351
- created_at_ms: now,
1352
- command_hash: commandHash,
1353
- attempt: 1,
1354
- leases: new Map(),
1355
- cancelled: false,
1356
- cancel_reason: null,
1357
- degraded: null,
1358
- deadline_ms: now + timeoutMs,
1359
- result_seqs: [],
1360
- /** v0.3.3: which machine actually executes under `broadcast`; null for the other modes. */
1361
- executor_machine_id: executorMachineId,
1362
- /**
1363
- * v0.3.3: the normalized stage list for `pipeline`, or null.
1364
- *
1365
- * Sent to the machine in the offer so it can run the whole chain. Kept on the task as well so
1366
- * a report can say how many stages were supposed to run even when the machine crashed before
1367
- * reporting any of them.
1368
- */
1369
- stages,
1370
- /**
1371
- * v0.4.0: where the task came from and under which P2P mode it was dispatched, or null.
1372
- *
1373
- * Kept on the task (not just passed to `emitOffer`) because an offer is emitted more than
1374
- * once: `redeliverPendingOffers` re-states unclaimed work on every reconnect and
1375
- * `_takeover` re-offers a freed index. A machine that reconnects must see the same routing
1376
- * facts as the machine that was connected at dispatch time, or the direct-push half of the
1377
- * feature would work exactly once and then quietly stop.
1378
- */
1379
- origin_machine_id: routing.origin_machine_id,
1380
- p2p: routing.p2p,
1381
- };
1382
-
1383
- /**
1384
- * Fleet scoping (v0.7.7).
1385
- *
1386
- * This replaces `targets = [...this.devices.values()]`, which reached **every device on the
1387
- * relay** with no filter — the finding that accounts were not isolated. The rule now lives in
1388
- * `fleet-wiring.mjs` so it is one function with its own exhaustive tests rather than three
1389
- * lines inside a two-thousand-line object.
1390
- *
1391
- * ## The migration is deliberate, and so is its limit
1392
- *
1393
- * The shared relay holds four devices that paired before fleets existed, and none of them has
1394
- * a `fleet_id`. `targetsFor` therefore has two modes: a relay with no fleet configured keeps
1395
- * exactly the old behaviour, and a relay that *is* scoped refuses a fleetless requester rather
1396
- * than handing it the run of the fleet. So deploying this cannot be an outage, and scoping it
1397
- * cannot be a silent grant.
1398
- *
1399
- * ## What this does NOT do, stated here rather than discovered later
1400
- *
1401
- * `POST /v1/task` authenticates with the **operator token**, and `authorizeTaskDispatch`
1402
- * returns `{ role: 'operator', machine_id: null }` for it — there is no device, so there is no
1403
- * fleet to scope by. Every device that authenticates with a **device token** gets full
1404
- * isolation from this change; an operator-token dispatch remains relay-wide, because the
1405
- * operator token *is* the relay-wide credential by design.
1406
- *
1407
- * `input.created_by` is the one device identity a task carries, and it is used when present:
1408
- * a task submitted on behalf of a machine is scoped to that machine's fleet.
1409
- */
1410
- const requester =
1411
- (typeof input.created_by === 'string' && this.devices.get(input.created_by)) ||
1412
- (typeof input.requester_machine_id === 'string' && this.devices.get(input.requester_machine_id)) ||
1413
- null;
1414
-
1415
- /**
1416
- * An operator-token dispatch, on a relay that has been scoped, acts **within that scope**.
1417
- *
1418
- * A first version passed `requester: null` and let it be refused -- which broke every
1419
- * operator dispatch on a scoped relay, and broke it in the least useful way: the operator had
1420
- * just configured the fleet, and configuring it made them unable to dispatch at all.
1421
- *
1422
- * The reasoning for the fix: the relay's own `fleetId` *is* an admin's statement about which
1423
- * fleet this relay serves. An operator dispatch therefore has an unambiguous scope, and
1424
- * refusing it would be refusing the case that was explicitly asked for.
1425
- *
1426
- * What stays genuinely ambiguous is an operator dispatch on a relay holding **several** fleets
1427
- * with no fleet of its own -- there is no way to tell which was meant, so `targetsFor` refuses
1428
- * that rather than guessing.
1429
- */
1430
- const scopedRequester =
1431
- requester ??
1432
- (this.fleetId === null ? null : { machine_id: null, fleet_id: this.fleetId, fleet_source: 'relay_scope' });
1433
-
1434
- const scoped = targetsFor({
1435
- devices: [...this.devices.values()],
1436
- requester: scopedRequester,
1437
- targetMachines: input.target_machines,
1438
- relayFleetId: this.fleetId ?? null,
1439
- // The existing code did not filter offline devices at selection time -- it offered to them and
1440
- // let the lease expire. Keeping that here avoids changing two behaviours at once, and the
1441
- // lease expiry is what `handoff.mjs` is for.
1442
- requireOnline: false,
1443
- });
1444
- if (!scoped.ok) {
1445
- /**
1446
- * `NO_ONLINE_DEVICE`, not a new code.
1447
- *
1448
- * A first version invented `NO_ELIGIBLE_DEVICE`, which broke the one relay test that pins
1449
- * this contract — and worse, it would have broken **callers**: `chatops/fleet.mjs` reads
1450
- * `NO_ONLINE_DEVICE` to produce *"start an agent on each machine"*, and a code it has never
1451
- * seen falls through to a generic refusal. The protocol's code table is part of the
1452
- * protocol, and adding a member to it is a decision for a protocol revision, not for a
1453
- * wiring change.
1454
- *
1455
- * What *is* new is why nothing was eligible, and that belongs in `detail`, where a caller
1456
- * can read it without a new code entering the table.
1457
- */
1458
- throw new ProtocolError('NO_ONLINE_DEVICE', scoped.text, {
1459
- reason: scoped.reason,
1460
- legacy: scoped.legacy,
1461
- fleet_scoped: scoped.legacy === false,
1462
- hint:
1463
- scoped.reason === 'requester_has_no_fleet'
1464
- ? 'this relay is bound to a fleet and the requesting device is in none; pair it with a code issued for that fleet'
1465
- : scoped.reason === 'target_outside_fleet'
1466
- ? 'the named target is not in this fleet'
1467
- : 'check that at least one device in this fleet is paired and streaming',
1468
- });
1469
- }
1470
- const targets = scoped.targets;
1471
- if (scoped.legacy) {
1472
- // Logged at debug so an operator reading logs during a migration can see which rule applied.
1473
- // "N targets" alone does not say whether isolation is on, and that is the question being asked.
1474
- this.log?.('debug', 'task targets selected without a fleet (legacy path)', {
1475
- task_candidates: targets.length,
1476
- relay_fleet_id: this.fleetId ?? null,
1477
- });
1478
- }
1479
-
1480
- // v0.3.3 `broadcast`: validated here, where `targets` is known. The executor id itself was
1481
- // resolved before the task literal because the literal records it.
1482
- if (mode === 'broadcast' && executorMachineId !== null && !targets.some((d) => d.machine_id === executorMachineId)) {
1483
- throw new ProtocolError('BAD_REQUEST', 'executor_machine_id is not among the target machines', {
1484
- executor_machine_id: executorMachineId,
1485
- targets: targets.map((d) => d.machine_id),
1486
- });
1487
- }
1488
-
1489
- if (targets.length === 0) {
1490
- throw new ProtocolError('NO_ONLINE_DEVICE', 'no paired device available for this task', {
1491
- known_devices: all.length,
1492
- });
1493
- }
1494
-
1495
- const dedupeKeyBase = computeDedupeKey(taskId, 0, commandHash, baseTree);
1496
- if (typeof dedupeKeyBase !== 'string' || dedupeKeyBase.length !== 64) {
1497
- throw new ProtocolError('INTERNAL', 'failed to compute dedupe_key');
1498
- }
1499
- let lastSeq = this.seq;
1500
- targets.forEach((device, i) => {
1501
- const index = mode === 'replicate' ? 0 : i % indexTotal;
1502
- const observing = mode === 'broadcast' && device.machine_id !== executorMachineId;
1503
- const lease = {
1504
- machine_id: device.machine_id,
1505
- machine_name: device.machine_name,
1506
- index,
1507
- attempt: 1,
1508
- // `observing` is a lease state, not an outcome: the machine holds a place in this task and
1509
- // will be told what happened, but it was never handed the command. `gateMachine` below is
1510
- // skipped for it because refusing to do work you were never asked to do is not a refusal.
1511
- state: observing ? 'observing' : 'queued',
1512
- refusal_reason: null,
1513
- refusal_detail: null,
1514
- dedupe_key: computeDedupeKey(taskId, index, commandHash, baseTree),
1515
- last_heartbeat_ms: now,
1516
- claimed_at_ms: now,
1517
- lease_until_ms: now + this.leaseWindowMs(),
1518
- phase: null,
1519
- progress: null,
1520
- offered_seq: null,
1521
- expired_at_ms: null,
1522
- finished_at_ms: null,
1523
- heartbeat_count: 0,
1524
- };
1525
- if (observing) {
1526
- // No capability gate and no offer: the machine is not being asked to run anything, so there
1527
- // is nothing to refuse and nothing to hand out. It is recorded so the task can tell it the
1528
- // outcome afterwards.
1529
- task.leases.set(device.machine_id, lease);
1530
- return;
1531
- }
1532
- const gate = gateMachine(task, device);
1533
- if (gate) {
1534
- lease.state = 'refused';
1535
- lease.refusal_reason = gate.refusal_reason;
1536
- lease.refusal_detail = gate.detail;
1537
- task.leases.set(device.machine_id, lease);
1538
- this.emit('notice', {
1539
- level: 'warn',
1540
- code: 'CAPABILITY_REFUSED',
1541
- message: `${device.machine_id} refused: ${gate.refusal_reason}`,
1542
- task_id: taskId,
1543
- });
1544
- return;
1545
- }
1546
- task.leases.set(device.machine_id, lease);
1547
- const grace = this._p2pOfferGraceMs(task);
1548
- if (grace > 0) {
1549
- // v0.4.0: the dispatcher is pushing this offer over the direct path right now. Hold the
1550
- // relay's copy for `grace` ms so the direct one can win; see `p2pOfferGraceMs`.
1551
- this._scheduleOffer(task, lease, grace);
1552
- } else {
1553
- const event = this.emitOffer(task, lease);
1554
- lastSeq = event.seq;
1555
- }
1556
- });
1557
-
1558
- this.tasks.set(taskId, task);
1559
-
1560
- this._ledger({
1561
- type: 'task.created',
1562
- task: {
1563
- task_id: task.task_id,
1564
- mode: task.mode,
1565
- command_argv: task.command_argv,
1566
- cwd_rel: task.cwd_rel,
1567
- index_total: task.index_total,
1568
- timeout_ms: task.timeout_ms,
1569
- write: task.write,
1570
- write_scope: task.write_scope,
1571
- require_exclusive_write: task.require_exclusive_write,
1572
- base_commit: task.base_commit,
1573
- base_tree: task.base_tree,
1574
- requirements: task.requirements,
1575
- compare_policy: task.compare_policy,
1576
- halt: task.halt,
1577
- created_by: task.created_by,
1578
- created_at_ms: task.created_at_ms,
1579
- command_hash: task.command_hash,
1580
- attempt: task.attempt,
1581
- deadline_ms: task.deadline_ms,
1582
- // v0.4.0: routing metadata, so a restarted relay re-offers with the same facts.
1583
- origin_machine_id: task.origin_machine_id ?? null,
1584
- p2p: task.p2p ?? null,
1585
- },
1586
- leases: [...task.leases.values()].map((l) => ({ ...l })),
1587
- });
1588
-
1589
- const refusedOnly = [...task.leases.values()].every((l) => l.state === 'refused');
1590
- if (refusedOnly) {
1591
- // §6.3 Step 0 territory: a task nobody may execute is still a valid, reportable task.
1592
- this.emit('notice', {
1593
- level: 'warn',
1594
- code: 'NO_ONLINE_DEVICE',
1595
- message: 'every candidate machine refused the capability gate',
1596
- task_id: taskId,
1597
- });
1598
- }
1599
-
1600
- return {
1601
- task_id: taskId,
1602
- leases: [...task.leases.values()].map((l) => ({
1603
- machine_id: l.machine_id,
1604
- index: l.index,
1605
- state: l.state,
1606
- // v0.4.0: the dispatcher pushes an offer over the direct path, and the executor dedupes it
1607
- // against the relay's copy by (task_id, attempt, dedupe_key). Without this field the
1608
- // dispatcher has to reproduce the §4.4 formula locally, which works for a single-command
1609
- // task and cannot work for a pipeline one -- the key covers the relay's normalised stage
1610
- // list. Publishing it makes the pushed copy recognisably the same work item, which is what
1611
- // keeps the direct path from turning exactly-once into at-least-twice.
1612
- dedupe_key: l.dedupe_key ?? null,
1613
- attempt: l.attempt,
1614
- ...(l.refusal_reason ? { refusal_reason: l.refusal_reason } : {}),
1615
- })),
1616
- seq: lastSeq,
1617
- };
1618
- }
1619
-
1620
- /**
1621
- * v0.4.0: should the relay hold its offer back for this task?
1622
- *
1623
- * Only when the dispatcher said two things: that it is a machine (`origin_machine_id`), and that
1624
- * the direct path is in play (`p2p.mode` of `auto` or `direct`). A `relay` mode or a v0.3.9
1625
- * dispatcher gets the old behaviour exactly, which is why this predicate is the only gate on the
1626
- * change and every pre-existing test still sees an immediate offer.
1627
- *
1628
- * @param {object} task
1629
- * @returns {number} milliseconds to hold, or 0 for "offer now"
1630
- */
1631
- _p2pOfferGraceMs(task) {
1632
- if (this.p2pOfferGraceMs <= 0) return 0;
1633
- const origin = task?.origin_machine_id;
1634
- if (typeof origin !== 'string' || origin === '') return 0;
1635
- const mode = task?.p2p?.mode;
1636
- if (mode !== 'auto' && mode !== 'direct') return 0;
1637
- return this.p2pOfferGraceMs;
1638
- }
1639
-
1640
- /**
1641
- * v0.4.0: emit `lease`'s offer after `delayMs`, unless the direct path got there first.
1642
- *
1643
- * "Got there first" is one fact and it is already tracked: the executor renews the lease
1644
- * (`POST /v1/heartbeat`) as its first act on any offer it accepts, and a renewal moves the lease
1645
- * out of `queued`. So the check is the lease's own state — no new message, no new endpoint, and
1646
- * nothing the executor has to know about.
1647
- *
1648
- * The timer is `unref()`ed: a relay with nothing else to do must still be able to exit, and a
1649
- * pending offer is not a reason to keep a process alive.
1650
- */
1651
- _scheduleOffer(task, lease, delayMs) {
1652
- const key = `${task.task_id}:${lease.machine_id}`;
1653
- const timer = setTimeout(() => {
1654
- this._pendingOffers.delete(key);
1655
- // A lease that was re-created (takeover) or swept is not this one any more.
1656
- if (task.leases.get(lease.machine_id) !== lease) return;
1657
- if (lease.state !== 'queued') return; // the direct copy won; nothing to fall back to
1658
- if (this.isLeaseExpired(lease, this.nowMs())) return; // the sweep owns it now
1659
- this.emitOffer(task, lease);
1660
- }, delayMs);
1661
- timer.unref?.();
1662
- this._pendingOffers.set(key, timer);
1663
- }
1664
-
1665
- /** Drop every deferred offer. Called by the relay's own teardown path. */
1666
- clearPendingOffers() {
1667
- for (const timer of this._pendingOffers.values()) clearTimeout(timer);
1668
- this._pendingOffers.clear();
1669
- }
1670
-
1671
- emitOffer(task, lease) {
1672
- const event = this.emit('task.offer', {
1673
- task_id: task.task_id,
1674
- attempt: lease.attempt,
1675
- mode: task.mode,
1676
- index: lease.index,
1677
- index_total: task.index_total,
1678
- command_argv: task.command_argv,
1679
- // v0.3.3: present only for `pipeline`, and only then does the machine run a chain. Omitted
1680
- // entirely otherwise, so an offer for a single-command task is byte-identical to before --
1681
- // which is what keeps a v0.2.3 agent working against this relay.
1682
- ...(task.stages ? { stages: task.stages } : {}),
1683
- /**
1684
- * v0.4.0: always present, `null` when the caller supplied nothing.
1685
- *
1686
- * Deliberately NOT conditional (unlike `stages` above, which is omitted so a single-command
1687
- * offer stays byte-identical for an old agent): the contract fixes the shape of these two as
1688
- * `string|null` and `{mode}|null`, and `null` is the honest value for "the dispatcher did not
1689
- * say" — which is exactly the v0.3.9 case. They are additive keys in an event whose consumers
1690
- * already ignore unknown fields.
1691
- */
1692
- origin_machine_id: task.origin_machine_id ?? null,
1693
- p2p: task.p2p ?? null,
1694
- cwd_rel: task.cwd_rel,
1695
- write: task.write,
1696
- timeout_ms: task.timeout_ms,
1697
- base_commit: task.base_commit,
1698
- base_tree: task.base_tree,
1699
- requirements: task.requirements,
1700
- compare_policy: task.compare_policy,
1701
- dedupe_key: lease.dedupe_key,
1702
- deadline: rfc3339(task.deadline_ms),
1703
- }, { target: lease.machine_id });
1704
- lease.state = 'offered';
1705
- lease.offered_seq = event.seq;
1706
- lease.last_heartbeat_ms = this.nowMs();
1707
- lease.lease_until_ms = lease.last_heartbeat_ms + this.leaseWindowMs();
1708
- return event;
1709
- }
1710
-
1711
- getTask(taskId) { return this.tasks.get(taskId) ?? null; }
1712
-
1713
- requireTask(taskId) {
1714
- const task = this.tasks.get(taskId);
1715
- if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
1716
- return task;
1717
- }
1718
-
1719
- /**
1720
- * Re-offer the work a machine was handed but has not yet started (v0.1.2 §8.6).
1721
- *
1722
- * WHY THIS EXISTS
1723
- *
1724
- * An offer is a point-in-time event on a stream. `createTask` publishes
1725
- * `task.offer` once, and if the target machine's stream happens to be down at
1726
- * that instant -- the ordinary situation after a relay restart, a tunnel drop,
1727
- * or any cross-network interruption -- the event is simply gone. The lease sits
1728
- * in `offered` until the sweep expires it, the task never runs, and the
1729
- * operator sees a timeout with no explanation. Measured: the machine is back
1730
- * online with a live stream within milliseconds, and still receives nothing.
1731
- *
1732
- * So the relay re-states unclaimed work when a machine connects. This is safe
1733
- * because the state it acts on is durable: the lease the offer refers to is
1734
- * still in the ledger.
1735
- *
1736
- * WHY ONLY `offered` AND `queued`
1737
- *
1738
- * `running` is deliberately excluded, and that exclusion is the whole safety
1739
- * argument. A running lease means a machine has already STARTED the command --
1740
- * it acknowledged the offer and is heartbeating. Re-offering it would make that
1741
- * machine execute the work a second time, so the fix for a lost offer would
1742
- * introduce the one failure this project is built to avoid: two machines, or
1743
- * one machine twice, running the same side-effecting command.
1744
- *
1745
- * A machine that dies mid-run therefore does not get its work back here; that
1746
- * path stays with the sweep, which expires the lease and hands the index to
1747
- * another machine with `attempt + 1`.
1748
- *
1749
- * Re-emitted offers are new events and are replayed like any other, so a client
1750
- * that reconnects repeatedly can see the same offer more than once. That is
1751
- * intentional: the agent already de-duplicates by `task_id` (it ignores an
1752
- * offer whose task is queued or running), and an at-least-once offer with a
1753
- * client-side dedupe is strictly better than an at-most-once offer that is
1754
- * silently dropped.
1755
- *
1756
- * @param {string} machineId
1757
- * @returns {{count: number, task_ids: string[]}}
1758
- */
1759
- redeliverPendingOffers(machineId) {
1760
- const taskIds = [];
1761
- const now = this.nowMs();
1762
-
1763
- for (const task of this.tasks.values()) {
1764
- if (task.cancelled === true) continue;
1765
- const lease = task.leases.get(machineId);
1766
- if (!lease) continue;
1767
- // Only work the machine has not begun. See the note above: `running` is
1768
- // excluded on purpose, not by omission.
1769
- if (lease.state !== 'offered' && lease.state !== 'queued') continue;
1770
- // Do not resurrect a lease the sweep has already written off; the takeover
1771
- // path owns that decision.
1772
- if (this.isLeaseExpired(lease, now)) continue;
1773
-
1774
- this.emitOffer(task, lease);
1775
- taskIds.push(task.task_id);
1776
- }
1777
-
1778
- return { count: taskIds.length, task_ids: taskIds };
1779
- }
1780
-
1781
- listTasks({ limit = 50 } = {}) {
1782
- const n = Number.isInteger(limit) && limit > 0 ? limit : 50;
1783
- return [...this.tasks.values()]
1784
- .sort((a, b) => b.created_at_ms - a.created_at_ms)
1785
- .slice(0, n)
1786
- .map((t) => this.taskSummary(t));
1787
- }
1788
-
1789
- taskSummary(task) {
1790
- return {
1791
- task_id: task.task_id,
1792
- mode: task.mode,
1793
- created_at: rfc3339(task.created_at_ms),
1794
- created_by: task.created_by,
1795
- index_total: task.index_total,
1796
- cancelled: task.cancelled,
1797
- degraded: task.degraded,
1798
- leases: [...task.leases.values()].map((l) => l.machine_id),
1799
- lease_states: Object.fromEntries([...task.leases.values()].map((l) => [l.machine_id, l.state])),
1800
- };
1801
- }
1802
-
1803
- cancelTask(taskId, reason = 'cancelled_by_operator') {
1804
- const task = this.requireTask(taskId);
1805
- if (task.cancelled) return task;
1806
- task.cancelled = true;
1807
- task.cancel_reason = reason;
1808
- for (const lease of task.leases.values()) {
1809
- if (ACTIVE_LEASE_STATES.includes(lease.state)) lease.state = 'cancelled';
1810
- }
1811
- this.emit('task.cancel', { task_id: taskId, reason });
1812
- this._ledger({ type: 'task.cancelled', task_id: taskId, reason });
1813
- return task;
1814
- }
1815
-
1816
- /* ---------------- leases / heartbeat (§4.3) ---------------- */
1817
-
1818
- /** Shared `phase`/`progress` validation for both heartbeat modes. */
1819
- _validatePhaseProgress(input) {
1820
- if (input.phase !== undefined && input.phase !== null && !LEASE_PHASES.includes(input.phase)) {
1821
- throw new ProtocolError('BAD_REQUEST', 'phase must be preparing|running|finalizing', { phase: input.phase });
1822
- }
1823
- if (input.progress !== undefined && input.progress !== null
1824
- && (typeof input.progress !== 'number' || input.progress < 0 || input.progress > 1)) {
1825
- throw new ProtocolError('BAD_REQUEST', 'progress must be a number in [0,1]', { progress: input.progress });
1826
- }
1827
- }
1828
-
1829
- /**
1830
- * v0.3.0 IDLE (diagnostic) heartbeat — an agent with no lease keeps its latency
1831
- * and liveness visible to other machines.
1832
- *
1833
- * This path deliberately NEVER looks at `this.tasks`. That is the whole safety
1834
- * argument, and it is structural rather than a matter of remembering a rule: an
1835
- * idle heartbeat cannot renew, resurrect or otherwise touch a lease because it
1836
- * never obtains one. Without that, an idle ping from a machine whose lease had
1837
- * expired would flip `expired` back to `running` and manufacture a live task out
1838
- * of nothing -- a fake "the work is still being done" signal, which is the most
1839
- * dangerous lie this relay can tell.
1840
- */
1841
- diagnosticHeartbeat(input = {}) {
1842
- const machineId = typeof input.machine_id === 'string' && input.machine_id !== ''
1843
- ? input.machine_id : null;
1844
- if (machineId === null) {
1845
- throw new ProtocolError('BAD_REQUEST', 'machine_id is required', {});
1846
- }
1847
- if (!this.devices.has(machineId)) {
1848
- throw new ProtocolError('NOT_FOUND', 'unknown machine_id', { machine_id: machineId });
1849
- }
1850
- this._validatePhaseProgress(input);
1851
-
1852
- const device = this.touchDevice(machineId);
1853
- this.recordHeartbeatMetrics(machineId, input);
1854
- return {
1855
- // No lease is involved, and the response says so instead of inventing one.
1856
- lease_until: null,
1857
- cancel: false,
1858
- diagnostic: true,
1859
- machine_id: machineId,
1860
- rtt_ms: device?.rtt_ms ?? null,
1861
- };
1862
- }
1863
-
1864
- /**
1865
- * §4.3 POST /v1/heartbeat — renews the lease using SERVER time only.
1866
- *
1867
- * Two modes since v0.3.0:
1868
- * - `task_id` present -> LEASE heartbeat (unchanged §4.3 behaviour)
1869
- * - `diagnostic: true`, no task -> IDLE heartbeat (diagnostics only, no lease)
1870
- * - neither -> 400. A heartbeat that silently renewed
1871
- * nothing would let an agent whose `task_id` went missing stop renewing while
1872
- * believing it was fine, ending in a falsely `expired` task. §0 forbids
1873
- * silent degradation, so this is loud.
1874
- *
1875
- * @returns {{lease_until:string|null, cancel:boolean, phase?:string, progress?:number, diagnostic?:boolean}}
1876
- */
1877
- heartbeat(input = {}) {
1878
- const hasTask = typeof input.task_id === 'string' && input.task_id !== '';
1879
- if (!hasTask) {
1880
- if (input.diagnostic !== true) {
1881
- throw new ProtocolError('BAD_REQUEST',
1882
- 'a heartbeat must carry task_id (lease renewal) or diagnostic:true (idle diagnostics); '
1883
- + 'refusing to treat a missing task_id as a no-op renewal', {
1884
- received_keys: Object.keys(input).sort(),
1885
- });
1886
- }
1887
- return this.diagnosticHeartbeat(input);
1888
- }
1889
- if (input.diagnostic === true) {
1890
- // task_id wins: a heartbeat that names a task IS a lease heartbeat.
1891
- // (documented; the flag is simply redundant here)
1892
- }
1893
- return this.leaseHeartbeat(input);
1894
- }
1895
-
1896
- leaseHeartbeat(input = {}) {
1897
- const task = this.requireTask(input.task_id);
1898
- const lease = task.leases.get(input.machine_id);
1899
- if (!lease) {
1900
- throw new ProtocolError('NOT_FOUND', 'no lease for this machine on this task', {
1901
- task_id: input.task_id, machine_id: input.machine_id,
1902
- });
1903
- }
1904
- this._validatePhaseProgress(input);
1905
-
1906
- const now = this.nowMs();
1907
- this.touchDevice(input.machine_id);
1908
- this.recordHeartbeatMetrics(input.machine_id, input, now);
1909
-
1910
- if (task.cancelled) {
1911
- return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: task.cancel_reason };
1912
- }
1913
-
1914
- // §4.3: once halt=now has handed this index to another attempt, the old owner
1915
- // must stop — a late heartbeat from the expired machine is not a valid re-claim.
1916
- if (lease.state === 'expired' && task.attempt > lease.attempt) {
1917
- return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: 'LEASE_EXPIRED_TAKEN_OVER' };
1918
- }
1919
-
1920
- const stateBefore = lease.state;
1921
- lease.last_heartbeat_ms = now;
1922
- lease.lease_until_ms = now + this.leaseWindowMs();
1923
- lease.heartbeat_count += 1;
1924
- if (input.phase !== undefined && input.phase !== null) lease.phase = input.phase;
1925
- if (input.progress !== undefined && input.progress !== null) lease.progress = input.progress;
1926
- if (Number.isInteger(input.attempt) && input.attempt >= 1) lease.attempt = input.attempt;
1927
- if (lease.state === 'queued' || lease.state === 'offered') lease.state = 'running';
1928
- // A live heartbeat is stronger evidence than a timer: with halt=never nobody took
1929
- // the work away, so the late-but-alive agent re-claims its own lease.
1930
- if (lease.state === 'expired') { lease.state = 'running'; lease.expired_at_ms = null; }
1931
-
1932
- // §7 ledger records lease STATE changes, not every 10s heartbeat.
1933
- if (lease.state !== stateBefore) this._ledgerLease(task, lease);
1934
-
1935
- return { lease_until: rfc3339(lease.lease_until_ms), cancel: false, phase: lease.phase, progress: lease.progress };
1936
- }
1937
-
1938
- /** §4.3: expiry is decided by Rabbit's clock only. */
1939
- isLeaseExpired(lease, nowMs = this.nowMs()) {
1940
- if (!ACTIVE_LEASE_STATES.includes(lease.state)) return false;
1941
- return nowMs >= lease.lease_until_ms;
1942
- }
1943
-
1944
- /**
1945
- * §4.3 expiry sweep.
1946
- * - halt=never → book-keeping only, task is marked `partial`
1947
- * - halt=now → notify remaining machines immediately that they may take over
1948
- */
1949
- sweepExpired(nowMs = this.nowMs()) {
1950
- const expired = [];
1951
- for (const task of this.tasks.values()) {
1952
- if (task.cancelled) continue;
1953
- let changed = false;
1954
- for (const lease of task.leases.values()) {
1955
- if (!this.isLeaseExpired(lease, nowMs)) continue;
1956
- lease.state = 'expired';
1957
- lease.expired_at_ms = nowMs;
1958
- changed = true;
1959
- expired.push({ task, lease });
1960
- this._ledgerLease(task, lease);
1961
- this.emit('notice', {
1962
- level: 'warn',
1963
- code: 'LEASE_EXPIRED',
1964
- message: `lease expired: machine=${lease.machine_id} task=${task.task_id} attempt=${lease.attempt}`,
1965
- task_id: task.task_id,
1966
- machine_id: lease.machine_id,
1967
- });
1968
- if (task.halt === 'now') {
1969
- task.attempt += 1;
1970
- this._takeover(task, lease, nowMs);
1971
- }
1972
- }
1973
- if (changed) task.degraded = 'partial';
1974
- }
1975
- return expired;
1976
- }
1977
-
1978
- /**
1979
- * §4.3 `halt=now`: "立即通知其他机器可接管".
1980
- *
1981
- * v0.0.1's PROTOCOL defines no per-index lease reassignment, so takeover is
1982
- * implemented as: re-offer the freed `index` (attempt+1) to every live machine
1983
- * that is idle — no lease yet, or a lease still queued/offered — and broadcast a
1984
- * notice. Machines already `running` are never re-offered, to avoid double
1985
- * execution of the same index.
1986
- */
1987
- _takeover(task, expiredLease, nowMs = this.nowMs()) {
1988
- /**
1989
- * v0.7.8: the two safety guards this path was missing.
1990
- *
1991
- * `_takeover` already existed and is invoked on `halt=now`, so hand-off was not absent — but it
1992
- * would re-offer a dead machine's index to **every idle device the relay holds**, with no fleet
1993
- * filter and no consideration of whether the task writes. Both are reachable defects:
1994
- *
1995
- * 1. **A device in another account could be handed the work.** On a relay serving several
1996
- * fleets, `this.devices` includes all of them, so a takeover crossed accounts — which is
1997
- * the isolation this release series is about, broken in the one path that is easy to miss
1998
- * because it only runs when something has already gone wrong.
1999
- * 2. **A writing task could be run a second time.** Hand-off makes execution at-least-once,
2000
- * which is harmless for `node --test` and is a duplicated write for anything else. The
2001
- * design forbids automatic hand-off for writing tasks until a fencing token exists.
2002
- *
2003
- * The re-offer shape itself is unchanged: PROTOCOL v0.0.1 defines no per-index lease
2004
- * reassignment, so takeover is expressed as a re-offer to idle machines. Narrowing that to a
2005
- * single successor is a protocol revision, not a wiring change, and is not made here.
2006
- */
2007
- if (task.write === true) {
2008
- this.emit('notice', {
2009
- level: 'warn',
2010
- code: 'LEASE_TAKEOVER_REFUSED',
2011
- message:
2012
- `task ${task.task_id} index ${expiredLease.index} is a writing task, so no machine takes it over: `
2013
- + 'a second run could apply the same change twice, and the caller cannot tell from the verdict',
2014
- task_id: task.task_id,
2015
- reason: 'write_not_idempotent',
2016
- });
2017
- return [];
2018
- }
2019
-
2020
- const requesterFleet = this.fleetId ?? null;
2021
-
2022
- const candidates = [];
2023
- for (const device of this.devices.values()) {
2024
- if (device.machine_id === expiredLease.machine_id) continue;
2025
- /**
2026
- * Fleet membership, when the relay is scoped.
2027
- *
2028
- * A device with its own `fleet_id` must match. A device with none cannot take over on a
2029
- * scoped relay, because there is no fleet to match it against — the same rule `targetsFor`
2030
- * applies to dispatch, kept identical here so the two paths cannot disagree.
2031
- */
2032
- if (requesterFleet !== null) {
2033
- const deviceFleet = typeof device.fleet_id === 'string' && device.fleet_id !== '' ? device.fleet_id : null;
2034
- if (deviceFleet !== requesterFleet) {
2035
- this.log?.('debug', 'takeover candidate skipped: not in this fleet', {
2036
- task_id: task.task_id,
2037
- machine_id: device.machine_id,
2038
- device_fleet: deviceFleet,
2039
- relay_fleet: requesterFleet,
2040
- });
2041
- continue;
2042
- }
2043
- }
2044
- const lease = task.leases.get(device.machine_id);
2045
- if (lease && !['queued', 'offered'].includes(lease.state)) continue;
2046
- if (!lease && gateMachine(task, device)) continue;
2047
- candidates.push({ device, lease: lease ?? null });
2048
- }
2049
-
2050
- for (const { device, lease } of candidates) {
2051
- const target = lease ?? {
2052
- machine_id: device.machine_id,
2053
- machine_name: device.machine_name,
2054
- index: expiredLease.index,
2055
- attempt: task.attempt,
2056
- state: 'queued',
2057
- refusal_reason: null,
2058
- refusal_detail: null,
2059
- dedupe_key: null,
2060
- last_heartbeat_ms: nowMs,
2061
- claimed_at_ms: nowMs,
2062
- lease_until_ms: nowMs + this.leaseWindowMs(),
2063
- phase: null,
2064
- progress: null,
2065
- offered_seq: null,
2066
- expired_at_ms: null,
2067
- finished_at_ms: null,
2068
- heartbeat_count: 0,
2069
- };
2070
- target.index = expiredLease.index;
2071
- target.attempt = task.attempt;
2072
- target.dedupe_key = computeDedupeKey(task.task_id, target.index, task.command_hash, task.base_tree);
2073
- task.leases.set(device.machine_id, target);
2074
- this.emitOffer(task, target);
2075
- this._ledgerLease(task, target);
2076
- }
2077
-
2078
- this.emit('notice', {
2079
- level: 'info',
2080
- code: candidates.length > 0 ? 'LEASE_TAKEOVER' : 'LEASE_TAKEOVER_UNAVAILABLE',
2081
- message: candidates.length > 0
2082
- ? `task ${task.task_id} index ${expiredLease.index} re-offered at attempt ${task.attempt} to: `
2083
- + candidates.map((c) => c.device.machine_id).join(',')
2084
- : `task ${task.task_id} index ${expiredLease.index} has no idle machine to take over`,
2085
- task_id: task.task_id,
2086
- });
2087
- return candidates.map((c) => c.device.machine_id);
2088
- }
2089
-
2090
- /* ---------------- results (§5) ---------------- */
2091
-
2092
- static instanceKey(machineId, dedupeKey, attempt) {
2093
- return `${machineId}${INSTANCE_SEP}${dedupeKey}${INSTANCE_SEP}${attempt}`;
2094
- }
2095
-
2096
- /**
2097
- * §5 result envelope validation.
2098
- * Missing §5.1 required fields do NOT reject the frame — they make the result
2099
- * `unverifiable` (§5.1 "缺一即 unverifiable").
2100
- */
2101
- static validateEnvelope(envelope) {
2102
- const missing = [];
2103
- if (!envelope || typeof envelope !== 'object' || Array.isArray(envelope)) {
2104
- return { missing: [...REQUIRED_ENVELOPE_FIELDS], malformed: true };
2105
- }
2106
- for (const field of REQUIRED_ENVELOPE_FIELDS) {
2107
- if (!(field in envelope) || envelope[field] === undefined) missing.push(field);
2108
- }
2109
- const problems = [];
2110
- if ('status' in envelope && !RESULT_STATUSES.includes(envelope.status)) {
2111
- problems.push(`status must be one of ${RESULT_STATUSES.join('|')}`);
2112
- }
2113
- if (envelope.status === 'refused' && !envelope.refusal_reason) {
2114
- problems.push('refusal_reason is required when status=refused');
2115
- }
2116
- if ('warnings' in envelope && !Array.isArray(envelope.warnings)) problems.push('warnings must be string[]');
2117
- return { missing, problems };
2118
- }
2119
-
2120
- /**
2121
- * §1 idempotent POST /v1/result.
2122
- *
2123
- * NOTE (contract ambiguity, see report): §4.4 `dedupe_key` deliberately excludes
2124
- * `machine_id`/`attempt`, but §6.3 aggregates results from *several* machines in
2125
- * replicate mode — they all share one dedupe_key. Deduping purely on dedupe_key
2126
- * would silently drop every machine's result but the first. We therefore key the
2127
- * idempotency check on (machine_id, dedupe_key, attempt): an identical redelivery
2128
- * is `deduped:true`, distinct machines/attempts are all recorded.
2129
- */
2130
- submitResult(envelope = {}) {
2131
- const { missing, problems, malformed } = RabbitState.validateEnvelope(envelope);
2132
- if (malformed) throw new ProtocolError('BAD_REQUEST', 'result must be a JSON object');
2133
-
2134
- /**
2135
- * v0.4.0: `transport` / `p2p` are OPTIONAL envelope fields, and they are validated as values
2136
- * rather than added to the §5.1 required set. A v0.3.9 agent sends neither and must keep
2137
- * working unchanged; an agent that sends `transport: 'satellite'` is refused by name instead of
2138
- * storing a value the report would then show as a path that does not exist.
2139
- */
2140
- RabbitState.validateTransportFields(envelope);
2141
-
2142
- const taskId = envelope.task_id;
2143
- if (typeof taskId !== 'string' || taskId.length === 0) {
2144
- throw new ProtocolError('BAD_REQUEST', 'task_id is required');
2145
- }
2146
- const task = this.tasks.get(taskId);
2147
- if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
2148
-
2149
- const machineId = typeof envelope.machine_id === 'string' ? envelope.machine_id : null;
2150
- const dedupeKey = typeof envelope.dedupe_key === 'string' && envelope.dedupe_key
2151
- ? envelope.dedupe_key : null;
2152
- const attempt = Number.isInteger(envelope.attempt) && envelope.attempt >= 1 ? envelope.attempt : 1;
2153
-
2154
- if (dedupeKey === null) {
2155
- // Cannot dedupe without a key; still record it, as unverifiable.
2156
- missing.push('dedupe_key');
2157
- }
2158
- const instanceKey = RabbitState.instanceKey(machineId ?? '<unknown>', dedupeKey ?? '<none>', attempt);
2159
-
2160
- // §4.4 sanity check against the lease this machine actually holds. Booked as a
2161
- // problem instead of a hard rejection so a buggy client cannot lose its result.
2162
- const leaseForMachine = machineId ? task.leases.get(machineId) : null;
2163
- if (leaseForMachine?.dedupe_key && dedupeKey && leaseForMachine.dedupe_key !== dedupeKey) {
2164
- problems.push('dedupe_key does not match the §4.4 formula for this lease');
2165
- }
2166
-
2167
- if (this.results.has(instanceKey)) {
2168
- const prior = this.results.get(instanceKey);
2169
- return { deduped: true, task_id: taskId, dedupe_key: dedupeKey, seq: prior.seq };
2170
- }
2171
-
2172
- const now = this.nowMs();
2173
- let envelopeShaOk = null;
2174
- if (typeof envelope.envelope_sha256 === 'string') {
2175
- const { envelope_sha256: _drop, ...rest } = envelope;
2176
- envelopeShaOk = sha256Hex(jcs(rest)) === envelope.envelope_sha256;
2177
- }
2178
-
2179
- const record = {
2180
- envelope,
2181
- machine_id: machineId,
2182
- dedupe_key: dedupeKey,
2183
- attempt,
2184
- instance_key: instanceKey,
2185
- received_at_ms: now,
2186
- validation: {
2187
- missing: [...new Set(missing)],
2188
- problems,
2189
- envelope_sha256_ok: envelopeShaOk,
2190
- },
2191
- };
2192
-
2193
- const effectiveStatus = record.validation.missing.length > 0 ? 'unverifiable' : envelope.status;
2194
-
2195
- const seq = this.nextSeq();
2196
- record.seq = seq;
2197
- record.effective_status = effectiveStatus;
2198
- this.results.set(instanceKey, record);
2199
-
2200
- if (machineId) {
2201
- const lease = task.leases.get(machineId);
2202
- if (lease) {
2203
- lease.state = effectiveStatus === 'refused' ? 'refused' : 'done';
2204
- if (effectiveStatus === 'refused' && !lease.refusal_reason) {
2205
- lease.refusal_reason = envelope.refusal_reason ?? 'REFUSED';
2206
- }
2207
- lease.finished_at_ms = now;
2208
- lease.attempt = attempt;
2209
- }
2210
- this.touchDevice(machineId);
2211
- }
2212
- task.result_seqs.push(seq);
2213
-
2214
- this.emit('task.result', {
2215
- task_id: taskId,
2216
- dedupe_key: dedupeKey,
2217
- attempt,
2218
- index: envelope.index ?? null,
2219
- status: effectiveStatus,
2220
- machine_id: machineId,
2221
- deduped: false,
2222
- validation_errors: record.validation.missing,
2223
- });
2224
-
2225
- // §7: results go into the ledger so the dedupe index survives a restart.
2226
- this._ledger({ type: 'result.stored', task_id: taskId, machine_id: machineId, result: record });
2227
- if (machineId) {
2228
- const changed = task.leases.get(machineId);
2229
- if (changed) this._ledgerLease(task, changed);
2230
- }
2231
-
2232
- return { deduped: false, task_id: taskId, dedupe_key: dedupeKey, seq, _instanceKey: instanceKey };
2233
- }
2234
-
2235
- /** All recorded results for a task, keyed by machine_id (later attempts win). */
2236
- taskResults(taskId) {
2237
- const task = this.tasks.get(taskId);
2238
- if (!task) return new Map();
2239
- const out = new Map();
2240
- for (const rec of this.results.values()) {
2241
- if (rec.envelope.task_id !== taskId) continue;
2242
- const prev = out.get(rec.machine_id);
2243
- if (!prev || rec.received_at_ms >= prev.received_at_ms) out.set(rec.machine_id, rec);
2244
- }
2245
- return out;
2246
- }
2247
-
2248
- /** Global dedupe book-keeping: dedupe_key -> instances seen. */
2249
- dedupeReport() {
2250
- const out = {};
2251
- for (const rec of this.results.values()) {
2252
- if (!rec.dedupe_key) continue;
2253
- out[rec.dedupe_key] = out[rec.dedupe_key] ?? [];
2254
- out[rec.dedupe_key].push({
2255
- machine_id: rec.machine_id,
2256
- attempt: rec.attempt,
2257
- received_at: rfc3339(rec.received_at_ms),
2258
- });
2259
- }
2260
- return out;
2261
- }
2262
-
2263
- /* ---------------- serialization ---------------- */
2264
-
2265
- stats() {
2266
- return {
2267
- protocol_version: PROTOCOL_VERSION,
2268
- seq: this.seq,
2269
- buffered_events: this.buffer.length,
2270
- devices: this.devices.size,
2271
- tasks: this.tasks.size,
2272
- results: this.results.size,
2273
- pairing_codes: [...this.pairingCodes.values()].filter((c) => !c.used && this.nowMs() < c.expires_at_ms).length,
2274
- uptime_ms: this.nowMs() - this.startedAtMs,
2275
- };
2276
- }
2277
- }
2278
-
2279
- export default RabbitState;
21
+ import { planJoin } from '../chatops/fleet-join.mjs';
22
+
23
+ export const PROTOCOL_VERSION = 1;
24
+
25
+ /**
26
+ * PROTOCOL §1 table + §7 code table, plus the two v0.1.2 additions:
27
+ * `OPERATOR_REQUIRED` (§5.2 → 401) and `RATE_LIMITED` (§6 → 429).
28
+ * A code missing from this map silently degrades to HTTP 500 via
29
+ * `ProtocolError`, which keeps the right error code but lies about the status.
30
+ */
31
+ export const ERROR_STATUS = Object.freeze({
32
+ UNAUTHORIZED: 401,
33
+ BAD_REQUEST: 400,
34
+ NOT_FOUND: 404,
35
+ PAIRING_INVALID: 400,
36
+ PAIRING_EXPIRED: 400,
37
+ TASK_EXISTS: 409,
38
+ FRAME_TOO_LARGE: 413,
39
+ NO_ONLINE_DEVICE: 503,
40
+ INTERNAL: 500,
41
+ OPERATOR_REQUIRED: 401, // v0.1.2 §5.2
42
+ RATE_LIMITED: 429, // v0.1.2 §6
43
+
44
+ /* v0.3.0 §9 request signing. Every client-side signature failure is a 401.
45
+ * `SIGNING_NOT_CONFIGURED` is deliberately NOT 401: the relay was started with
46
+ * `--require-signature` but no secret, so the fault is the relay's own
47
+ * configuration. Reporting that as 401 would send every operator to debug the
48
+ * client -- the one side that is not broken. 500 rather than 503 because
49
+ * retrying never fixes a missing configuration, so nothing should be invited to
50
+ * retry. */
51
+ SIGNATURE_REQUIRED: 401,
52
+ SIGNATURE_INCOMPLETE: 401,
53
+ SIGNATURE_MISMATCH: 401,
54
+ SIGNATURE_BAD_TIMESTAMP: 401,
55
+ SIGNATURE_BAD_NONCE: 401,
56
+ SIGNATURE_EXPIRED: 401,
57
+ SIGNATURE_REPLAY: 401,
58
+ SIGNING_NOT_CONFIGURED: 500,
59
+ });
60
+
61
+ /** §5.1 required envelope fields — missing any one means `unverifiable`. */
62
+ export const REQUIRED_ENVELOPE_FIELDS = Object.freeze([
63
+ 'envelope_version', 'task_id', 'attempt', 'dedupe_key', 'machine_id', 'machine_name',
64
+ 'platform', 'caps', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
65
+ 'pre_tree_fingerprint', 'post_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error',
66
+ 'head_commit', 'dirty_before', 'command_argv', 'command_hash', 'shell_id',
67
+ 'started_at', 'ended_at', 'duration_ms', 'exit_code', 'status', 'refusal_reason',
68
+ 'stdout_sha256', 'stdout_bytes', 'stderr_sha256', 'stderr_bytes', 'warnings',
69
+ 'envelope_sha256',
70
+ ]);
71
+
72
+ /** §5.3 comparable fields — the ONLY fields that take part in consistency judgement. */
73
+ export const COMPARABLE_FIELDS = Object.freeze([
74
+ 'task_id', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
75
+ 'pre_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error', 'head_commit',
76
+ 'command_hash', 'shell_id', 'exit_code', 'status', 'refusal_reason',
77
+ 'stdout_sha256', 'stdout_bytes', 'stdout_normalized_sha256',
78
+ 'stderr_sha256', 'artifacts', 'diff_numstat', 'tests', 'semantic_counts', 'warnings',
79
+ ]);
80
+
81
+ /** §5.1 status enum. */
82
+ export const RESULT_STATUSES = Object.freeze([
83
+ 'ok', 'nonzero_exit', 'timeout', 'crashed', 'refused', 'unverifiable',
84
+ ]);
85
+
86
+ export const FAILURE_STATUSES = Object.freeze(['nonzero_exit', 'timeout', 'crashed']);
87
+
88
+ /** §4.3 phase enum. */
89
+ export const LEASE_PHASES = Object.freeze(['preparing', 'running', 'finalizing']);
90
+
91
+ /** Lease states that can still be expired / renewed. */
92
+ export const ACTIVE_LEASE_STATES = Object.freeze(['queued', 'offered', 'running']);
93
+
94
+ export const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000; // §4.3: agent heartbeats every 10s
95
+ export const DEFAULT_MISSED_HEARTBEATS = 2; // §4.3: 2 missed cycles = 20s
96
+ export const DEFAULT_LEASE_GRACE_MS = 30_000; // §4.3: + grace 30s
97
+ /**
98
+ * v0.4.0: the bounded hold on the relay's own offer when a dispatcher is pushing it directly.
99
+ *
100
+ * Sized against what it has to cover: one punch round trip (measured 15-370 ms on the reference
101
+ * networks), the frame's delivery, and the executor's first heartbeat POST — all on a host that may
102
+ * be busy. 1.2 s was the first guess and it was too tight: measured on a 1-vCPU shared host, two
103
+ * agent tests saw the relay's copy overtake the direct one, which for `p2pMode: direct` means a
104
+ * refusal of an offer that was about to arrive. 3 s covers a slow host with room to spare.
105
+ *
106
+ * The cost is honest and bounded: a punch that fails delays the relay's copy by up to this much,
107
+ * once per task. Nothing else waits on it.
108
+ */
109
+ export const DEFAULT_P2P_OFFER_GRACE_MS = 3_000;
110
+ export const DEFAULT_PAIRING_TTL_MS = 24 * 60 * 60 * 1000; // §2.2: 24h
111
+ export const DEFAULT_EVENT_BUFFER_SIZE = 1000; // §requirement: keep last 1000 events
112
+ export const MAX_FRAME_BYTES = 64 * 1024; // 64 KiB single frame
113
+ export const SHELL_ID_DIRECT = 'direct-exec'; // §5.1: v0.0.1 only direct exec
114
+ /** v0.3.0: RTT sanity ceiling — anything above 24h is treated as a broken value. */
115
+ export const MAX_RTT_MS = 24 * 60 * 60 * 1000;
116
+ /** v0.3.0: an RTT measurement older than this is reported as stale.
117
+ * Must stay comfortably ABOVE the idle-heartbeat interval (60s): at 1x the value
118
+ * would flip stale just before every refresh and flap, which would make
119
+ * `rtt_stale === false` useless as a "trust this number" predicate. 3x gives two
120
+ * consecutive missed idle heartbeats of margin. See PROTOCOL-v0.3.0.md §7. */
121
+ export const DEFAULT_RTT_STALE_MS = 180_000;
122
+ /** v0.3.0: recommended cadence for an agent with no lease (reference for the agent side). */
123
+ export const DEFAULT_IDLE_HEARTBEAT_MS = 60_000;
124
+ export const ENVELOPE_VERSION = '1.0';
125
+ export const FINGERPRINT_ALGO = 'git-temp-index-tree/v1';
126
+
127
+ /* ------------------------------------------------------------------ */
128
+ /* Errors */
129
+ /* ------------------------------------------------------------------ */
130
+
131
+ export class ProtocolError extends Error {
132
+ constructor(code, message, detail = {}) {
133
+ super(message);
134
+ this.name = 'ProtocolError';
135
+ this.code = code;
136
+ this.detail = detail;
137
+ this.status = ERROR_STATUS[code] ?? 500;
138
+ }
139
+ toBody() {
140
+ return { error: { code: this.code, message: this.message, detail: this.detail } };
141
+ }
142
+ }
143
+
144
+ /* ------------------------------------------------------------------ */
145
+ /* Small pure helpers */
146
+ /* ------------------------------------------------------------------ */
147
+
148
+ export function sha256Hex(input) {
149
+ return createHash('sha256').update(input).digest('hex');
150
+ }
151
+
152
+ /** RFC 8785-ish canonical JSON (JCS): sorted keys, no whitespace. */
153
+ export function jcs(value) {
154
+ if (value === null) return 'null';
155
+ const t = typeof value;
156
+ if (t === 'number') return Number.isFinite(value) ? JSON.stringify(value) : 'null';
157
+ if (t === 'boolean') return value ? 'true' : 'false';
158
+ if (t === 'string') return JSON.stringify(value);
159
+ if (t === 'undefined' || t === 'function' || t === 'symbol') return 'null';
160
+ if (Array.isArray(value)) return `[${value.map((v) => (v === undefined ? 'null' : jcs(v))).join(',')}]`;
161
+ if (t === 'object') {
162
+ const keys = Object.keys(value).filter((k) => value[k] !== undefined).sort();
163
+ return `{${keys.map((k) => `${JSON.stringify(k)}:${jcs(value[k])}`).join(',')}}`;
164
+ }
165
+ return JSON.stringify(String(value));
166
+ }
167
+
168
+ /**
169
+ * Upper bound on the stages a `pipeline` may contain.
170
+ *
171
+ * Each stage is another command run on someone else's machine, so an unbounded list is an unbounded
172
+ * amount of work created by one request. The limit is generous for the intended use (build, test,
173
+ * package) and exists so the failure is a clear refusal rather than a fleet-wide surprise.
174
+ */
175
+ export const MAX_PIPELINE_STAGES = 16;
176
+
177
+ /**
178
+ * v0.4.0: upper bound on `POST /v1/task`'s `origin_machine_id`.
179
+ *
180
+ * A machine id is an identifier, not a payload: 128 characters is far beyond any id this system
181
+ * mints, and the ceiling is what stops the field from becoming an unbounded free-text channel into
182
+ * every offer frame the relay emits.
183
+ */
184
+ export const MAX_ORIGIN_MACHINE_ID_LENGTH = 128;
185
+
186
+ /**
187
+ * v0.4.0: the P2P modes a task may be dispatched under (§1 of the frozen v0.4.0 contract).
188
+ *
189
+ * Mirrored here rather than imported from `src/agent/p2p-node.mjs`: the relay is a standalone
190
+ * process that must not depend on agent code, and a mode list is three strings. A value outside
191
+ * this list is a configuration error, never a silent fall back to the default.
192
+ */
193
+ export const P2P_MODES = Object.freeze(['auto', 'direct', 'relay']);
194
+
195
+ /**
196
+ * v0.4.0: how a result travelled — the direct P2P channel or the relay.
197
+ *
198
+ * A **transport is not a result**: how an envelope arrived must never take part in the consistency
199
+ * judgement (§5 of the contract), or two machines that produced byte-identical output would be
200
+ * reported `divergent` merely because one of them punched a hole. Hence: not required, not
201
+ * comparable, and absent on every v0.3.9 envelope.
202
+ */
203
+ export const TRANSPORT_VALUES = Object.freeze(['p2p', 'relay']);
204
+
205
+ /** §4.4 helper: command_hash = sha256(JCS(argv)+"|"+shell_id+"|"+cwd_rel). */
206
+ export function computeCommandHash(commandArgv, shellId = SHELL_ID_DIRECT, cwdRel = '.') {
207
+ return sha256Hex(`${jcs(commandArgv)}|${shellId}|${cwdRel}`);
208
+ }
209
+
210
+ /**
211
+ * `command_hash` for a whole pipeline.
212
+ *
213
+ * The hash must bind **every** stage, not just the first. Bound to stage 0 alone, two pipelines that
214
+ * differ after the first stage would carry the same hash -- so an envelope produced by the wrong
215
+ * chain would still verify, and making "the machine ran what I asked" checkable is the entire point
216
+ * of this field. Each stage's cwd participates for the same reason.
217
+ *
218
+ * The `|pipeline` suffix means this can never collide with a single-command hash: the two hash over
219
+ * different shapes, and that difference is deliberate rather than incidental.
220
+ */
221
+ export function computePipelineCommandHash(stages, shellId = SHELL_ID_DIRECT) {
222
+ const canonical = stages.map((s) => ({
223
+ command_argv: s.command_argv,
224
+ cwd_rel: s.cwd_rel ?? '.',
225
+ continue_on_failure: s.continue_on_failure === true,
226
+ }));
227
+ return sha256Hex(`${jcs(canonical)}|${shellId}|pipeline`);
228
+ }
229
+
230
+ /** §4.4: dedupe_key = sha256(task_id + "|" + index + "|" + command_hash + "|" + base_tree). */
231
+ export function computeDedupeKey(taskId, index, commandHash, baseTree) {
232
+ return sha256Hex(`${taskId}|${index}|${commandHash}|${baseTree}`);
233
+ }
234
+
235
+ /** Epoch ms -> RFC3339 UTC, second precision (matches the PROTOCOL examples). */
236
+ export function rfc3339(ms) {
237
+ return new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
238
+ }
239
+
240
+ export function isRfc3339(value) {
241
+ return typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/.test(value);
242
+ }
243
+
244
+ export function deepEqual(a, b) {
245
+ return jcs(a === undefined ? null : a) === jcs(b === undefined ? null : b);
246
+ }
247
+
248
+ function normalizeNow(now) {
249
+ if (typeof now !== 'function') return () => Date.now();
250
+ return () => {
251
+ const v = now();
252
+ if (typeof v === 'number') return v;
253
+ if (v instanceof Date) return v.getTime();
254
+ return Date.parse(v);
255
+ };
256
+ }
257
+
258
+ /* ------------------------------------------------------------------ */
259
+ /* ULID */
260
+ /* ------------------------------------------------------------------ */
261
+
262
+ const CROCKFORD = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
263
+
264
+ function encodeBig(value, len) {
265
+ let out = '';
266
+ let v = value;
267
+ for (let i = 0; i < len; i += 1) {
268
+ out = CROCKFORD[Number(v & 31n)] + out;
269
+ v >>= 5n;
270
+ }
271
+ return out;
272
+ }
273
+
274
+ export function createIdFactory() {
275
+ let lastMs = -1;
276
+ let lastRand = 0n;
277
+ return function nextId(nowMs = Date.now()) {
278
+ const bytes = randomBytes(10);
279
+ let rand = 0n;
280
+ for (const b of bytes) rand = (rand << 8n) | BigInt(b);
281
+ if (nowMs === lastMs) rand = (lastRand + 1n) & ((1n << 80n) - 1n);
282
+ lastMs = nowMs;
283
+ lastRand = rand;
284
+ return encodeBig(BigInt(nowMs), 10) + encodeBig(rand, 16);
285
+ };
286
+ }
287
+
288
+ export function ulid(nowMs = Date.now()) {
289
+ return createIdFactory()(nowMs);
290
+ }
291
+
292
+ /* ------------------------------------------------------------------ */
293
+ /* Version / capability gate (§6.1) */
294
+ /* ------------------------------------------------------------------ */
295
+
296
+ function parseVersion(text) {
297
+ if (typeof text !== 'string') return null;
298
+ const m = text.trim().match(/^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:\.(\d+))?/);
299
+ if (!m) return null;
300
+ return [Number(m[1]), Number(m[2] ?? 0), Number(m[3] ?? 0), Number(m[4] ?? 0)];
301
+ }
302
+
303
+ function cmpVersion(a, b) {
304
+ for (let i = 0; i < 4; i += 1) {
305
+ if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1;
306
+ }
307
+ return 0;
308
+ }
309
+
310
+ /**
311
+ * Minimal semver-range check used by the §6.1 capability gate.
312
+ * Supports bare versions and comma/space separated `>= > <= < = ==` conjunctions.
313
+ * Unparseable ranges are treated as NOT satisfied (never silently pass).
314
+ */
315
+ export function satisfiesRange(version, range) {
316
+ const v = parseVersion(version);
317
+ if (v === null) return false;
318
+ if (typeof range !== 'string') return false;
319
+ const parts = range.split(/[\s,]+/).filter(Boolean);
320
+ if (parts.length === 0) return false;
321
+ for (const part of parts) {
322
+ const m = part.match(/^(>=|<=|==|=|>|<)?\s*(.+)$/);
323
+ if (!m) return false;
324
+ const op = m[1] ?? '=';
325
+ const target = parseVersion(m[2]);
326
+ if (target === null) return false;
327
+ const c = cmpVersion(v, target);
328
+ if (op === '>=' && c < 0) return false;
329
+ if (op === '<=' && c > 0) return false;
330
+ if (op === '>' && c <= 0) return false;
331
+ if (op === '<' && c >= 0) return false;
332
+ if ((op === '=' || op === '==') && c !== 0) return false;
333
+ }
334
+ return true;
335
+ }
336
+
337
+ function machineWritable(caps) {
338
+ if (!caps || typeof caps !== 'object') return false;
339
+ if (caps.write === false || caps.writable === false || caps.write_allowed === false) return false;
340
+ if (caps.read_only === true || caps.readonly === true || caps.ro_mount === true) return false;
341
+ return true;
342
+ }
343
+
344
+ /**
345
+ * §6.1 capability gate, evaluated per machine BEFORE dispatch.
346
+ * @returns {null | {status:'refused', refusal_reason:string, detail:object}}
347
+ */
348
+ export function gateMachine(task, machine) {
349
+ const caps = machine?.caps ?? {};
350
+ const req = task.requirements ?? {};
351
+
352
+ if (req.platform && Array.isArray(req.platform) && req.platform.length > 0) {
353
+ const os = String(machine?.platform?.os ?? '').toLowerCase();
354
+ if (!req.platform.map((p) => String(p).toLowerCase()).includes(os)) {
355
+ return { status: 'refused', refusal_reason: 'PLATFORM_MISMATCH', detail: { os, want: req.platform } };
356
+ }
357
+ }
358
+
359
+ const toolchain = req.toolchain ?? {};
360
+ for (const [tool, range] of Object.entries(toolchain)) {
361
+ const declared = caps[tool];
362
+ if (declared === null || declared === undefined || declared === '') {
363
+ return {
364
+ status: 'refused',
365
+ refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
366
+ detail: { tool, want: range, have: declared ?? null },
367
+ };
368
+ }
369
+ if (typeof range === 'string' && !satisfiesRange(declared, range)) {
370
+ return {
371
+ status: 'refused',
372
+ refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
373
+ detail: { tool, want: range, have: declared },
374
+ };
375
+ }
376
+ }
377
+
378
+ if (task.write === true && !machineWritable(caps)) {
379
+ return { status: 'refused', refusal_reason: 'READ_ONLY_MACHINE', detail: { caps } };
380
+ }
381
+
382
+ return null;
383
+ }
384
+
385
+ /* ------------------------------------------------------------------ */
386
+ /* RabbitState */
387
+ /* ------------------------------------------------------------------ */
388
+
389
+ const INSTANCE_SEP = '\u0000';
390
+
391
+ export class RabbitState {
392
+ constructor(options = {}) {
393
+ this._now = normalizeNow(options.now);
394
+ this._idFactory = options.idFactory ?? createIdFactory();
395
+ this._randomToken = options.tokenFactory ?? (() => randomBytes(24).toString('hex'));
396
+ this._randomChars = options.randomChars
397
+ ?? ((n) => { const a = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; const b = randomBytes(n); let s = ''; for (let i = 0; i < n; i += 1) s += a[b[i] % a.length]; return s; });
398
+
399
+ this.protocolVersion = PROTOCOL_VERSION;
400
+ this.eventBufferSize = options.eventBufferSize ?? DEFAULT_EVENT_BUFFER_SIZE;
401
+ /**
402
+ * v0.7.7: the fleet this relay serves, or null for a relay that serves every device.
403
+ *
404
+ * **Null is the migration-safe default and is what the shared relay runs today.** With it,
405
+ * task targeting behaves exactly as it did before this release, so deploying cannot be an
406
+ * outage. Set it (via `--fleet <id>` or `fleetId`) and the relay becomes scoped: a device
407
+ * with a `fleet_id` reaches only its own fleet, and a device with none reaches nothing.
408
+ *
409
+ * This is the relay's *own* fleet, not a device's. A device carries its own `fleet_id`, which
410
+ * the relay wrote at pairing time from the pairing code.
411
+ */
412
+ this.fleetId = typeof options.fleetId === 'string' && options.fleetId !== '' ? options.fleetId : null;
413
+ this.pairingTtlMs = options.pairingTtlMs ?? DEFAULT_PAIRING_TTL_MS;
414
+ /** When true one code stays valid for its whole TTL instead of rotating. */
415
+ this.pairingCodeReusable = options.pairingCodeReusable === true;
416
+ this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
417
+ this.missedHeartbeats = options.missedHeartbeats ?? DEFAULT_MISSED_HEARTBEATS;
418
+ this.leaseGraceMs = options.leaseGraceMs ?? DEFAULT_LEASE_GRACE_MS;
419
+ /**
420
+ * v0.4.0: how long the relay holds its own offer back when the dispatcher says it will push the
421
+ * offer over the direct path (`origin_machine_id` + `p2p.mode`).
422
+ *
423
+ * Without this, the direct path could never win: the relay's offer goes out over an SSE stream
424
+ * that is already open, while a punch costs at least one round trip, so the relay copy would
425
+ * arrive first essentially always and `transport` would read `relay` even on a network where
426
+ * the punch succeeded. Measured on loopback, and it is exactly what the first version of this
427
+ * release did.
428
+ *
429
+ * The hold is bounded and self-healing: the executor renews its lease (a heartbeat) the moment
430
+ * it starts work on the direct copy, which moves the lease out of `queued`, and the deferred
431
+ * offer is then dropped. If the punch fails, nothing renews the lease and the relay offers
432
+ * normally once the grace expires -- so a failed punch costs this much latency and nothing else.
433
+ */
434
+ this.p2pOfferGraceMs = Number.isFinite(options.p2pOfferGraceMs) && options.p2pOfferGraceMs >= 0
435
+ ? options.p2pOfferGraceMs
436
+ : DEFAULT_P2P_OFFER_GRACE_MS;
437
+ /** Deferred offers, keyed `task_id:machine_id`, so they can be cleared on close(). */
438
+ this._pendingOffers = new Map();
439
+ /** v0.3.0: age beyond which a reported RTT is flagged stale. */
440
+ this.rttStaleMs = Number.isFinite(options.rttStaleMs) ? options.rttStaleMs : DEFAULT_RTT_STALE_MS;
441
+
442
+ this.startedAtMs = this._now();
443
+ /** v0.1.2 §8.3: random per-process id; a change tells clients the relay restarted. */
444
+ this.relayId = options.relayId ?? randomBytes(8).toString('hex');
445
+ /** v0.1.2 §7: optional injected Persistence instance (state.mjs never touches fs itself). */
446
+ this._persist = options.persistence ?? null;
447
+ /** True while replaying a ledger — suppresses event emission and re-persistence. */
448
+ this._restoring = false;
449
+ /** Ledger entries that could not be replayed. */
450
+ this.replayWarnings = [];
451
+ this.seq = 0;
452
+ /** @type {{seq:number,event:object,target:string|null}[]} ring buffer, last N events */
453
+ this.buffer = [];
454
+ this.devices = new Map(); // machine_id -> device
455
+ this.tokens = new Map(); // device_token -> machine_id
456
+ this.tasks = new Map(); // task_id -> task
457
+ this.results = new Map(); // instanceKey -> {envelope, machine_id, received_at_ms, ...}
458
+ this.pairingCodes = new Map(); // code -> {code, created_at_ms, expires_at_ms, used}
459
+ this._listeners = new Set();
460
+ }
461
+
462
+ /* ---------------- clock / ids ---------------- */
463
+
464
+ nowMs() { return this._now(); }
465
+ nowIso() { return rfc3339(this.nowMs()); }
466
+ nextId() { return this._idFactory(this.nowMs()); }
467
+ newToken() { return this._randomToken(); }
468
+
469
+ /** §4.3 lease window = 2 missed heartbeats (20s) + grace (30s) = 50s. */
470
+ leaseWindowMs() { return this.heartbeatIntervalMs * this.missedHeartbeats + this.leaseGraceMs; }
471
+
472
+ /* ---------------- persistence hooks (v0.1.2 §7) ---------------- */
473
+
474
+ /** Inject/replace the persistence backend (also used by tests). */
475
+ setPersistence(persistence) { this._persist = persistence ?? null; return this._persist; }
476
+
477
+ /** Snapshot the device table; failures must never break the request path. */
478
+ _saveDevices() {
479
+ if (this._restoring || !this._persist) return false;
480
+ try {
481
+ return this._persist.saveDevices(this.devices);
482
+ } catch (err) {
483
+ this._persist.warn?.('devices.json', `snapshot failed: ${err.message}`);
484
+ return false;
485
+ }
486
+ }
487
+
488
+ /** Append one ledger entry; failures must never break the request path. */
489
+ _ledger(entry) {
490
+ if (this._restoring || !this._persist) return false;
491
+ try {
492
+ const appended = this._persist.append(entry);
493
+ // v0.6.2: a successful append is what clears a degraded relay, so the outcome has to be
494
+ // reported back rather than assumed -- otherwise the state could only ever be entered.
495
+ if (appended) this._persist.noteAppendSuccess?.();
496
+ return appended;
497
+ } catch (err) {
498
+ // Recorded as a state, not merely as a line: this is the one relay failure that keeps serving,
499
+ // and so the one that is invisible from every other angle (v0.6.2).
500
+ if (typeof this._persist.noteAppendFailure === 'function') this._persist.noteAppendFailure(err.message);
501
+ else this._persist.warn?.('ledger.jsonl', `append failed: ${err.message}`);
502
+ return false;
503
+ }
504
+ }
505
+
506
+ _ledgerLease(task, lease) {
507
+ this._ledger({ type: 'lease.changed', task_id: task.task_id, machine_id: lease.machine_id, lease: { ...lease } });
508
+ }
509
+
510
+ /**
511
+ * v0.1.2 §7 startup recovery: rebuild the device table, token index, tasks,
512
+ * leases and the result/dedupe index from a devices snapshot + ledger replay.
513
+ * Never emits events and never writes back what it just read.
514
+ */
515
+ restore({ devices = {}, ledger = [] } = {}) {
516
+ this._restoring = true;
517
+ try {
518
+ for (const [machineId, d] of Object.entries(devices)) {
519
+ if (!d || typeof d !== 'object') continue;
520
+ const pairedAt = Date.parse(d.paired_at ?? '');
521
+ const device = {
522
+ machine_id: machineId,
523
+ machine_name: typeof d.machine_name === 'string' && d.machine_name ? d.machine_name : machineId,
524
+ platform: d.platform ?? {},
525
+ caps: d.caps ?? {},
526
+ user_id: d.user_id ?? null,
527
+ device_token: typeof d.device_token === 'string' ? d.device_token : null,
528
+ paired_at_ms: Number.isFinite(pairedAt) ? pairedAt : this.nowMs(),
529
+ re_paired: d.re_paired === true,
530
+ last_seen_at_ms: this.nowMs(),
531
+ // a snapshot proves the device exists, not that it is connected
532
+ online: false,
533
+ streams: 0,
534
+ // v0.3.0: RTT is volatile and is NEVER restored. Even if a snapshot on
535
+ // disk happens to carry `rtt_ms`, it is dropped here: after a restart the
536
+ // relay genuinely does not know the latency any more, and reporting a
537
+ // pre-restart number as current would be a lie that no reader can detect.
538
+ rtt_ms: null,
539
+ rtt_at_ms: null,
540
+ last_heartbeat_at_ms: null,
541
+ stream_connects: 0,
542
+ };
543
+ this.devices.set(machineId, device);
544
+ if (device.device_token) this.tokens.set(device.device_token, machineId);
545
+ }
546
+
547
+ let revivedTasks = 0;
548
+ for (const entry of ledger) {
549
+ try {
550
+ if (this._applyLedgerEntry(entry) === true) revivedTasks += 1;
551
+ } catch (err) {
552
+ this.replayWarnings.push({ type: entry?.type ?? '<unknown>', message: err.message });
553
+ }
554
+ }
555
+ return { revived_devices: this.devices.size, revived_tasks: revivedTasks };
556
+ } finally {
557
+ this._restoring = false;
558
+ }
559
+ }
560
+
561
+ /** @returns {boolean} true when a new task was created. */
562
+ _applyLedgerEntry(entry) {
563
+ if (!entry || typeof entry.type !== 'string') return false;
564
+ switch (entry.type) {
565
+ case 'task.created': {
566
+ const t = entry.task;
567
+ if (!t || typeof t.task_id !== 'string') return false;
568
+ const createdAt = Number.isFinite(t.created_at_ms) ? t.created_at_ms : Date.parse(entry.ts ?? '');
569
+ const task = {
570
+ task_id: t.task_id,
571
+ mode: t.mode ?? 'replicate',
572
+ command_argv: t.command_argv ?? [],
573
+ cwd_rel: t.cwd_rel ?? '.',
574
+ index_total: t.index_total ?? 1,
575
+ timeout_ms: t.timeout_ms ?? 300_000,
576
+ write: t.write === true,
577
+ write_scope: t.write_scope ?? [],
578
+ require_exclusive_write: t.require_exclusive_write === true,
579
+ base_commit: t.base_commit ?? null,
580
+ base_tree: t.base_tree ?? null,
581
+ requirements: t.requirements ?? {},
582
+ compare_policy: t.compare_policy ?? {},
583
+ halt: t.halt ?? 'never',
584
+ created_by: t.created_by ?? null,
585
+ created_at_ms: Number.isFinite(createdAt) ? createdAt : this.nowMs(),
586
+ command_hash: t.command_hash ?? null,
587
+ attempt: t.attempt ?? 1,
588
+ leases: new Map(),
589
+ cancelled: false,
590
+ cancel_reason: null,
591
+ degraded: t.degraded ?? null,
592
+ deadline_ms: t.deadline_ms ?? 0,
593
+ // v0.4.0 routing metadata. Read defensively: a ledger written by v0.3.9 has neither key,
594
+ // and a replayed task with `null` for both is exactly a v0.3.9 task.
595
+ origin_machine_id: t.origin_machine_id ?? null,
596
+ p2p: t.p2p ?? null,
597
+ result_seqs: [],
598
+ };
599
+ for (const lease of entry.leases ?? []) {
600
+ if (lease && typeof lease.machine_id === 'string') task.leases.set(lease.machine_id, { ...lease });
601
+ }
602
+ this.tasks.set(task.task_id, task);
603
+ return true;
604
+ }
605
+ case 'lease.changed': {
606
+ const task = this.tasks.get(entry.task_id);
607
+ const lease = task?.leases.get(entry.machine_id);
608
+ if (!lease) return false;
609
+ Object.assign(lease, entry.lease ?? {});
610
+ return false;
611
+ }
612
+ case 'result.stored': {
613
+ const record = entry.result;
614
+ if (!record || typeof record.instance_key !== 'string') return false;
615
+ this.results.set(record.instance_key, { ...record });
616
+ const task = this.tasks.get(record.envelope?.task_id);
617
+ if (task && Number.isInteger(record.seq)) task.result_seqs.push(record.seq);
618
+ return false;
619
+ }
620
+ case 'task.cancelled': {
621
+ const task = this.tasks.get(entry.task_id);
622
+ if (task) {
623
+ task.cancelled = true;
624
+ task.cancel_reason = entry.reason ?? null;
625
+ }
626
+ return false;
627
+ }
628
+ default:
629
+ return false; // unknown entry type: forward compatible, ignored
630
+ }
631
+ }
632
+
633
+ /* ---------------- events (§3) ---------------- */
634
+
635
+ nextSeq() { this.seq += 1; return this.seq; }
636
+
637
+ /** Reserve a seq WITHOUT buffering (used for the per-connection `ready` frame). */
638
+ reserveSeq() { return this.nextSeq(); }
639
+
640
+ onEvent(listener) {
641
+ this._listeners.add(listener);
642
+ return () => this._listeners.delete(listener);
643
+ }
644
+
645
+ /**
646
+ * Emit an event: allocates the next global seq, buffers it (ring of N) and
647
+ * notifies subscribers. `target` is the machine_id the event is addressed to
648
+ * (null = broadcast). The target is NOT serialized except as `machine_id`.
649
+ */
650
+ emit(type, fields = {}, opts = {}) {
651
+ const target = opts.target ?? null;
652
+ const seq = this.nextSeq();
653
+ const event = {
654
+ type,
655
+ seq,
656
+ rabbit_time: this.nowIso(),
657
+ ...(target ? { machine_id: target } : {}),
658
+ ...fields,
659
+ };
660
+ const entry = { seq, event, target };
661
+ this.buffer.push(entry);
662
+ if (this.buffer.length > this.eventBufferSize) {
663
+ this.buffer.splice(0, this.buffer.length - this.eventBufferSize);
664
+ }
665
+ for (const listener of this._listeners) {
666
+ try { listener(entry); } catch { /* subscriber errors must never break state */ }
667
+ }
668
+ return event;
669
+ }
670
+
671
+ oldestBufferedSeq() { return this.buffer.length ? this.buffer[0].seq : this.seq + 1; }
672
+ /**
673
+ * Highest seq present in the replay ring (0 when empty).
674
+ *
675
+ * Deliberately different from `lastSeq()`: `ready` frames allocate seq numbers
676
+ * without being buffered, so `lastSeq()` can run ahead of anything a client can
677
+ * actually replay. Diagnostics that describe the REPLAY WINDOW must use this one.
678
+ */
679
+ lastBufferedSeq() { return this.buffer.length ? this.buffer[this.buffer.length - 1].seq : 0; }
680
+ lastSeq() { return this.seq; }
681
+
682
+ /** Replay entries with seq >= from (used by GET /v1/stream?seq=|Last-Event-ID). */
683
+ replayEntries(from) {
684
+ return this.buffer.filter((e) => e.seq >= from);
685
+ }
686
+
687
+ eventsSince(from) { return this.replayEntries(from).map((e) => e.event); }
688
+
689
+ /* ---------------- pairing (§2) ---------------- */
690
+
691
+ /**
692
+ * Mint a pairing code.
693
+ *
694
+ * **v0.7.7: the code carries the relay's fleet.** This is the link that makes the scheme work,
695
+ * and it is easy to leave out: `pair()` reads the fleet *from the code*, so a code with no
696
+ * fleet pairs a device into no fleet -- and a device in no fleet reaches nothing on a scoped
697
+ * relay. `--fleet X` would then have had no effect on anything, silently.
698
+ *
699
+ * Defaulting to `this.fleetId` rather than requiring the caller to pass it means there is no
700
+ * path that mints a fleetless code on a scoped relay. An explicit `fleetId` overrides, which
701
+ * is what a relay serving several fleets would need.
702
+ */
703
+ createPairingCode({ ttlMs = this.pairingTtlMs, fleetId = this.fleetId } = {}) {
704
+ let code;
705
+ do { code = `PAIR-${this._randomChars(8)}`; } while (this.pairingCodes.has(code));
706
+ const now = this.nowMs();
707
+ this.pairingCodes.set(code, {
708
+ code,
709
+ created_at_ms: now,
710
+ expires_at_ms: now + ttlMs,
711
+ used: false,
712
+ // Present as an explicit key even when null, so a code dump shows whether a fleet was
713
+ // attached rather than leaving it to be inferred from an absent field.
714
+ fleet_id: typeof fleetId === 'string' && fleetId !== '' ? fleetId : null,
715
+ });
716
+ return code;
717
+ }
718
+
719
+ /** Revoke every outstanding code and mint a new one. */
720
+ rotatePairingCode(opts) {
721
+ for (const rec of this.pairingCodes.values()) rec.used = true;
722
+ return this.createPairingCode(opts);
723
+ }
724
+
725
+ /**
726
+ * Register an externally supplied code (`--pairing-code <c>`).
727
+ *
728
+ * v0.1.2 BUG-1: assigning the code to `relay.pairingCode` without registering it
729
+ * advertised a code that `/v1/pair` then rejected with PAIRING_INVALID, so every
730
+ * documented scripted-onboarding flow failed. "Which code do we use" and "is the
731
+ * code registered" are two different questions; this answers the second.
732
+ */
733
+ registerPairingCode(code, { ttlMs = this.pairingTtlMs } = {}) {
734
+ if (typeof code !== 'string' || code.trim() === '') {
735
+ throw new ProtocolError('BAD_REQUEST', 'pairing code must be a non-empty string', { pairing_code: code });
736
+ }
737
+ const value = code.trim();
738
+ const now = this.nowMs();
739
+ this.pairingCodes.set(value, { code: value, created_at_ms: now, expires_at_ms: now + ttlMs, used: false });
740
+ return value;
741
+ }
742
+
743
+ /** §2.2 step 3: validate a one-time code. */
744
+ validatePairingCode(code) {
745
+ if (typeof code !== 'string' || code.length === 0) {
746
+ throw new ProtocolError('PAIRING_INVALID', 'pairing_code is required');
747
+ }
748
+ const rec = this.pairingCodes.get(code);
749
+ if (!rec) throw new ProtocolError('PAIRING_INVALID', 'unknown pairing code', { pairing_code: code });
750
+ if (rec.used) throw new ProtocolError('PAIRING_INVALID', 'pairing code already used', { pairing_code: code });
751
+ if (this.nowMs() >= rec.expires_at_ms) {
752
+ throw new ProtocolError('PAIRING_EXPIRED', 'pairing code expired', { pairing_code: code, expired_at: rfc3339(rec.expires_at_ms) });
753
+ }
754
+ return rec;
755
+ }
756
+
757
+ /** §2.2: exchange a one-time code for a device_token. §1: idempotent. */
758
+ pair(request = {}) {
759
+ /**
760
+ * v0.8.1: a pairing code **or** an account-derived fleet key.
761
+ *
762
+ * The code path is unchanged. The key path is what lets a second machine on one account join
763
+ * with no human step and no relay configuration — the gap that left a new PC invisible beside
764
+ * its own fleet.
765
+ *
766
+ * ## Why a key may stand in for a code
767
+ *
768
+ * They authorise different things. A **code** is the operator saying *"let this machine in"*,
769
+ * and it is the only way to place a device in a fleet the device could not compute for itself.
770
+ * A **key** is the account saying *"this machine is mine"*, and it can only ever place the
771
+ * device in the fleet derived from that key — so it cannot be used to reach a fleet its holder
772
+ * is not already entitled to.
773
+ *
774
+ * ## Why a malformed key refuses rather than falling back
775
+ *
776
+ * A request with a `fleet_key` is a request that *meant* to join by account. Quietly demanding a
777
+ * code instead would surface at the client as an unexplained `PAIRING_INVALID`, which is the
778
+ * silent-wrong-answer shape this codebase keeps producing; the reason is named instead.
779
+ */
780
+ const hasKey = typeof request.fleet_key === 'string' && request.fleet_key !== '';
781
+ let rec = null;
782
+ if (hasKey) {
783
+ const join = planJoin({ fleetKeyEncoded: request.fleet_key });
784
+ if (!join.ok) {
785
+ throw new ProtocolError('PAIRING_INVALID', join.text, { reason: join.reason });
786
+ }
787
+ // A synthetic code record so the fleet binding below reads from one place. `fleet_id` is the
788
+ // DERIVED value, never anything the request claimed.
789
+ rec = { code: null, fleet_id: join.fleetId, from_fleet_key: true };
790
+ } else {
791
+ rec = this.validatePairingCode(request.pairing_code);
792
+ }
793
+
794
+ const machineId = request.machine_id;
795
+ if (typeof machineId !== 'string' || machineId.length === 0) {
796
+ throw new ProtocolError('BAD_REQUEST', 'machine_id is required');
797
+ }
798
+ const now = this.nowMs();
799
+ const token = this.newToken();
800
+ const existing = this.devices.get(machineId);
801
+ if (existing?.device_token) this.tokens.delete(existing.device_token);
802
+
803
+ const device = {
804
+ machine_id: machineId,
805
+ machine_name: typeof request.machine_name === 'string' && request.machine_name
806
+ ? request.machine_name : machineId.slice(0, 12),
807
+ platform: request.platform ?? {},
808
+ caps: request.caps ?? {},
809
+ user_id: request.user_id ?? null,
810
+ // v0.7.7: which fleet this device belongs to, and where that fact came from.
811
+ //
812
+ // **The fleet comes from the pairing code, not from the request.** A device that could
813
+ // name its own fleet would join any fleet, and every signature would faithfully attest
814
+ // to it. `deviceFleetFields` is the single place that binding is made, and it records
815
+ // its provenance so a fleet that arrived from a request is visible as such in
816
+ // `devices.json` rather than looking identical to one that came from a code.
817
+ //
818
+ // v0.8.1 adds the third source: an account-derived `fleet_key`. That is what lets a second
819
+ // machine on one account join with **no human step and no relay configuration** -- the gap
820
+ // that left a new PC invisible beside its own fleet. The relay does not trust a claimed
821
+ // fleet id; it *derives* one from the key, so a device can only land in the fleet whose key
822
+ // it holds and anything else forms a separate, empty fleet.
823
+ ...deviceFleetFields({
824
+ codeRecord: rec,
825
+ fleetKeyEncoded: typeof request.fleet_key === 'string' ? request.fleet_key : null,
826
+ requestedFleet: null,
827
+ }),
828
+ /**
829
+ * v0.7.10: the device's Ed25519 public key, if it offered one.
830
+ *
831
+ * A public key is not a secret, so storing it costs nothing and it is the whole basis of
832
+ * per-device verification: the relay can then check a signature only *this* machine could
833
+ * have made, rather than a shared secret every machine holds.
834
+ *
835
+ * Not validated here, deliberately. A malformed key fails at verification with a named
836
+ * reason, while rejecting a pairing over it would lock out every existing device the
837
+ * moment this field was added -- and devices that send no key keep working, with
838
+ * `public_key: null` meaning "signs with the shared secret or not at all".
839
+ */
840
+ public_key: typeof request.public_key === 'string' && request.public_key !== '' ? request.public_key : null,
841
+ device_token: token,
842
+ paired_at_ms: now,
843
+ re_paired: Boolean(existing),
844
+ last_seen_at_ms: now,
845
+ online: true,
846
+ streams: existing?.streams ?? 0,
847
+ // v0.3.0 RTT: starts unknown. `null` is "never measured"; 0 would mean "instant".
848
+ rtt_ms: null,
849
+ rtt_at_ms: null,
850
+ last_heartbeat_at_ms: null,
851
+ stream_connects: existing?.stream_connects ?? 0,
852
+ };
853
+ this.devices.set(machineId, device);
854
+ this.tokens.set(token, machineId);
855
+ rec.used = true;
856
+
857
+ // A group has more than one machine, and a single-use code would leave the
858
+ // operator unable to pair the second one without restarting the relay. So
859
+ // consuming a code immediately mints the next one and announces it: the
860
+ // operator pairs machine A, sees a fresh code, and pairs machine B with it.
861
+ // `pairingCodeReusable` keeps one code alive for the whole window instead,
862
+ // for scripted onboarding where nobody is watching stdout.
863
+ let nextCode = null;
864
+ const reusable = this.pairingCodeReusable === true;
865
+ if (reusable) {
866
+ rec.used = false;
867
+ } else {
868
+ nextCode = this.rotatePairingCode();
869
+ }
870
+
871
+ this.emit('peer.hello', {
872
+ machine_id: machineId,
873
+ machine_name: device.machine_name,
874
+ platform: device.platform,
875
+ });
876
+
877
+ if (nextCode) {
878
+ this.emit('notice', {
879
+ level: 'info',
880
+ code: 'PAIRING_CODE_ROTATED',
881
+ message: 'pairing code consumed; a new one is available',
882
+ pairing_code: nextCode,
883
+ });
884
+ }
885
+
886
+ // §7: a device change is snapshotted immediately.
887
+ this._saveDevices();
888
+ this._ledger({
889
+ type: 'device.paired',
890
+ machine_id: machineId,
891
+ machine_name: device.machine_name,
892
+ re_paired: device.re_paired,
893
+ });
894
+
895
+ return {
896
+ device_token: token,
897
+ protocol_version: PROTOCOL_VERSION,
898
+ rabbit_time: rfc3339(now),
899
+ ...(nextCode ? { next_pairing_code: nextCode } : {}),
900
+ };
901
+ }
902
+
903
+ /** @returns {object|null} device for a bearer token. */
904
+ authenticate(token) {
905
+ if (typeof token !== 'string' || token.length === 0) return null;
906
+ const machineId = this.tokens.get(token);
907
+ if (!machineId) return null;
908
+ return this.devices.get(machineId) ?? null;
909
+ }
910
+
911
+ touchDevice(machineId, { online = true } = {}) {
912
+ const device = this.devices.get(machineId);
913
+ if (!device) return null;
914
+ device.last_seen_at_ms = this.nowMs();
915
+ if (online) device.online = true;
916
+ return device;
917
+ }
918
+
919
+ setOnline(machineId, online, reason = null) {
920
+ const device = this.devices.get(machineId);
921
+ if (!device) return null;
922
+ if (device.online === online) return device;
923
+ device.online = online;
924
+ if (online) {
925
+ this.touchDevice(machineId);
926
+ this.emit('peer.hello', {
927
+ machine_id: machineId,
928
+ machine_name: device.machine_name,
929
+ platform: device.platform,
930
+ });
931
+ } else {
932
+ this.emit('peer.bye', { machine_id: machineId, reason: reason ?? 'stream_closed' });
933
+ }
934
+ return device;
935
+ }
936
+
937
+ /* ---------------- RTT (v0.3.0 §R) ---------------- */
938
+
939
+ /**
940
+ * Validate an inbound `rtt_ms`.
941
+ *
942
+ * The contract is deliberately liberal: a MISSING or INVALID value must never
943
+ * reject the heartbeat and must never clobber a previously known good value.
944
+ * RTT is a diagnostic; it must not be able to break lease renewal.
945
+ *
946
+ * Sanity ceiling: anything above `MAX_RTT_MS` (24h) is treated as invalid. A
947
+ * single garbage value would otherwise poison the fleet aggregate.
948
+ */
949
+ static normalizeRtt(value) {
950
+ if (value === undefined) return { provided: false, valid: false };
951
+ if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
952
+ return { provided: true, valid: false };
953
+ }
954
+ if (value > MAX_RTT_MS) return { provided: true, valid: false };
955
+ return { provided: true, valid: true, value };
956
+ }
957
+
958
+ /**
959
+ * RTT is a VOLATILE diagnostic: it is intentionally NOT written to
960
+ * `devices.json` and NOT replayed from the ledger, so a restarted relay reports
961
+ * `rtt_ms: null` until a machine heartbeats again. A persisted value would be
962
+ * indistinguishable from a fresh one and would let the relay claim knowledge it
963
+ * lost when the process died.
964
+ */
965
+ rttStaleAfterMs() { return this.rttStaleMs; }
966
+
967
+ /**
968
+ * v0.3.0: absorb the optional `rtt_ms` carried by a heartbeat.
969
+ *
970
+ * Deliberately liberal, because RTT is a diagnostic and must never be able to
971
+ * break lease renewal:
972
+ * - the field is optional; a v0.2.3 agent never sends it
973
+ * - a MISSING or INVALID value is IGNORED, keeping the previous value, and the
974
+ * heartbeat still succeeds
975
+ * - unknown extra fields are ignored entirely (forward compatibility)
976
+ *
977
+ * `last_heartbeat_at` advances on EVERY accepted heartbeat, with or without a
978
+ * usable RTT: it answers "is the machine alive", while `rtt_at` answers "when
979
+ * was this number measured". Keeping them separate is what lets a reader tell a
980
+ * genuinely slow machine from one that is simply gone.
981
+ */
982
+ recordHeartbeatMetrics(machineId, input = {}, nowMs = this.nowMs()) {
983
+ const device = this.devices.get(machineId);
984
+ if (!device) return null;
985
+ device.last_heartbeat_at_ms = nowMs;
986
+ const rtt = RabbitState.normalizeRtt(input.rtt_ms);
987
+ if (rtt.valid) {
988
+ device.rtt_ms = rtt.value;
989
+ device.rtt_at_ms = nowMs;
990
+ }
991
+ return device;
992
+ }
993
+
994
+ /** True when there is no value, or the value is older than the staleness window. */
995
+ isRttStale(device, nowMs = this.nowMs()) {
996
+ if (!device || device.rtt_ms === null || device.rtt_ms === undefined) return true;
997
+ if (device.rtt_at_ms === null || device.rtt_at_ms === undefined) return true;
998
+ return nowMs - device.rtt_at_ms > this.rttStaleMs;
999
+ }
1000
+
1001
+ /** How old the RTT measurement is, in ms; null when there is none. */
1002
+ rttAgeMs(device, nowMs = this.nowMs()) {
1003
+ if (!device || device.rtt_at_ms === null || device.rtt_at_ms === undefined) return null;
1004
+ return Math.max(0, nowMs - device.rtt_at_ms);
1005
+ }
1006
+
1007
+ /** Record that a machine attached a stream (a reconnect is any attach after the first). */
1008
+ noteStreamConnect(machineId) {
1009
+ const device = this.devices.get(machineId);
1010
+ if (!device) return null;
1011
+ device.stream_connects = (device.stream_connects ?? 0) + 1;
1012
+ return device.stream_connects;
1013
+ }
1014
+
1015
+ /**
1016
+ * v0.3.0 aggregate for `/healthz`.
1017
+ *
1018
+ * Only FRESH measurements count. A machine that has been gone for a day must not
1019
+ * drag min/max/avg around, and with no fresh data every numeric field is null --
1020
+ * never 0, which would read as "instantaneous".
1021
+ */
1022
+ rttSummary(nowMs = this.nowMs()) {
1023
+ const fresh = [];
1024
+ let stale = 0;
1025
+ let unknown = 0;
1026
+ for (const device of this.devices.values()) {
1027
+ if (device.rtt_ms === null || device.rtt_ms === undefined) { unknown += 1; continue; }
1028
+ if (this.isRttStale(device, nowMs)) { stale += 1; continue; }
1029
+ fresh.push(device.rtt_ms);
1030
+ }
1031
+ if (fresh.length === 0) {
1032
+ return {
1033
+ machines_reporting: 0,
1034
+ min_ms: null,
1035
+ max_ms: null,
1036
+ avg_ms: null,
1037
+ machines_stale: stale,
1038
+ machines_unknown: unknown,
1039
+ };
1040
+ }
1041
+ const sum = fresh.reduce((a, b) => a + b, 0);
1042
+ return {
1043
+ machines_reporting: fresh.length,
1044
+ min_ms: Math.min(...fresh),
1045
+ max_ms: Math.max(...fresh),
1046
+ avg_ms: Math.round((sum / fresh.length) * 10) / 10,
1047
+ machines_stale: stale,
1048
+ machines_unknown: unknown,
1049
+ };
1050
+ }
1051
+
1052
+ /**
1053
+ * v0.3.0 single-machine status view.
1054
+ *
1055
+ * `last_heartbeat_at` and `rtt_at` answer different questions, and the gap
1056
+ * between them is the whole point: a machine can be heartbeating (alive) while
1057
+ * its RTT number is old (a v0.2.3 agent that never sends `rtt_ms`), and a
1058
+ * machine can be gone while a plausible-looking RTT is still on file.
1059
+ */
1060
+ deviceStatus(machineId, nowMs = this.nowMs()) {
1061
+ const device = this.devices.get(machineId);
1062
+ if (!device) return null;
1063
+ const rttAt = device.rtt_at_ms ?? null;
1064
+ const lastHeartbeat = device.last_heartbeat_at_ms ?? null;
1065
+ return {
1066
+ machine_id: device.machine_id,
1067
+ machine_name: device.machine_name,
1068
+ relay_id: this.relayId,
1069
+ connected: (device.streams ?? 0) > 0,
1070
+ streams: device.streams ?? 0,
1071
+ online: device.online === true,
1072
+ rtt_ms: device.rtt_ms ?? null,
1073
+ rtt_at: rttAt === null ? null : rfc3339(rttAt),
1074
+ rtt_age_ms: this.rttAgeMs(device, nowMs),
1075
+ rtt_stale: this.isRttStale(device, nowMs),
1076
+ last_heartbeat_at: lastHeartbeat === null ? null : rfc3339(lastHeartbeat),
1077
+ last_seen_at: device.last_seen_at_ms === null || device.last_seen_at_ms === undefined
1078
+ ? null : rfc3339(device.last_seen_at_ms),
1079
+ reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
1080
+ stream_connects: device.stream_connects ?? 0,
1081
+ paired_at: device.paired_at_ms === null || device.paired_at_ms === undefined
1082
+ ? null : rfc3339(device.paired_at_ms),
1083
+ platform: device.platform,
1084
+ caps: device.caps,
1085
+ user_id: device.user_id ?? null,
1086
+ now: rfc3339(nowMs),
1087
+ };
1088
+ }
1089
+
1090
+ /** Public device view — never leaks device_token. */
1091
+ publicDevice(device) {
1092
+ return {
1093
+ machine_id: device.machine_id,
1094
+ machine_name: device.machine_name,
1095
+ platform: device.platform,
1096
+ caps: device.caps,
1097
+ user_id: device.user_id,
1098
+ /**
1099
+ * v0.7.7: the fleet this device belongs to, and where the fact came from.
1100
+ *
1101
+ * **Carried in the public shape on purpose, and it has to be**, because `devices.json`
1102
+ * is written from this shape -- a fleet that lived only on the in-memory record would
1103
+ * vanish on the next restart, which is precisely when an operator is looking at it. A
1104
+ * first version omitted these and the live test caught the fleet disappearing from
1105
+ * `listDevices()`.
1106
+ *
1107
+ * Safe to expose: `fleet_id` is a truncated hash of the account id, not the account id,
1108
+ * and devices on one relay can already enumerate each other. Narrowing `/v1/devices` to
1109
+ * a device's own fleet is a further step, recorded as such rather than implied.
1110
+ */
1111
+ fleet_id: device.fleet_id ?? null,
1112
+ fleet_source: device.fleet_source ?? null,
1113
+ // v0.7.10: a boolean rather than the key itself. The key is not a secret, but a device
1114
+ // list is read by every device, and there is no reason to hand every machine every other
1115
+ // machine's key when the verification happens relay-side.
1116
+ public_key_present: typeof device.public_key === 'string' && device.public_key !== '',
1117
+ online: device.online,
1118
+ streams: device.streams,
1119
+ paired_at: rfc3339(device.paired_at_ms),
1120
+ last_seen_at: rfc3339(device.last_seen_at_ms),
1121
+ // v0.3.0: `null` means "never measured", which is NOT the same as 0ms.
1122
+ rtt_ms: device.rtt_ms ?? null,
1123
+ rtt_at: device.rtt_at_ms === null || device.rtt_at_ms === undefined ? null : rfc3339(device.rtt_at_ms),
1124
+ rtt_age_ms: this.rttAgeMs(device),
1125
+ rtt_stale: this.isRttStale(device),
1126
+ last_heartbeat_at: device.last_heartbeat_at_ms === null || device.last_heartbeat_at_ms === undefined
1127
+ ? null : rfc3339(device.last_heartbeat_at_ms),
1128
+ reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
1129
+ };
1130
+ }
1131
+
1132
+ listDevices() { return [...this.devices.values()].map((d) => this.publicDevice(d)); }
1133
+
1134
+ /* ---------------- tasks (§4) ---------------- */
1135
+
1136
+ /**
1137
+ * v0.4.0: validate the routing metadata a `POST /v1/task` body may carry.
1138
+ *
1139
+ * A machine, not a mode, is `origin_machine_id`: where the task was typed (`device.json`), so an
1140
+ * offer can carry that fact to the machine that has to run it. `p2p.mode` is the dispatcher's
1141
+ * resolved mode for this particular task — the plugin may be configured `direct` while a
1142
+ * scheduled run is `relay`.
1143
+ *
1144
+ * Validation is deliberately shallow and strictly typed, in the house style of this file: a
1145
+ * missing value is the v0.3.9 shape and stays valid, while a present-but-wrong value is refused
1146
+ * by name. Accepting a non-string here would put a number into `origin_machine_id` and only fail
1147
+ * later, on a machine that cannot tell a bad field from a bad relay.
1148
+ *
1149
+ * @returns {{origin_machine_id:string|null, p2p:{mode:string}|null}}
1150
+ */
1151
+ static validateTaskRouting(input = {}) {
1152
+ let originMachineId = null;
1153
+ if (input.origin_machine_id !== undefined && input.origin_machine_id !== null) {
1154
+ if (typeof input.origin_machine_id !== 'string'
1155
+ || input.origin_machine_id.length === 0
1156
+ || input.origin_machine_id.length > MAX_ORIGIN_MACHINE_ID_LENGTH) {
1157
+ throw new ProtocolError(
1158
+ 'BAD_REQUEST',
1159
+ `origin_machine_id must be a non-empty string of at most ${MAX_ORIGIN_MACHINE_ID_LENGTH} characters`,
1160
+ { origin_machine_id: input.origin_machine_id },
1161
+ );
1162
+ }
1163
+ originMachineId = input.origin_machine_id;
1164
+ }
1165
+
1166
+ let p2p = null;
1167
+ if (input.p2p !== undefined && input.p2p !== null) {
1168
+ if (typeof input.p2p !== 'object' || Array.isArray(input.p2p)) {
1169
+ throw new ProtocolError('BAD_REQUEST', 'p2p must be an object with a `mode` field', { p2p: input.p2p });
1170
+ }
1171
+ if (!P2P_MODES.includes(input.p2p.mode)) {
1172
+ throw new ProtocolError(
1173
+ 'BAD_REQUEST',
1174
+ `p2p.mode must be one of ${P2P_MODES.join('|')}`,
1175
+ { 'p2p.mode': input.p2p.mode ?? null },
1176
+ );
1177
+ }
1178
+ // Copied rather than referenced: the recorded task must not be mutable through the request
1179
+ // body the caller still holds.
1180
+ p2p = { ...input.p2p };
1181
+ }
1182
+
1183
+ return { origin_machine_id: originMachineId, p2p };
1184
+ }
1185
+
1186
+ /**
1187
+ * v0.4.0 §5.2: validate the optional transport fields of a result envelope.
1188
+ *
1189
+ * Same reasoning as `validateTaskRouting`: `transport` is an enum and `p2p` is an opaque object,
1190
+ * so anything else is refused by name at the door. Both fields stay OUT of
1191
+ * `REQUIRED_ENVELOPE_FIELDS` (a missing one is a v0.3.9 envelope, not an incomplete one) and out
1192
+ * of `COMPARABLE_FIELDS` (a path is not a result).
1193
+ */
1194
+ static validateTransportFields(envelope = {}) {
1195
+ if (envelope.transport !== undefined && envelope.transport !== null) {
1196
+ if (!TRANSPORT_VALUES.includes(envelope.transport)) {
1197
+ throw new ProtocolError(
1198
+ 'BAD_REQUEST',
1199
+ `transport must be one of ${TRANSPORT_VALUES.join('|')}`,
1200
+ { transport: envelope.transport },
1201
+ );
1202
+ }
1203
+ }
1204
+ if (envelope.p2p !== undefined && envelope.p2p !== null
1205
+ && (typeof envelope.p2p !== 'object' || Array.isArray(envelope.p2p))) {
1206
+ throw new ProtocolError('BAD_REQUEST', 'p2p must be an object', { p2p: envelope.p2p });
1207
+ }
1208
+ }
1209
+
1210
+ /**
1211
+ * §4.1 POST /v1/task.
1212
+ * Machine selection is not specified by the PROTOCOL; we implement:
1213
+ * - `target_machines` (optional explicit list), else every paired device
1214
+ * - capability gate §6.1 per machine (failures become `refused` leases)
1215
+ * - index assignment: replicate -> 0, split -> round-robin modulo index_total
1216
+ */
1217
+ createTask(input = {}) {
1218
+ // v0.4.0: refused BEFORE the task literal is built and before any offer is emitted, so a bad
1219
+ // value cannot leave a half-created task behind. `POST /v1/task` is all-or-nothing.
1220
+ const routing = RabbitState.validateTaskRouting(input);
1221
+
1222
+ const mode = input.mode ?? 'replicate';
1223
+ if (mode !== 'replicate' && mode !== 'split' && mode !== 'broadcast' && mode !== 'pipeline' && mode !== 'compose') {
1224
+ throw new ProtocolError(
1225
+ 'BAD_REQUEST',
1226
+ 'mode must be replicate|split|broadcast|pipeline|compose',
1227
+ { mode },
1228
+ );
1229
+ }
1230
+ /**
1231
+ * v0.3.3 `pipeline`: the whole chain runs on each machine, stage after stage.
1232
+ *
1233
+ * The semantics were chosen against the alternative (fan out stage 1, converge its outputs into
1234
+ * stage 2) because that one has no well-defined input: on a fleet, stage 1 produces n different
1235
+ * outputs, and "feed them all to stage 2" does not say in what order, or whether stage 2 then
1236
+ * runs once or n times. Worse, two machines finishing in a different order would produce
1237
+ * different stage-2 inputs, so one task_id would no longer describe one reproducible execution --
1238
+ * which is the property the rest of this system is built on. See docs/PIPELINE-DESIGN.md.
1239
+ *
1240
+ * So: a pipeline is `replicate` applied to a sequence. Every machine runs the whole chain, there
1241
+ * is no cross-machine data flow, and nothing new can be silently dropped in transit.
1242
+ */
1243
+ const rawStages = Array.isArray(input.stages) ? input.stages : null;
1244
+ const isPipeline = mode === 'pipeline';
1245
+ if (isPipeline) {
1246
+ if (rawStages === null || rawStages.length === 0) {
1247
+ throw new ProtocolError('BAD_REQUEST', 'mode=pipeline requires a non-empty `stages` array', {
1248
+ stages: rawStages === null ? null : rawStages.length,
1249
+ });
1250
+ }
1251
+ if (rawStages.length > MAX_PIPELINE_STAGES) {
1252
+ throw new ProtocolError('BAD_REQUEST', `stages must have at most ${MAX_PIPELINE_STAGES} entries`, {
1253
+ stages: rawStages.length,
1254
+ });
1255
+ }
1256
+ rawStages.forEach((stage, i) => {
1257
+ const argv = stage?.command_argv;
1258
+ if (!Array.isArray(argv) || argv.length === 0 || argv.some((a) => typeof a !== 'string')) {
1259
+ throw new ProtocolError('BAD_REQUEST', `stages[${i}].command_argv must be a non-empty string[]`, { index: i });
1260
+ }
1261
+ });
1262
+ // A chain is one execution on one machine; sharding it would mean each machine ran a chain of
1263
+ // its own slice, which is a different feature and not what the caller asked for.
1264
+ if (Number.isInteger(input.index_total) && input.index_total !== 1) {
1265
+ throw new ProtocolError('BAD_REQUEST', 'mode=pipeline runs the whole chain per machine, so index_total must be 1', {
1266
+ index_total: input.index_total,
1267
+ });
1268
+ }
1269
+ } else if (rawStages !== null) {
1270
+ // Refused rather than ignored: the caller believes they described a sequence, and running only
1271
+ // `command_argv` would silently drop every stage after the first.
1272
+ throw new ProtocolError('BAD_REQUEST', '`stages` is only valid with mode=pipeline', { mode, stages: rawStages.length });
1273
+ }
1274
+
1275
+ /**
1276
+ * Normalized stage list. For every mode there is at least one stage, so downstream code has a
1277
+ * single shape to reason about and `pipeline` is not a special case at the execution layer.
1278
+ */
1279
+ const stages = isPipeline
1280
+ ? rawStages.map((s, i) => ({
1281
+ index: i,
1282
+ command_argv: [...s.command_argv],
1283
+ cwd_rel: typeof s.cwd_rel === 'string' && s.cwd_rel ? s.cwd_rel : (typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.'),
1284
+ timeout_ms: Number.isInteger(s.timeout_ms) ? s.timeout_ms : null,
1285
+ continue_on_failure: s.continue_on_failure === true,
1286
+ }))
1287
+ : null;
1288
+
1289
+ const commandArgv = isPipeline ? [...rawStages[0].command_argv] : input.command_argv;
1290
+ if (!Array.isArray(commandArgv) || commandArgv.length === 0 || commandArgv.some((a) => typeof a !== 'string')) {
1291
+ throw new ProtocolError('BAD_REQUEST', 'command_argv must be a non-empty string[]');
1292
+ }
1293
+ const indexTotal = Number.isInteger(input.index_total) ? input.index_total : 1;
1294
+ if (indexTotal < 1) throw new ProtocolError('BAD_REQUEST', 'index_total must be >= 1');
1295
+
1296
+ const halt = input.halt ?? 'never';
1297
+ if (halt !== 'never' && halt !== 'now') throw new ProtocolError('BAD_REQUEST', 'halt must be never|now', { halt });
1298
+
1299
+ const taskId = input.task_id ?? this.nextId();
1300
+ if (this.tasks.has(taskId)) {
1301
+ throw new ProtocolError('TASK_EXISTS', 'task_id already exists', { task_id: taskId });
1302
+ }
1303
+
1304
+ const now = this.nowMs();
1305
+ const cwdRel = typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.';
1306
+ const baseTree = input.base_tree ?? null;
1307
+ const commandHash = input.command_hash ?? (isPipeline
1308
+ ? computePipelineCommandHash(stages, SHELL_ID_DIRECT)
1309
+ : computeCommandHash(commandArgv, SHELL_ID_DIRECT, cwdRel));
1310
+ const timeoutMs = Number.isInteger(input.timeout_ms) ? input.timeout_ms : 300_000;
1311
+
1312
+ // v0.3.3 `broadcast`: one machine runs the command, every other machine is told the outcome.
1313
+ //
1314
+ // The executor choice is deterministic rather than random. A random leader makes a task
1315
+ // impossible to reproduce from its own record -- the same task id would run somewhere else on a
1316
+ // retry -- and this project's whole value rests on being able to say exactly where something ran.
1317
+ // The first target in the relay's own device order is stable for a given device table.
1318
+ //
1319
+ // Observers get a lease so they are tracked and notified, but their lease is `observing`, which
1320
+ // every aggregation step treats as "did not run". That distinction is the point: a machine that
1321
+ // never executed anything agreeing with a result would be a fabrication, and it is exactly the
1322
+ // fabrication this mode could most easily produce.
1323
+ //
1324
+ // Resolved before the task literal because the literal records it. (`targets` is computed below,
1325
+ // so the executor is re-validated against it after that.)
1326
+ const targetIds = Array.isArray(input.target_machines) && input.target_machines.length > 0
1327
+ ? input.target_machines
1328
+ : [...this.devices.keys()];
1329
+ const executorMachineId = mode === 'broadcast'
1330
+ ? (typeof input.executor_machine_id === 'string' && input.executor_machine_id !== ''
1331
+ ? input.executor_machine_id
1332
+ : targetIds[0] ?? null)
1333
+ : null;
1334
+
1335
+ const task = {
1336
+ task_id: taskId,
1337
+ mode,
1338
+ command_argv: commandArgv,
1339
+ cwd_rel: cwdRel,
1340
+ index_total: indexTotal,
1341
+ timeout_ms: timeoutMs,
1342
+ write: input.write === true,
1343
+ write_scope: input.write_scope ?? [],
1344
+ require_exclusive_write: input.require_exclusive_write === true,
1345
+ base_commit: input.base_commit ?? null,
1346
+ base_tree: baseTree,
1347
+ requirements: input.requirements ?? {},
1348
+ compare_policy: input.compare_policy ?? {},
1349
+ halt,
1350
+ created_by: input.created_by ?? null,
1351
+ created_at_ms: now,
1352
+ command_hash: commandHash,
1353
+ attempt: 1,
1354
+ leases: new Map(),
1355
+ cancelled: false,
1356
+ cancel_reason: null,
1357
+ degraded: null,
1358
+ deadline_ms: now + timeoutMs,
1359
+ result_seqs: [],
1360
+ /** v0.3.3: which machine actually executes under `broadcast`; null for the other modes. */
1361
+ executor_machine_id: executorMachineId,
1362
+ /**
1363
+ * v0.3.3: the normalized stage list for `pipeline`, or null.
1364
+ *
1365
+ * Sent to the machine in the offer so it can run the whole chain. Kept on the task as well so
1366
+ * a report can say how many stages were supposed to run even when the machine crashed before
1367
+ * reporting any of them.
1368
+ */
1369
+ stages,
1370
+ /**
1371
+ * v0.4.0: where the task came from and under which P2P mode it was dispatched, or null.
1372
+ *
1373
+ * Kept on the task (not just passed to `emitOffer`) because an offer is emitted more than
1374
+ * once: `redeliverPendingOffers` re-states unclaimed work on every reconnect and
1375
+ * `_takeover` re-offers a freed index. A machine that reconnects must see the same routing
1376
+ * facts as the machine that was connected at dispatch time, or the direct-push half of the
1377
+ * feature would work exactly once and then quietly stop.
1378
+ */
1379
+ origin_machine_id: routing.origin_machine_id,
1380
+ p2p: routing.p2p,
1381
+ };
1382
+
1383
+ /**
1384
+ * Fleet scoping (v0.7.7).
1385
+ *
1386
+ * This replaces `targets = [...this.devices.values()]`, which reached **every device on the
1387
+ * relay** with no filter — the finding that accounts were not isolated. The rule now lives in
1388
+ * `fleet-wiring.mjs` so it is one function with its own exhaustive tests rather than three
1389
+ * lines inside a two-thousand-line object.
1390
+ *
1391
+ * ## The migration is deliberate, and so is its limit
1392
+ *
1393
+ * The shared relay holds four devices that paired before fleets existed, and none of them has
1394
+ * a `fleet_id`. `targetsFor` therefore has two modes: a relay with no fleet configured keeps
1395
+ * exactly the old behaviour, and a relay that *is* scoped refuses a fleetless requester rather
1396
+ * than handing it the run of the fleet. So deploying this cannot be an outage, and scoping it
1397
+ * cannot be a silent grant.
1398
+ *
1399
+ * ## What this does NOT do, stated here rather than discovered later
1400
+ *
1401
+ * `POST /v1/task` authenticates with the **operator token**, and `authorizeTaskDispatch`
1402
+ * returns `{ role: 'operator', machine_id: null }` for it — there is no device, so there is no
1403
+ * fleet to scope by. Every device that authenticates with a **device token** gets full
1404
+ * isolation from this change; an operator-token dispatch remains relay-wide, because the
1405
+ * operator token *is* the relay-wide credential by design.
1406
+ *
1407
+ * `input.created_by` is the one device identity a task carries, and it is used when present:
1408
+ * a task submitted on behalf of a machine is scoped to that machine's fleet.
1409
+ */
1410
+ const requester =
1411
+ (typeof input.created_by === 'string' && this.devices.get(input.created_by)) ||
1412
+ (typeof input.requester_machine_id === 'string' && this.devices.get(input.requester_machine_id)) ||
1413
+ null;
1414
+
1415
+ /**
1416
+ * An operator-token dispatch, on a relay that has been scoped, acts **within that scope**.
1417
+ *
1418
+ * A first version passed `requester: null` and let it be refused -- which broke every
1419
+ * operator dispatch on a scoped relay, and broke it in the least useful way: the operator had
1420
+ * just configured the fleet, and configuring it made them unable to dispatch at all.
1421
+ *
1422
+ * The reasoning for the fix: the relay's own `fleetId` *is* an admin's statement about which
1423
+ * fleet this relay serves. An operator dispatch therefore has an unambiguous scope, and
1424
+ * refusing it would be refusing the case that was explicitly asked for.
1425
+ *
1426
+ * What stays genuinely ambiguous is an operator dispatch on a relay holding **several** fleets
1427
+ * with no fleet of its own -- there is no way to tell which was meant, so `targetsFor` refuses
1428
+ * that rather than guessing.
1429
+ */
1430
+ const scopedRequester =
1431
+ requester ??
1432
+ (this.fleetId === null ? null : { machine_id: null, fleet_id: this.fleetId, fleet_source: 'relay_scope' });
1433
+
1434
+ const scoped = targetsFor({
1435
+ devices: [...this.devices.values()],
1436
+ requester: scopedRequester,
1437
+ targetMachines: input.target_machines,
1438
+ relayFleetId: this.fleetId ?? null,
1439
+ // The existing code did not filter offline devices at selection time -- it offered to them and
1440
+ // let the lease expire. Keeping that here avoids changing two behaviours at once, and the
1441
+ // lease expiry is what `handoff.mjs` is for.
1442
+ requireOnline: false,
1443
+ });
1444
+ if (!scoped.ok) {
1445
+ /**
1446
+ * `NO_ONLINE_DEVICE`, not a new code.
1447
+ *
1448
+ * A first version invented `NO_ELIGIBLE_DEVICE`, which broke the one relay test that pins
1449
+ * this contract — and worse, it would have broken **callers**: `chatops/fleet.mjs` reads
1450
+ * `NO_ONLINE_DEVICE` to produce *"start an agent on each machine"*, and a code it has never
1451
+ * seen falls through to a generic refusal. The protocol's code table is part of the
1452
+ * protocol, and adding a member to it is a decision for a protocol revision, not for a
1453
+ * wiring change.
1454
+ *
1455
+ * What *is* new is why nothing was eligible, and that belongs in `detail`, where a caller
1456
+ * can read it without a new code entering the table.
1457
+ */
1458
+ throw new ProtocolError('NO_ONLINE_DEVICE', scoped.text, {
1459
+ reason: scoped.reason,
1460
+ legacy: scoped.legacy,
1461
+ fleet_scoped: scoped.legacy === false,
1462
+ hint:
1463
+ scoped.reason === 'requester_has_no_fleet'
1464
+ ? 'this relay is bound to a fleet and the requesting device is in none; pair it with a code issued for that fleet'
1465
+ : scoped.reason === 'target_outside_fleet'
1466
+ ? 'the named target is not in this fleet'
1467
+ : 'check that at least one device in this fleet is paired and streaming',
1468
+ });
1469
+ }
1470
+ const targets = scoped.targets;
1471
+ if (scoped.legacy) {
1472
+ // Logged at debug so an operator reading logs during a migration can see which rule applied.
1473
+ // "N targets" alone does not say whether isolation is on, and that is the question being asked.
1474
+ this.log?.('debug', 'task targets selected without a fleet (legacy path)', {
1475
+ task_candidates: targets.length,
1476
+ relay_fleet_id: this.fleetId ?? null,
1477
+ });
1478
+ }
1479
+
1480
+ // v0.3.3 `broadcast`: validated here, where `targets` is known. The executor id itself was
1481
+ // resolved before the task literal because the literal records it.
1482
+ if (mode === 'broadcast' && executorMachineId !== null && !targets.some((d) => d.machine_id === executorMachineId)) {
1483
+ throw new ProtocolError('BAD_REQUEST', 'executor_machine_id is not among the target machines', {
1484
+ executor_machine_id: executorMachineId,
1485
+ targets: targets.map((d) => d.machine_id),
1486
+ });
1487
+ }
1488
+
1489
+ if (targets.length === 0) {
1490
+ throw new ProtocolError('NO_ONLINE_DEVICE', 'no paired device available for this task', {
1491
+ known_devices: all.length,
1492
+ });
1493
+ }
1494
+
1495
+ const dedupeKeyBase = computeDedupeKey(taskId, 0, commandHash, baseTree);
1496
+ if (typeof dedupeKeyBase !== 'string' || dedupeKeyBase.length !== 64) {
1497
+ throw new ProtocolError('INTERNAL', 'failed to compute dedupe_key');
1498
+ }
1499
+ let lastSeq = this.seq;
1500
+ targets.forEach((device, i) => {
1501
+ const index = mode === 'replicate' ? 0 : i % indexTotal;
1502
+ const observing = mode === 'broadcast' && device.machine_id !== executorMachineId;
1503
+ const lease = {
1504
+ machine_id: device.machine_id,
1505
+ machine_name: device.machine_name,
1506
+ index,
1507
+ attempt: 1,
1508
+ // `observing` is a lease state, not an outcome: the machine holds a place in this task and
1509
+ // will be told what happened, but it was never handed the command. `gateMachine` below is
1510
+ // skipped for it because refusing to do work you were never asked to do is not a refusal.
1511
+ state: observing ? 'observing' : 'queued',
1512
+ refusal_reason: null,
1513
+ refusal_detail: null,
1514
+ dedupe_key: computeDedupeKey(taskId, index, commandHash, baseTree),
1515
+ last_heartbeat_ms: now,
1516
+ claimed_at_ms: now,
1517
+ lease_until_ms: now + this.leaseWindowMs(),
1518
+ phase: null,
1519
+ progress: null,
1520
+ offered_seq: null,
1521
+ expired_at_ms: null,
1522
+ finished_at_ms: null,
1523
+ heartbeat_count: 0,
1524
+ };
1525
+ if (observing) {
1526
+ // No capability gate and no offer: the machine is not being asked to run anything, so there
1527
+ // is nothing to refuse and nothing to hand out. It is recorded so the task can tell it the
1528
+ // outcome afterwards.
1529
+ task.leases.set(device.machine_id, lease);
1530
+ return;
1531
+ }
1532
+ const gate = gateMachine(task, device);
1533
+ if (gate) {
1534
+ lease.state = 'refused';
1535
+ lease.refusal_reason = gate.refusal_reason;
1536
+ lease.refusal_detail = gate.detail;
1537
+ task.leases.set(device.machine_id, lease);
1538
+ this.emit('notice', {
1539
+ level: 'warn',
1540
+ code: 'CAPABILITY_REFUSED',
1541
+ message: `${device.machine_id} refused: ${gate.refusal_reason}`,
1542
+ task_id: taskId,
1543
+ });
1544
+ return;
1545
+ }
1546
+ task.leases.set(device.machine_id, lease);
1547
+ const grace = this._p2pOfferGraceMs(task);
1548
+ if (grace > 0) {
1549
+ // v0.4.0: the dispatcher is pushing this offer over the direct path right now. Hold the
1550
+ // relay's copy for `grace` ms so the direct one can win; see `p2pOfferGraceMs`.
1551
+ this._scheduleOffer(task, lease, grace);
1552
+ } else {
1553
+ const event = this.emitOffer(task, lease);
1554
+ lastSeq = event.seq;
1555
+ }
1556
+ });
1557
+
1558
+ this.tasks.set(taskId, task);
1559
+
1560
+ this._ledger({
1561
+ type: 'task.created',
1562
+ task: {
1563
+ task_id: task.task_id,
1564
+ mode: task.mode,
1565
+ command_argv: task.command_argv,
1566
+ cwd_rel: task.cwd_rel,
1567
+ index_total: task.index_total,
1568
+ timeout_ms: task.timeout_ms,
1569
+ write: task.write,
1570
+ write_scope: task.write_scope,
1571
+ require_exclusive_write: task.require_exclusive_write,
1572
+ base_commit: task.base_commit,
1573
+ base_tree: task.base_tree,
1574
+ requirements: task.requirements,
1575
+ compare_policy: task.compare_policy,
1576
+ halt: task.halt,
1577
+ created_by: task.created_by,
1578
+ created_at_ms: task.created_at_ms,
1579
+ command_hash: task.command_hash,
1580
+ attempt: task.attempt,
1581
+ deadline_ms: task.deadline_ms,
1582
+ // v0.4.0: routing metadata, so a restarted relay re-offers with the same facts.
1583
+ origin_machine_id: task.origin_machine_id ?? null,
1584
+ p2p: task.p2p ?? null,
1585
+ },
1586
+ leases: [...task.leases.values()].map((l) => ({ ...l })),
1587
+ });
1588
+
1589
+ const refusedOnly = [...task.leases.values()].every((l) => l.state === 'refused');
1590
+ if (refusedOnly) {
1591
+ // §6.3 Step 0 territory: a task nobody may execute is still a valid, reportable task.
1592
+ this.emit('notice', {
1593
+ level: 'warn',
1594
+ code: 'NO_ONLINE_DEVICE',
1595
+ message: 'every candidate machine refused the capability gate',
1596
+ task_id: taskId,
1597
+ });
1598
+ }
1599
+
1600
+ return {
1601
+ task_id: taskId,
1602
+ leases: [...task.leases.values()].map((l) => ({
1603
+ machine_id: l.machine_id,
1604
+ index: l.index,
1605
+ state: l.state,
1606
+ // v0.4.0: the dispatcher pushes an offer over the direct path, and the executor dedupes it
1607
+ // against the relay's copy by (task_id, attempt, dedupe_key). Without this field the
1608
+ // dispatcher has to reproduce the §4.4 formula locally, which works for a single-command
1609
+ // task and cannot work for a pipeline one -- the key covers the relay's normalised stage
1610
+ // list. Publishing it makes the pushed copy recognisably the same work item, which is what
1611
+ // keeps the direct path from turning exactly-once into at-least-twice.
1612
+ dedupe_key: l.dedupe_key ?? null,
1613
+ attempt: l.attempt,
1614
+ ...(l.refusal_reason ? { refusal_reason: l.refusal_reason } : {}),
1615
+ })),
1616
+ seq: lastSeq,
1617
+ };
1618
+ }
1619
+
1620
+ /**
1621
+ * v0.4.0: should the relay hold its offer back for this task?
1622
+ *
1623
+ * Only when the dispatcher said two things: that it is a machine (`origin_machine_id`), and that
1624
+ * the direct path is in play (`p2p.mode` of `auto` or `direct`). A `relay` mode or a v0.3.9
1625
+ * dispatcher gets the old behaviour exactly, which is why this predicate is the only gate on the
1626
+ * change and every pre-existing test still sees an immediate offer.
1627
+ *
1628
+ * @param {object} task
1629
+ * @returns {number} milliseconds to hold, or 0 for "offer now"
1630
+ */
1631
+ _p2pOfferGraceMs(task) {
1632
+ if (this.p2pOfferGraceMs <= 0) return 0;
1633
+ const origin = task?.origin_machine_id;
1634
+ if (typeof origin !== 'string' || origin === '') return 0;
1635
+ const mode = task?.p2p?.mode;
1636
+ if (mode !== 'auto' && mode !== 'direct') return 0;
1637
+ return this.p2pOfferGraceMs;
1638
+ }
1639
+
1640
+ /**
1641
+ * v0.4.0: emit `lease`'s offer after `delayMs`, unless the direct path got there first.
1642
+ *
1643
+ * "Got there first" is one fact and it is already tracked: the executor renews the lease
1644
+ * (`POST /v1/heartbeat`) as its first act on any offer it accepts, and a renewal moves the lease
1645
+ * out of `queued`. So the check is the lease's own state — no new message, no new endpoint, and
1646
+ * nothing the executor has to know about.
1647
+ *
1648
+ * The timer is `unref()`ed: a relay with nothing else to do must still be able to exit, and a
1649
+ * pending offer is not a reason to keep a process alive.
1650
+ */
1651
+ _scheduleOffer(task, lease, delayMs) {
1652
+ const key = `${task.task_id}:${lease.machine_id}`;
1653
+ const timer = setTimeout(() => {
1654
+ this._pendingOffers.delete(key);
1655
+ // A lease that was re-created (takeover) or swept is not this one any more.
1656
+ if (task.leases.get(lease.machine_id) !== lease) return;
1657
+ if (lease.state !== 'queued') return; // the direct copy won; nothing to fall back to
1658
+ if (this.isLeaseExpired(lease, this.nowMs())) return; // the sweep owns it now
1659
+ this.emitOffer(task, lease);
1660
+ }, delayMs);
1661
+ timer.unref?.();
1662
+ this._pendingOffers.set(key, timer);
1663
+ }
1664
+
1665
+ /** Drop every deferred offer. Called by the relay's own teardown path. */
1666
+ clearPendingOffers() {
1667
+ for (const timer of this._pendingOffers.values()) clearTimeout(timer);
1668
+ this._pendingOffers.clear();
1669
+ }
1670
+
1671
+ emitOffer(task, lease) {
1672
+ const event = this.emit('task.offer', {
1673
+ task_id: task.task_id,
1674
+ attempt: lease.attempt,
1675
+ mode: task.mode,
1676
+ index: lease.index,
1677
+ index_total: task.index_total,
1678
+ command_argv: task.command_argv,
1679
+ // v0.3.3: present only for `pipeline`, and only then does the machine run a chain. Omitted
1680
+ // entirely otherwise, so an offer for a single-command task is byte-identical to before --
1681
+ // which is what keeps a v0.2.3 agent working against this relay.
1682
+ ...(task.stages ? { stages: task.stages } : {}),
1683
+ /**
1684
+ * v0.4.0: always present, `null` when the caller supplied nothing.
1685
+ *
1686
+ * Deliberately NOT conditional (unlike `stages` above, which is omitted so a single-command
1687
+ * offer stays byte-identical for an old agent): the contract fixes the shape of these two as
1688
+ * `string|null` and `{mode}|null`, and `null` is the honest value for "the dispatcher did not
1689
+ * say" — which is exactly the v0.3.9 case. They are additive keys in an event whose consumers
1690
+ * already ignore unknown fields.
1691
+ */
1692
+ origin_machine_id: task.origin_machine_id ?? null,
1693
+ p2p: task.p2p ?? null,
1694
+ cwd_rel: task.cwd_rel,
1695
+ write: task.write,
1696
+ timeout_ms: task.timeout_ms,
1697
+ base_commit: task.base_commit,
1698
+ base_tree: task.base_tree,
1699
+ requirements: task.requirements,
1700
+ compare_policy: task.compare_policy,
1701
+ dedupe_key: lease.dedupe_key,
1702
+ deadline: rfc3339(task.deadline_ms),
1703
+ }, { target: lease.machine_id });
1704
+ lease.state = 'offered';
1705
+ lease.offered_seq = event.seq;
1706
+ lease.last_heartbeat_ms = this.nowMs();
1707
+ lease.lease_until_ms = lease.last_heartbeat_ms + this.leaseWindowMs();
1708
+ return event;
1709
+ }
1710
+
1711
+ getTask(taskId) { return this.tasks.get(taskId) ?? null; }
1712
+
1713
+ requireTask(taskId) {
1714
+ const task = this.tasks.get(taskId);
1715
+ if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
1716
+ return task;
1717
+ }
1718
+
1719
+ /**
1720
+ * Re-offer the work a machine was handed but has not yet started (v0.1.2 §8.6).
1721
+ *
1722
+ * WHY THIS EXISTS
1723
+ *
1724
+ * An offer is a point-in-time event on a stream. `createTask` publishes
1725
+ * `task.offer` once, and if the target machine's stream happens to be down at
1726
+ * that instant -- the ordinary situation after a relay restart, a tunnel drop,
1727
+ * or any cross-network interruption -- the event is simply gone. The lease sits
1728
+ * in `offered` until the sweep expires it, the task never runs, and the
1729
+ * operator sees a timeout with no explanation. Measured: the machine is back
1730
+ * online with a live stream within milliseconds, and still receives nothing.
1731
+ *
1732
+ * So the relay re-states unclaimed work when a machine connects. This is safe
1733
+ * because the state it acts on is durable: the lease the offer refers to is
1734
+ * still in the ledger.
1735
+ *
1736
+ * WHY ONLY `offered` AND `queued`
1737
+ *
1738
+ * `running` is deliberately excluded, and that exclusion is the whole safety
1739
+ * argument. A running lease means a machine has already STARTED the command --
1740
+ * it acknowledged the offer and is heartbeating. Re-offering it would make that
1741
+ * machine execute the work a second time, so the fix for a lost offer would
1742
+ * introduce the one failure this project is built to avoid: two machines, or
1743
+ * one machine twice, running the same side-effecting command.
1744
+ *
1745
+ * A machine that dies mid-run therefore does not get its work back here; that
1746
+ * path stays with the sweep, which expires the lease and hands the index to
1747
+ * another machine with `attempt + 1`.
1748
+ *
1749
+ * Re-emitted offers are new events and are replayed like any other, so a client
1750
+ * that reconnects repeatedly can see the same offer more than once. That is
1751
+ * intentional: the agent already de-duplicates by `task_id` (it ignores an
1752
+ * offer whose task is queued or running), and an at-least-once offer with a
1753
+ * client-side dedupe is strictly better than an at-most-once offer that is
1754
+ * silently dropped.
1755
+ *
1756
+ * @param {string} machineId
1757
+ * @returns {{count: number, task_ids: string[]}}
1758
+ */
1759
+ redeliverPendingOffers(machineId) {
1760
+ const taskIds = [];
1761
+ const now = this.nowMs();
1762
+
1763
+ for (const task of this.tasks.values()) {
1764
+ if (task.cancelled === true) continue;
1765
+ const lease = task.leases.get(machineId);
1766
+ if (!lease) continue;
1767
+ // Only work the machine has not begun. See the note above: `running` is
1768
+ // excluded on purpose, not by omission.
1769
+ if (lease.state !== 'offered' && lease.state !== 'queued') continue;
1770
+ // Do not resurrect a lease the sweep has already written off; the takeover
1771
+ // path owns that decision.
1772
+ if (this.isLeaseExpired(lease, now)) continue;
1773
+
1774
+ this.emitOffer(task, lease);
1775
+ taskIds.push(task.task_id);
1776
+ }
1777
+
1778
+ return { count: taskIds.length, task_ids: taskIds };
1779
+ }
1780
+
1781
+ listTasks({ limit = 50 } = {}) {
1782
+ const n = Number.isInteger(limit) && limit > 0 ? limit : 50;
1783
+ return [...this.tasks.values()]
1784
+ .sort((a, b) => b.created_at_ms - a.created_at_ms)
1785
+ .slice(0, n)
1786
+ .map((t) => this.taskSummary(t));
1787
+ }
1788
+
1789
+ taskSummary(task) {
1790
+ return {
1791
+ task_id: task.task_id,
1792
+ mode: task.mode,
1793
+ created_at: rfc3339(task.created_at_ms),
1794
+ created_by: task.created_by,
1795
+ index_total: task.index_total,
1796
+ cancelled: task.cancelled,
1797
+ degraded: task.degraded,
1798
+ leases: [...task.leases.values()].map((l) => l.machine_id),
1799
+ lease_states: Object.fromEntries([...task.leases.values()].map((l) => [l.machine_id, l.state])),
1800
+ };
1801
+ }
1802
+
1803
+ cancelTask(taskId, reason = 'cancelled_by_operator') {
1804
+ const task = this.requireTask(taskId);
1805
+ if (task.cancelled) return task;
1806
+ task.cancelled = true;
1807
+ task.cancel_reason = reason;
1808
+ for (const lease of task.leases.values()) {
1809
+ if (ACTIVE_LEASE_STATES.includes(lease.state)) lease.state = 'cancelled';
1810
+ }
1811
+ this.emit('task.cancel', { task_id: taskId, reason });
1812
+ this._ledger({ type: 'task.cancelled', task_id: taskId, reason });
1813
+ return task;
1814
+ }
1815
+
1816
+ /* ---------------- leases / heartbeat (§4.3) ---------------- */
1817
+
1818
+ /** Shared `phase`/`progress` validation for both heartbeat modes. */
1819
+ _validatePhaseProgress(input) {
1820
+ if (input.phase !== undefined && input.phase !== null && !LEASE_PHASES.includes(input.phase)) {
1821
+ throw new ProtocolError('BAD_REQUEST', 'phase must be preparing|running|finalizing', { phase: input.phase });
1822
+ }
1823
+ if (input.progress !== undefined && input.progress !== null
1824
+ && (typeof input.progress !== 'number' || input.progress < 0 || input.progress > 1)) {
1825
+ throw new ProtocolError('BAD_REQUEST', 'progress must be a number in [0,1]', { progress: input.progress });
1826
+ }
1827
+ }
1828
+
1829
+ /**
1830
+ * v0.3.0 IDLE (diagnostic) heartbeat — an agent with no lease keeps its latency
1831
+ * and liveness visible to other machines.
1832
+ *
1833
+ * This path deliberately NEVER looks at `this.tasks`. That is the whole safety
1834
+ * argument, and it is structural rather than a matter of remembering a rule: an
1835
+ * idle heartbeat cannot renew, resurrect or otherwise touch a lease because it
1836
+ * never obtains one. Without that, an idle ping from a machine whose lease had
1837
+ * expired would flip `expired` back to `running` and manufacture a live task out
1838
+ * of nothing -- a fake "the work is still being done" signal, which is the most
1839
+ * dangerous lie this relay can tell.
1840
+ */
1841
+ diagnosticHeartbeat(input = {}) {
1842
+ const machineId = typeof input.machine_id === 'string' && input.machine_id !== ''
1843
+ ? input.machine_id : null;
1844
+ if (machineId === null) {
1845
+ throw new ProtocolError('BAD_REQUEST', 'machine_id is required', {});
1846
+ }
1847
+ if (!this.devices.has(machineId)) {
1848
+ throw new ProtocolError('NOT_FOUND', 'unknown machine_id', { machine_id: machineId });
1849
+ }
1850
+ this._validatePhaseProgress(input);
1851
+
1852
+ const device = this.touchDevice(machineId);
1853
+ this.recordHeartbeatMetrics(machineId, input);
1854
+ return {
1855
+ // No lease is involved, and the response says so instead of inventing one.
1856
+ lease_until: null,
1857
+ cancel: false,
1858
+ diagnostic: true,
1859
+ machine_id: machineId,
1860
+ rtt_ms: device?.rtt_ms ?? null,
1861
+ };
1862
+ }
1863
+
1864
+ /**
1865
+ * §4.3 POST /v1/heartbeat — renews the lease using SERVER time only.
1866
+ *
1867
+ * Two modes since v0.3.0:
1868
+ * - `task_id` present -> LEASE heartbeat (unchanged §4.3 behaviour)
1869
+ * - `diagnostic: true`, no task -> IDLE heartbeat (diagnostics only, no lease)
1870
+ * - neither -> 400. A heartbeat that silently renewed
1871
+ * nothing would let an agent whose `task_id` went missing stop renewing while
1872
+ * believing it was fine, ending in a falsely `expired` task. §0 forbids
1873
+ * silent degradation, so this is loud.
1874
+ *
1875
+ * @returns {{lease_until:string|null, cancel:boolean, phase?:string, progress?:number, diagnostic?:boolean}}
1876
+ */
1877
+ heartbeat(input = {}) {
1878
+ const hasTask = typeof input.task_id === 'string' && input.task_id !== '';
1879
+ if (!hasTask) {
1880
+ if (input.diagnostic !== true) {
1881
+ throw new ProtocolError('BAD_REQUEST',
1882
+ 'a heartbeat must carry task_id (lease renewal) or diagnostic:true (idle diagnostics); '
1883
+ + 'refusing to treat a missing task_id as a no-op renewal', {
1884
+ received_keys: Object.keys(input).sort(),
1885
+ });
1886
+ }
1887
+ return this.diagnosticHeartbeat(input);
1888
+ }
1889
+ if (input.diagnostic === true) {
1890
+ // task_id wins: a heartbeat that names a task IS a lease heartbeat.
1891
+ // (documented; the flag is simply redundant here)
1892
+ }
1893
+ return this.leaseHeartbeat(input);
1894
+ }
1895
+
1896
+ leaseHeartbeat(input = {}) {
1897
+ const task = this.requireTask(input.task_id);
1898
+ const lease = task.leases.get(input.machine_id);
1899
+ if (!lease) {
1900
+ throw new ProtocolError('NOT_FOUND', 'no lease for this machine on this task', {
1901
+ task_id: input.task_id, machine_id: input.machine_id,
1902
+ });
1903
+ }
1904
+ this._validatePhaseProgress(input);
1905
+
1906
+ const now = this.nowMs();
1907
+ this.touchDevice(input.machine_id);
1908
+ this.recordHeartbeatMetrics(input.machine_id, input, now);
1909
+
1910
+ if (task.cancelled) {
1911
+ return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: task.cancel_reason };
1912
+ }
1913
+
1914
+ // §4.3: once halt=now has handed this index to another attempt, the old owner
1915
+ // must stop — a late heartbeat from the expired machine is not a valid re-claim.
1916
+ if (lease.state === 'expired' && task.attempt > lease.attempt) {
1917
+ return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: 'LEASE_EXPIRED_TAKEN_OVER' };
1918
+ }
1919
+
1920
+ const stateBefore = lease.state;
1921
+ lease.last_heartbeat_ms = now;
1922
+ lease.lease_until_ms = now + this.leaseWindowMs();
1923
+ lease.heartbeat_count += 1;
1924
+ if (input.phase !== undefined && input.phase !== null) lease.phase = input.phase;
1925
+ if (input.progress !== undefined && input.progress !== null) lease.progress = input.progress;
1926
+ if (Number.isInteger(input.attempt) && input.attempt >= 1) lease.attempt = input.attempt;
1927
+ if (lease.state === 'queued' || lease.state === 'offered') lease.state = 'running';
1928
+ // A live heartbeat is stronger evidence than a timer: with halt=never nobody took
1929
+ // the work away, so the late-but-alive agent re-claims its own lease.
1930
+ if (lease.state === 'expired') { lease.state = 'running'; lease.expired_at_ms = null; }
1931
+
1932
+ // §7 ledger records lease STATE changes, not every 10s heartbeat.
1933
+ if (lease.state !== stateBefore) this._ledgerLease(task, lease);
1934
+
1935
+ return { lease_until: rfc3339(lease.lease_until_ms), cancel: false, phase: lease.phase, progress: lease.progress };
1936
+ }
1937
+
1938
+ /** §4.3: expiry is decided by Rabbit's clock only. */
1939
+ isLeaseExpired(lease, nowMs = this.nowMs()) {
1940
+ if (!ACTIVE_LEASE_STATES.includes(lease.state)) return false;
1941
+ return nowMs >= lease.lease_until_ms;
1942
+ }
1943
+
1944
+ /**
1945
+ * §4.3 expiry sweep.
1946
+ * - halt=never → book-keeping only, task is marked `partial`
1947
+ * - halt=now → notify remaining machines immediately that they may take over
1948
+ */
1949
+ sweepExpired(nowMs = this.nowMs()) {
1950
+ const expired = [];
1951
+ for (const task of this.tasks.values()) {
1952
+ if (task.cancelled) continue;
1953
+ let changed = false;
1954
+ for (const lease of task.leases.values()) {
1955
+ if (!this.isLeaseExpired(lease, nowMs)) continue;
1956
+ lease.state = 'expired';
1957
+ lease.expired_at_ms = nowMs;
1958
+ changed = true;
1959
+ expired.push({ task, lease });
1960
+ this._ledgerLease(task, lease);
1961
+ this.emit('notice', {
1962
+ level: 'warn',
1963
+ code: 'LEASE_EXPIRED',
1964
+ message: `lease expired: machine=${lease.machine_id} task=${task.task_id} attempt=${lease.attempt}`,
1965
+ task_id: task.task_id,
1966
+ machine_id: lease.machine_id,
1967
+ });
1968
+ if (task.halt === 'now') {
1969
+ task.attempt += 1;
1970
+ this._takeover(task, lease, nowMs);
1971
+ }
1972
+ }
1973
+ if (changed) task.degraded = 'partial';
1974
+ }
1975
+ return expired;
1976
+ }
1977
+
1978
+ /**
1979
+ * §4.3 `halt=now`: "立即通知其他机器可接管".
1980
+ *
1981
+ * v0.0.1's PROTOCOL defines no per-index lease reassignment, so takeover is
1982
+ * implemented as: re-offer the freed `index` (attempt+1) to every live machine
1983
+ * that is idle — no lease yet, or a lease still queued/offered — and broadcast a
1984
+ * notice. Machines already `running` are never re-offered, to avoid double
1985
+ * execution of the same index.
1986
+ */
1987
+ _takeover(task, expiredLease, nowMs = this.nowMs()) {
1988
+ /**
1989
+ * v0.7.8: the two safety guards this path was missing.
1990
+ *
1991
+ * `_takeover` already existed and is invoked on `halt=now`, so hand-off was not absent — but it
1992
+ * would re-offer a dead machine's index to **every idle device the relay holds**, with no fleet
1993
+ * filter and no consideration of whether the task writes. Both are reachable defects:
1994
+ *
1995
+ * 1. **A device in another account could be handed the work.** On a relay serving several
1996
+ * fleets, `this.devices` includes all of them, so a takeover crossed accounts — which is
1997
+ * the isolation this release series is about, broken in the one path that is easy to miss
1998
+ * because it only runs when something has already gone wrong.
1999
+ * 2. **A writing task could be run a second time.** Hand-off makes execution at-least-once,
2000
+ * which is harmless for `node --test` and is a duplicated write for anything else. The
2001
+ * design forbids automatic hand-off for writing tasks until a fencing token exists.
2002
+ *
2003
+ * The re-offer shape itself is unchanged: PROTOCOL v0.0.1 defines no per-index lease
2004
+ * reassignment, so takeover is expressed as a re-offer to idle machines. Narrowing that to a
2005
+ * single successor is a protocol revision, not a wiring change, and is not made here.
2006
+ */
2007
+ if (task.write === true) {
2008
+ this.emit('notice', {
2009
+ level: 'warn',
2010
+ code: 'LEASE_TAKEOVER_REFUSED',
2011
+ message:
2012
+ `task ${task.task_id} index ${expiredLease.index} is a writing task, so no machine takes it over: `
2013
+ + 'a second run could apply the same change twice, and the caller cannot tell from the verdict',
2014
+ task_id: task.task_id,
2015
+ reason: 'write_not_idempotent',
2016
+ });
2017
+ return [];
2018
+ }
2019
+
2020
+ const requesterFleet = this.fleetId ?? null;
2021
+
2022
+ const candidates = [];
2023
+ for (const device of this.devices.values()) {
2024
+ if (device.machine_id === expiredLease.machine_id) continue;
2025
+ /**
2026
+ * Fleet membership, when the relay is scoped.
2027
+ *
2028
+ * A device with its own `fleet_id` must match. A device with none cannot take over on a
2029
+ * scoped relay, because there is no fleet to match it against — the same rule `targetsFor`
2030
+ * applies to dispatch, kept identical here so the two paths cannot disagree.
2031
+ */
2032
+ if (requesterFleet !== null) {
2033
+ const deviceFleet = typeof device.fleet_id === 'string' && device.fleet_id !== '' ? device.fleet_id : null;
2034
+ if (deviceFleet !== requesterFleet) {
2035
+ this.log?.('debug', 'takeover candidate skipped: not in this fleet', {
2036
+ task_id: task.task_id,
2037
+ machine_id: device.machine_id,
2038
+ device_fleet: deviceFleet,
2039
+ relay_fleet: requesterFleet,
2040
+ });
2041
+ continue;
2042
+ }
2043
+ }
2044
+ const lease = task.leases.get(device.machine_id);
2045
+ if (lease && !['queued', 'offered'].includes(lease.state)) continue;
2046
+ if (!lease && gateMachine(task, device)) continue;
2047
+ candidates.push({ device, lease: lease ?? null });
2048
+ }
2049
+
2050
+ for (const { device, lease } of candidates) {
2051
+ const target = lease ?? {
2052
+ machine_id: device.machine_id,
2053
+ machine_name: device.machine_name,
2054
+ index: expiredLease.index,
2055
+ attempt: task.attempt,
2056
+ state: 'queued',
2057
+ refusal_reason: null,
2058
+ refusal_detail: null,
2059
+ dedupe_key: null,
2060
+ last_heartbeat_ms: nowMs,
2061
+ claimed_at_ms: nowMs,
2062
+ lease_until_ms: nowMs + this.leaseWindowMs(),
2063
+ phase: null,
2064
+ progress: null,
2065
+ offered_seq: null,
2066
+ expired_at_ms: null,
2067
+ finished_at_ms: null,
2068
+ heartbeat_count: 0,
2069
+ };
2070
+ target.index = expiredLease.index;
2071
+ target.attempt = task.attempt;
2072
+ target.dedupe_key = computeDedupeKey(task.task_id, target.index, task.command_hash, task.base_tree);
2073
+ task.leases.set(device.machine_id, target);
2074
+ this.emitOffer(task, target);
2075
+ this._ledgerLease(task, target);
2076
+ }
2077
+
2078
+ this.emit('notice', {
2079
+ level: 'info',
2080
+ code: candidates.length > 0 ? 'LEASE_TAKEOVER' : 'LEASE_TAKEOVER_UNAVAILABLE',
2081
+ message: candidates.length > 0
2082
+ ? `task ${task.task_id} index ${expiredLease.index} re-offered at attempt ${task.attempt} to: `
2083
+ + candidates.map((c) => c.device.machine_id).join(',')
2084
+ : `task ${task.task_id} index ${expiredLease.index} has no idle machine to take over`,
2085
+ task_id: task.task_id,
2086
+ });
2087
+ return candidates.map((c) => c.device.machine_id);
2088
+ }
2089
+
2090
+ /* ---------------- results (§5) ---------------- */
2091
+
2092
+ static instanceKey(machineId, dedupeKey, attempt) {
2093
+ return `${machineId}${INSTANCE_SEP}${dedupeKey}${INSTANCE_SEP}${attempt}`;
2094
+ }
2095
+
2096
+ /**
2097
+ * §5 result envelope validation.
2098
+ * Missing §5.1 required fields do NOT reject the frame — they make the result
2099
+ * `unverifiable` (§5.1 "缺一即 unverifiable").
2100
+ */
2101
+ static validateEnvelope(envelope) {
2102
+ const missing = [];
2103
+ if (!envelope || typeof envelope !== 'object' || Array.isArray(envelope)) {
2104
+ return { missing: [...REQUIRED_ENVELOPE_FIELDS], malformed: true };
2105
+ }
2106
+ for (const field of REQUIRED_ENVELOPE_FIELDS) {
2107
+ if (!(field in envelope) || envelope[field] === undefined) missing.push(field);
2108
+ }
2109
+ const problems = [];
2110
+ if ('status' in envelope && !RESULT_STATUSES.includes(envelope.status)) {
2111
+ problems.push(`status must be one of ${RESULT_STATUSES.join('|')}`);
2112
+ }
2113
+ if (envelope.status === 'refused' && !envelope.refusal_reason) {
2114
+ problems.push('refusal_reason is required when status=refused');
2115
+ }
2116
+ if ('warnings' in envelope && !Array.isArray(envelope.warnings)) problems.push('warnings must be string[]');
2117
+ return { missing, problems };
2118
+ }
2119
+
2120
+ /**
2121
+ * §1 idempotent POST /v1/result.
2122
+ *
2123
+ * NOTE (contract ambiguity, see report): §4.4 `dedupe_key` deliberately excludes
2124
+ * `machine_id`/`attempt`, but §6.3 aggregates results from *several* machines in
2125
+ * replicate mode — they all share one dedupe_key. Deduping purely on dedupe_key
2126
+ * would silently drop every machine's result but the first. We therefore key the
2127
+ * idempotency check on (machine_id, dedupe_key, attempt): an identical redelivery
2128
+ * is `deduped:true`, distinct machines/attempts are all recorded.
2129
+ */
2130
+ submitResult(envelope = {}) {
2131
+ const { missing, problems, malformed } = RabbitState.validateEnvelope(envelope);
2132
+ if (malformed) throw new ProtocolError('BAD_REQUEST', 'result must be a JSON object');
2133
+
2134
+ /**
2135
+ * v0.4.0: `transport` / `p2p` are OPTIONAL envelope fields, and they are validated as values
2136
+ * rather than added to the §5.1 required set. A v0.3.9 agent sends neither and must keep
2137
+ * working unchanged; an agent that sends `transport: 'satellite'` is refused by name instead of
2138
+ * storing a value the report would then show as a path that does not exist.
2139
+ */
2140
+ RabbitState.validateTransportFields(envelope);
2141
+
2142
+ const taskId = envelope.task_id;
2143
+ if (typeof taskId !== 'string' || taskId.length === 0) {
2144
+ throw new ProtocolError('BAD_REQUEST', 'task_id is required');
2145
+ }
2146
+ const task = this.tasks.get(taskId);
2147
+ if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
2148
+
2149
+ const machineId = typeof envelope.machine_id === 'string' ? envelope.machine_id : null;
2150
+ const dedupeKey = typeof envelope.dedupe_key === 'string' && envelope.dedupe_key
2151
+ ? envelope.dedupe_key : null;
2152
+ const attempt = Number.isInteger(envelope.attempt) && envelope.attempt >= 1 ? envelope.attempt : 1;
2153
+
2154
+ if (dedupeKey === null) {
2155
+ // Cannot dedupe without a key; still record it, as unverifiable.
2156
+ missing.push('dedupe_key');
2157
+ }
2158
+ const instanceKey = RabbitState.instanceKey(machineId ?? '<unknown>', dedupeKey ?? '<none>', attempt);
2159
+
2160
+ // §4.4 sanity check against the lease this machine actually holds. Booked as a
2161
+ // problem instead of a hard rejection so a buggy client cannot lose its result.
2162
+ const leaseForMachine = machineId ? task.leases.get(machineId) : null;
2163
+ if (leaseForMachine?.dedupe_key && dedupeKey && leaseForMachine.dedupe_key !== dedupeKey) {
2164
+ problems.push('dedupe_key does not match the §4.4 formula for this lease');
2165
+ }
2166
+
2167
+ if (this.results.has(instanceKey)) {
2168
+ const prior = this.results.get(instanceKey);
2169
+ return { deduped: true, task_id: taskId, dedupe_key: dedupeKey, seq: prior.seq };
2170
+ }
2171
+
2172
+ const now = this.nowMs();
2173
+ let envelopeShaOk = null;
2174
+ if (typeof envelope.envelope_sha256 === 'string') {
2175
+ const { envelope_sha256: _drop, ...rest } = envelope;
2176
+ envelopeShaOk = sha256Hex(jcs(rest)) === envelope.envelope_sha256;
2177
+ }
2178
+
2179
+ const record = {
2180
+ envelope,
2181
+ machine_id: machineId,
2182
+ dedupe_key: dedupeKey,
2183
+ attempt,
2184
+ instance_key: instanceKey,
2185
+ received_at_ms: now,
2186
+ validation: {
2187
+ missing: [...new Set(missing)],
2188
+ problems,
2189
+ envelope_sha256_ok: envelopeShaOk,
2190
+ },
2191
+ };
2192
+
2193
+ const effectiveStatus = record.validation.missing.length > 0 ? 'unverifiable' : envelope.status;
2194
+
2195
+ const seq = this.nextSeq();
2196
+ record.seq = seq;
2197
+ record.effective_status = effectiveStatus;
2198
+ this.results.set(instanceKey, record);
2199
+
2200
+ if (machineId) {
2201
+ const lease = task.leases.get(machineId);
2202
+ if (lease) {
2203
+ lease.state = effectiveStatus === 'refused' ? 'refused' : 'done';
2204
+ if (effectiveStatus === 'refused' && !lease.refusal_reason) {
2205
+ lease.refusal_reason = envelope.refusal_reason ?? 'REFUSED';
2206
+ }
2207
+ lease.finished_at_ms = now;
2208
+ lease.attempt = attempt;
2209
+ }
2210
+ this.touchDevice(machineId);
2211
+ }
2212
+ task.result_seqs.push(seq);
2213
+
2214
+ this.emit('task.result', {
2215
+ task_id: taskId,
2216
+ dedupe_key: dedupeKey,
2217
+ attempt,
2218
+ index: envelope.index ?? null,
2219
+ status: effectiveStatus,
2220
+ machine_id: machineId,
2221
+ deduped: false,
2222
+ validation_errors: record.validation.missing,
2223
+ });
2224
+
2225
+ // §7: results go into the ledger so the dedupe index survives a restart.
2226
+ this._ledger({ type: 'result.stored', task_id: taskId, machine_id: machineId, result: record });
2227
+ if (machineId) {
2228
+ const changed = task.leases.get(machineId);
2229
+ if (changed) this._ledgerLease(task, changed);
2230
+ }
2231
+
2232
+ return { deduped: false, task_id: taskId, dedupe_key: dedupeKey, seq, _instanceKey: instanceKey };
2233
+ }
2234
+
2235
+ /** All recorded results for a task, keyed by machine_id (later attempts win). */
2236
+ taskResults(taskId) {
2237
+ const task = this.tasks.get(taskId);
2238
+ if (!task) return new Map();
2239
+ const out = new Map();
2240
+ for (const rec of this.results.values()) {
2241
+ if (rec.envelope.task_id !== taskId) continue;
2242
+ const prev = out.get(rec.machine_id);
2243
+ if (!prev || rec.received_at_ms >= prev.received_at_ms) out.set(rec.machine_id, rec);
2244
+ }
2245
+ return out;
2246
+ }
2247
+
2248
+ /** Global dedupe book-keeping: dedupe_key -> instances seen. */
2249
+ dedupeReport() {
2250
+ const out = {};
2251
+ for (const rec of this.results.values()) {
2252
+ if (!rec.dedupe_key) continue;
2253
+ out[rec.dedupe_key] = out[rec.dedupe_key] ?? [];
2254
+ out[rec.dedupe_key].push({
2255
+ machine_id: rec.machine_id,
2256
+ attempt: rec.attempt,
2257
+ received_at: rfc3339(rec.received_at_ms),
2258
+ });
2259
+ }
2260
+ return out;
2261
+ }
2262
+
2263
+ /* ---------------- serialization ---------------- */
2264
+
2265
+ stats() {
2266
+ return {
2267
+ protocol_version: PROTOCOL_VERSION,
2268
+ seq: this.seq,
2269
+ buffered_events: this.buffer.length,
2270
+ devices: this.devices.size,
2271
+ tasks: this.tasks.size,
2272
+ results: this.results.size,
2273
+ pairing_codes: [...this.pairingCodes.values()].filter((c) => !c.used && this.nowMs() < c.expires_at_ms).length,
2274
+ uptime_ms: this.nowMs() - this.startedAtMs,
2275
+ };
2276
+ }
2277
+ }
2278
+
2279
+ export default RabbitState;