@twinsearth/w2m-dsh-plugin 0.8.10 → 0.8.99-tokenprobe.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +122 -0
- package/bin/w2m-localside.mjs +325 -325
- package/package.json +3 -2
- package/src/chatops/account.mjs +432 -412
- package/src/chatops/index.mjs +440 -440
- package/src/plugin/auto-update.mjs +31 -2
- package/src/plugin/tools.mjs +5207 -4878
- package/src/relay/state.mjs +2277 -2277
package/src/relay/state.mjs
CHANGED
|
@@ -1,2279 +1,2279 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W2M Rabbit relay — in-memory state machine.
|
|
3
|
-
*
|
|
4
|
-
* Contract: E:\DS\_work\w2m\PROTOCOL.md (protocol_version = 1, frozen).
|
|
5
|
-
* This module is intentionally PURE LOGIC:
|
|
6
|
-
* - no network, no filesystem, no timers
|
|
7
|
-
* - time is injectable via `now()` (required for lease-expiry unit tests)
|
|
8
|
-
* - randomness is injectable for deterministic tests
|
|
9
|
-
*
|
|
10
|
-
* Persistence: none. All state is in-memory and is lost on restart (v0.0.1 scope);
|
|
11
|
-
* there is intentionally no snapshot/serialization hook yet.
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
import { createHash, randomBytes } from 'node:crypto';
|
|
15
|
-
|
|
16
|
-
// v0.7.7: fleet scoping. Imported rather than inlined so the rule has one home with its own
|
|
17
|
-
// exhaustive tests — three lines inside this object would be untestable without a live relay.
|
|
18
|
-
import { targetsFor } from '../chatops/fleet-wiring.mjs';
|
|
1
|
+
/**
|
|
2
|
+
* W2M Rabbit relay — in-memory state machine.
|
|
3
|
+
*
|
|
4
|
+
* Contract: E:\DS\_work\w2m\PROTOCOL.md (protocol_version = 1, frozen).
|
|
5
|
+
* This module is intentionally PURE LOGIC:
|
|
6
|
+
* - no network, no filesystem, no timers
|
|
7
|
+
* - time is injectable via `now()` (required for lease-expiry unit tests)
|
|
8
|
+
* - randomness is injectable for deterministic tests
|
|
9
|
+
*
|
|
10
|
+
* Persistence: none. All state is in-memory and is lost on restart (v0.0.1 scope);
|
|
11
|
+
* there is intentionally no snapshot/serialization hook yet.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
15
|
+
|
|
16
|
+
// v0.7.7: fleet scoping. Imported rather than inlined so the rule has one home with its own
|
|
17
|
+
// exhaustive tests — three lines inside this object would be untestable without a live relay.
|
|
18
|
+
import { targetsFor } from '../chatops/fleet-wiring.mjs';
|
|
19
19
|
import { fleetForPairingCode, deviceFleetFields } from '../chatops/fleet-wiring.mjs';
|
|
20
20
|
// v0.8.1: the account-derived fleet key, so a machine can join with no pairing code.
|
|
21
|
-
import { planJoin } from '../chatops/fleet-join.mjs';
|
|
22
|
-
|
|
23
|
-
export const PROTOCOL_VERSION = 1;
|
|
24
|
-
|
|
25
|
-
/**
|
|
26
|
-
* PROTOCOL §1 table + §7 code table, plus the two v0.1.2 additions:
|
|
27
|
-
* `OPERATOR_REQUIRED` (§5.2 → 401) and `RATE_LIMITED` (§6 → 429).
|
|
28
|
-
* A code missing from this map silently degrades to HTTP 500 via
|
|
29
|
-
* `ProtocolError`, which keeps the right error code but lies about the status.
|
|
30
|
-
*/
|
|
31
|
-
export const ERROR_STATUS = Object.freeze({
|
|
32
|
-
UNAUTHORIZED: 401,
|
|
33
|
-
BAD_REQUEST: 400,
|
|
34
|
-
NOT_FOUND: 404,
|
|
35
|
-
PAIRING_INVALID: 400,
|
|
36
|
-
PAIRING_EXPIRED: 400,
|
|
37
|
-
TASK_EXISTS: 409,
|
|
38
|
-
FRAME_TOO_LARGE: 413,
|
|
39
|
-
NO_ONLINE_DEVICE: 503,
|
|
40
|
-
INTERNAL: 500,
|
|
41
|
-
OPERATOR_REQUIRED: 401, // v0.1.2 §5.2
|
|
42
|
-
RATE_LIMITED: 429, // v0.1.2 §6
|
|
43
|
-
|
|
44
|
-
/* v0.3.0 §9 request signing. Every client-side signature failure is a 401.
|
|
45
|
-
* `SIGNING_NOT_CONFIGURED` is deliberately NOT 401: the relay was started with
|
|
46
|
-
* `--require-signature` but no secret, so the fault is the relay's own
|
|
47
|
-
* configuration. Reporting that as 401 would send every operator to debug the
|
|
48
|
-
* client -- the one side that is not broken. 500 rather than 503 because
|
|
49
|
-
* retrying never fixes a missing configuration, so nothing should be invited to
|
|
50
|
-
* retry. */
|
|
51
|
-
SIGNATURE_REQUIRED: 401,
|
|
52
|
-
SIGNATURE_INCOMPLETE: 401,
|
|
53
|
-
SIGNATURE_MISMATCH: 401,
|
|
54
|
-
SIGNATURE_BAD_TIMESTAMP: 401,
|
|
55
|
-
SIGNATURE_BAD_NONCE: 401,
|
|
56
|
-
SIGNATURE_EXPIRED: 401,
|
|
57
|
-
SIGNATURE_REPLAY: 401,
|
|
58
|
-
SIGNING_NOT_CONFIGURED: 500,
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
/** §5.1 required envelope fields — missing any one means `unverifiable`. */
|
|
62
|
-
export const REQUIRED_ENVELOPE_FIELDS = Object.freeze([
|
|
63
|
-
'envelope_version', 'task_id', 'attempt', 'dedupe_key', 'machine_id', 'machine_name',
|
|
64
|
-
'platform', 'caps', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
|
|
65
|
-
'pre_tree_fingerprint', 'post_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error',
|
|
66
|
-
'head_commit', 'dirty_before', 'command_argv', 'command_hash', 'shell_id',
|
|
67
|
-
'started_at', 'ended_at', 'duration_ms', 'exit_code', 'status', 'refusal_reason',
|
|
68
|
-
'stdout_sha256', 'stdout_bytes', 'stderr_sha256', 'stderr_bytes', 'warnings',
|
|
69
|
-
'envelope_sha256',
|
|
70
|
-
]);
|
|
71
|
-
|
|
72
|
-
/** §5.3 comparable fields — the ONLY fields that take part in consistency judgement. */
|
|
73
|
-
export const COMPARABLE_FIELDS = Object.freeze([
|
|
74
|
-
'task_id', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
|
|
75
|
-
'pre_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error', 'head_commit',
|
|
76
|
-
'command_hash', 'shell_id', 'exit_code', 'status', 'refusal_reason',
|
|
77
|
-
'stdout_sha256', 'stdout_bytes', 'stdout_normalized_sha256',
|
|
78
|
-
'stderr_sha256', 'artifacts', 'diff_numstat', 'tests', 'semantic_counts', 'warnings',
|
|
79
|
-
]);
|
|
80
|
-
|
|
81
|
-
/** §5.1 status enum. */
|
|
82
|
-
export const RESULT_STATUSES = Object.freeze([
|
|
83
|
-
'ok', 'nonzero_exit', 'timeout', 'crashed', 'refused', 'unverifiable',
|
|
84
|
-
]);
|
|
85
|
-
|
|
86
|
-
export const FAILURE_STATUSES = Object.freeze(['nonzero_exit', 'timeout', 'crashed']);
|
|
87
|
-
|
|
88
|
-
/** §4.3 phase enum. */
|
|
89
|
-
export const LEASE_PHASES = Object.freeze(['preparing', 'running', 'finalizing']);
|
|
90
|
-
|
|
91
|
-
/** Lease states that can still be expired / renewed. */
|
|
92
|
-
export const ACTIVE_LEASE_STATES = Object.freeze(['queued', 'offered', 'running']);
|
|
93
|
-
|
|
94
|
-
export const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000; // §4.3: agent heartbeats every 10s
|
|
95
|
-
export const DEFAULT_MISSED_HEARTBEATS = 2; // §4.3: 2 missed cycles = 20s
|
|
96
|
-
export const DEFAULT_LEASE_GRACE_MS = 30_000; // §4.3: + grace 30s
|
|
97
|
-
/**
|
|
98
|
-
* v0.4.0: the bounded hold on the relay's own offer when a dispatcher is pushing it directly.
|
|
99
|
-
*
|
|
100
|
-
* Sized against what it has to cover: one punch round trip (measured 15-370 ms on the reference
|
|
101
|
-
* networks), the frame's delivery, and the executor's first heartbeat POST — all on a host that may
|
|
102
|
-
* be busy. 1.2 s was the first guess and it was too tight: measured on a 1-vCPU shared host, two
|
|
103
|
-
* agent tests saw the relay's copy overtake the direct one, which for `p2pMode: direct` means a
|
|
104
|
-
* refusal of an offer that was about to arrive. 3 s covers a slow host with room to spare.
|
|
105
|
-
*
|
|
106
|
-
* The cost is honest and bounded: a punch that fails delays the relay's copy by up to this much,
|
|
107
|
-
* once per task. Nothing else waits on it.
|
|
108
|
-
*/
|
|
109
|
-
export const DEFAULT_P2P_OFFER_GRACE_MS = 3_000;
|
|
110
|
-
export const DEFAULT_PAIRING_TTL_MS = 24 * 60 * 60 * 1000; // §2.2: 24h
|
|
111
|
-
export const DEFAULT_EVENT_BUFFER_SIZE = 1000; // §requirement: keep last 1000 events
|
|
112
|
-
export const MAX_FRAME_BYTES = 64 * 1024; // 64 KiB single frame
|
|
113
|
-
export const SHELL_ID_DIRECT = 'direct-exec'; // §5.1: v0.0.1 only direct exec
|
|
114
|
-
/** v0.3.0: RTT sanity ceiling — anything above 24h is treated as a broken value. */
|
|
115
|
-
export const MAX_RTT_MS = 24 * 60 * 60 * 1000;
|
|
116
|
-
/** v0.3.0: an RTT measurement older than this is reported as stale.
|
|
117
|
-
* Must stay comfortably ABOVE the idle-heartbeat interval (60s): at 1x the value
|
|
118
|
-
* would flip stale just before every refresh and flap, which would make
|
|
119
|
-
* `rtt_stale === false` useless as a "trust this number" predicate. 3x gives two
|
|
120
|
-
* consecutive missed idle heartbeats of margin. See PROTOCOL-v0.3.0.md §7. */
|
|
121
|
-
export const DEFAULT_RTT_STALE_MS = 180_000;
|
|
122
|
-
/** v0.3.0: recommended cadence for an agent with no lease (reference for the agent side). */
|
|
123
|
-
export const DEFAULT_IDLE_HEARTBEAT_MS = 60_000;
|
|
124
|
-
export const ENVELOPE_VERSION = '1.0';
|
|
125
|
-
export const FINGERPRINT_ALGO = 'git-temp-index-tree/v1';
|
|
126
|
-
|
|
127
|
-
/* ------------------------------------------------------------------ */
|
|
128
|
-
/* Errors */
|
|
129
|
-
/* ------------------------------------------------------------------ */
|
|
130
|
-
|
|
131
|
-
export class ProtocolError extends Error {
|
|
132
|
-
constructor(code, message, detail = {}) {
|
|
133
|
-
super(message);
|
|
134
|
-
this.name = 'ProtocolError';
|
|
135
|
-
this.code = code;
|
|
136
|
-
this.detail = detail;
|
|
137
|
-
this.status = ERROR_STATUS[code] ?? 500;
|
|
138
|
-
}
|
|
139
|
-
toBody() {
|
|
140
|
-
return { error: { code: this.code, message: this.message, detail: this.detail } };
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
/* ------------------------------------------------------------------ */
|
|
145
|
-
/* Small pure helpers */
|
|
146
|
-
/* ------------------------------------------------------------------ */
|
|
147
|
-
|
|
148
|
-
export function sha256Hex(input) {
|
|
149
|
-
return createHash('sha256').update(input).digest('hex');
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
/** RFC 8785-ish canonical JSON (JCS): sorted keys, no whitespace. */
|
|
153
|
-
export function jcs(value) {
|
|
154
|
-
if (value === null) return 'null';
|
|
155
|
-
const t = typeof value;
|
|
156
|
-
if (t === 'number') return Number.isFinite(value) ? JSON.stringify(value) : 'null';
|
|
157
|
-
if (t === 'boolean') return value ? 'true' : 'false';
|
|
158
|
-
if (t === 'string') return JSON.stringify(value);
|
|
159
|
-
if (t === 'undefined' || t === 'function' || t === 'symbol') return 'null';
|
|
160
|
-
if (Array.isArray(value)) return `[${value.map((v) => (v === undefined ? 'null' : jcs(v))).join(',')}]`;
|
|
161
|
-
if (t === 'object') {
|
|
162
|
-
const keys = Object.keys(value).filter((k) => value[k] !== undefined).sort();
|
|
163
|
-
return `{${keys.map((k) => `${JSON.stringify(k)}:${jcs(value[k])}`).join(',')}}`;
|
|
164
|
-
}
|
|
165
|
-
return JSON.stringify(String(value));
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
/**
|
|
169
|
-
* Upper bound on the stages a `pipeline` may contain.
|
|
170
|
-
*
|
|
171
|
-
* Each stage is another command run on someone else's machine, so an unbounded list is an unbounded
|
|
172
|
-
* amount of work created by one request. The limit is generous for the intended use (build, test,
|
|
173
|
-
* package) and exists so the failure is a clear refusal rather than a fleet-wide surprise.
|
|
174
|
-
*/
|
|
175
|
-
export const MAX_PIPELINE_STAGES = 16;
|
|
176
|
-
|
|
177
|
-
/**
|
|
178
|
-
* v0.4.0: upper bound on `POST /v1/task`'s `origin_machine_id`.
|
|
179
|
-
*
|
|
180
|
-
* A machine id is an identifier, not a payload: 128 characters is far beyond any id this system
|
|
181
|
-
* mints, and the ceiling is what stops the field from becoming an unbounded free-text channel into
|
|
182
|
-
* every offer frame the relay emits.
|
|
183
|
-
*/
|
|
184
|
-
export const MAX_ORIGIN_MACHINE_ID_LENGTH = 128;
|
|
185
|
-
|
|
186
|
-
/**
|
|
187
|
-
* v0.4.0: the P2P modes a task may be dispatched under (§1 of the frozen v0.4.0 contract).
|
|
188
|
-
*
|
|
189
|
-
* Mirrored here rather than imported from `src/agent/p2p-node.mjs`: the relay is a standalone
|
|
190
|
-
* process that must not depend on agent code, and a mode list is three strings. A value outside
|
|
191
|
-
* this list is a configuration error, never a silent fall back to the default.
|
|
192
|
-
*/
|
|
193
|
-
export const P2P_MODES = Object.freeze(['auto', 'direct', 'relay']);
|
|
194
|
-
|
|
195
|
-
/**
|
|
196
|
-
* v0.4.0: how a result travelled — the direct P2P channel or the relay.
|
|
197
|
-
*
|
|
198
|
-
* A **transport is not a result**: how an envelope arrived must never take part in the consistency
|
|
199
|
-
* judgement (§5 of the contract), or two machines that produced byte-identical output would be
|
|
200
|
-
* reported `divergent` merely because one of them punched a hole. Hence: not required, not
|
|
201
|
-
* comparable, and absent on every v0.3.9 envelope.
|
|
202
|
-
*/
|
|
203
|
-
export const TRANSPORT_VALUES = Object.freeze(['p2p', 'relay']);
|
|
204
|
-
|
|
205
|
-
/** §4.4 helper: command_hash = sha256(JCS(argv)+"|"+shell_id+"|"+cwd_rel). */
|
|
206
|
-
export function computeCommandHash(commandArgv, shellId = SHELL_ID_DIRECT, cwdRel = '.') {
|
|
207
|
-
return sha256Hex(`${jcs(commandArgv)}|${shellId}|${cwdRel}`);
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
/**
|
|
211
|
-
* `command_hash` for a whole pipeline.
|
|
212
|
-
*
|
|
213
|
-
* The hash must bind **every** stage, not just the first. Bound to stage 0 alone, two pipelines that
|
|
214
|
-
* differ after the first stage would carry the same hash -- so an envelope produced by the wrong
|
|
215
|
-
* chain would still verify, and making "the machine ran what I asked" checkable is the entire point
|
|
216
|
-
* of this field. Each stage's cwd participates for the same reason.
|
|
217
|
-
*
|
|
218
|
-
* The `|pipeline` suffix means this can never collide with a single-command hash: the two hash over
|
|
219
|
-
* different shapes, and that difference is deliberate rather than incidental.
|
|
220
|
-
*/
|
|
221
|
-
export function computePipelineCommandHash(stages, shellId = SHELL_ID_DIRECT) {
|
|
222
|
-
const canonical = stages.map((s) => ({
|
|
223
|
-
command_argv: s.command_argv,
|
|
224
|
-
cwd_rel: s.cwd_rel ?? '.',
|
|
225
|
-
continue_on_failure: s.continue_on_failure === true,
|
|
226
|
-
}));
|
|
227
|
-
return sha256Hex(`${jcs(canonical)}|${shellId}|pipeline`);
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
/** §4.4: dedupe_key = sha256(task_id + "|" + index + "|" + command_hash + "|" + base_tree). */
|
|
231
|
-
export function computeDedupeKey(taskId, index, commandHash, baseTree) {
|
|
232
|
-
return sha256Hex(`${taskId}|${index}|${commandHash}|${baseTree}`);
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
/** Epoch ms -> RFC3339 UTC, second precision (matches the PROTOCOL examples). */
|
|
236
|
-
export function rfc3339(ms) {
|
|
237
|
-
return new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
export function isRfc3339(value) {
|
|
241
|
-
return typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/.test(value);
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
export function deepEqual(a, b) {
|
|
245
|
-
return jcs(a === undefined ? null : a) === jcs(b === undefined ? null : b);
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
function normalizeNow(now) {
|
|
249
|
-
if (typeof now !== 'function') return () => Date.now();
|
|
250
|
-
return () => {
|
|
251
|
-
const v = now();
|
|
252
|
-
if (typeof v === 'number') return v;
|
|
253
|
-
if (v instanceof Date) return v.getTime();
|
|
254
|
-
return Date.parse(v);
|
|
255
|
-
};
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
/* ------------------------------------------------------------------ */
|
|
259
|
-
/* ULID */
|
|
260
|
-
/* ------------------------------------------------------------------ */
|
|
261
|
-
|
|
262
|
-
const CROCKFORD = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
|
|
263
|
-
|
|
264
|
-
function encodeBig(value, len) {
|
|
265
|
-
let out = '';
|
|
266
|
-
let v = value;
|
|
267
|
-
for (let i = 0; i < len; i += 1) {
|
|
268
|
-
out = CROCKFORD[Number(v & 31n)] + out;
|
|
269
|
-
v >>= 5n;
|
|
270
|
-
}
|
|
271
|
-
return out;
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
export function createIdFactory() {
|
|
275
|
-
let lastMs = -1;
|
|
276
|
-
let lastRand = 0n;
|
|
277
|
-
return function nextId(nowMs = Date.now()) {
|
|
278
|
-
const bytes = randomBytes(10);
|
|
279
|
-
let rand = 0n;
|
|
280
|
-
for (const b of bytes) rand = (rand << 8n) | BigInt(b);
|
|
281
|
-
if (nowMs === lastMs) rand = (lastRand + 1n) & ((1n << 80n) - 1n);
|
|
282
|
-
lastMs = nowMs;
|
|
283
|
-
lastRand = rand;
|
|
284
|
-
return encodeBig(BigInt(nowMs), 10) + encodeBig(rand, 16);
|
|
285
|
-
};
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
export function ulid(nowMs = Date.now()) {
|
|
289
|
-
return createIdFactory()(nowMs);
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
/* ------------------------------------------------------------------ */
|
|
293
|
-
/* Version / capability gate (§6.1) */
|
|
294
|
-
/* ------------------------------------------------------------------ */
|
|
295
|
-
|
|
296
|
-
function parseVersion(text) {
|
|
297
|
-
if (typeof text !== 'string') return null;
|
|
298
|
-
const m = text.trim().match(/^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:\.(\d+))?/);
|
|
299
|
-
if (!m) return null;
|
|
300
|
-
return [Number(m[1]), Number(m[2] ?? 0), Number(m[3] ?? 0), Number(m[4] ?? 0)];
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
function cmpVersion(a, b) {
|
|
304
|
-
for (let i = 0; i < 4; i += 1) {
|
|
305
|
-
if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1;
|
|
306
|
-
}
|
|
307
|
-
return 0;
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
/**
|
|
311
|
-
* Minimal semver-range check used by the §6.1 capability gate.
|
|
312
|
-
* Supports bare versions and comma/space separated `>= > <= < = ==` conjunctions.
|
|
313
|
-
* Unparseable ranges are treated as NOT satisfied (never silently pass).
|
|
314
|
-
*/
|
|
315
|
-
export function satisfiesRange(version, range) {
|
|
316
|
-
const v = parseVersion(version);
|
|
317
|
-
if (v === null) return false;
|
|
318
|
-
if (typeof range !== 'string') return false;
|
|
319
|
-
const parts = range.split(/[\s,]+/).filter(Boolean);
|
|
320
|
-
if (parts.length === 0) return false;
|
|
321
|
-
for (const part of parts) {
|
|
322
|
-
const m = part.match(/^(>=|<=|==|=|>|<)?\s*(.+)$/);
|
|
323
|
-
if (!m) return false;
|
|
324
|
-
const op = m[1] ?? '=';
|
|
325
|
-
const target = parseVersion(m[2]);
|
|
326
|
-
if (target === null) return false;
|
|
327
|
-
const c = cmpVersion(v, target);
|
|
328
|
-
if (op === '>=' && c < 0) return false;
|
|
329
|
-
if (op === '<=' && c > 0) return false;
|
|
330
|
-
if (op === '>' && c <= 0) return false;
|
|
331
|
-
if (op === '<' && c >= 0) return false;
|
|
332
|
-
if ((op === '=' || op === '==') && c !== 0) return false;
|
|
333
|
-
}
|
|
334
|
-
return true;
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
function machineWritable(caps) {
|
|
338
|
-
if (!caps || typeof caps !== 'object') return false;
|
|
339
|
-
if (caps.write === false || caps.writable === false || caps.write_allowed === false) return false;
|
|
340
|
-
if (caps.read_only === true || caps.readonly === true || caps.ro_mount === true) return false;
|
|
341
|
-
return true;
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
/**
|
|
345
|
-
* §6.1 capability gate, evaluated per machine BEFORE dispatch.
|
|
346
|
-
* @returns {null | {status:'refused', refusal_reason:string, detail:object}}
|
|
347
|
-
*/
|
|
348
|
-
export function gateMachine(task, machine) {
|
|
349
|
-
const caps = machine?.caps ?? {};
|
|
350
|
-
const req = task.requirements ?? {};
|
|
351
|
-
|
|
352
|
-
if (req.platform && Array.isArray(req.platform) && req.platform.length > 0) {
|
|
353
|
-
const os = String(machine?.platform?.os ?? '').toLowerCase();
|
|
354
|
-
if (!req.platform.map((p) => String(p).toLowerCase()).includes(os)) {
|
|
355
|
-
return { status: 'refused', refusal_reason: 'PLATFORM_MISMATCH', detail: { os, want: req.platform } };
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
const toolchain = req.toolchain ?? {};
|
|
360
|
-
for (const [tool, range] of Object.entries(toolchain)) {
|
|
361
|
-
const declared = caps[tool];
|
|
362
|
-
if (declared === null || declared === undefined || declared === '') {
|
|
363
|
-
return {
|
|
364
|
-
status: 'refused',
|
|
365
|
-
refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
|
|
366
|
-
detail: { tool, want: range, have: declared ?? null },
|
|
367
|
-
};
|
|
368
|
-
}
|
|
369
|
-
if (typeof range === 'string' && !satisfiesRange(declared, range)) {
|
|
370
|
-
return {
|
|
371
|
-
status: 'refused',
|
|
372
|
-
refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
|
|
373
|
-
detail: { tool, want: range, have: declared },
|
|
374
|
-
};
|
|
375
|
-
}
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
if (task.write === true && !machineWritable(caps)) {
|
|
379
|
-
return { status: 'refused', refusal_reason: 'READ_ONLY_MACHINE', detail: { caps } };
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
return null;
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
/* ------------------------------------------------------------------ */
|
|
386
|
-
/* RabbitState */
|
|
387
|
-
/* ------------------------------------------------------------------ */
|
|
388
|
-
|
|
389
|
-
const INSTANCE_SEP = '\u0000';
|
|
390
|
-
|
|
391
|
-
export class RabbitState {
|
|
392
|
-
constructor(options = {}) {
|
|
393
|
-
this._now = normalizeNow(options.now);
|
|
394
|
-
this._idFactory = options.idFactory ?? createIdFactory();
|
|
395
|
-
this._randomToken = options.tokenFactory ?? (() => randomBytes(24).toString('hex'));
|
|
396
|
-
this._randomChars = options.randomChars
|
|
397
|
-
?? ((n) => { const a = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; const b = randomBytes(n); let s = ''; for (let i = 0; i < n; i += 1) s += a[b[i] % a.length]; return s; });
|
|
398
|
-
|
|
399
|
-
this.protocolVersion = PROTOCOL_VERSION;
|
|
400
|
-
this.eventBufferSize = options.eventBufferSize ?? DEFAULT_EVENT_BUFFER_SIZE;
|
|
401
|
-
/**
|
|
402
|
-
* v0.7.7: the fleet this relay serves, or null for a relay that serves every device.
|
|
403
|
-
*
|
|
404
|
-
* **Null is the migration-safe default and is what the shared relay runs today.** With it,
|
|
405
|
-
* task targeting behaves exactly as it did before this release, so deploying cannot be an
|
|
406
|
-
* outage. Set it (via `--fleet <id>` or `fleetId`) and the relay becomes scoped: a device
|
|
407
|
-
* with a `fleet_id` reaches only its own fleet, and a device with none reaches nothing.
|
|
408
|
-
*
|
|
409
|
-
* This is the relay's *own* fleet, not a device's. A device carries its own `fleet_id`, which
|
|
410
|
-
* the relay wrote at pairing time from the pairing code.
|
|
411
|
-
*/
|
|
412
|
-
this.fleetId = typeof options.fleetId === 'string' && options.fleetId !== '' ? options.fleetId : null;
|
|
413
|
-
this.pairingTtlMs = options.pairingTtlMs ?? DEFAULT_PAIRING_TTL_MS;
|
|
414
|
-
/** When true one code stays valid for its whole TTL instead of rotating. */
|
|
415
|
-
this.pairingCodeReusable = options.pairingCodeReusable === true;
|
|
416
|
-
this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
|
|
417
|
-
this.missedHeartbeats = options.missedHeartbeats ?? DEFAULT_MISSED_HEARTBEATS;
|
|
418
|
-
this.leaseGraceMs = options.leaseGraceMs ?? DEFAULT_LEASE_GRACE_MS;
|
|
419
|
-
/**
|
|
420
|
-
* v0.4.0: how long the relay holds its own offer back when the dispatcher says it will push the
|
|
421
|
-
* offer over the direct path (`origin_machine_id` + `p2p.mode`).
|
|
422
|
-
*
|
|
423
|
-
* Without this, the direct path could never win: the relay's offer goes out over an SSE stream
|
|
424
|
-
* that is already open, while a punch costs at least one round trip, so the relay copy would
|
|
425
|
-
* arrive first essentially always and `transport` would read `relay` even on a network where
|
|
426
|
-
* the punch succeeded. Measured on loopback, and it is exactly what the first version of this
|
|
427
|
-
* release did.
|
|
428
|
-
*
|
|
429
|
-
* The hold is bounded and self-healing: the executor renews its lease (a heartbeat) the moment
|
|
430
|
-
* it starts work on the direct copy, which moves the lease out of `queued`, and the deferred
|
|
431
|
-
* offer is then dropped. If the punch fails, nothing renews the lease and the relay offers
|
|
432
|
-
* normally once the grace expires -- so a failed punch costs this much latency and nothing else.
|
|
433
|
-
*/
|
|
434
|
-
this.p2pOfferGraceMs = Number.isFinite(options.p2pOfferGraceMs) && options.p2pOfferGraceMs >= 0
|
|
435
|
-
? options.p2pOfferGraceMs
|
|
436
|
-
: DEFAULT_P2P_OFFER_GRACE_MS;
|
|
437
|
-
/** Deferred offers, keyed `task_id:machine_id`, so they can be cleared on close(). */
|
|
438
|
-
this._pendingOffers = new Map();
|
|
439
|
-
/** v0.3.0: age beyond which a reported RTT is flagged stale. */
|
|
440
|
-
this.rttStaleMs = Number.isFinite(options.rttStaleMs) ? options.rttStaleMs : DEFAULT_RTT_STALE_MS;
|
|
441
|
-
|
|
442
|
-
this.startedAtMs = this._now();
|
|
443
|
-
/** v0.1.2 §8.3: random per-process id; a change tells clients the relay restarted. */
|
|
444
|
-
this.relayId = options.relayId ?? randomBytes(8).toString('hex');
|
|
445
|
-
/** v0.1.2 §7: optional injected Persistence instance (state.mjs never touches fs itself). */
|
|
446
|
-
this._persist = options.persistence ?? null;
|
|
447
|
-
/** True while replaying a ledger — suppresses event emission and re-persistence. */
|
|
448
|
-
this._restoring = false;
|
|
449
|
-
/** Ledger entries that could not be replayed. */
|
|
450
|
-
this.replayWarnings = [];
|
|
451
|
-
this.seq = 0;
|
|
452
|
-
/** @type {{seq:number,event:object,target:string|null}[]} ring buffer, last N events */
|
|
453
|
-
this.buffer = [];
|
|
454
|
-
this.devices = new Map(); // machine_id -> device
|
|
455
|
-
this.tokens = new Map(); // device_token -> machine_id
|
|
456
|
-
this.tasks = new Map(); // task_id -> task
|
|
457
|
-
this.results = new Map(); // instanceKey -> {envelope, machine_id, received_at_ms, ...}
|
|
458
|
-
this.pairingCodes = new Map(); // code -> {code, created_at_ms, expires_at_ms, used}
|
|
459
|
-
this._listeners = new Set();
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
/* ---------------- clock / ids ---------------- */
|
|
463
|
-
|
|
464
|
-
nowMs() { return this._now(); }
|
|
465
|
-
nowIso() { return rfc3339(this.nowMs()); }
|
|
466
|
-
nextId() { return this._idFactory(this.nowMs()); }
|
|
467
|
-
newToken() { return this._randomToken(); }
|
|
468
|
-
|
|
469
|
-
/** §4.3 lease window = 2 missed heartbeats (20s) + grace (30s) = 50s. */
|
|
470
|
-
leaseWindowMs() { return this.heartbeatIntervalMs * this.missedHeartbeats + this.leaseGraceMs; }
|
|
471
|
-
|
|
472
|
-
/* ---------------- persistence hooks (v0.1.2 §7) ---------------- */
|
|
473
|
-
|
|
474
|
-
/** Inject/replace the persistence backend (also used by tests). */
|
|
475
|
-
setPersistence(persistence) { this._persist = persistence ?? null; return this._persist; }
|
|
476
|
-
|
|
477
|
-
/** Snapshot the device table; failures must never break the request path. */
|
|
478
|
-
_saveDevices() {
|
|
479
|
-
if (this._restoring || !this._persist) return false;
|
|
480
|
-
try {
|
|
481
|
-
return this._persist.saveDevices(this.devices);
|
|
482
|
-
} catch (err) {
|
|
483
|
-
this._persist.warn?.('devices.json', `snapshot failed: ${err.message}`);
|
|
484
|
-
return false;
|
|
485
|
-
}
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
/** Append one ledger entry; failures must never break the request path. */
|
|
489
|
-
_ledger(entry) {
|
|
490
|
-
if (this._restoring || !this._persist) return false;
|
|
491
|
-
try {
|
|
492
|
-
const appended = this._persist.append(entry);
|
|
493
|
-
// v0.6.2: a successful append is what clears a degraded relay, so the outcome has to be
|
|
494
|
-
// reported back rather than assumed -- otherwise the state could only ever be entered.
|
|
495
|
-
if (appended) this._persist.noteAppendSuccess?.();
|
|
496
|
-
return appended;
|
|
497
|
-
} catch (err) {
|
|
498
|
-
// Recorded as a state, not merely as a line: this is the one relay failure that keeps serving,
|
|
499
|
-
// and so the one that is invisible from every other angle (v0.6.2).
|
|
500
|
-
if (typeof this._persist.noteAppendFailure === 'function') this._persist.noteAppendFailure(err.message);
|
|
501
|
-
else this._persist.warn?.('ledger.jsonl', `append failed: ${err.message}`);
|
|
502
|
-
return false;
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
_ledgerLease(task, lease) {
|
|
507
|
-
this._ledger({ type: 'lease.changed', task_id: task.task_id, machine_id: lease.machine_id, lease: { ...lease } });
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
/**
|
|
511
|
-
* v0.1.2 §7 startup recovery: rebuild the device table, token index, tasks,
|
|
512
|
-
* leases and the result/dedupe index from a devices snapshot + ledger replay.
|
|
513
|
-
* Never emits events and never writes back what it just read.
|
|
514
|
-
*/
|
|
515
|
-
restore({ devices = {}, ledger = [] } = {}) {
|
|
516
|
-
this._restoring = true;
|
|
517
|
-
try {
|
|
518
|
-
for (const [machineId, d] of Object.entries(devices)) {
|
|
519
|
-
if (!d || typeof d !== 'object') continue;
|
|
520
|
-
const pairedAt = Date.parse(d.paired_at ?? '');
|
|
521
|
-
const device = {
|
|
522
|
-
machine_id: machineId,
|
|
523
|
-
machine_name: typeof d.machine_name === 'string' && d.machine_name ? d.machine_name : machineId,
|
|
524
|
-
platform: d.platform ?? {},
|
|
525
|
-
caps: d.caps ?? {},
|
|
526
|
-
user_id: d.user_id ?? null,
|
|
527
|
-
device_token: typeof d.device_token === 'string' ? d.device_token : null,
|
|
528
|
-
paired_at_ms: Number.isFinite(pairedAt) ? pairedAt : this.nowMs(),
|
|
529
|
-
re_paired: d.re_paired === true,
|
|
530
|
-
last_seen_at_ms: this.nowMs(),
|
|
531
|
-
// a snapshot proves the device exists, not that it is connected
|
|
532
|
-
online: false,
|
|
533
|
-
streams: 0,
|
|
534
|
-
// v0.3.0: RTT is volatile and is NEVER restored. Even if a snapshot on
|
|
535
|
-
// disk happens to carry `rtt_ms`, it is dropped here: after a restart the
|
|
536
|
-
// relay genuinely does not know the latency any more, and reporting a
|
|
537
|
-
// pre-restart number as current would be a lie that no reader can detect.
|
|
538
|
-
rtt_ms: null,
|
|
539
|
-
rtt_at_ms: null,
|
|
540
|
-
last_heartbeat_at_ms: null,
|
|
541
|
-
stream_connects: 0,
|
|
542
|
-
};
|
|
543
|
-
this.devices.set(machineId, device);
|
|
544
|
-
if (device.device_token) this.tokens.set(device.device_token, machineId);
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
let revivedTasks = 0;
|
|
548
|
-
for (const entry of ledger) {
|
|
549
|
-
try {
|
|
550
|
-
if (this._applyLedgerEntry(entry) === true) revivedTasks += 1;
|
|
551
|
-
} catch (err) {
|
|
552
|
-
this.replayWarnings.push({ type: entry?.type ?? '<unknown>', message: err.message });
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
|
-
return { revived_devices: this.devices.size, revived_tasks: revivedTasks };
|
|
556
|
-
} finally {
|
|
557
|
-
this._restoring = false;
|
|
558
|
-
}
|
|
559
|
-
}
|
|
560
|
-
|
|
561
|
-
/** @returns {boolean} true when a new task was created. */
|
|
562
|
-
_applyLedgerEntry(entry) {
|
|
563
|
-
if (!entry || typeof entry.type !== 'string') return false;
|
|
564
|
-
switch (entry.type) {
|
|
565
|
-
case 'task.created': {
|
|
566
|
-
const t = entry.task;
|
|
567
|
-
if (!t || typeof t.task_id !== 'string') return false;
|
|
568
|
-
const createdAt = Number.isFinite(t.created_at_ms) ? t.created_at_ms : Date.parse(entry.ts ?? '');
|
|
569
|
-
const task = {
|
|
570
|
-
task_id: t.task_id,
|
|
571
|
-
mode: t.mode ?? 'replicate',
|
|
572
|
-
command_argv: t.command_argv ?? [],
|
|
573
|
-
cwd_rel: t.cwd_rel ?? '.',
|
|
574
|
-
index_total: t.index_total ?? 1,
|
|
575
|
-
timeout_ms: t.timeout_ms ?? 300_000,
|
|
576
|
-
write: t.write === true,
|
|
577
|
-
write_scope: t.write_scope ?? [],
|
|
578
|
-
require_exclusive_write: t.require_exclusive_write === true,
|
|
579
|
-
base_commit: t.base_commit ?? null,
|
|
580
|
-
base_tree: t.base_tree ?? null,
|
|
581
|
-
requirements: t.requirements ?? {},
|
|
582
|
-
compare_policy: t.compare_policy ?? {},
|
|
583
|
-
halt: t.halt ?? 'never',
|
|
584
|
-
created_by: t.created_by ?? null,
|
|
585
|
-
created_at_ms: Number.isFinite(createdAt) ? createdAt : this.nowMs(),
|
|
586
|
-
command_hash: t.command_hash ?? null,
|
|
587
|
-
attempt: t.attempt ?? 1,
|
|
588
|
-
leases: new Map(),
|
|
589
|
-
cancelled: false,
|
|
590
|
-
cancel_reason: null,
|
|
591
|
-
degraded: t.degraded ?? null,
|
|
592
|
-
deadline_ms: t.deadline_ms ?? 0,
|
|
593
|
-
// v0.4.0 routing metadata. Read defensively: a ledger written by v0.3.9 has neither key,
|
|
594
|
-
// and a replayed task with `null` for both is exactly a v0.3.9 task.
|
|
595
|
-
origin_machine_id: t.origin_machine_id ?? null,
|
|
596
|
-
p2p: t.p2p ?? null,
|
|
597
|
-
result_seqs: [],
|
|
598
|
-
};
|
|
599
|
-
for (const lease of entry.leases ?? []) {
|
|
600
|
-
if (lease && typeof lease.machine_id === 'string') task.leases.set(lease.machine_id, { ...lease });
|
|
601
|
-
}
|
|
602
|
-
this.tasks.set(task.task_id, task);
|
|
603
|
-
return true;
|
|
604
|
-
}
|
|
605
|
-
case 'lease.changed': {
|
|
606
|
-
const task = this.tasks.get(entry.task_id);
|
|
607
|
-
const lease = task?.leases.get(entry.machine_id);
|
|
608
|
-
if (!lease) return false;
|
|
609
|
-
Object.assign(lease, entry.lease ?? {});
|
|
610
|
-
return false;
|
|
611
|
-
}
|
|
612
|
-
case 'result.stored': {
|
|
613
|
-
const record = entry.result;
|
|
614
|
-
if (!record || typeof record.instance_key !== 'string') return false;
|
|
615
|
-
this.results.set(record.instance_key, { ...record });
|
|
616
|
-
const task = this.tasks.get(record.envelope?.task_id);
|
|
617
|
-
if (task && Number.isInteger(record.seq)) task.result_seqs.push(record.seq);
|
|
618
|
-
return false;
|
|
619
|
-
}
|
|
620
|
-
case 'task.cancelled': {
|
|
621
|
-
const task = this.tasks.get(entry.task_id);
|
|
622
|
-
if (task) {
|
|
623
|
-
task.cancelled = true;
|
|
624
|
-
task.cancel_reason = entry.reason ?? null;
|
|
625
|
-
}
|
|
626
|
-
return false;
|
|
627
|
-
}
|
|
628
|
-
default:
|
|
629
|
-
return false; // unknown entry type: forward compatible, ignored
|
|
630
|
-
}
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
/* ---------------- events (§3) ---------------- */
|
|
634
|
-
|
|
635
|
-
nextSeq() { this.seq += 1; return this.seq; }
|
|
636
|
-
|
|
637
|
-
/** Reserve a seq WITHOUT buffering (used for the per-connection `ready` frame). */
|
|
638
|
-
reserveSeq() { return this.nextSeq(); }
|
|
639
|
-
|
|
640
|
-
onEvent(listener) {
|
|
641
|
-
this._listeners.add(listener);
|
|
642
|
-
return () => this._listeners.delete(listener);
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
/**
|
|
646
|
-
* Emit an event: allocates the next global seq, buffers it (ring of N) and
|
|
647
|
-
* notifies subscribers. `target` is the machine_id the event is addressed to
|
|
648
|
-
* (null = broadcast). The target is NOT serialized except as `machine_id`.
|
|
649
|
-
*/
|
|
650
|
-
emit(type, fields = {}, opts = {}) {
|
|
651
|
-
const target = opts.target ?? null;
|
|
652
|
-
const seq = this.nextSeq();
|
|
653
|
-
const event = {
|
|
654
|
-
type,
|
|
655
|
-
seq,
|
|
656
|
-
rabbit_time: this.nowIso(),
|
|
657
|
-
...(target ? { machine_id: target } : {}),
|
|
658
|
-
...fields,
|
|
659
|
-
};
|
|
660
|
-
const entry = { seq, event, target };
|
|
661
|
-
this.buffer.push(entry);
|
|
662
|
-
if (this.buffer.length > this.eventBufferSize) {
|
|
663
|
-
this.buffer.splice(0, this.buffer.length - this.eventBufferSize);
|
|
664
|
-
}
|
|
665
|
-
for (const listener of this._listeners) {
|
|
666
|
-
try { listener(entry); } catch { /* subscriber errors must never break state */ }
|
|
667
|
-
}
|
|
668
|
-
return event;
|
|
669
|
-
}
|
|
670
|
-
|
|
671
|
-
oldestBufferedSeq() { return this.buffer.length ? this.buffer[0].seq : this.seq + 1; }
|
|
672
|
-
/**
|
|
673
|
-
* Highest seq present in the replay ring (0 when empty).
|
|
674
|
-
*
|
|
675
|
-
* Deliberately different from `lastSeq()`: `ready` frames allocate seq numbers
|
|
676
|
-
* without being buffered, so `lastSeq()` can run ahead of anything a client can
|
|
677
|
-
* actually replay. Diagnostics that describe the REPLAY WINDOW must use this one.
|
|
678
|
-
*/
|
|
679
|
-
lastBufferedSeq() { return this.buffer.length ? this.buffer[this.buffer.length - 1].seq : 0; }
|
|
680
|
-
lastSeq() { return this.seq; }
|
|
681
|
-
|
|
682
|
-
/** Replay entries with seq >= from (used by GET /v1/stream?seq=|Last-Event-ID). */
|
|
683
|
-
replayEntries(from) {
|
|
684
|
-
return this.buffer.filter((e) => e.seq >= from);
|
|
685
|
-
}
|
|
686
|
-
|
|
687
|
-
eventsSince(from) { return this.replayEntries(from).map((e) => e.event); }
|
|
688
|
-
|
|
689
|
-
/* ---------------- pairing (§2) ---------------- */
|
|
690
|
-
|
|
691
|
-
/**
|
|
692
|
-
* Mint a pairing code.
|
|
693
|
-
*
|
|
694
|
-
* **v0.7.7: the code carries the relay's fleet.** This is the link that makes the scheme work,
|
|
695
|
-
* and it is easy to leave out: `pair()` reads the fleet *from the code*, so a code with no
|
|
696
|
-
* fleet pairs a device into no fleet -- and a device in no fleet reaches nothing on a scoped
|
|
697
|
-
* relay. `--fleet X` would then have had no effect on anything, silently.
|
|
698
|
-
*
|
|
699
|
-
* Defaulting to `this.fleetId` rather than requiring the caller to pass it means there is no
|
|
700
|
-
* path that mints a fleetless code on a scoped relay. An explicit `fleetId` overrides, which
|
|
701
|
-
* is what a relay serving several fleets would need.
|
|
702
|
-
*/
|
|
703
|
-
createPairingCode({ ttlMs = this.pairingTtlMs, fleetId = this.fleetId } = {}) {
|
|
704
|
-
let code;
|
|
705
|
-
do { code = `PAIR-${this._randomChars(8)}`; } while (this.pairingCodes.has(code));
|
|
706
|
-
const now = this.nowMs();
|
|
707
|
-
this.pairingCodes.set(code, {
|
|
708
|
-
code,
|
|
709
|
-
created_at_ms: now,
|
|
710
|
-
expires_at_ms: now + ttlMs,
|
|
711
|
-
used: false,
|
|
712
|
-
// Present as an explicit key even when null, so a code dump shows whether a fleet was
|
|
713
|
-
// attached rather than leaving it to be inferred from an absent field.
|
|
714
|
-
fleet_id: typeof fleetId === 'string' && fleetId !== '' ? fleetId : null,
|
|
715
|
-
});
|
|
716
|
-
return code;
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
/** Revoke every outstanding code and mint a new one. */
|
|
720
|
-
rotatePairingCode(opts) {
|
|
721
|
-
for (const rec of this.pairingCodes.values()) rec.used = true;
|
|
722
|
-
return this.createPairingCode(opts);
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
/**
|
|
726
|
-
* Register an externally supplied code (`--pairing-code <c>`).
|
|
727
|
-
*
|
|
728
|
-
* v0.1.2 BUG-1: assigning the code to `relay.pairingCode` without registering it
|
|
729
|
-
* advertised a code that `/v1/pair` then rejected with PAIRING_INVALID, so every
|
|
730
|
-
* documented scripted-onboarding flow failed. "Which code do we use" and "is the
|
|
731
|
-
* code registered" are two different questions; this answers the second.
|
|
732
|
-
*/
|
|
733
|
-
registerPairingCode(code, { ttlMs = this.pairingTtlMs } = {}) {
|
|
734
|
-
if (typeof code !== 'string' || code.trim() === '') {
|
|
735
|
-
throw new ProtocolError('BAD_REQUEST', 'pairing code must be a non-empty string', { pairing_code: code });
|
|
736
|
-
}
|
|
737
|
-
const value = code.trim();
|
|
738
|
-
const now = this.nowMs();
|
|
739
|
-
this.pairingCodes.set(value, { code: value, created_at_ms: now, expires_at_ms: now + ttlMs, used: false });
|
|
740
|
-
return value;
|
|
741
|
-
}
|
|
742
|
-
|
|
743
|
-
/** §2.2 step 3: validate a one-time code. */
|
|
744
|
-
validatePairingCode(code) {
|
|
745
|
-
if (typeof code !== 'string' || code.length === 0) {
|
|
746
|
-
throw new ProtocolError('PAIRING_INVALID', 'pairing_code is required');
|
|
747
|
-
}
|
|
748
|
-
const rec = this.pairingCodes.get(code);
|
|
749
|
-
if (!rec) throw new ProtocolError('PAIRING_INVALID', 'unknown pairing code', { pairing_code: code });
|
|
750
|
-
if (rec.used) throw new ProtocolError('PAIRING_INVALID', 'pairing code already used', { pairing_code: code });
|
|
751
|
-
if (this.nowMs() >= rec.expires_at_ms) {
|
|
752
|
-
throw new ProtocolError('PAIRING_EXPIRED', 'pairing code expired', { pairing_code: code, expired_at: rfc3339(rec.expires_at_ms) });
|
|
753
|
-
}
|
|
754
|
-
return rec;
|
|
755
|
-
}
|
|
756
|
-
|
|
757
|
-
/** §2.2: exchange a one-time code for a device_token. §1: idempotent. */
|
|
758
|
-
pair(request = {}) {
|
|
759
|
-
/**
|
|
760
|
-
* v0.8.1: a pairing code **or** an account-derived fleet key.
|
|
761
|
-
*
|
|
762
|
-
* The code path is unchanged. The key path is what lets a second machine on one account join
|
|
763
|
-
* with no human step and no relay configuration — the gap that left a new PC invisible beside
|
|
764
|
-
* its own fleet.
|
|
765
|
-
*
|
|
766
|
-
* ## Why a key may stand in for a code
|
|
767
|
-
*
|
|
768
|
-
* They authorise different things. A **code** is the operator saying *"let this machine in"*,
|
|
769
|
-
* and it is the only way to place a device in a fleet the device could not compute for itself.
|
|
770
|
-
* A **key** is the account saying *"this machine is mine"*, and it can only ever place the
|
|
771
|
-
* device in the fleet derived from that key — so it cannot be used to reach a fleet its holder
|
|
772
|
-
* is not already entitled to.
|
|
773
|
-
*
|
|
774
|
-
* ## Why a malformed key refuses rather than falling back
|
|
775
|
-
*
|
|
776
|
-
* A request with a `fleet_key` is a request that *meant* to join by account. Quietly demanding a
|
|
777
|
-
* code instead would surface at the client as an unexplained `PAIRING_INVALID`, which is the
|
|
778
|
-
* silent-wrong-answer shape this codebase keeps producing; the reason is named instead.
|
|
779
|
-
*/
|
|
780
|
-
const hasKey = typeof request.fleet_key === 'string' && request.fleet_key !== '';
|
|
781
|
-
let rec = null;
|
|
782
|
-
if (hasKey) {
|
|
783
|
-
const join = planJoin({ fleetKeyEncoded: request.fleet_key });
|
|
784
|
-
if (!join.ok) {
|
|
785
|
-
throw new ProtocolError('PAIRING_INVALID', join.text, { reason: join.reason });
|
|
786
|
-
}
|
|
787
|
-
// A synthetic code record so the fleet binding below reads from one place. `fleet_id` is the
|
|
788
|
-
// DERIVED value, never anything the request claimed.
|
|
789
|
-
rec = { code: null, fleet_id: join.fleetId, from_fleet_key: true };
|
|
790
|
-
} else {
|
|
791
|
-
rec = this.validatePairingCode(request.pairing_code);
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
const machineId = request.machine_id;
|
|
795
|
-
if (typeof machineId !== 'string' || machineId.length === 0) {
|
|
796
|
-
throw new ProtocolError('BAD_REQUEST', 'machine_id is required');
|
|
797
|
-
}
|
|
798
|
-
const now = this.nowMs();
|
|
799
|
-
const token = this.newToken();
|
|
800
|
-
const existing = this.devices.get(machineId);
|
|
801
|
-
if (existing?.device_token) this.tokens.delete(existing.device_token);
|
|
802
|
-
|
|
803
|
-
const device = {
|
|
804
|
-
machine_id: machineId,
|
|
805
|
-
machine_name: typeof request.machine_name === 'string' && request.machine_name
|
|
806
|
-
? request.machine_name : machineId.slice(0, 12),
|
|
807
|
-
platform: request.platform ?? {},
|
|
808
|
-
caps: request.caps ?? {},
|
|
809
|
-
user_id: request.user_id ?? null,
|
|
810
|
-
// v0.7.7: which fleet this device belongs to, and where that fact came from.
|
|
811
|
-
//
|
|
812
|
-
// **The fleet comes from the pairing code, not from the request.** A device that could
|
|
813
|
-
// name its own fleet would join any fleet, and every signature would faithfully attest
|
|
814
|
-
// to it. `deviceFleetFields` is the single place that binding is made, and it records
|
|
815
|
-
// its provenance so a fleet that arrived from a request is visible as such in
|
|
816
|
-
// `devices.json` rather than looking identical to one that came from a code.
|
|
817
|
-
//
|
|
818
|
-
// v0.8.1 adds the third source: an account-derived `fleet_key`. That is what lets a second
|
|
819
|
-
// machine on one account join with **no human step and no relay configuration** -- the gap
|
|
820
|
-
// that left a new PC invisible beside its own fleet. The relay does not trust a claimed
|
|
821
|
-
// fleet id; it *derives* one from the key, so a device can only land in the fleet whose key
|
|
822
|
-
// it holds and anything else forms a separate, empty fleet.
|
|
823
|
-
...deviceFleetFields({
|
|
824
|
-
codeRecord: rec,
|
|
825
|
-
fleetKeyEncoded: typeof request.fleet_key === 'string' ? request.fleet_key : null,
|
|
826
|
-
requestedFleet: null,
|
|
827
|
-
}),
|
|
828
|
-
/**
|
|
829
|
-
* v0.7.10: the device's Ed25519 public key, if it offered one.
|
|
830
|
-
*
|
|
831
|
-
* A public key is not a secret, so storing it costs nothing and it is the whole basis of
|
|
832
|
-
* per-device verification: the relay can then check a signature only *this* machine could
|
|
833
|
-
* have made, rather than a shared secret every machine holds.
|
|
834
|
-
*
|
|
835
|
-
* Not validated here, deliberately. A malformed key fails at verification with a named
|
|
836
|
-
* reason, while rejecting a pairing over it would lock out every existing device the
|
|
837
|
-
* moment this field was added -- and devices that send no key keep working, with
|
|
838
|
-
* `public_key: null` meaning "signs with the shared secret or not at all".
|
|
839
|
-
*/
|
|
840
|
-
public_key: typeof request.public_key === 'string' && request.public_key !== '' ? request.public_key : null,
|
|
841
|
-
device_token: token,
|
|
842
|
-
paired_at_ms: now,
|
|
843
|
-
re_paired: Boolean(existing),
|
|
844
|
-
last_seen_at_ms: now,
|
|
845
|
-
online: true,
|
|
846
|
-
streams: existing?.streams ?? 0,
|
|
847
|
-
// v0.3.0 RTT: starts unknown. `null` is "never measured"; 0 would mean "instant".
|
|
848
|
-
rtt_ms: null,
|
|
849
|
-
rtt_at_ms: null,
|
|
850
|
-
last_heartbeat_at_ms: null,
|
|
851
|
-
stream_connects: existing?.stream_connects ?? 0,
|
|
852
|
-
};
|
|
853
|
-
this.devices.set(machineId, device);
|
|
854
|
-
this.tokens.set(token, machineId);
|
|
855
|
-
rec.used = true;
|
|
856
|
-
|
|
857
|
-
// A group has more than one machine, and a single-use code would leave the
|
|
858
|
-
// operator unable to pair the second one without restarting the relay. So
|
|
859
|
-
// consuming a code immediately mints the next one and announces it: the
|
|
860
|
-
// operator pairs machine A, sees a fresh code, and pairs machine B with it.
|
|
861
|
-
// `pairingCodeReusable` keeps one code alive for the whole window instead,
|
|
862
|
-
// for scripted onboarding where nobody is watching stdout.
|
|
863
|
-
let nextCode = null;
|
|
864
|
-
const reusable = this.pairingCodeReusable === true;
|
|
865
|
-
if (reusable) {
|
|
866
|
-
rec.used = false;
|
|
867
|
-
} else {
|
|
868
|
-
nextCode = this.rotatePairingCode();
|
|
869
|
-
}
|
|
870
|
-
|
|
871
|
-
this.emit('peer.hello', {
|
|
872
|
-
machine_id: machineId,
|
|
873
|
-
machine_name: device.machine_name,
|
|
874
|
-
platform: device.platform,
|
|
875
|
-
});
|
|
876
|
-
|
|
877
|
-
if (nextCode) {
|
|
878
|
-
this.emit('notice', {
|
|
879
|
-
level: 'info',
|
|
880
|
-
code: 'PAIRING_CODE_ROTATED',
|
|
881
|
-
message: 'pairing code consumed; a new one is available',
|
|
882
|
-
pairing_code: nextCode,
|
|
883
|
-
});
|
|
884
|
-
}
|
|
885
|
-
|
|
886
|
-
// §7: a device change is snapshotted immediately.
|
|
887
|
-
this._saveDevices();
|
|
888
|
-
this._ledger({
|
|
889
|
-
type: 'device.paired',
|
|
890
|
-
machine_id: machineId,
|
|
891
|
-
machine_name: device.machine_name,
|
|
892
|
-
re_paired: device.re_paired,
|
|
893
|
-
});
|
|
894
|
-
|
|
895
|
-
return {
|
|
896
|
-
device_token: token,
|
|
897
|
-
protocol_version: PROTOCOL_VERSION,
|
|
898
|
-
rabbit_time: rfc3339(now),
|
|
899
|
-
...(nextCode ? { next_pairing_code: nextCode } : {}),
|
|
900
|
-
};
|
|
901
|
-
}
|
|
902
|
-
|
|
903
|
-
/** @returns {object|null} device for a bearer token. */
|
|
904
|
-
authenticate(token) {
|
|
905
|
-
if (typeof token !== 'string' || token.length === 0) return null;
|
|
906
|
-
const machineId = this.tokens.get(token);
|
|
907
|
-
if (!machineId) return null;
|
|
908
|
-
return this.devices.get(machineId) ?? null;
|
|
909
|
-
}
|
|
910
|
-
|
|
911
|
-
touchDevice(machineId, { online = true } = {}) {
|
|
912
|
-
const device = this.devices.get(machineId);
|
|
913
|
-
if (!device) return null;
|
|
914
|
-
device.last_seen_at_ms = this.nowMs();
|
|
915
|
-
if (online) device.online = true;
|
|
916
|
-
return device;
|
|
917
|
-
}
|
|
918
|
-
|
|
919
|
-
setOnline(machineId, online, reason = null) {
|
|
920
|
-
const device = this.devices.get(machineId);
|
|
921
|
-
if (!device) return null;
|
|
922
|
-
if (device.online === online) return device;
|
|
923
|
-
device.online = online;
|
|
924
|
-
if (online) {
|
|
925
|
-
this.touchDevice(machineId);
|
|
926
|
-
this.emit('peer.hello', {
|
|
927
|
-
machine_id: machineId,
|
|
928
|
-
machine_name: device.machine_name,
|
|
929
|
-
platform: device.platform,
|
|
930
|
-
});
|
|
931
|
-
} else {
|
|
932
|
-
this.emit('peer.bye', { machine_id: machineId, reason: reason ?? 'stream_closed' });
|
|
933
|
-
}
|
|
934
|
-
return device;
|
|
935
|
-
}
|
|
936
|
-
|
|
937
|
-
/* ---------------- RTT (v0.3.0 §R) ---------------- */
|
|
938
|
-
|
|
939
|
-
/**
|
|
940
|
-
* Validate an inbound `rtt_ms`.
|
|
941
|
-
*
|
|
942
|
-
* The contract is deliberately liberal: a MISSING or INVALID value must never
|
|
943
|
-
* reject the heartbeat and must never clobber a previously known good value.
|
|
944
|
-
* RTT is a diagnostic; it must not be able to break lease renewal.
|
|
945
|
-
*
|
|
946
|
-
* Sanity ceiling: anything above `MAX_RTT_MS` (24h) is treated as invalid. A
|
|
947
|
-
* single garbage value would otherwise poison the fleet aggregate.
|
|
948
|
-
*/
|
|
949
|
-
static normalizeRtt(value) {
|
|
950
|
-
if (value === undefined) return { provided: false, valid: false };
|
|
951
|
-
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
952
|
-
return { provided: true, valid: false };
|
|
953
|
-
}
|
|
954
|
-
if (value > MAX_RTT_MS) return { provided: true, valid: false };
|
|
955
|
-
return { provided: true, valid: true, value };
|
|
956
|
-
}
|
|
957
|
-
|
|
958
|
-
/**
|
|
959
|
-
* RTT is a VOLATILE diagnostic: it is intentionally NOT written to
|
|
960
|
-
* `devices.json` and NOT replayed from the ledger, so a restarted relay reports
|
|
961
|
-
* `rtt_ms: null` until a machine heartbeats again. A persisted value would be
|
|
962
|
-
* indistinguishable from a fresh one and would let the relay claim knowledge it
|
|
963
|
-
* lost when the process died.
|
|
964
|
-
*/
|
|
965
|
-
rttStaleAfterMs() { return this.rttStaleMs; }
|
|
966
|
-
|
|
967
|
-
/**
|
|
968
|
-
* v0.3.0: absorb the optional `rtt_ms` carried by a heartbeat.
|
|
969
|
-
*
|
|
970
|
-
* Deliberately liberal, because RTT is a diagnostic and must never be able to
|
|
971
|
-
* break lease renewal:
|
|
972
|
-
* - the field is optional; a v0.2.3 agent never sends it
|
|
973
|
-
* - a MISSING or INVALID value is IGNORED, keeping the previous value, and the
|
|
974
|
-
* heartbeat still succeeds
|
|
975
|
-
* - unknown extra fields are ignored entirely (forward compatibility)
|
|
976
|
-
*
|
|
977
|
-
* `last_heartbeat_at` advances on EVERY accepted heartbeat, with or without a
|
|
978
|
-
* usable RTT: it answers "is the machine alive", while `rtt_at` answers "when
|
|
979
|
-
* was this number measured". Keeping them separate is what lets a reader tell a
|
|
980
|
-
* genuinely slow machine from one that is simply gone.
|
|
981
|
-
*/
|
|
982
|
-
recordHeartbeatMetrics(machineId, input = {}, nowMs = this.nowMs()) {
|
|
983
|
-
const device = this.devices.get(machineId);
|
|
984
|
-
if (!device) return null;
|
|
985
|
-
device.last_heartbeat_at_ms = nowMs;
|
|
986
|
-
const rtt = RabbitState.normalizeRtt(input.rtt_ms);
|
|
987
|
-
if (rtt.valid) {
|
|
988
|
-
device.rtt_ms = rtt.value;
|
|
989
|
-
device.rtt_at_ms = nowMs;
|
|
990
|
-
}
|
|
991
|
-
return device;
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
/** True when there is no value, or the value is older than the staleness window. */
|
|
995
|
-
isRttStale(device, nowMs = this.nowMs()) {
|
|
996
|
-
if (!device || device.rtt_ms === null || device.rtt_ms === undefined) return true;
|
|
997
|
-
if (device.rtt_at_ms === null || device.rtt_at_ms === undefined) return true;
|
|
998
|
-
return nowMs - device.rtt_at_ms > this.rttStaleMs;
|
|
999
|
-
}
|
|
1000
|
-
|
|
1001
|
-
/** How old the RTT measurement is, in ms; null when there is none. */
|
|
1002
|
-
rttAgeMs(device, nowMs = this.nowMs()) {
|
|
1003
|
-
if (!device || device.rtt_at_ms === null || device.rtt_at_ms === undefined) return null;
|
|
1004
|
-
return Math.max(0, nowMs - device.rtt_at_ms);
|
|
1005
|
-
}
|
|
1006
|
-
|
|
1007
|
-
/** Record that a machine attached a stream (a reconnect is any attach after the first). */
|
|
1008
|
-
noteStreamConnect(machineId) {
|
|
1009
|
-
const device = this.devices.get(machineId);
|
|
1010
|
-
if (!device) return null;
|
|
1011
|
-
device.stream_connects = (device.stream_connects ?? 0) + 1;
|
|
1012
|
-
return device.stream_connects;
|
|
1013
|
-
}
|
|
1014
|
-
|
|
1015
|
-
/**
|
|
1016
|
-
* v0.3.0 aggregate for `/healthz`.
|
|
1017
|
-
*
|
|
1018
|
-
* Only FRESH measurements count. A machine that has been gone for a day must not
|
|
1019
|
-
* drag min/max/avg around, and with no fresh data every numeric field is null --
|
|
1020
|
-
* never 0, which would read as "instantaneous".
|
|
1021
|
-
*/
|
|
1022
|
-
rttSummary(nowMs = this.nowMs()) {
|
|
1023
|
-
const fresh = [];
|
|
1024
|
-
let stale = 0;
|
|
1025
|
-
let unknown = 0;
|
|
1026
|
-
for (const device of this.devices.values()) {
|
|
1027
|
-
if (device.rtt_ms === null || device.rtt_ms === undefined) { unknown += 1; continue; }
|
|
1028
|
-
if (this.isRttStale(device, nowMs)) { stale += 1; continue; }
|
|
1029
|
-
fresh.push(device.rtt_ms);
|
|
1030
|
-
}
|
|
1031
|
-
if (fresh.length === 0) {
|
|
1032
|
-
return {
|
|
1033
|
-
machines_reporting: 0,
|
|
1034
|
-
min_ms: null,
|
|
1035
|
-
max_ms: null,
|
|
1036
|
-
avg_ms: null,
|
|
1037
|
-
machines_stale: stale,
|
|
1038
|
-
machines_unknown: unknown,
|
|
1039
|
-
};
|
|
1040
|
-
}
|
|
1041
|
-
const sum = fresh.reduce((a, b) => a + b, 0);
|
|
1042
|
-
return {
|
|
1043
|
-
machines_reporting: fresh.length,
|
|
1044
|
-
min_ms: Math.min(...fresh),
|
|
1045
|
-
max_ms: Math.max(...fresh),
|
|
1046
|
-
avg_ms: Math.round((sum / fresh.length) * 10) / 10,
|
|
1047
|
-
machines_stale: stale,
|
|
1048
|
-
machines_unknown: unknown,
|
|
1049
|
-
};
|
|
1050
|
-
}
|
|
1051
|
-
|
|
1052
|
-
/**
|
|
1053
|
-
* v0.3.0 single-machine status view.
|
|
1054
|
-
*
|
|
1055
|
-
* `last_heartbeat_at` and `rtt_at` answer different questions, and the gap
|
|
1056
|
-
* between them is the whole point: a machine can be heartbeating (alive) while
|
|
1057
|
-
* its RTT number is old (a v0.2.3 agent that never sends `rtt_ms`), and a
|
|
1058
|
-
* machine can be gone while a plausible-looking RTT is still on file.
|
|
1059
|
-
*/
|
|
1060
|
-
deviceStatus(machineId, nowMs = this.nowMs()) {
|
|
1061
|
-
const device = this.devices.get(machineId);
|
|
1062
|
-
if (!device) return null;
|
|
1063
|
-
const rttAt = device.rtt_at_ms ?? null;
|
|
1064
|
-
const lastHeartbeat = device.last_heartbeat_at_ms ?? null;
|
|
1065
|
-
return {
|
|
1066
|
-
machine_id: device.machine_id,
|
|
1067
|
-
machine_name: device.machine_name,
|
|
1068
|
-
relay_id: this.relayId,
|
|
1069
|
-
connected: (device.streams ?? 0) > 0,
|
|
1070
|
-
streams: device.streams ?? 0,
|
|
1071
|
-
online: device.online === true,
|
|
1072
|
-
rtt_ms: device.rtt_ms ?? null,
|
|
1073
|
-
rtt_at: rttAt === null ? null : rfc3339(rttAt),
|
|
1074
|
-
rtt_age_ms: this.rttAgeMs(device, nowMs),
|
|
1075
|
-
rtt_stale: this.isRttStale(device, nowMs),
|
|
1076
|
-
last_heartbeat_at: lastHeartbeat === null ? null : rfc3339(lastHeartbeat),
|
|
1077
|
-
last_seen_at: device.last_seen_at_ms === null || device.last_seen_at_ms === undefined
|
|
1078
|
-
? null : rfc3339(device.last_seen_at_ms),
|
|
1079
|
-
reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
|
|
1080
|
-
stream_connects: device.stream_connects ?? 0,
|
|
1081
|
-
paired_at: device.paired_at_ms === null || device.paired_at_ms === undefined
|
|
1082
|
-
? null : rfc3339(device.paired_at_ms),
|
|
1083
|
-
platform: device.platform,
|
|
1084
|
-
caps: device.caps,
|
|
1085
|
-
user_id: device.user_id ?? null,
|
|
1086
|
-
now: rfc3339(nowMs),
|
|
1087
|
-
};
|
|
1088
|
-
}
|
|
1089
|
-
|
|
1090
|
-
/** Public device view — never leaks device_token. */
|
|
1091
|
-
publicDevice(device) {
|
|
1092
|
-
return {
|
|
1093
|
-
machine_id: device.machine_id,
|
|
1094
|
-
machine_name: device.machine_name,
|
|
1095
|
-
platform: device.platform,
|
|
1096
|
-
caps: device.caps,
|
|
1097
|
-
user_id: device.user_id,
|
|
1098
|
-
/**
|
|
1099
|
-
* v0.7.7: the fleet this device belongs to, and where the fact came from.
|
|
1100
|
-
*
|
|
1101
|
-
* **Carried in the public shape on purpose, and it has to be**, because `devices.json`
|
|
1102
|
-
* is written from this shape -- a fleet that lived only on the in-memory record would
|
|
1103
|
-
* vanish on the next restart, which is precisely when an operator is looking at it. A
|
|
1104
|
-
* first version omitted these and the live test caught the fleet disappearing from
|
|
1105
|
-
* `listDevices()`.
|
|
1106
|
-
*
|
|
1107
|
-
* Safe to expose: `fleet_id` is a truncated hash of the account id, not the account id,
|
|
1108
|
-
* and devices on one relay can already enumerate each other. Narrowing `/v1/devices` to
|
|
1109
|
-
* a device's own fleet is a further step, recorded as such rather than implied.
|
|
1110
|
-
*/
|
|
1111
|
-
fleet_id: device.fleet_id ?? null,
|
|
1112
|
-
fleet_source: device.fleet_source ?? null,
|
|
1113
|
-
// v0.7.10: a boolean rather than the key itself. The key is not a secret, but a device
|
|
1114
|
-
// list is read by every device, and there is no reason to hand every machine every other
|
|
1115
|
-
// machine's key when the verification happens relay-side.
|
|
1116
|
-
public_key_present: typeof device.public_key === 'string' && device.public_key !== '',
|
|
1117
|
-
online: device.online,
|
|
1118
|
-
streams: device.streams,
|
|
1119
|
-
paired_at: rfc3339(device.paired_at_ms),
|
|
1120
|
-
last_seen_at: rfc3339(device.last_seen_at_ms),
|
|
1121
|
-
// v0.3.0: `null` means "never measured", which is NOT the same as 0ms.
|
|
1122
|
-
rtt_ms: device.rtt_ms ?? null,
|
|
1123
|
-
rtt_at: device.rtt_at_ms === null || device.rtt_at_ms === undefined ? null : rfc3339(device.rtt_at_ms),
|
|
1124
|
-
rtt_age_ms: this.rttAgeMs(device),
|
|
1125
|
-
rtt_stale: this.isRttStale(device),
|
|
1126
|
-
last_heartbeat_at: device.last_heartbeat_at_ms === null || device.last_heartbeat_at_ms === undefined
|
|
1127
|
-
? null : rfc3339(device.last_heartbeat_at_ms),
|
|
1128
|
-
reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
|
|
1129
|
-
};
|
|
1130
|
-
}
|
|
1131
|
-
|
|
1132
|
-
listDevices() { return [...this.devices.values()].map((d) => this.publicDevice(d)); }
|
|
1133
|
-
|
|
1134
|
-
/* ---------------- tasks (§4) ---------------- */
|
|
1135
|
-
|
|
1136
|
-
/**
|
|
1137
|
-
* v0.4.0: validate the routing metadata a `POST /v1/task` body may carry.
|
|
1138
|
-
*
|
|
1139
|
-
* A machine, not a mode, is `origin_machine_id`: where the task was typed (`device.json`), so an
|
|
1140
|
-
* offer can carry that fact to the machine that has to run it. `p2p.mode` is the dispatcher's
|
|
1141
|
-
* resolved mode for this particular task — the plugin may be configured `direct` while a
|
|
1142
|
-
* scheduled run is `relay`.
|
|
1143
|
-
*
|
|
1144
|
-
* Validation is deliberately shallow and strictly typed, in the house style of this file: a
|
|
1145
|
-
* missing value is the v0.3.9 shape and stays valid, while a present-but-wrong value is refused
|
|
1146
|
-
* by name. Accepting a non-string here would put a number into `origin_machine_id` and only fail
|
|
1147
|
-
* later, on a machine that cannot tell a bad field from a bad relay.
|
|
1148
|
-
*
|
|
1149
|
-
* @returns {{origin_machine_id:string|null, p2p:{mode:string}|null}}
|
|
1150
|
-
*/
|
|
1151
|
-
static validateTaskRouting(input = {}) {
|
|
1152
|
-
let originMachineId = null;
|
|
1153
|
-
if (input.origin_machine_id !== undefined && input.origin_machine_id !== null) {
|
|
1154
|
-
if (typeof input.origin_machine_id !== 'string'
|
|
1155
|
-
|| input.origin_machine_id.length === 0
|
|
1156
|
-
|| input.origin_machine_id.length > MAX_ORIGIN_MACHINE_ID_LENGTH) {
|
|
1157
|
-
throw new ProtocolError(
|
|
1158
|
-
'BAD_REQUEST',
|
|
1159
|
-
`origin_machine_id must be a non-empty string of at most ${MAX_ORIGIN_MACHINE_ID_LENGTH} characters`,
|
|
1160
|
-
{ origin_machine_id: input.origin_machine_id },
|
|
1161
|
-
);
|
|
1162
|
-
}
|
|
1163
|
-
originMachineId = input.origin_machine_id;
|
|
1164
|
-
}
|
|
1165
|
-
|
|
1166
|
-
let p2p = null;
|
|
1167
|
-
if (input.p2p !== undefined && input.p2p !== null) {
|
|
1168
|
-
if (typeof input.p2p !== 'object' || Array.isArray(input.p2p)) {
|
|
1169
|
-
throw new ProtocolError('BAD_REQUEST', 'p2p must be an object with a `mode` field', { p2p: input.p2p });
|
|
1170
|
-
}
|
|
1171
|
-
if (!P2P_MODES.includes(input.p2p.mode)) {
|
|
1172
|
-
throw new ProtocolError(
|
|
1173
|
-
'BAD_REQUEST',
|
|
1174
|
-
`p2p.mode must be one of ${P2P_MODES.join('|')}`,
|
|
1175
|
-
{ 'p2p.mode': input.p2p.mode ?? null },
|
|
1176
|
-
);
|
|
1177
|
-
}
|
|
1178
|
-
// Copied rather than referenced: the recorded task must not be mutable through the request
|
|
1179
|
-
// body the caller still holds.
|
|
1180
|
-
p2p = { ...input.p2p };
|
|
1181
|
-
}
|
|
1182
|
-
|
|
1183
|
-
return { origin_machine_id: originMachineId, p2p };
|
|
1184
|
-
}
|
|
1185
|
-
|
|
1186
|
-
/**
|
|
1187
|
-
* v0.4.0 §5.2: validate the optional transport fields of a result envelope.
|
|
1188
|
-
*
|
|
1189
|
-
* Same reasoning as `validateTaskRouting`: `transport` is an enum and `p2p` is an opaque object,
|
|
1190
|
-
* so anything else is refused by name at the door. Both fields stay OUT of
|
|
1191
|
-
* `REQUIRED_ENVELOPE_FIELDS` (a missing one is a v0.3.9 envelope, not an incomplete one) and out
|
|
1192
|
-
* of `COMPARABLE_FIELDS` (a path is not a result).
|
|
1193
|
-
*/
|
|
1194
|
-
static validateTransportFields(envelope = {}) {
|
|
1195
|
-
if (envelope.transport !== undefined && envelope.transport !== null) {
|
|
1196
|
-
if (!TRANSPORT_VALUES.includes(envelope.transport)) {
|
|
1197
|
-
throw new ProtocolError(
|
|
1198
|
-
'BAD_REQUEST',
|
|
1199
|
-
`transport must be one of ${TRANSPORT_VALUES.join('|')}`,
|
|
1200
|
-
{ transport: envelope.transport },
|
|
1201
|
-
);
|
|
1202
|
-
}
|
|
1203
|
-
}
|
|
1204
|
-
if (envelope.p2p !== undefined && envelope.p2p !== null
|
|
1205
|
-
&& (typeof envelope.p2p !== 'object' || Array.isArray(envelope.p2p))) {
|
|
1206
|
-
throw new ProtocolError('BAD_REQUEST', 'p2p must be an object', { p2p: envelope.p2p });
|
|
1207
|
-
}
|
|
1208
|
-
}
|
|
1209
|
-
|
|
1210
|
-
/**
|
|
1211
|
-
* §4.1 POST /v1/task.
|
|
1212
|
-
* Machine selection is not specified by the PROTOCOL; we implement:
|
|
1213
|
-
* - `target_machines` (optional explicit list), else every paired device
|
|
1214
|
-
* - capability gate §6.1 per machine (failures become `refused` leases)
|
|
1215
|
-
* - index assignment: replicate -> 0, split -> round-robin modulo index_total
|
|
1216
|
-
*/
|
|
1217
|
-
createTask(input = {}) {
|
|
1218
|
-
// v0.4.0: refused BEFORE the task literal is built and before any offer is emitted, so a bad
|
|
1219
|
-
// value cannot leave a half-created task behind. `POST /v1/task` is all-or-nothing.
|
|
1220
|
-
const routing = RabbitState.validateTaskRouting(input);
|
|
1221
|
-
|
|
1222
|
-
const mode = input.mode ?? 'replicate';
|
|
1223
|
-
if (mode !== 'replicate' && mode !== 'split' && mode !== 'broadcast' && mode !== 'pipeline' && mode !== 'compose') {
|
|
1224
|
-
throw new ProtocolError(
|
|
1225
|
-
'BAD_REQUEST',
|
|
1226
|
-
'mode must be replicate|split|broadcast|pipeline|compose',
|
|
1227
|
-
{ mode },
|
|
1228
|
-
);
|
|
1229
|
-
}
|
|
1230
|
-
/**
|
|
1231
|
-
* v0.3.3 `pipeline`: the whole chain runs on each machine, stage after stage.
|
|
1232
|
-
*
|
|
1233
|
-
* The semantics were chosen against the alternative (fan out stage 1, converge its outputs into
|
|
1234
|
-
* stage 2) because that one has no well-defined input: on a fleet, stage 1 produces n different
|
|
1235
|
-
* outputs, and "feed them all to stage 2" does not say in what order, or whether stage 2 then
|
|
1236
|
-
* runs once or n times. Worse, two machines finishing in a different order would produce
|
|
1237
|
-
* different stage-2 inputs, so one task_id would no longer describe one reproducible execution --
|
|
1238
|
-
* which is the property the rest of this system is built on. See docs/PIPELINE-DESIGN.md.
|
|
1239
|
-
*
|
|
1240
|
-
* So: a pipeline is `replicate` applied to a sequence. Every machine runs the whole chain, there
|
|
1241
|
-
* is no cross-machine data flow, and nothing new can be silently dropped in transit.
|
|
1242
|
-
*/
|
|
1243
|
-
const rawStages = Array.isArray(input.stages) ? input.stages : null;
|
|
1244
|
-
const isPipeline = mode === 'pipeline';
|
|
1245
|
-
if (isPipeline) {
|
|
1246
|
-
if (rawStages === null || rawStages.length === 0) {
|
|
1247
|
-
throw new ProtocolError('BAD_REQUEST', 'mode=pipeline requires a non-empty `stages` array', {
|
|
1248
|
-
stages: rawStages === null ? null : rawStages.length,
|
|
1249
|
-
});
|
|
1250
|
-
}
|
|
1251
|
-
if (rawStages.length > MAX_PIPELINE_STAGES) {
|
|
1252
|
-
throw new ProtocolError('BAD_REQUEST', `stages must have at most ${MAX_PIPELINE_STAGES} entries`, {
|
|
1253
|
-
stages: rawStages.length,
|
|
1254
|
-
});
|
|
1255
|
-
}
|
|
1256
|
-
rawStages.forEach((stage, i) => {
|
|
1257
|
-
const argv = stage?.command_argv;
|
|
1258
|
-
if (!Array.isArray(argv) || argv.length === 0 || argv.some((a) => typeof a !== 'string')) {
|
|
1259
|
-
throw new ProtocolError('BAD_REQUEST', `stages[${i}].command_argv must be a non-empty string[]`, { index: i });
|
|
1260
|
-
}
|
|
1261
|
-
});
|
|
1262
|
-
// A chain is one execution on one machine; sharding it would mean each machine ran a chain of
|
|
1263
|
-
// its own slice, which is a different feature and not what the caller asked for.
|
|
1264
|
-
if (Number.isInteger(input.index_total) && input.index_total !== 1) {
|
|
1265
|
-
throw new ProtocolError('BAD_REQUEST', 'mode=pipeline runs the whole chain per machine, so index_total must be 1', {
|
|
1266
|
-
index_total: input.index_total,
|
|
1267
|
-
});
|
|
1268
|
-
}
|
|
1269
|
-
} else if (rawStages !== null) {
|
|
1270
|
-
// Refused rather than ignored: the caller believes they described a sequence, and running only
|
|
1271
|
-
// `command_argv` would silently drop every stage after the first.
|
|
1272
|
-
throw new ProtocolError('BAD_REQUEST', '`stages` is only valid with mode=pipeline', { mode, stages: rawStages.length });
|
|
1273
|
-
}
|
|
1274
|
-
|
|
1275
|
-
/**
|
|
1276
|
-
* Normalized stage list. For every mode there is at least one stage, so downstream code has a
|
|
1277
|
-
* single shape to reason about and `pipeline` is not a special case at the execution layer.
|
|
1278
|
-
*/
|
|
1279
|
-
const stages = isPipeline
|
|
1280
|
-
? rawStages.map((s, i) => ({
|
|
1281
|
-
index: i,
|
|
1282
|
-
command_argv: [...s.command_argv],
|
|
1283
|
-
cwd_rel: typeof s.cwd_rel === 'string' && s.cwd_rel ? s.cwd_rel : (typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.'),
|
|
1284
|
-
timeout_ms: Number.isInteger(s.timeout_ms) ? s.timeout_ms : null,
|
|
1285
|
-
continue_on_failure: s.continue_on_failure === true,
|
|
1286
|
-
}))
|
|
1287
|
-
: null;
|
|
1288
|
-
|
|
1289
|
-
const commandArgv = isPipeline ? [...rawStages[0].command_argv] : input.command_argv;
|
|
1290
|
-
if (!Array.isArray(commandArgv) || commandArgv.length === 0 || commandArgv.some((a) => typeof a !== 'string')) {
|
|
1291
|
-
throw new ProtocolError('BAD_REQUEST', 'command_argv must be a non-empty string[]');
|
|
1292
|
-
}
|
|
1293
|
-
const indexTotal = Number.isInteger(input.index_total) ? input.index_total : 1;
|
|
1294
|
-
if (indexTotal < 1) throw new ProtocolError('BAD_REQUEST', 'index_total must be >= 1');
|
|
1295
|
-
|
|
1296
|
-
const halt = input.halt ?? 'never';
|
|
1297
|
-
if (halt !== 'never' && halt !== 'now') throw new ProtocolError('BAD_REQUEST', 'halt must be never|now', { halt });
|
|
1298
|
-
|
|
1299
|
-
const taskId = input.task_id ?? this.nextId();
|
|
1300
|
-
if (this.tasks.has(taskId)) {
|
|
1301
|
-
throw new ProtocolError('TASK_EXISTS', 'task_id already exists', { task_id: taskId });
|
|
1302
|
-
}
|
|
1303
|
-
|
|
1304
|
-
const now = this.nowMs();
|
|
1305
|
-
const cwdRel = typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.';
|
|
1306
|
-
const baseTree = input.base_tree ?? null;
|
|
1307
|
-
const commandHash = input.command_hash ?? (isPipeline
|
|
1308
|
-
? computePipelineCommandHash(stages, SHELL_ID_DIRECT)
|
|
1309
|
-
: computeCommandHash(commandArgv, SHELL_ID_DIRECT, cwdRel));
|
|
1310
|
-
const timeoutMs = Number.isInteger(input.timeout_ms) ? input.timeout_ms : 300_000;
|
|
1311
|
-
|
|
1312
|
-
// v0.3.3 `broadcast`: one machine runs the command, every other machine is told the outcome.
|
|
1313
|
-
//
|
|
1314
|
-
// The executor choice is deterministic rather than random. A random leader makes a task
|
|
1315
|
-
// impossible to reproduce from its own record -- the same task id would run somewhere else on a
|
|
1316
|
-
// retry -- and this project's whole value rests on being able to say exactly where something ran.
|
|
1317
|
-
// The first target in the relay's own device order is stable for a given device table.
|
|
1318
|
-
//
|
|
1319
|
-
// Observers get a lease so they are tracked and notified, but their lease is `observing`, which
|
|
1320
|
-
// every aggregation step treats as "did not run". That distinction is the point: a machine that
|
|
1321
|
-
// never executed anything agreeing with a result would be a fabrication, and it is exactly the
|
|
1322
|
-
// fabrication this mode could most easily produce.
|
|
1323
|
-
//
|
|
1324
|
-
// Resolved before the task literal because the literal records it. (`targets` is computed below,
|
|
1325
|
-
// so the executor is re-validated against it after that.)
|
|
1326
|
-
const targetIds = Array.isArray(input.target_machines) && input.target_machines.length > 0
|
|
1327
|
-
? input.target_machines
|
|
1328
|
-
: [...this.devices.keys()];
|
|
1329
|
-
const executorMachineId = mode === 'broadcast'
|
|
1330
|
-
? (typeof input.executor_machine_id === 'string' && input.executor_machine_id !== ''
|
|
1331
|
-
? input.executor_machine_id
|
|
1332
|
-
: targetIds[0] ?? null)
|
|
1333
|
-
: null;
|
|
1334
|
-
|
|
1335
|
-
const task = {
|
|
1336
|
-
task_id: taskId,
|
|
1337
|
-
mode,
|
|
1338
|
-
command_argv: commandArgv,
|
|
1339
|
-
cwd_rel: cwdRel,
|
|
1340
|
-
index_total: indexTotal,
|
|
1341
|
-
timeout_ms: timeoutMs,
|
|
1342
|
-
write: input.write === true,
|
|
1343
|
-
write_scope: input.write_scope ?? [],
|
|
1344
|
-
require_exclusive_write: input.require_exclusive_write === true,
|
|
1345
|
-
base_commit: input.base_commit ?? null,
|
|
1346
|
-
base_tree: baseTree,
|
|
1347
|
-
requirements: input.requirements ?? {},
|
|
1348
|
-
compare_policy: input.compare_policy ?? {},
|
|
1349
|
-
halt,
|
|
1350
|
-
created_by: input.created_by ?? null,
|
|
1351
|
-
created_at_ms: now,
|
|
1352
|
-
command_hash: commandHash,
|
|
1353
|
-
attempt: 1,
|
|
1354
|
-
leases: new Map(),
|
|
1355
|
-
cancelled: false,
|
|
1356
|
-
cancel_reason: null,
|
|
1357
|
-
degraded: null,
|
|
1358
|
-
deadline_ms: now + timeoutMs,
|
|
1359
|
-
result_seqs: [],
|
|
1360
|
-
/** v0.3.3: which machine actually executes under `broadcast`; null for the other modes. */
|
|
1361
|
-
executor_machine_id: executorMachineId,
|
|
1362
|
-
/**
|
|
1363
|
-
* v0.3.3: the normalized stage list for `pipeline`, or null.
|
|
1364
|
-
*
|
|
1365
|
-
* Sent to the machine in the offer so it can run the whole chain. Kept on the task as well so
|
|
1366
|
-
* a report can say how many stages were supposed to run even when the machine crashed before
|
|
1367
|
-
* reporting any of them.
|
|
1368
|
-
*/
|
|
1369
|
-
stages,
|
|
1370
|
-
/**
|
|
1371
|
-
* v0.4.0: where the task came from and under which P2P mode it was dispatched, or null.
|
|
1372
|
-
*
|
|
1373
|
-
* Kept on the task (not just passed to `emitOffer`) because an offer is emitted more than
|
|
1374
|
-
* once: `redeliverPendingOffers` re-states unclaimed work on every reconnect and
|
|
1375
|
-
* `_takeover` re-offers a freed index. A machine that reconnects must see the same routing
|
|
1376
|
-
* facts as the machine that was connected at dispatch time, or the direct-push half of the
|
|
1377
|
-
* feature would work exactly once and then quietly stop.
|
|
1378
|
-
*/
|
|
1379
|
-
origin_machine_id: routing.origin_machine_id,
|
|
1380
|
-
p2p: routing.p2p,
|
|
1381
|
-
};
|
|
1382
|
-
|
|
1383
|
-
/**
|
|
1384
|
-
* Fleet scoping (v0.7.7).
|
|
1385
|
-
*
|
|
1386
|
-
* This replaces `targets = [...this.devices.values()]`, which reached **every device on the
|
|
1387
|
-
* relay** with no filter — the finding that accounts were not isolated. The rule now lives in
|
|
1388
|
-
* `fleet-wiring.mjs` so it is one function with its own exhaustive tests rather than three
|
|
1389
|
-
* lines inside a two-thousand-line object.
|
|
1390
|
-
*
|
|
1391
|
-
* ## The migration is deliberate, and so is its limit
|
|
1392
|
-
*
|
|
1393
|
-
* The shared relay holds four devices that paired before fleets existed, and none of them has
|
|
1394
|
-
* a `fleet_id`. `targetsFor` therefore has two modes: a relay with no fleet configured keeps
|
|
1395
|
-
* exactly the old behaviour, and a relay that *is* scoped refuses a fleetless requester rather
|
|
1396
|
-
* than handing it the run of the fleet. So deploying this cannot be an outage, and scoping it
|
|
1397
|
-
* cannot be a silent grant.
|
|
1398
|
-
*
|
|
1399
|
-
* ## What this does NOT do, stated here rather than discovered later
|
|
1400
|
-
*
|
|
1401
|
-
* `POST /v1/task` authenticates with the **operator token**, and `authorizeTaskDispatch`
|
|
1402
|
-
* returns `{ role: 'operator', machine_id: null }` for it — there is no device, so there is no
|
|
1403
|
-
* fleet to scope by. Every device that authenticates with a **device token** gets full
|
|
1404
|
-
* isolation from this change; an operator-token dispatch remains relay-wide, because the
|
|
1405
|
-
* operator token *is* the relay-wide credential by design.
|
|
1406
|
-
*
|
|
1407
|
-
* `input.created_by` is the one device identity a task carries, and it is used when present:
|
|
1408
|
-
* a task submitted on behalf of a machine is scoped to that machine's fleet.
|
|
1409
|
-
*/
|
|
1410
|
-
const requester =
|
|
1411
|
-
(typeof input.created_by === 'string' && this.devices.get(input.created_by)) ||
|
|
1412
|
-
(typeof input.requester_machine_id === 'string' && this.devices.get(input.requester_machine_id)) ||
|
|
1413
|
-
null;
|
|
1414
|
-
|
|
1415
|
-
/**
|
|
1416
|
-
* An operator-token dispatch, on a relay that has been scoped, acts **within that scope**.
|
|
1417
|
-
*
|
|
1418
|
-
* A first version passed `requester: null` and let it be refused -- which broke every
|
|
1419
|
-
* operator dispatch on a scoped relay, and broke it in the least useful way: the operator had
|
|
1420
|
-
* just configured the fleet, and configuring it made them unable to dispatch at all.
|
|
1421
|
-
*
|
|
1422
|
-
* The reasoning for the fix: the relay's own `fleetId` *is* an admin's statement about which
|
|
1423
|
-
* fleet this relay serves. An operator dispatch therefore has an unambiguous scope, and
|
|
1424
|
-
* refusing it would be refusing the case that was explicitly asked for.
|
|
1425
|
-
*
|
|
1426
|
-
* What stays genuinely ambiguous is an operator dispatch on a relay holding **several** fleets
|
|
1427
|
-
* with no fleet of its own -- there is no way to tell which was meant, so `targetsFor` refuses
|
|
1428
|
-
* that rather than guessing.
|
|
1429
|
-
*/
|
|
1430
|
-
const scopedRequester =
|
|
1431
|
-
requester ??
|
|
1432
|
-
(this.fleetId === null ? null : { machine_id: null, fleet_id: this.fleetId, fleet_source: 'relay_scope' });
|
|
1433
|
-
|
|
1434
|
-
const scoped = targetsFor({
|
|
1435
|
-
devices: [...this.devices.values()],
|
|
1436
|
-
requester: scopedRequester,
|
|
1437
|
-
targetMachines: input.target_machines,
|
|
1438
|
-
relayFleetId: this.fleetId ?? null,
|
|
1439
|
-
// The existing code did not filter offline devices at selection time -- it offered to them and
|
|
1440
|
-
// let the lease expire. Keeping that here avoids changing two behaviours at once, and the
|
|
1441
|
-
// lease expiry is what `handoff.mjs` is for.
|
|
1442
|
-
requireOnline: false,
|
|
1443
|
-
});
|
|
1444
|
-
if (!scoped.ok) {
|
|
1445
|
-
/**
|
|
1446
|
-
* `NO_ONLINE_DEVICE`, not a new code.
|
|
1447
|
-
*
|
|
1448
|
-
* A first version invented `NO_ELIGIBLE_DEVICE`, which broke the one relay test that pins
|
|
1449
|
-
* this contract — and worse, it would have broken **callers**: `chatops/fleet.mjs` reads
|
|
1450
|
-
* `NO_ONLINE_DEVICE` to produce *"start an agent on each machine"*, and a code it has never
|
|
1451
|
-
* seen falls through to a generic refusal. The protocol's code table is part of the
|
|
1452
|
-
* protocol, and adding a member to it is a decision for a protocol revision, not for a
|
|
1453
|
-
* wiring change.
|
|
1454
|
-
*
|
|
1455
|
-
* What *is* new is why nothing was eligible, and that belongs in `detail`, where a caller
|
|
1456
|
-
* can read it without a new code entering the table.
|
|
1457
|
-
*/
|
|
1458
|
-
throw new ProtocolError('NO_ONLINE_DEVICE', scoped.text, {
|
|
1459
|
-
reason: scoped.reason,
|
|
1460
|
-
legacy: scoped.legacy,
|
|
1461
|
-
fleet_scoped: scoped.legacy === false,
|
|
1462
|
-
hint:
|
|
1463
|
-
scoped.reason === 'requester_has_no_fleet'
|
|
1464
|
-
? 'this relay is bound to a fleet and the requesting device is in none; pair it with a code issued for that fleet'
|
|
1465
|
-
: scoped.reason === 'target_outside_fleet'
|
|
1466
|
-
? 'the named target is not in this fleet'
|
|
1467
|
-
: 'check that at least one device in this fleet is paired and streaming',
|
|
1468
|
-
});
|
|
1469
|
-
}
|
|
1470
|
-
const targets = scoped.targets;
|
|
1471
|
-
if (scoped.legacy) {
|
|
1472
|
-
// Logged at debug so an operator reading logs during a migration can see which rule applied.
|
|
1473
|
-
// "N targets" alone does not say whether isolation is on, and that is the question being asked.
|
|
1474
|
-
this.log?.('debug', 'task targets selected without a fleet (legacy path)', {
|
|
1475
|
-
task_candidates: targets.length,
|
|
1476
|
-
relay_fleet_id: this.fleetId ?? null,
|
|
1477
|
-
});
|
|
1478
|
-
}
|
|
1479
|
-
|
|
1480
|
-
// v0.3.3 `broadcast`: validated here, where `targets` is known. The executor id itself was
|
|
1481
|
-
// resolved before the task literal because the literal records it.
|
|
1482
|
-
if (mode === 'broadcast' && executorMachineId !== null && !targets.some((d) => d.machine_id === executorMachineId)) {
|
|
1483
|
-
throw new ProtocolError('BAD_REQUEST', 'executor_machine_id is not among the target machines', {
|
|
1484
|
-
executor_machine_id: executorMachineId,
|
|
1485
|
-
targets: targets.map((d) => d.machine_id),
|
|
1486
|
-
});
|
|
1487
|
-
}
|
|
1488
|
-
|
|
1489
|
-
if (targets.length === 0) {
|
|
1490
|
-
throw new ProtocolError('NO_ONLINE_DEVICE', 'no paired device available for this task', {
|
|
1491
|
-
known_devices: all.length,
|
|
1492
|
-
});
|
|
1493
|
-
}
|
|
1494
|
-
|
|
1495
|
-
const dedupeKeyBase = computeDedupeKey(taskId, 0, commandHash, baseTree);
|
|
1496
|
-
if (typeof dedupeKeyBase !== 'string' || dedupeKeyBase.length !== 64) {
|
|
1497
|
-
throw new ProtocolError('INTERNAL', 'failed to compute dedupe_key');
|
|
1498
|
-
}
|
|
1499
|
-
let lastSeq = this.seq;
|
|
1500
|
-
targets.forEach((device, i) => {
|
|
1501
|
-
const index = mode === 'replicate' ? 0 : i % indexTotal;
|
|
1502
|
-
const observing = mode === 'broadcast' && device.machine_id !== executorMachineId;
|
|
1503
|
-
const lease = {
|
|
1504
|
-
machine_id: device.machine_id,
|
|
1505
|
-
machine_name: device.machine_name,
|
|
1506
|
-
index,
|
|
1507
|
-
attempt: 1,
|
|
1508
|
-
// `observing` is a lease state, not an outcome: the machine holds a place in this task and
|
|
1509
|
-
// will be told what happened, but it was never handed the command. `gateMachine` below is
|
|
1510
|
-
// skipped for it because refusing to do work you were never asked to do is not a refusal.
|
|
1511
|
-
state: observing ? 'observing' : 'queued',
|
|
1512
|
-
refusal_reason: null,
|
|
1513
|
-
refusal_detail: null,
|
|
1514
|
-
dedupe_key: computeDedupeKey(taskId, index, commandHash, baseTree),
|
|
1515
|
-
last_heartbeat_ms: now,
|
|
1516
|
-
claimed_at_ms: now,
|
|
1517
|
-
lease_until_ms: now + this.leaseWindowMs(),
|
|
1518
|
-
phase: null,
|
|
1519
|
-
progress: null,
|
|
1520
|
-
offered_seq: null,
|
|
1521
|
-
expired_at_ms: null,
|
|
1522
|
-
finished_at_ms: null,
|
|
1523
|
-
heartbeat_count: 0,
|
|
1524
|
-
};
|
|
1525
|
-
if (observing) {
|
|
1526
|
-
// No capability gate and no offer: the machine is not being asked to run anything, so there
|
|
1527
|
-
// is nothing to refuse and nothing to hand out. It is recorded so the task can tell it the
|
|
1528
|
-
// outcome afterwards.
|
|
1529
|
-
task.leases.set(device.machine_id, lease);
|
|
1530
|
-
return;
|
|
1531
|
-
}
|
|
1532
|
-
const gate = gateMachine(task, device);
|
|
1533
|
-
if (gate) {
|
|
1534
|
-
lease.state = 'refused';
|
|
1535
|
-
lease.refusal_reason = gate.refusal_reason;
|
|
1536
|
-
lease.refusal_detail = gate.detail;
|
|
1537
|
-
task.leases.set(device.machine_id, lease);
|
|
1538
|
-
this.emit('notice', {
|
|
1539
|
-
level: 'warn',
|
|
1540
|
-
code: 'CAPABILITY_REFUSED',
|
|
1541
|
-
message: `${device.machine_id} refused: ${gate.refusal_reason}`,
|
|
1542
|
-
task_id: taskId,
|
|
1543
|
-
});
|
|
1544
|
-
return;
|
|
1545
|
-
}
|
|
1546
|
-
task.leases.set(device.machine_id, lease);
|
|
1547
|
-
const grace = this._p2pOfferGraceMs(task);
|
|
1548
|
-
if (grace > 0) {
|
|
1549
|
-
// v0.4.0: the dispatcher is pushing this offer over the direct path right now. Hold the
|
|
1550
|
-
// relay's copy for `grace` ms so the direct one can win; see `p2pOfferGraceMs`.
|
|
1551
|
-
this._scheduleOffer(task, lease, grace);
|
|
1552
|
-
} else {
|
|
1553
|
-
const event = this.emitOffer(task, lease);
|
|
1554
|
-
lastSeq = event.seq;
|
|
1555
|
-
}
|
|
1556
|
-
});
|
|
1557
|
-
|
|
1558
|
-
this.tasks.set(taskId, task);
|
|
1559
|
-
|
|
1560
|
-
this._ledger({
|
|
1561
|
-
type: 'task.created',
|
|
1562
|
-
task: {
|
|
1563
|
-
task_id: task.task_id,
|
|
1564
|
-
mode: task.mode,
|
|
1565
|
-
command_argv: task.command_argv,
|
|
1566
|
-
cwd_rel: task.cwd_rel,
|
|
1567
|
-
index_total: task.index_total,
|
|
1568
|
-
timeout_ms: task.timeout_ms,
|
|
1569
|
-
write: task.write,
|
|
1570
|
-
write_scope: task.write_scope,
|
|
1571
|
-
require_exclusive_write: task.require_exclusive_write,
|
|
1572
|
-
base_commit: task.base_commit,
|
|
1573
|
-
base_tree: task.base_tree,
|
|
1574
|
-
requirements: task.requirements,
|
|
1575
|
-
compare_policy: task.compare_policy,
|
|
1576
|
-
halt: task.halt,
|
|
1577
|
-
created_by: task.created_by,
|
|
1578
|
-
created_at_ms: task.created_at_ms,
|
|
1579
|
-
command_hash: task.command_hash,
|
|
1580
|
-
attempt: task.attempt,
|
|
1581
|
-
deadline_ms: task.deadline_ms,
|
|
1582
|
-
// v0.4.0: routing metadata, so a restarted relay re-offers with the same facts.
|
|
1583
|
-
origin_machine_id: task.origin_machine_id ?? null,
|
|
1584
|
-
p2p: task.p2p ?? null,
|
|
1585
|
-
},
|
|
1586
|
-
leases: [...task.leases.values()].map((l) => ({ ...l })),
|
|
1587
|
-
});
|
|
1588
|
-
|
|
1589
|
-
const refusedOnly = [...task.leases.values()].every((l) => l.state === 'refused');
|
|
1590
|
-
if (refusedOnly) {
|
|
1591
|
-
// §6.3 Step 0 territory: a task nobody may execute is still a valid, reportable task.
|
|
1592
|
-
this.emit('notice', {
|
|
1593
|
-
level: 'warn',
|
|
1594
|
-
code: 'NO_ONLINE_DEVICE',
|
|
1595
|
-
message: 'every candidate machine refused the capability gate',
|
|
1596
|
-
task_id: taskId,
|
|
1597
|
-
});
|
|
1598
|
-
}
|
|
1599
|
-
|
|
1600
|
-
return {
|
|
1601
|
-
task_id: taskId,
|
|
1602
|
-
leases: [...task.leases.values()].map((l) => ({
|
|
1603
|
-
machine_id: l.machine_id,
|
|
1604
|
-
index: l.index,
|
|
1605
|
-
state: l.state,
|
|
1606
|
-
// v0.4.0: the dispatcher pushes an offer over the direct path, and the executor dedupes it
|
|
1607
|
-
// against the relay's copy by (task_id, attempt, dedupe_key). Without this field the
|
|
1608
|
-
// dispatcher has to reproduce the §4.4 formula locally, which works for a single-command
|
|
1609
|
-
// task and cannot work for a pipeline one -- the key covers the relay's normalised stage
|
|
1610
|
-
// list. Publishing it makes the pushed copy recognisably the same work item, which is what
|
|
1611
|
-
// keeps the direct path from turning exactly-once into at-least-twice.
|
|
1612
|
-
dedupe_key: l.dedupe_key ?? null,
|
|
1613
|
-
attempt: l.attempt,
|
|
1614
|
-
...(l.refusal_reason ? { refusal_reason: l.refusal_reason } : {}),
|
|
1615
|
-
})),
|
|
1616
|
-
seq: lastSeq,
|
|
1617
|
-
};
|
|
1618
|
-
}
|
|
1619
|
-
|
|
1620
|
-
/**
|
|
1621
|
-
* v0.4.0: should the relay hold its offer back for this task?
|
|
1622
|
-
*
|
|
1623
|
-
* Only when the dispatcher said two things: that it is a machine (`origin_machine_id`), and that
|
|
1624
|
-
* the direct path is in play (`p2p.mode` of `auto` or `direct`). A `relay` mode or a v0.3.9
|
|
1625
|
-
* dispatcher gets the old behaviour exactly, which is why this predicate is the only gate on the
|
|
1626
|
-
* change and every pre-existing test still sees an immediate offer.
|
|
1627
|
-
*
|
|
1628
|
-
* @param {object} task
|
|
1629
|
-
* @returns {number} milliseconds to hold, or 0 for "offer now"
|
|
1630
|
-
*/
|
|
1631
|
-
_p2pOfferGraceMs(task) {
|
|
1632
|
-
if (this.p2pOfferGraceMs <= 0) return 0;
|
|
1633
|
-
const origin = task?.origin_machine_id;
|
|
1634
|
-
if (typeof origin !== 'string' || origin === '') return 0;
|
|
1635
|
-
const mode = task?.p2p?.mode;
|
|
1636
|
-
if (mode !== 'auto' && mode !== 'direct') return 0;
|
|
1637
|
-
return this.p2pOfferGraceMs;
|
|
1638
|
-
}
|
|
1639
|
-
|
|
1640
|
-
/**
|
|
1641
|
-
* v0.4.0: emit `lease`'s offer after `delayMs`, unless the direct path got there first.
|
|
1642
|
-
*
|
|
1643
|
-
* "Got there first" is one fact and it is already tracked: the executor renews the lease
|
|
1644
|
-
* (`POST /v1/heartbeat`) as its first act on any offer it accepts, and a renewal moves the lease
|
|
1645
|
-
* out of `queued`. So the check is the lease's own state — no new message, no new endpoint, and
|
|
1646
|
-
* nothing the executor has to know about.
|
|
1647
|
-
*
|
|
1648
|
-
* The timer is `unref()`ed: a relay with nothing else to do must still be able to exit, and a
|
|
1649
|
-
* pending offer is not a reason to keep a process alive.
|
|
1650
|
-
*/
|
|
1651
|
-
_scheduleOffer(task, lease, delayMs) {
|
|
1652
|
-
const key = `${task.task_id}:${lease.machine_id}`;
|
|
1653
|
-
const timer = setTimeout(() => {
|
|
1654
|
-
this._pendingOffers.delete(key);
|
|
1655
|
-
// A lease that was re-created (takeover) or swept is not this one any more.
|
|
1656
|
-
if (task.leases.get(lease.machine_id) !== lease) return;
|
|
1657
|
-
if (lease.state !== 'queued') return; // the direct copy won; nothing to fall back to
|
|
1658
|
-
if (this.isLeaseExpired(lease, this.nowMs())) return; // the sweep owns it now
|
|
1659
|
-
this.emitOffer(task, lease);
|
|
1660
|
-
}, delayMs);
|
|
1661
|
-
timer.unref?.();
|
|
1662
|
-
this._pendingOffers.set(key, timer);
|
|
1663
|
-
}
|
|
1664
|
-
|
|
1665
|
-
/** Drop every deferred offer. Called by the relay's own teardown path. */
|
|
1666
|
-
clearPendingOffers() {
|
|
1667
|
-
for (const timer of this._pendingOffers.values()) clearTimeout(timer);
|
|
1668
|
-
this._pendingOffers.clear();
|
|
1669
|
-
}
|
|
1670
|
-
|
|
1671
|
-
emitOffer(task, lease) {
|
|
1672
|
-
const event = this.emit('task.offer', {
|
|
1673
|
-
task_id: task.task_id,
|
|
1674
|
-
attempt: lease.attempt,
|
|
1675
|
-
mode: task.mode,
|
|
1676
|
-
index: lease.index,
|
|
1677
|
-
index_total: task.index_total,
|
|
1678
|
-
command_argv: task.command_argv,
|
|
1679
|
-
// v0.3.3: present only for `pipeline`, and only then does the machine run a chain. Omitted
|
|
1680
|
-
// entirely otherwise, so an offer for a single-command task is byte-identical to before --
|
|
1681
|
-
// which is what keeps a v0.2.3 agent working against this relay.
|
|
1682
|
-
...(task.stages ? { stages: task.stages } : {}),
|
|
1683
|
-
/**
|
|
1684
|
-
* v0.4.0: always present, `null` when the caller supplied nothing.
|
|
1685
|
-
*
|
|
1686
|
-
* Deliberately NOT conditional (unlike `stages` above, which is omitted so a single-command
|
|
1687
|
-
* offer stays byte-identical for an old agent): the contract fixes the shape of these two as
|
|
1688
|
-
* `string|null` and `{mode}|null`, and `null` is the honest value for "the dispatcher did not
|
|
1689
|
-
* say" — which is exactly the v0.3.9 case. They are additive keys in an event whose consumers
|
|
1690
|
-
* already ignore unknown fields.
|
|
1691
|
-
*/
|
|
1692
|
-
origin_machine_id: task.origin_machine_id ?? null,
|
|
1693
|
-
p2p: task.p2p ?? null,
|
|
1694
|
-
cwd_rel: task.cwd_rel,
|
|
1695
|
-
write: task.write,
|
|
1696
|
-
timeout_ms: task.timeout_ms,
|
|
1697
|
-
base_commit: task.base_commit,
|
|
1698
|
-
base_tree: task.base_tree,
|
|
1699
|
-
requirements: task.requirements,
|
|
1700
|
-
compare_policy: task.compare_policy,
|
|
1701
|
-
dedupe_key: lease.dedupe_key,
|
|
1702
|
-
deadline: rfc3339(task.deadline_ms),
|
|
1703
|
-
}, { target: lease.machine_id });
|
|
1704
|
-
lease.state = 'offered';
|
|
1705
|
-
lease.offered_seq = event.seq;
|
|
1706
|
-
lease.last_heartbeat_ms = this.nowMs();
|
|
1707
|
-
lease.lease_until_ms = lease.last_heartbeat_ms + this.leaseWindowMs();
|
|
1708
|
-
return event;
|
|
1709
|
-
}
|
|
1710
|
-
|
|
1711
|
-
getTask(taskId) { return this.tasks.get(taskId) ?? null; }
|
|
1712
|
-
|
|
1713
|
-
requireTask(taskId) {
|
|
1714
|
-
const task = this.tasks.get(taskId);
|
|
1715
|
-
if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
|
|
1716
|
-
return task;
|
|
1717
|
-
}
|
|
1718
|
-
|
|
1719
|
-
/**
|
|
1720
|
-
* Re-offer the work a machine was handed but has not yet started (v0.1.2 §8.6).
|
|
1721
|
-
*
|
|
1722
|
-
* WHY THIS EXISTS
|
|
1723
|
-
*
|
|
1724
|
-
* An offer is a point-in-time event on a stream. `createTask` publishes
|
|
1725
|
-
* `task.offer` once, and if the target machine's stream happens to be down at
|
|
1726
|
-
* that instant -- the ordinary situation after a relay restart, a tunnel drop,
|
|
1727
|
-
* or any cross-network interruption -- the event is simply gone. The lease sits
|
|
1728
|
-
* in `offered` until the sweep expires it, the task never runs, and the
|
|
1729
|
-
* operator sees a timeout with no explanation. Measured: the machine is back
|
|
1730
|
-
* online with a live stream within milliseconds, and still receives nothing.
|
|
1731
|
-
*
|
|
1732
|
-
* So the relay re-states unclaimed work when a machine connects. This is safe
|
|
1733
|
-
* because the state it acts on is durable: the lease the offer refers to is
|
|
1734
|
-
* still in the ledger.
|
|
1735
|
-
*
|
|
1736
|
-
* WHY ONLY `offered` AND `queued`
|
|
1737
|
-
*
|
|
1738
|
-
* `running` is deliberately excluded, and that exclusion is the whole safety
|
|
1739
|
-
* argument. A running lease means a machine has already STARTED the command --
|
|
1740
|
-
* it acknowledged the offer and is heartbeating. Re-offering it would make that
|
|
1741
|
-
* machine execute the work a second time, so the fix for a lost offer would
|
|
1742
|
-
* introduce the one failure this project is built to avoid: two machines, or
|
|
1743
|
-
* one machine twice, running the same side-effecting command.
|
|
1744
|
-
*
|
|
1745
|
-
* A machine that dies mid-run therefore does not get its work back here; that
|
|
1746
|
-
* path stays with the sweep, which expires the lease and hands the index to
|
|
1747
|
-
* another machine with `attempt + 1`.
|
|
1748
|
-
*
|
|
1749
|
-
* Re-emitted offers are new events and are replayed like any other, so a client
|
|
1750
|
-
* that reconnects repeatedly can see the same offer more than once. That is
|
|
1751
|
-
* intentional: the agent already de-duplicates by `task_id` (it ignores an
|
|
1752
|
-
* offer whose task is queued or running), and an at-least-once offer with a
|
|
1753
|
-
* client-side dedupe is strictly better than an at-most-once offer that is
|
|
1754
|
-
* silently dropped.
|
|
1755
|
-
*
|
|
1756
|
-
* @param {string} machineId
|
|
1757
|
-
* @returns {{count: number, task_ids: string[]}}
|
|
1758
|
-
*/
|
|
1759
|
-
redeliverPendingOffers(machineId) {
|
|
1760
|
-
const taskIds = [];
|
|
1761
|
-
const now = this.nowMs();
|
|
1762
|
-
|
|
1763
|
-
for (const task of this.tasks.values()) {
|
|
1764
|
-
if (task.cancelled === true) continue;
|
|
1765
|
-
const lease = task.leases.get(machineId);
|
|
1766
|
-
if (!lease) continue;
|
|
1767
|
-
// Only work the machine has not begun. See the note above: `running` is
|
|
1768
|
-
// excluded on purpose, not by omission.
|
|
1769
|
-
if (lease.state !== 'offered' && lease.state !== 'queued') continue;
|
|
1770
|
-
// Do not resurrect a lease the sweep has already written off; the takeover
|
|
1771
|
-
// path owns that decision.
|
|
1772
|
-
if (this.isLeaseExpired(lease, now)) continue;
|
|
1773
|
-
|
|
1774
|
-
this.emitOffer(task, lease);
|
|
1775
|
-
taskIds.push(task.task_id);
|
|
1776
|
-
}
|
|
1777
|
-
|
|
1778
|
-
return { count: taskIds.length, task_ids: taskIds };
|
|
1779
|
-
}
|
|
1780
|
-
|
|
1781
|
-
listTasks({ limit = 50 } = {}) {
|
|
1782
|
-
const n = Number.isInteger(limit) && limit > 0 ? limit : 50;
|
|
1783
|
-
return [...this.tasks.values()]
|
|
1784
|
-
.sort((a, b) => b.created_at_ms - a.created_at_ms)
|
|
1785
|
-
.slice(0, n)
|
|
1786
|
-
.map((t) => this.taskSummary(t));
|
|
1787
|
-
}
|
|
1788
|
-
|
|
1789
|
-
taskSummary(task) {
|
|
1790
|
-
return {
|
|
1791
|
-
task_id: task.task_id,
|
|
1792
|
-
mode: task.mode,
|
|
1793
|
-
created_at: rfc3339(task.created_at_ms),
|
|
1794
|
-
created_by: task.created_by,
|
|
1795
|
-
index_total: task.index_total,
|
|
1796
|
-
cancelled: task.cancelled,
|
|
1797
|
-
degraded: task.degraded,
|
|
1798
|
-
leases: [...task.leases.values()].map((l) => l.machine_id),
|
|
1799
|
-
lease_states: Object.fromEntries([...task.leases.values()].map((l) => [l.machine_id, l.state])),
|
|
1800
|
-
};
|
|
1801
|
-
}
|
|
1802
|
-
|
|
1803
|
-
cancelTask(taskId, reason = 'cancelled_by_operator') {
|
|
1804
|
-
const task = this.requireTask(taskId);
|
|
1805
|
-
if (task.cancelled) return task;
|
|
1806
|
-
task.cancelled = true;
|
|
1807
|
-
task.cancel_reason = reason;
|
|
1808
|
-
for (const lease of task.leases.values()) {
|
|
1809
|
-
if (ACTIVE_LEASE_STATES.includes(lease.state)) lease.state = 'cancelled';
|
|
1810
|
-
}
|
|
1811
|
-
this.emit('task.cancel', { task_id: taskId, reason });
|
|
1812
|
-
this._ledger({ type: 'task.cancelled', task_id: taskId, reason });
|
|
1813
|
-
return task;
|
|
1814
|
-
}
|
|
1815
|
-
|
|
1816
|
-
/* ---------------- leases / heartbeat (§4.3) ---------------- */
|
|
1817
|
-
|
|
1818
|
-
/** Shared `phase`/`progress` validation for both heartbeat modes. */
|
|
1819
|
-
_validatePhaseProgress(input) {
|
|
1820
|
-
if (input.phase !== undefined && input.phase !== null && !LEASE_PHASES.includes(input.phase)) {
|
|
1821
|
-
throw new ProtocolError('BAD_REQUEST', 'phase must be preparing|running|finalizing', { phase: input.phase });
|
|
1822
|
-
}
|
|
1823
|
-
if (input.progress !== undefined && input.progress !== null
|
|
1824
|
-
&& (typeof input.progress !== 'number' || input.progress < 0 || input.progress > 1)) {
|
|
1825
|
-
throw new ProtocolError('BAD_REQUEST', 'progress must be a number in [0,1]', { progress: input.progress });
|
|
1826
|
-
}
|
|
1827
|
-
}
|
|
1828
|
-
|
|
1829
|
-
/**
|
|
1830
|
-
* v0.3.0 IDLE (diagnostic) heartbeat — an agent with no lease keeps its latency
|
|
1831
|
-
* and liveness visible to other machines.
|
|
1832
|
-
*
|
|
1833
|
-
* This path deliberately NEVER looks at `this.tasks`. That is the whole safety
|
|
1834
|
-
* argument, and it is structural rather than a matter of remembering a rule: an
|
|
1835
|
-
* idle heartbeat cannot renew, resurrect or otherwise touch a lease because it
|
|
1836
|
-
* never obtains one. Without that, an idle ping from a machine whose lease had
|
|
1837
|
-
* expired would flip `expired` back to `running` and manufacture a live task out
|
|
1838
|
-
* of nothing -- a fake "the work is still being done" signal, which is the most
|
|
1839
|
-
* dangerous lie this relay can tell.
|
|
1840
|
-
*/
|
|
1841
|
-
diagnosticHeartbeat(input = {}) {
|
|
1842
|
-
const machineId = typeof input.machine_id === 'string' && input.machine_id !== ''
|
|
1843
|
-
? input.machine_id : null;
|
|
1844
|
-
if (machineId === null) {
|
|
1845
|
-
throw new ProtocolError('BAD_REQUEST', 'machine_id is required', {});
|
|
1846
|
-
}
|
|
1847
|
-
if (!this.devices.has(machineId)) {
|
|
1848
|
-
throw new ProtocolError('NOT_FOUND', 'unknown machine_id', { machine_id: machineId });
|
|
1849
|
-
}
|
|
1850
|
-
this._validatePhaseProgress(input);
|
|
1851
|
-
|
|
1852
|
-
const device = this.touchDevice(machineId);
|
|
1853
|
-
this.recordHeartbeatMetrics(machineId, input);
|
|
1854
|
-
return {
|
|
1855
|
-
// No lease is involved, and the response says so instead of inventing one.
|
|
1856
|
-
lease_until: null,
|
|
1857
|
-
cancel: false,
|
|
1858
|
-
diagnostic: true,
|
|
1859
|
-
machine_id: machineId,
|
|
1860
|
-
rtt_ms: device?.rtt_ms ?? null,
|
|
1861
|
-
};
|
|
1862
|
-
}
|
|
1863
|
-
|
|
1864
|
-
/**
|
|
1865
|
-
* §4.3 POST /v1/heartbeat — renews the lease using SERVER time only.
|
|
1866
|
-
*
|
|
1867
|
-
* Two modes since v0.3.0:
|
|
1868
|
-
* - `task_id` present -> LEASE heartbeat (unchanged §4.3 behaviour)
|
|
1869
|
-
* - `diagnostic: true`, no task -> IDLE heartbeat (diagnostics only, no lease)
|
|
1870
|
-
* - neither -> 400. A heartbeat that silently renewed
|
|
1871
|
-
* nothing would let an agent whose `task_id` went missing stop renewing while
|
|
1872
|
-
* believing it was fine, ending in a falsely `expired` task. §0 forbids
|
|
1873
|
-
* silent degradation, so this is loud.
|
|
1874
|
-
*
|
|
1875
|
-
* @returns {{lease_until:string|null, cancel:boolean, phase?:string, progress?:number, diagnostic?:boolean}}
|
|
1876
|
-
*/
|
|
1877
|
-
heartbeat(input = {}) {
|
|
1878
|
-
const hasTask = typeof input.task_id === 'string' && input.task_id !== '';
|
|
1879
|
-
if (!hasTask) {
|
|
1880
|
-
if (input.diagnostic !== true) {
|
|
1881
|
-
throw new ProtocolError('BAD_REQUEST',
|
|
1882
|
-
'a heartbeat must carry task_id (lease renewal) or diagnostic:true (idle diagnostics); '
|
|
1883
|
-
+ 'refusing to treat a missing task_id as a no-op renewal', {
|
|
1884
|
-
received_keys: Object.keys(input).sort(),
|
|
1885
|
-
});
|
|
1886
|
-
}
|
|
1887
|
-
return this.diagnosticHeartbeat(input);
|
|
1888
|
-
}
|
|
1889
|
-
if (input.diagnostic === true) {
|
|
1890
|
-
// task_id wins: a heartbeat that names a task IS a lease heartbeat.
|
|
1891
|
-
// (documented; the flag is simply redundant here)
|
|
1892
|
-
}
|
|
1893
|
-
return this.leaseHeartbeat(input);
|
|
1894
|
-
}
|
|
1895
|
-
|
|
1896
|
-
leaseHeartbeat(input = {}) {
|
|
1897
|
-
const task = this.requireTask(input.task_id);
|
|
1898
|
-
const lease = task.leases.get(input.machine_id);
|
|
1899
|
-
if (!lease) {
|
|
1900
|
-
throw new ProtocolError('NOT_FOUND', 'no lease for this machine on this task', {
|
|
1901
|
-
task_id: input.task_id, machine_id: input.machine_id,
|
|
1902
|
-
});
|
|
1903
|
-
}
|
|
1904
|
-
this._validatePhaseProgress(input);
|
|
1905
|
-
|
|
1906
|
-
const now = this.nowMs();
|
|
1907
|
-
this.touchDevice(input.machine_id);
|
|
1908
|
-
this.recordHeartbeatMetrics(input.machine_id, input, now);
|
|
1909
|
-
|
|
1910
|
-
if (task.cancelled) {
|
|
1911
|
-
return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: task.cancel_reason };
|
|
1912
|
-
}
|
|
1913
|
-
|
|
1914
|
-
// §4.3: once halt=now has handed this index to another attempt, the old owner
|
|
1915
|
-
// must stop — a late heartbeat from the expired machine is not a valid re-claim.
|
|
1916
|
-
if (lease.state === 'expired' && task.attempt > lease.attempt) {
|
|
1917
|
-
return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: 'LEASE_EXPIRED_TAKEN_OVER' };
|
|
1918
|
-
}
|
|
1919
|
-
|
|
1920
|
-
const stateBefore = lease.state;
|
|
1921
|
-
lease.last_heartbeat_ms = now;
|
|
1922
|
-
lease.lease_until_ms = now + this.leaseWindowMs();
|
|
1923
|
-
lease.heartbeat_count += 1;
|
|
1924
|
-
if (input.phase !== undefined && input.phase !== null) lease.phase = input.phase;
|
|
1925
|
-
if (input.progress !== undefined && input.progress !== null) lease.progress = input.progress;
|
|
1926
|
-
if (Number.isInteger(input.attempt) && input.attempt >= 1) lease.attempt = input.attempt;
|
|
1927
|
-
if (lease.state === 'queued' || lease.state === 'offered') lease.state = 'running';
|
|
1928
|
-
// A live heartbeat is stronger evidence than a timer: with halt=never nobody took
|
|
1929
|
-
// the work away, so the late-but-alive agent re-claims its own lease.
|
|
1930
|
-
if (lease.state === 'expired') { lease.state = 'running'; lease.expired_at_ms = null; }
|
|
1931
|
-
|
|
1932
|
-
// §7 ledger records lease STATE changes, not every 10s heartbeat.
|
|
1933
|
-
if (lease.state !== stateBefore) this._ledgerLease(task, lease);
|
|
1934
|
-
|
|
1935
|
-
return { lease_until: rfc3339(lease.lease_until_ms), cancel: false, phase: lease.phase, progress: lease.progress };
|
|
1936
|
-
}
|
|
1937
|
-
|
|
1938
|
-
/** §4.3: expiry is decided by Rabbit's clock only. */
|
|
1939
|
-
isLeaseExpired(lease, nowMs = this.nowMs()) {
|
|
1940
|
-
if (!ACTIVE_LEASE_STATES.includes(lease.state)) return false;
|
|
1941
|
-
return nowMs >= lease.lease_until_ms;
|
|
1942
|
-
}
|
|
1943
|
-
|
|
1944
|
-
/**
|
|
1945
|
-
* §4.3 expiry sweep.
|
|
1946
|
-
* - halt=never → book-keeping only, task is marked `partial`
|
|
1947
|
-
* - halt=now → notify remaining machines immediately that they may take over
|
|
1948
|
-
*/
|
|
1949
|
-
sweepExpired(nowMs = this.nowMs()) {
|
|
1950
|
-
const expired = [];
|
|
1951
|
-
for (const task of this.tasks.values()) {
|
|
1952
|
-
if (task.cancelled) continue;
|
|
1953
|
-
let changed = false;
|
|
1954
|
-
for (const lease of task.leases.values()) {
|
|
1955
|
-
if (!this.isLeaseExpired(lease, nowMs)) continue;
|
|
1956
|
-
lease.state = 'expired';
|
|
1957
|
-
lease.expired_at_ms = nowMs;
|
|
1958
|
-
changed = true;
|
|
1959
|
-
expired.push({ task, lease });
|
|
1960
|
-
this._ledgerLease(task, lease);
|
|
1961
|
-
this.emit('notice', {
|
|
1962
|
-
level: 'warn',
|
|
1963
|
-
code: 'LEASE_EXPIRED',
|
|
1964
|
-
message: `lease expired: machine=${lease.machine_id} task=${task.task_id} attempt=${lease.attempt}`,
|
|
1965
|
-
task_id: task.task_id,
|
|
1966
|
-
machine_id: lease.machine_id,
|
|
1967
|
-
});
|
|
1968
|
-
if (task.halt === 'now') {
|
|
1969
|
-
task.attempt += 1;
|
|
1970
|
-
this._takeover(task, lease, nowMs);
|
|
1971
|
-
}
|
|
1972
|
-
}
|
|
1973
|
-
if (changed) task.degraded = 'partial';
|
|
1974
|
-
}
|
|
1975
|
-
return expired;
|
|
1976
|
-
}
|
|
1977
|
-
|
|
1978
|
-
/**
|
|
1979
|
-
* §4.3 `halt=now`: "立即通知其他机器可接管".
|
|
1980
|
-
*
|
|
1981
|
-
* v0.0.1's PROTOCOL defines no per-index lease reassignment, so takeover is
|
|
1982
|
-
* implemented as: re-offer the freed `index` (attempt+1) to every live machine
|
|
1983
|
-
* that is idle — no lease yet, or a lease still queued/offered — and broadcast a
|
|
1984
|
-
* notice. Machines already `running` are never re-offered, to avoid double
|
|
1985
|
-
* execution of the same index.
|
|
1986
|
-
*/
|
|
1987
|
-
_takeover(task, expiredLease, nowMs = this.nowMs()) {
|
|
1988
|
-
/**
|
|
1989
|
-
* v0.7.8: the two safety guards this path was missing.
|
|
1990
|
-
*
|
|
1991
|
-
* `_takeover` already existed and is invoked on `halt=now`, so hand-off was not absent — but it
|
|
1992
|
-
* would re-offer a dead machine's index to **every idle device the relay holds**, with no fleet
|
|
1993
|
-
* filter and no consideration of whether the task writes. Both are reachable defects:
|
|
1994
|
-
*
|
|
1995
|
-
* 1. **A device in another account could be handed the work.** On a relay serving several
|
|
1996
|
-
* fleets, `this.devices` includes all of them, so a takeover crossed accounts — which is
|
|
1997
|
-
* the isolation this release series is about, broken in the one path that is easy to miss
|
|
1998
|
-
* because it only runs when something has already gone wrong.
|
|
1999
|
-
* 2. **A writing task could be run a second time.** Hand-off makes execution at-least-once,
|
|
2000
|
-
* which is harmless for `node --test` and is a duplicated write for anything else. The
|
|
2001
|
-
* design forbids automatic hand-off for writing tasks until a fencing token exists.
|
|
2002
|
-
*
|
|
2003
|
-
* The re-offer shape itself is unchanged: PROTOCOL v0.0.1 defines no per-index lease
|
|
2004
|
-
* reassignment, so takeover is expressed as a re-offer to idle machines. Narrowing that to a
|
|
2005
|
-
* single successor is a protocol revision, not a wiring change, and is not made here.
|
|
2006
|
-
*/
|
|
2007
|
-
if (task.write === true) {
|
|
2008
|
-
this.emit('notice', {
|
|
2009
|
-
level: 'warn',
|
|
2010
|
-
code: 'LEASE_TAKEOVER_REFUSED',
|
|
2011
|
-
message:
|
|
2012
|
-
`task ${task.task_id} index ${expiredLease.index} is a writing task, so no machine takes it over: `
|
|
2013
|
-
+ 'a second run could apply the same change twice, and the caller cannot tell from the verdict',
|
|
2014
|
-
task_id: task.task_id,
|
|
2015
|
-
reason: 'write_not_idempotent',
|
|
2016
|
-
});
|
|
2017
|
-
return [];
|
|
2018
|
-
}
|
|
2019
|
-
|
|
2020
|
-
const requesterFleet = this.fleetId ?? null;
|
|
2021
|
-
|
|
2022
|
-
const candidates = [];
|
|
2023
|
-
for (const device of this.devices.values()) {
|
|
2024
|
-
if (device.machine_id === expiredLease.machine_id) continue;
|
|
2025
|
-
/**
|
|
2026
|
-
* Fleet membership, when the relay is scoped.
|
|
2027
|
-
*
|
|
2028
|
-
* A device with its own `fleet_id` must match. A device with none cannot take over on a
|
|
2029
|
-
* scoped relay, because there is no fleet to match it against — the same rule `targetsFor`
|
|
2030
|
-
* applies to dispatch, kept identical here so the two paths cannot disagree.
|
|
2031
|
-
*/
|
|
2032
|
-
if (requesterFleet !== null) {
|
|
2033
|
-
const deviceFleet = typeof device.fleet_id === 'string' && device.fleet_id !== '' ? device.fleet_id : null;
|
|
2034
|
-
if (deviceFleet !== requesterFleet) {
|
|
2035
|
-
this.log?.('debug', 'takeover candidate skipped: not in this fleet', {
|
|
2036
|
-
task_id: task.task_id,
|
|
2037
|
-
machine_id: device.machine_id,
|
|
2038
|
-
device_fleet: deviceFleet,
|
|
2039
|
-
relay_fleet: requesterFleet,
|
|
2040
|
-
});
|
|
2041
|
-
continue;
|
|
2042
|
-
}
|
|
2043
|
-
}
|
|
2044
|
-
const lease = task.leases.get(device.machine_id);
|
|
2045
|
-
if (lease && !['queued', 'offered'].includes(lease.state)) continue;
|
|
2046
|
-
if (!lease && gateMachine(task, device)) continue;
|
|
2047
|
-
candidates.push({ device, lease: lease ?? null });
|
|
2048
|
-
}
|
|
2049
|
-
|
|
2050
|
-
for (const { device, lease } of candidates) {
|
|
2051
|
-
const target = lease ?? {
|
|
2052
|
-
machine_id: device.machine_id,
|
|
2053
|
-
machine_name: device.machine_name,
|
|
2054
|
-
index: expiredLease.index,
|
|
2055
|
-
attempt: task.attempt,
|
|
2056
|
-
state: 'queued',
|
|
2057
|
-
refusal_reason: null,
|
|
2058
|
-
refusal_detail: null,
|
|
2059
|
-
dedupe_key: null,
|
|
2060
|
-
last_heartbeat_ms: nowMs,
|
|
2061
|
-
claimed_at_ms: nowMs,
|
|
2062
|
-
lease_until_ms: nowMs + this.leaseWindowMs(),
|
|
2063
|
-
phase: null,
|
|
2064
|
-
progress: null,
|
|
2065
|
-
offered_seq: null,
|
|
2066
|
-
expired_at_ms: null,
|
|
2067
|
-
finished_at_ms: null,
|
|
2068
|
-
heartbeat_count: 0,
|
|
2069
|
-
};
|
|
2070
|
-
target.index = expiredLease.index;
|
|
2071
|
-
target.attempt = task.attempt;
|
|
2072
|
-
target.dedupe_key = computeDedupeKey(task.task_id, target.index, task.command_hash, task.base_tree);
|
|
2073
|
-
task.leases.set(device.machine_id, target);
|
|
2074
|
-
this.emitOffer(task, target);
|
|
2075
|
-
this._ledgerLease(task, target);
|
|
2076
|
-
}
|
|
2077
|
-
|
|
2078
|
-
this.emit('notice', {
|
|
2079
|
-
level: 'info',
|
|
2080
|
-
code: candidates.length > 0 ? 'LEASE_TAKEOVER' : 'LEASE_TAKEOVER_UNAVAILABLE',
|
|
2081
|
-
message: candidates.length > 0
|
|
2082
|
-
? `task ${task.task_id} index ${expiredLease.index} re-offered at attempt ${task.attempt} to: `
|
|
2083
|
-
+ candidates.map((c) => c.device.machine_id).join(',')
|
|
2084
|
-
: `task ${task.task_id} index ${expiredLease.index} has no idle machine to take over`,
|
|
2085
|
-
task_id: task.task_id,
|
|
2086
|
-
});
|
|
2087
|
-
return candidates.map((c) => c.device.machine_id);
|
|
2088
|
-
}
|
|
2089
|
-
|
|
2090
|
-
/* ---------------- results (§5) ---------------- */
|
|
2091
|
-
|
|
2092
|
-
static instanceKey(machineId, dedupeKey, attempt) {
|
|
2093
|
-
return `${machineId}${INSTANCE_SEP}${dedupeKey}${INSTANCE_SEP}${attempt}`;
|
|
2094
|
-
}
|
|
2095
|
-
|
|
2096
|
-
/**
|
|
2097
|
-
* §5 result envelope validation.
|
|
2098
|
-
* Missing §5.1 required fields do NOT reject the frame — they make the result
|
|
2099
|
-
* `unverifiable` (§5.1 "缺一即 unverifiable").
|
|
2100
|
-
*/
|
|
2101
|
-
static validateEnvelope(envelope) {
|
|
2102
|
-
const missing = [];
|
|
2103
|
-
if (!envelope || typeof envelope !== 'object' || Array.isArray(envelope)) {
|
|
2104
|
-
return { missing: [...REQUIRED_ENVELOPE_FIELDS], malformed: true };
|
|
2105
|
-
}
|
|
2106
|
-
for (const field of REQUIRED_ENVELOPE_FIELDS) {
|
|
2107
|
-
if (!(field in envelope) || envelope[field] === undefined) missing.push(field);
|
|
2108
|
-
}
|
|
2109
|
-
const problems = [];
|
|
2110
|
-
if ('status' in envelope && !RESULT_STATUSES.includes(envelope.status)) {
|
|
2111
|
-
problems.push(`status must be one of ${RESULT_STATUSES.join('|')}`);
|
|
2112
|
-
}
|
|
2113
|
-
if (envelope.status === 'refused' && !envelope.refusal_reason) {
|
|
2114
|
-
problems.push('refusal_reason is required when status=refused');
|
|
2115
|
-
}
|
|
2116
|
-
if ('warnings' in envelope && !Array.isArray(envelope.warnings)) problems.push('warnings must be string[]');
|
|
2117
|
-
return { missing, problems };
|
|
2118
|
-
}
|
|
2119
|
-
|
|
2120
|
-
/**
|
|
2121
|
-
* §1 idempotent POST /v1/result.
|
|
2122
|
-
*
|
|
2123
|
-
* NOTE (contract ambiguity, see report): §4.4 `dedupe_key` deliberately excludes
|
|
2124
|
-
* `machine_id`/`attempt`, but §6.3 aggregates results from *several* machines in
|
|
2125
|
-
* replicate mode — they all share one dedupe_key. Deduping purely on dedupe_key
|
|
2126
|
-
* would silently drop every machine's result but the first. We therefore key the
|
|
2127
|
-
* idempotency check on (machine_id, dedupe_key, attempt): an identical redelivery
|
|
2128
|
-
* is `deduped:true`, distinct machines/attempts are all recorded.
|
|
2129
|
-
*/
|
|
2130
|
-
submitResult(envelope = {}) {
|
|
2131
|
-
const { missing, problems, malformed } = RabbitState.validateEnvelope(envelope);
|
|
2132
|
-
if (malformed) throw new ProtocolError('BAD_REQUEST', 'result must be a JSON object');
|
|
2133
|
-
|
|
2134
|
-
/**
|
|
2135
|
-
* v0.4.0: `transport` / `p2p` are OPTIONAL envelope fields, and they are validated as values
|
|
2136
|
-
* rather than added to the §5.1 required set. A v0.3.9 agent sends neither and must keep
|
|
2137
|
-
* working unchanged; an agent that sends `transport: 'satellite'` is refused by name instead of
|
|
2138
|
-
* storing a value the report would then show as a path that does not exist.
|
|
2139
|
-
*/
|
|
2140
|
-
RabbitState.validateTransportFields(envelope);
|
|
2141
|
-
|
|
2142
|
-
const taskId = envelope.task_id;
|
|
2143
|
-
if (typeof taskId !== 'string' || taskId.length === 0) {
|
|
2144
|
-
throw new ProtocolError('BAD_REQUEST', 'task_id is required');
|
|
2145
|
-
}
|
|
2146
|
-
const task = this.tasks.get(taskId);
|
|
2147
|
-
if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
|
|
2148
|
-
|
|
2149
|
-
const machineId = typeof envelope.machine_id === 'string' ? envelope.machine_id : null;
|
|
2150
|
-
const dedupeKey = typeof envelope.dedupe_key === 'string' && envelope.dedupe_key
|
|
2151
|
-
? envelope.dedupe_key : null;
|
|
2152
|
-
const attempt = Number.isInteger(envelope.attempt) && envelope.attempt >= 1 ? envelope.attempt : 1;
|
|
2153
|
-
|
|
2154
|
-
if (dedupeKey === null) {
|
|
2155
|
-
// Cannot dedupe without a key; still record it, as unverifiable.
|
|
2156
|
-
missing.push('dedupe_key');
|
|
2157
|
-
}
|
|
2158
|
-
const instanceKey = RabbitState.instanceKey(machineId ?? '<unknown>', dedupeKey ?? '<none>', attempt);
|
|
2159
|
-
|
|
2160
|
-
// §4.4 sanity check against the lease this machine actually holds. Booked as a
|
|
2161
|
-
// problem instead of a hard rejection so a buggy client cannot lose its result.
|
|
2162
|
-
const leaseForMachine = machineId ? task.leases.get(machineId) : null;
|
|
2163
|
-
if (leaseForMachine?.dedupe_key && dedupeKey && leaseForMachine.dedupe_key !== dedupeKey) {
|
|
2164
|
-
problems.push('dedupe_key does not match the §4.4 formula for this lease');
|
|
2165
|
-
}
|
|
2166
|
-
|
|
2167
|
-
if (this.results.has(instanceKey)) {
|
|
2168
|
-
const prior = this.results.get(instanceKey);
|
|
2169
|
-
return { deduped: true, task_id: taskId, dedupe_key: dedupeKey, seq: prior.seq };
|
|
2170
|
-
}
|
|
2171
|
-
|
|
2172
|
-
const now = this.nowMs();
|
|
2173
|
-
let envelopeShaOk = null;
|
|
2174
|
-
if (typeof envelope.envelope_sha256 === 'string') {
|
|
2175
|
-
const { envelope_sha256: _drop, ...rest } = envelope;
|
|
2176
|
-
envelopeShaOk = sha256Hex(jcs(rest)) === envelope.envelope_sha256;
|
|
2177
|
-
}
|
|
2178
|
-
|
|
2179
|
-
const record = {
|
|
2180
|
-
envelope,
|
|
2181
|
-
machine_id: machineId,
|
|
2182
|
-
dedupe_key: dedupeKey,
|
|
2183
|
-
attempt,
|
|
2184
|
-
instance_key: instanceKey,
|
|
2185
|
-
received_at_ms: now,
|
|
2186
|
-
validation: {
|
|
2187
|
-
missing: [...new Set(missing)],
|
|
2188
|
-
problems,
|
|
2189
|
-
envelope_sha256_ok: envelopeShaOk,
|
|
2190
|
-
},
|
|
2191
|
-
};
|
|
2192
|
-
|
|
2193
|
-
const effectiveStatus = record.validation.missing.length > 0 ? 'unverifiable' : envelope.status;
|
|
2194
|
-
|
|
2195
|
-
const seq = this.nextSeq();
|
|
2196
|
-
record.seq = seq;
|
|
2197
|
-
record.effective_status = effectiveStatus;
|
|
2198
|
-
this.results.set(instanceKey, record);
|
|
2199
|
-
|
|
2200
|
-
if (machineId) {
|
|
2201
|
-
const lease = task.leases.get(machineId);
|
|
2202
|
-
if (lease) {
|
|
2203
|
-
lease.state = effectiveStatus === 'refused' ? 'refused' : 'done';
|
|
2204
|
-
if (effectiveStatus === 'refused' && !lease.refusal_reason) {
|
|
2205
|
-
lease.refusal_reason = envelope.refusal_reason ?? 'REFUSED';
|
|
2206
|
-
}
|
|
2207
|
-
lease.finished_at_ms = now;
|
|
2208
|
-
lease.attempt = attempt;
|
|
2209
|
-
}
|
|
2210
|
-
this.touchDevice(machineId);
|
|
2211
|
-
}
|
|
2212
|
-
task.result_seqs.push(seq);
|
|
2213
|
-
|
|
2214
|
-
this.emit('task.result', {
|
|
2215
|
-
task_id: taskId,
|
|
2216
|
-
dedupe_key: dedupeKey,
|
|
2217
|
-
attempt,
|
|
2218
|
-
index: envelope.index ?? null,
|
|
2219
|
-
status: effectiveStatus,
|
|
2220
|
-
machine_id: machineId,
|
|
2221
|
-
deduped: false,
|
|
2222
|
-
validation_errors: record.validation.missing,
|
|
2223
|
-
});
|
|
2224
|
-
|
|
2225
|
-
// §7: results go into the ledger so the dedupe index survives a restart.
|
|
2226
|
-
this._ledger({ type: 'result.stored', task_id: taskId, machine_id: machineId, result: record });
|
|
2227
|
-
if (machineId) {
|
|
2228
|
-
const changed = task.leases.get(machineId);
|
|
2229
|
-
if (changed) this._ledgerLease(task, changed);
|
|
2230
|
-
}
|
|
2231
|
-
|
|
2232
|
-
return { deduped: false, task_id: taskId, dedupe_key: dedupeKey, seq, _instanceKey: instanceKey };
|
|
2233
|
-
}
|
|
2234
|
-
|
|
2235
|
-
/** All recorded results for a task, keyed by machine_id (later attempts win). */
|
|
2236
|
-
taskResults(taskId) {
|
|
2237
|
-
const task = this.tasks.get(taskId);
|
|
2238
|
-
if (!task) return new Map();
|
|
2239
|
-
const out = new Map();
|
|
2240
|
-
for (const rec of this.results.values()) {
|
|
2241
|
-
if (rec.envelope.task_id !== taskId) continue;
|
|
2242
|
-
const prev = out.get(rec.machine_id);
|
|
2243
|
-
if (!prev || rec.received_at_ms >= prev.received_at_ms) out.set(rec.machine_id, rec);
|
|
2244
|
-
}
|
|
2245
|
-
return out;
|
|
2246
|
-
}
|
|
2247
|
-
|
|
2248
|
-
/** Global dedupe book-keeping: dedupe_key -> instances seen. */
|
|
2249
|
-
dedupeReport() {
|
|
2250
|
-
const out = {};
|
|
2251
|
-
for (const rec of this.results.values()) {
|
|
2252
|
-
if (!rec.dedupe_key) continue;
|
|
2253
|
-
out[rec.dedupe_key] = out[rec.dedupe_key] ?? [];
|
|
2254
|
-
out[rec.dedupe_key].push({
|
|
2255
|
-
machine_id: rec.machine_id,
|
|
2256
|
-
attempt: rec.attempt,
|
|
2257
|
-
received_at: rfc3339(rec.received_at_ms),
|
|
2258
|
-
});
|
|
2259
|
-
}
|
|
2260
|
-
return out;
|
|
2261
|
-
}
|
|
2262
|
-
|
|
2263
|
-
/* ---------------- serialization ---------------- */
|
|
2264
|
-
|
|
2265
|
-
stats() {
|
|
2266
|
-
return {
|
|
2267
|
-
protocol_version: PROTOCOL_VERSION,
|
|
2268
|
-
seq: this.seq,
|
|
2269
|
-
buffered_events: this.buffer.length,
|
|
2270
|
-
devices: this.devices.size,
|
|
2271
|
-
tasks: this.tasks.size,
|
|
2272
|
-
results: this.results.size,
|
|
2273
|
-
pairing_codes: [...this.pairingCodes.values()].filter((c) => !c.used && this.nowMs() < c.expires_at_ms).length,
|
|
2274
|
-
uptime_ms: this.nowMs() - this.startedAtMs,
|
|
2275
|
-
};
|
|
2276
|
-
}
|
|
2277
|
-
}
|
|
2278
|
-
|
|
2279
|
-
export default RabbitState;
|
|
21
|
+
import { planJoin } from '../chatops/fleet-join.mjs';
|
|
22
|
+
|
|
23
|
+
export const PROTOCOL_VERSION = 1;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* PROTOCOL §1 table + §7 code table, plus the two v0.1.2 additions:
|
|
27
|
+
* `OPERATOR_REQUIRED` (§5.2 → 401) and `RATE_LIMITED` (§6 → 429).
|
|
28
|
+
* A code missing from this map silently degrades to HTTP 500 via
|
|
29
|
+
* `ProtocolError`, which keeps the right error code but lies about the status.
|
|
30
|
+
*/
|
|
31
|
+
export const ERROR_STATUS = Object.freeze({
|
|
32
|
+
UNAUTHORIZED: 401,
|
|
33
|
+
BAD_REQUEST: 400,
|
|
34
|
+
NOT_FOUND: 404,
|
|
35
|
+
PAIRING_INVALID: 400,
|
|
36
|
+
PAIRING_EXPIRED: 400,
|
|
37
|
+
TASK_EXISTS: 409,
|
|
38
|
+
FRAME_TOO_LARGE: 413,
|
|
39
|
+
NO_ONLINE_DEVICE: 503,
|
|
40
|
+
INTERNAL: 500,
|
|
41
|
+
OPERATOR_REQUIRED: 401, // v0.1.2 §5.2
|
|
42
|
+
RATE_LIMITED: 429, // v0.1.2 §6
|
|
43
|
+
|
|
44
|
+
/* v0.3.0 §9 request signing. Every client-side signature failure is a 401.
|
|
45
|
+
* `SIGNING_NOT_CONFIGURED` is deliberately NOT 401: the relay was started with
|
|
46
|
+
* `--require-signature` but no secret, so the fault is the relay's own
|
|
47
|
+
* configuration. Reporting that as 401 would send every operator to debug the
|
|
48
|
+
* client -- the one side that is not broken. 500 rather than 503 because
|
|
49
|
+
* retrying never fixes a missing configuration, so nothing should be invited to
|
|
50
|
+
* retry. */
|
|
51
|
+
SIGNATURE_REQUIRED: 401,
|
|
52
|
+
SIGNATURE_INCOMPLETE: 401,
|
|
53
|
+
SIGNATURE_MISMATCH: 401,
|
|
54
|
+
SIGNATURE_BAD_TIMESTAMP: 401,
|
|
55
|
+
SIGNATURE_BAD_NONCE: 401,
|
|
56
|
+
SIGNATURE_EXPIRED: 401,
|
|
57
|
+
SIGNATURE_REPLAY: 401,
|
|
58
|
+
SIGNING_NOT_CONFIGURED: 500,
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
/** §5.1 required envelope fields — missing any one means `unverifiable`. */
|
|
62
|
+
export const REQUIRED_ENVELOPE_FIELDS = Object.freeze([
|
|
63
|
+
'envelope_version', 'task_id', 'attempt', 'dedupe_key', 'machine_id', 'machine_name',
|
|
64
|
+
'platform', 'caps', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
|
|
65
|
+
'pre_tree_fingerprint', 'post_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error',
|
|
66
|
+
'head_commit', 'dirty_before', 'command_argv', 'command_hash', 'shell_id',
|
|
67
|
+
'started_at', 'ended_at', 'duration_ms', 'exit_code', 'status', 'refusal_reason',
|
|
68
|
+
'stdout_sha256', 'stdout_bytes', 'stderr_sha256', 'stderr_bytes', 'warnings',
|
|
69
|
+
'envelope_sha256',
|
|
70
|
+
]);
|
|
71
|
+
|
|
72
|
+
/** §5.3 comparable fields — the ONLY fields that take part in consistency judgement. */
|
|
73
|
+
export const COMPARABLE_FIELDS = Object.freeze([
|
|
74
|
+
'task_id', 'index', 'index_total', 'mode', 'cwd_rel', 'base_commit', 'base_tree',
|
|
75
|
+
'pre_tree_fingerprint', 'fingerprint_algo', 'fingerprint_error', 'head_commit',
|
|
76
|
+
'command_hash', 'shell_id', 'exit_code', 'status', 'refusal_reason',
|
|
77
|
+
'stdout_sha256', 'stdout_bytes', 'stdout_normalized_sha256',
|
|
78
|
+
'stderr_sha256', 'artifacts', 'diff_numstat', 'tests', 'semantic_counts', 'warnings',
|
|
79
|
+
]);
|
|
80
|
+
|
|
81
|
+
/** §5.1 status enum. */
|
|
82
|
+
export const RESULT_STATUSES = Object.freeze([
|
|
83
|
+
'ok', 'nonzero_exit', 'timeout', 'crashed', 'refused', 'unverifiable',
|
|
84
|
+
]);
|
|
85
|
+
|
|
86
|
+
export const FAILURE_STATUSES = Object.freeze(['nonzero_exit', 'timeout', 'crashed']);
|
|
87
|
+
|
|
88
|
+
/** §4.3 phase enum. */
|
|
89
|
+
export const LEASE_PHASES = Object.freeze(['preparing', 'running', 'finalizing']);
|
|
90
|
+
|
|
91
|
+
/** Lease states that can still be expired / renewed. */
|
|
92
|
+
export const ACTIVE_LEASE_STATES = Object.freeze(['queued', 'offered', 'running']);
|
|
93
|
+
|
|
94
|
+
export const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000; // §4.3: agent heartbeats every 10s
|
|
95
|
+
export const DEFAULT_MISSED_HEARTBEATS = 2; // §4.3: 2 missed cycles = 20s
|
|
96
|
+
export const DEFAULT_LEASE_GRACE_MS = 30_000; // §4.3: + grace 30s
|
|
97
|
+
/**
|
|
98
|
+
* v0.4.0: the bounded hold on the relay's own offer when a dispatcher is pushing it directly.
|
|
99
|
+
*
|
|
100
|
+
* Sized against what it has to cover: one punch round trip (measured 15-370 ms on the reference
|
|
101
|
+
* networks), the frame's delivery, and the executor's first heartbeat POST — all on a host that may
|
|
102
|
+
* be busy. 1.2 s was the first guess and it was too tight: measured on a 1-vCPU shared host, two
|
|
103
|
+
* agent tests saw the relay's copy overtake the direct one, which for `p2pMode: direct` means a
|
|
104
|
+
* refusal of an offer that was about to arrive. 3 s covers a slow host with room to spare.
|
|
105
|
+
*
|
|
106
|
+
* The cost is honest and bounded: a punch that fails delays the relay's copy by up to this much,
|
|
107
|
+
* once per task. Nothing else waits on it.
|
|
108
|
+
*/
|
|
109
|
+
export const DEFAULT_P2P_OFFER_GRACE_MS = 3_000;
|
|
110
|
+
export const DEFAULT_PAIRING_TTL_MS = 24 * 60 * 60 * 1000; // §2.2: 24h
|
|
111
|
+
export const DEFAULT_EVENT_BUFFER_SIZE = 1000; // §requirement: keep last 1000 events
|
|
112
|
+
export const MAX_FRAME_BYTES = 64 * 1024; // 64 KiB single frame
|
|
113
|
+
export const SHELL_ID_DIRECT = 'direct-exec'; // §5.1: v0.0.1 only direct exec
|
|
114
|
+
/** v0.3.0: RTT sanity ceiling — anything above 24h is treated as a broken value. */
|
|
115
|
+
export const MAX_RTT_MS = 24 * 60 * 60 * 1000;
|
|
116
|
+
/** v0.3.0: an RTT measurement older than this is reported as stale.
|
|
117
|
+
* Must stay comfortably ABOVE the idle-heartbeat interval (60s): at 1x the value
|
|
118
|
+
* would flip stale just before every refresh and flap, which would make
|
|
119
|
+
* `rtt_stale === false` useless as a "trust this number" predicate. 3x gives two
|
|
120
|
+
* consecutive missed idle heartbeats of margin. See PROTOCOL-v0.3.0.md §7. */
|
|
121
|
+
export const DEFAULT_RTT_STALE_MS = 180_000;
|
|
122
|
+
/** v0.3.0: recommended cadence for an agent with no lease (reference for the agent side). */
|
|
123
|
+
export const DEFAULT_IDLE_HEARTBEAT_MS = 60_000;
|
|
124
|
+
export const ENVELOPE_VERSION = '1.0';
|
|
125
|
+
export const FINGERPRINT_ALGO = 'git-temp-index-tree/v1';
|
|
126
|
+
|
|
127
|
+
/* ------------------------------------------------------------------ */
|
|
128
|
+
/* Errors */
|
|
129
|
+
/* ------------------------------------------------------------------ */
|
|
130
|
+
|
|
131
|
+
export class ProtocolError extends Error {
|
|
132
|
+
constructor(code, message, detail = {}) {
|
|
133
|
+
super(message);
|
|
134
|
+
this.name = 'ProtocolError';
|
|
135
|
+
this.code = code;
|
|
136
|
+
this.detail = detail;
|
|
137
|
+
this.status = ERROR_STATUS[code] ?? 500;
|
|
138
|
+
}
|
|
139
|
+
toBody() {
|
|
140
|
+
return { error: { code: this.code, message: this.message, detail: this.detail } };
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/* ------------------------------------------------------------------ */
|
|
145
|
+
/* Small pure helpers */
|
|
146
|
+
/* ------------------------------------------------------------------ */
|
|
147
|
+
|
|
148
|
+
export function sha256Hex(input) {
|
|
149
|
+
return createHash('sha256').update(input).digest('hex');
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** RFC 8785-ish canonical JSON (JCS): sorted keys, no whitespace. */
|
|
153
|
+
export function jcs(value) {
|
|
154
|
+
if (value === null) return 'null';
|
|
155
|
+
const t = typeof value;
|
|
156
|
+
if (t === 'number') return Number.isFinite(value) ? JSON.stringify(value) : 'null';
|
|
157
|
+
if (t === 'boolean') return value ? 'true' : 'false';
|
|
158
|
+
if (t === 'string') return JSON.stringify(value);
|
|
159
|
+
if (t === 'undefined' || t === 'function' || t === 'symbol') return 'null';
|
|
160
|
+
if (Array.isArray(value)) return `[${value.map((v) => (v === undefined ? 'null' : jcs(v))).join(',')}]`;
|
|
161
|
+
if (t === 'object') {
|
|
162
|
+
const keys = Object.keys(value).filter((k) => value[k] !== undefined).sort();
|
|
163
|
+
return `{${keys.map((k) => `${JSON.stringify(k)}:${jcs(value[k])}`).join(',')}}`;
|
|
164
|
+
}
|
|
165
|
+
return JSON.stringify(String(value));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Upper bound on the stages a `pipeline` may contain.
|
|
170
|
+
*
|
|
171
|
+
* Each stage is another command run on someone else's machine, so an unbounded list is an unbounded
|
|
172
|
+
* amount of work created by one request. The limit is generous for the intended use (build, test,
|
|
173
|
+
* package) and exists so the failure is a clear refusal rather than a fleet-wide surprise.
|
|
174
|
+
*/
|
|
175
|
+
export const MAX_PIPELINE_STAGES = 16;
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* v0.4.0: upper bound on `POST /v1/task`'s `origin_machine_id`.
|
|
179
|
+
*
|
|
180
|
+
* A machine id is an identifier, not a payload: 128 characters is far beyond any id this system
|
|
181
|
+
* mints, and the ceiling is what stops the field from becoming an unbounded free-text channel into
|
|
182
|
+
* every offer frame the relay emits.
|
|
183
|
+
*/
|
|
184
|
+
export const MAX_ORIGIN_MACHINE_ID_LENGTH = 128;
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* v0.4.0: the P2P modes a task may be dispatched under (§1 of the frozen v0.4.0 contract).
|
|
188
|
+
*
|
|
189
|
+
* Mirrored here rather than imported from `src/agent/p2p-node.mjs`: the relay is a standalone
|
|
190
|
+
* process that must not depend on agent code, and a mode list is three strings. A value outside
|
|
191
|
+
* this list is a configuration error, never a silent fall back to the default.
|
|
192
|
+
*/
|
|
193
|
+
export const P2P_MODES = Object.freeze(['auto', 'direct', 'relay']);
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* v0.4.0: how a result travelled — the direct P2P channel or the relay.
|
|
197
|
+
*
|
|
198
|
+
* A **transport is not a result**: how an envelope arrived must never take part in the consistency
|
|
199
|
+
* judgement (§5 of the contract), or two machines that produced byte-identical output would be
|
|
200
|
+
* reported `divergent` merely because one of them punched a hole. Hence: not required, not
|
|
201
|
+
* comparable, and absent on every v0.3.9 envelope.
|
|
202
|
+
*/
|
|
203
|
+
export const TRANSPORT_VALUES = Object.freeze(['p2p', 'relay']);
|
|
204
|
+
|
|
205
|
+
/** §4.4 helper: command_hash = sha256(JCS(argv)+"|"+shell_id+"|"+cwd_rel). */
|
|
206
|
+
export function computeCommandHash(commandArgv, shellId = SHELL_ID_DIRECT, cwdRel = '.') {
|
|
207
|
+
return sha256Hex(`${jcs(commandArgv)}|${shellId}|${cwdRel}`);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* `command_hash` for a whole pipeline.
|
|
212
|
+
*
|
|
213
|
+
* The hash must bind **every** stage, not just the first. Bound to stage 0 alone, two pipelines that
|
|
214
|
+
* differ after the first stage would carry the same hash -- so an envelope produced by the wrong
|
|
215
|
+
* chain would still verify, and making "the machine ran what I asked" checkable is the entire point
|
|
216
|
+
* of this field. Each stage's cwd participates for the same reason.
|
|
217
|
+
*
|
|
218
|
+
* The `|pipeline` suffix means this can never collide with a single-command hash: the two hash over
|
|
219
|
+
* different shapes, and that difference is deliberate rather than incidental.
|
|
220
|
+
*/
|
|
221
|
+
export function computePipelineCommandHash(stages, shellId = SHELL_ID_DIRECT) {
|
|
222
|
+
const canonical = stages.map((s) => ({
|
|
223
|
+
command_argv: s.command_argv,
|
|
224
|
+
cwd_rel: s.cwd_rel ?? '.',
|
|
225
|
+
continue_on_failure: s.continue_on_failure === true,
|
|
226
|
+
}));
|
|
227
|
+
return sha256Hex(`${jcs(canonical)}|${shellId}|pipeline`);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** §4.4: dedupe_key = sha256(task_id + "|" + index + "|" + command_hash + "|" + base_tree). */
|
|
231
|
+
export function computeDedupeKey(taskId, index, commandHash, baseTree) {
|
|
232
|
+
return sha256Hex(`${taskId}|${index}|${commandHash}|${baseTree}`);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Epoch ms -> RFC3339 UTC, second precision (matches the PROTOCOL examples). */
|
|
236
|
+
export function rfc3339(ms) {
|
|
237
|
+
return new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
export function isRfc3339(value) {
|
|
241
|
+
return typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/.test(value);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export function deepEqual(a, b) {
|
|
245
|
+
return jcs(a === undefined ? null : a) === jcs(b === undefined ? null : b);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function normalizeNow(now) {
|
|
249
|
+
if (typeof now !== 'function') return () => Date.now();
|
|
250
|
+
return () => {
|
|
251
|
+
const v = now();
|
|
252
|
+
if (typeof v === 'number') return v;
|
|
253
|
+
if (v instanceof Date) return v.getTime();
|
|
254
|
+
return Date.parse(v);
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/* ------------------------------------------------------------------ */
|
|
259
|
+
/* ULID */
|
|
260
|
+
/* ------------------------------------------------------------------ */
|
|
261
|
+
|
|
262
|
+
const CROCKFORD = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
|
|
263
|
+
|
|
264
|
+
function encodeBig(value, len) {
|
|
265
|
+
let out = '';
|
|
266
|
+
let v = value;
|
|
267
|
+
for (let i = 0; i < len; i += 1) {
|
|
268
|
+
out = CROCKFORD[Number(v & 31n)] + out;
|
|
269
|
+
v >>= 5n;
|
|
270
|
+
}
|
|
271
|
+
return out;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
export function createIdFactory() {
|
|
275
|
+
let lastMs = -1;
|
|
276
|
+
let lastRand = 0n;
|
|
277
|
+
return function nextId(nowMs = Date.now()) {
|
|
278
|
+
const bytes = randomBytes(10);
|
|
279
|
+
let rand = 0n;
|
|
280
|
+
for (const b of bytes) rand = (rand << 8n) | BigInt(b);
|
|
281
|
+
if (nowMs === lastMs) rand = (lastRand + 1n) & ((1n << 80n) - 1n);
|
|
282
|
+
lastMs = nowMs;
|
|
283
|
+
lastRand = rand;
|
|
284
|
+
return encodeBig(BigInt(nowMs), 10) + encodeBig(rand, 16);
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
export function ulid(nowMs = Date.now()) {
|
|
289
|
+
return createIdFactory()(nowMs);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/* ------------------------------------------------------------------ */
|
|
293
|
+
/* Version / capability gate (§6.1) */
|
|
294
|
+
/* ------------------------------------------------------------------ */
|
|
295
|
+
|
|
296
|
+
function parseVersion(text) {
|
|
297
|
+
if (typeof text !== 'string') return null;
|
|
298
|
+
const m = text.trim().match(/^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:\.(\d+))?/);
|
|
299
|
+
if (!m) return null;
|
|
300
|
+
return [Number(m[1]), Number(m[2] ?? 0), Number(m[3] ?? 0), Number(m[4] ?? 0)];
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function cmpVersion(a, b) {
|
|
304
|
+
for (let i = 0; i < 4; i += 1) {
|
|
305
|
+
if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1;
|
|
306
|
+
}
|
|
307
|
+
return 0;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Minimal semver-range check used by the §6.1 capability gate.
|
|
312
|
+
* Supports bare versions and comma/space separated `>= > <= < = ==` conjunctions.
|
|
313
|
+
* Unparseable ranges are treated as NOT satisfied (never silently pass).
|
|
314
|
+
*/
|
|
315
|
+
export function satisfiesRange(version, range) {
|
|
316
|
+
const v = parseVersion(version);
|
|
317
|
+
if (v === null) return false;
|
|
318
|
+
if (typeof range !== 'string') return false;
|
|
319
|
+
const parts = range.split(/[\s,]+/).filter(Boolean);
|
|
320
|
+
if (parts.length === 0) return false;
|
|
321
|
+
for (const part of parts) {
|
|
322
|
+
const m = part.match(/^(>=|<=|==|=|>|<)?\s*(.+)$/);
|
|
323
|
+
if (!m) return false;
|
|
324
|
+
const op = m[1] ?? '=';
|
|
325
|
+
const target = parseVersion(m[2]);
|
|
326
|
+
if (target === null) return false;
|
|
327
|
+
const c = cmpVersion(v, target);
|
|
328
|
+
if (op === '>=' && c < 0) return false;
|
|
329
|
+
if (op === '<=' && c > 0) return false;
|
|
330
|
+
if (op === '>' && c <= 0) return false;
|
|
331
|
+
if (op === '<' && c >= 0) return false;
|
|
332
|
+
if ((op === '=' || op === '==') && c !== 0) return false;
|
|
333
|
+
}
|
|
334
|
+
return true;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
function machineWritable(caps) {
|
|
338
|
+
if (!caps || typeof caps !== 'object') return false;
|
|
339
|
+
if (caps.write === false || caps.writable === false || caps.write_allowed === false) return false;
|
|
340
|
+
if (caps.read_only === true || caps.readonly === true || caps.ro_mount === true) return false;
|
|
341
|
+
return true;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* §6.1 capability gate, evaluated per machine BEFORE dispatch.
|
|
346
|
+
* @returns {null | {status:'refused', refusal_reason:string, detail:object}}
|
|
347
|
+
*/
|
|
348
|
+
export function gateMachine(task, machine) {
|
|
349
|
+
const caps = machine?.caps ?? {};
|
|
350
|
+
const req = task.requirements ?? {};
|
|
351
|
+
|
|
352
|
+
if (req.platform && Array.isArray(req.platform) && req.platform.length > 0) {
|
|
353
|
+
const os = String(machine?.platform?.os ?? '').toLowerCase();
|
|
354
|
+
if (!req.platform.map((p) => String(p).toLowerCase()).includes(os)) {
|
|
355
|
+
return { status: 'refused', refusal_reason: 'PLATFORM_MISMATCH', detail: { os, want: req.platform } };
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
const toolchain = req.toolchain ?? {};
|
|
360
|
+
for (const [tool, range] of Object.entries(toolchain)) {
|
|
361
|
+
const declared = caps[tool];
|
|
362
|
+
if (declared === null || declared === undefined || declared === '') {
|
|
363
|
+
return {
|
|
364
|
+
status: 'refused',
|
|
365
|
+
refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
|
|
366
|
+
detail: { tool, want: range, have: declared ?? null },
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
if (typeof range === 'string' && !satisfiesRange(declared, range)) {
|
|
370
|
+
return {
|
|
371
|
+
status: 'refused',
|
|
372
|
+
refusal_reason: `MISSING_${String(tool).toUpperCase()}`,
|
|
373
|
+
detail: { tool, want: range, have: declared },
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
if (task.write === true && !machineWritable(caps)) {
|
|
379
|
+
return { status: 'refused', refusal_reason: 'READ_ONLY_MACHINE', detail: { caps } };
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
return null;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/* ------------------------------------------------------------------ */
|
|
386
|
+
/* RabbitState */
|
|
387
|
+
/* ------------------------------------------------------------------ */
|
|
388
|
+
|
|
389
|
+
const INSTANCE_SEP = '\u0000';
|
|
390
|
+
|
|
391
|
+
export class RabbitState {
|
|
392
|
+
constructor(options = {}) {
|
|
393
|
+
this._now = normalizeNow(options.now);
|
|
394
|
+
this._idFactory = options.idFactory ?? createIdFactory();
|
|
395
|
+
this._randomToken = options.tokenFactory ?? (() => randomBytes(24).toString('hex'));
|
|
396
|
+
this._randomChars = options.randomChars
|
|
397
|
+
?? ((n) => { const a = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; const b = randomBytes(n); let s = ''; for (let i = 0; i < n; i += 1) s += a[b[i] % a.length]; return s; });
|
|
398
|
+
|
|
399
|
+
this.protocolVersion = PROTOCOL_VERSION;
|
|
400
|
+
this.eventBufferSize = options.eventBufferSize ?? DEFAULT_EVENT_BUFFER_SIZE;
|
|
401
|
+
/**
|
|
402
|
+
* v0.7.7: the fleet this relay serves, or null for a relay that serves every device.
|
|
403
|
+
*
|
|
404
|
+
* **Null is the migration-safe default and is what the shared relay runs today.** With it,
|
|
405
|
+
* task targeting behaves exactly as it did before this release, so deploying cannot be an
|
|
406
|
+
* outage. Set it (via `--fleet <id>` or `fleetId`) and the relay becomes scoped: a device
|
|
407
|
+
* with a `fleet_id` reaches only its own fleet, and a device with none reaches nothing.
|
|
408
|
+
*
|
|
409
|
+
* This is the relay's *own* fleet, not a device's. A device carries its own `fleet_id`, which
|
|
410
|
+
* the relay wrote at pairing time from the pairing code.
|
|
411
|
+
*/
|
|
412
|
+
this.fleetId = typeof options.fleetId === 'string' && options.fleetId !== '' ? options.fleetId : null;
|
|
413
|
+
this.pairingTtlMs = options.pairingTtlMs ?? DEFAULT_PAIRING_TTL_MS;
|
|
414
|
+
/** When true one code stays valid for its whole TTL instead of rotating. */
|
|
415
|
+
this.pairingCodeReusable = options.pairingCodeReusable === true;
|
|
416
|
+
this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
|
|
417
|
+
this.missedHeartbeats = options.missedHeartbeats ?? DEFAULT_MISSED_HEARTBEATS;
|
|
418
|
+
this.leaseGraceMs = options.leaseGraceMs ?? DEFAULT_LEASE_GRACE_MS;
|
|
419
|
+
/**
|
|
420
|
+
* v0.4.0: how long the relay holds its own offer back when the dispatcher says it will push the
|
|
421
|
+
* offer over the direct path (`origin_machine_id` + `p2p.mode`).
|
|
422
|
+
*
|
|
423
|
+
* Without this, the direct path could never win: the relay's offer goes out over an SSE stream
|
|
424
|
+
* that is already open, while a punch costs at least one round trip, so the relay copy would
|
|
425
|
+
* arrive first essentially always and `transport` would read `relay` even on a network where
|
|
426
|
+
* the punch succeeded. Measured on loopback, and it is exactly what the first version of this
|
|
427
|
+
* release did.
|
|
428
|
+
*
|
|
429
|
+
* The hold is bounded and self-healing: the executor renews its lease (a heartbeat) the moment
|
|
430
|
+
* it starts work on the direct copy, which moves the lease out of `queued`, and the deferred
|
|
431
|
+
* offer is then dropped. If the punch fails, nothing renews the lease and the relay offers
|
|
432
|
+
* normally once the grace expires -- so a failed punch costs this much latency and nothing else.
|
|
433
|
+
*/
|
|
434
|
+
this.p2pOfferGraceMs = Number.isFinite(options.p2pOfferGraceMs) && options.p2pOfferGraceMs >= 0
|
|
435
|
+
? options.p2pOfferGraceMs
|
|
436
|
+
: DEFAULT_P2P_OFFER_GRACE_MS;
|
|
437
|
+
/** Deferred offers, keyed `task_id:machine_id`, so they can be cleared on close(). */
|
|
438
|
+
this._pendingOffers = new Map();
|
|
439
|
+
/** v0.3.0: age beyond which a reported RTT is flagged stale. */
|
|
440
|
+
this.rttStaleMs = Number.isFinite(options.rttStaleMs) ? options.rttStaleMs : DEFAULT_RTT_STALE_MS;
|
|
441
|
+
|
|
442
|
+
this.startedAtMs = this._now();
|
|
443
|
+
/** v0.1.2 §8.3: random per-process id; a change tells clients the relay restarted. */
|
|
444
|
+
this.relayId = options.relayId ?? randomBytes(8).toString('hex');
|
|
445
|
+
/** v0.1.2 §7: optional injected Persistence instance (state.mjs never touches fs itself). */
|
|
446
|
+
this._persist = options.persistence ?? null;
|
|
447
|
+
/** True while replaying a ledger — suppresses event emission and re-persistence. */
|
|
448
|
+
this._restoring = false;
|
|
449
|
+
/** Ledger entries that could not be replayed. */
|
|
450
|
+
this.replayWarnings = [];
|
|
451
|
+
this.seq = 0;
|
|
452
|
+
/** @type {{seq:number,event:object,target:string|null}[]} ring buffer, last N events */
|
|
453
|
+
this.buffer = [];
|
|
454
|
+
this.devices = new Map(); // machine_id -> device
|
|
455
|
+
this.tokens = new Map(); // device_token -> machine_id
|
|
456
|
+
this.tasks = new Map(); // task_id -> task
|
|
457
|
+
this.results = new Map(); // instanceKey -> {envelope, machine_id, received_at_ms, ...}
|
|
458
|
+
this.pairingCodes = new Map(); // code -> {code, created_at_ms, expires_at_ms, used}
|
|
459
|
+
this._listeners = new Set();
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/* ---------------- clock / ids ---------------- */
|
|
463
|
+
|
|
464
|
+
nowMs() { return this._now(); }
|
|
465
|
+
nowIso() { return rfc3339(this.nowMs()); }
|
|
466
|
+
nextId() { return this._idFactory(this.nowMs()); }
|
|
467
|
+
newToken() { return this._randomToken(); }
|
|
468
|
+
|
|
469
|
+
/** §4.3 lease window = 2 missed heartbeats (20s) + grace (30s) = 50s. */
|
|
470
|
+
leaseWindowMs() { return this.heartbeatIntervalMs * this.missedHeartbeats + this.leaseGraceMs; }
|
|
471
|
+
|
|
472
|
+
/* ---------------- persistence hooks (v0.1.2 §7) ---------------- */
|
|
473
|
+
|
|
474
|
+
/** Inject/replace the persistence backend (also used by tests). */
|
|
475
|
+
setPersistence(persistence) { this._persist = persistence ?? null; return this._persist; }
|
|
476
|
+
|
|
477
|
+
/** Snapshot the device table; failures must never break the request path. */
|
|
478
|
+
_saveDevices() {
|
|
479
|
+
if (this._restoring || !this._persist) return false;
|
|
480
|
+
try {
|
|
481
|
+
return this._persist.saveDevices(this.devices);
|
|
482
|
+
} catch (err) {
|
|
483
|
+
this._persist.warn?.('devices.json', `snapshot failed: ${err.message}`);
|
|
484
|
+
return false;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/** Append one ledger entry; failures must never break the request path. */
|
|
489
|
+
_ledger(entry) {
|
|
490
|
+
if (this._restoring || !this._persist) return false;
|
|
491
|
+
try {
|
|
492
|
+
const appended = this._persist.append(entry);
|
|
493
|
+
// v0.6.2: a successful append is what clears a degraded relay, so the outcome has to be
|
|
494
|
+
// reported back rather than assumed -- otherwise the state could only ever be entered.
|
|
495
|
+
if (appended) this._persist.noteAppendSuccess?.();
|
|
496
|
+
return appended;
|
|
497
|
+
} catch (err) {
|
|
498
|
+
// Recorded as a state, not merely as a line: this is the one relay failure that keeps serving,
|
|
499
|
+
// and so the one that is invisible from every other angle (v0.6.2).
|
|
500
|
+
if (typeof this._persist.noteAppendFailure === 'function') this._persist.noteAppendFailure(err.message);
|
|
501
|
+
else this._persist.warn?.('ledger.jsonl', `append failed: ${err.message}`);
|
|
502
|
+
return false;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
_ledgerLease(task, lease) {
|
|
507
|
+
this._ledger({ type: 'lease.changed', task_id: task.task_id, machine_id: lease.machine_id, lease: { ...lease } });
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/**
|
|
511
|
+
* v0.1.2 §7 startup recovery: rebuild the device table, token index, tasks,
|
|
512
|
+
* leases and the result/dedupe index from a devices snapshot + ledger replay.
|
|
513
|
+
* Never emits events and never writes back what it just read.
|
|
514
|
+
*/
|
|
515
|
+
restore({ devices = {}, ledger = [] } = {}) {
|
|
516
|
+
this._restoring = true;
|
|
517
|
+
try {
|
|
518
|
+
for (const [machineId, d] of Object.entries(devices)) {
|
|
519
|
+
if (!d || typeof d !== 'object') continue;
|
|
520
|
+
const pairedAt = Date.parse(d.paired_at ?? '');
|
|
521
|
+
const device = {
|
|
522
|
+
machine_id: machineId,
|
|
523
|
+
machine_name: typeof d.machine_name === 'string' && d.machine_name ? d.machine_name : machineId,
|
|
524
|
+
platform: d.platform ?? {},
|
|
525
|
+
caps: d.caps ?? {},
|
|
526
|
+
user_id: d.user_id ?? null,
|
|
527
|
+
device_token: typeof d.device_token === 'string' ? d.device_token : null,
|
|
528
|
+
paired_at_ms: Number.isFinite(pairedAt) ? pairedAt : this.nowMs(),
|
|
529
|
+
re_paired: d.re_paired === true,
|
|
530
|
+
last_seen_at_ms: this.nowMs(),
|
|
531
|
+
// a snapshot proves the device exists, not that it is connected
|
|
532
|
+
online: false,
|
|
533
|
+
streams: 0,
|
|
534
|
+
// v0.3.0: RTT is volatile and is NEVER restored. Even if a snapshot on
|
|
535
|
+
// disk happens to carry `rtt_ms`, it is dropped here: after a restart the
|
|
536
|
+
// relay genuinely does not know the latency any more, and reporting a
|
|
537
|
+
// pre-restart number as current would be a lie that no reader can detect.
|
|
538
|
+
rtt_ms: null,
|
|
539
|
+
rtt_at_ms: null,
|
|
540
|
+
last_heartbeat_at_ms: null,
|
|
541
|
+
stream_connects: 0,
|
|
542
|
+
};
|
|
543
|
+
this.devices.set(machineId, device);
|
|
544
|
+
if (device.device_token) this.tokens.set(device.device_token, machineId);
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
let revivedTasks = 0;
|
|
548
|
+
for (const entry of ledger) {
|
|
549
|
+
try {
|
|
550
|
+
if (this._applyLedgerEntry(entry) === true) revivedTasks += 1;
|
|
551
|
+
} catch (err) {
|
|
552
|
+
this.replayWarnings.push({ type: entry?.type ?? '<unknown>', message: err.message });
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
return { revived_devices: this.devices.size, revived_tasks: revivedTasks };
|
|
556
|
+
} finally {
|
|
557
|
+
this._restoring = false;
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
/** @returns {boolean} true when a new task was created. */
|
|
562
|
+
_applyLedgerEntry(entry) {
|
|
563
|
+
if (!entry || typeof entry.type !== 'string') return false;
|
|
564
|
+
switch (entry.type) {
|
|
565
|
+
case 'task.created': {
|
|
566
|
+
const t = entry.task;
|
|
567
|
+
if (!t || typeof t.task_id !== 'string') return false;
|
|
568
|
+
const createdAt = Number.isFinite(t.created_at_ms) ? t.created_at_ms : Date.parse(entry.ts ?? '');
|
|
569
|
+
const task = {
|
|
570
|
+
task_id: t.task_id,
|
|
571
|
+
mode: t.mode ?? 'replicate',
|
|
572
|
+
command_argv: t.command_argv ?? [],
|
|
573
|
+
cwd_rel: t.cwd_rel ?? '.',
|
|
574
|
+
index_total: t.index_total ?? 1,
|
|
575
|
+
timeout_ms: t.timeout_ms ?? 300_000,
|
|
576
|
+
write: t.write === true,
|
|
577
|
+
write_scope: t.write_scope ?? [],
|
|
578
|
+
require_exclusive_write: t.require_exclusive_write === true,
|
|
579
|
+
base_commit: t.base_commit ?? null,
|
|
580
|
+
base_tree: t.base_tree ?? null,
|
|
581
|
+
requirements: t.requirements ?? {},
|
|
582
|
+
compare_policy: t.compare_policy ?? {},
|
|
583
|
+
halt: t.halt ?? 'never',
|
|
584
|
+
created_by: t.created_by ?? null,
|
|
585
|
+
created_at_ms: Number.isFinite(createdAt) ? createdAt : this.nowMs(),
|
|
586
|
+
command_hash: t.command_hash ?? null,
|
|
587
|
+
attempt: t.attempt ?? 1,
|
|
588
|
+
leases: new Map(),
|
|
589
|
+
cancelled: false,
|
|
590
|
+
cancel_reason: null,
|
|
591
|
+
degraded: t.degraded ?? null,
|
|
592
|
+
deadline_ms: t.deadline_ms ?? 0,
|
|
593
|
+
// v0.4.0 routing metadata. Read defensively: a ledger written by v0.3.9 has neither key,
|
|
594
|
+
// and a replayed task with `null` for both is exactly a v0.3.9 task.
|
|
595
|
+
origin_machine_id: t.origin_machine_id ?? null,
|
|
596
|
+
p2p: t.p2p ?? null,
|
|
597
|
+
result_seqs: [],
|
|
598
|
+
};
|
|
599
|
+
for (const lease of entry.leases ?? []) {
|
|
600
|
+
if (lease && typeof lease.machine_id === 'string') task.leases.set(lease.machine_id, { ...lease });
|
|
601
|
+
}
|
|
602
|
+
this.tasks.set(task.task_id, task);
|
|
603
|
+
return true;
|
|
604
|
+
}
|
|
605
|
+
case 'lease.changed': {
|
|
606
|
+
const task = this.tasks.get(entry.task_id);
|
|
607
|
+
const lease = task?.leases.get(entry.machine_id);
|
|
608
|
+
if (!lease) return false;
|
|
609
|
+
Object.assign(lease, entry.lease ?? {});
|
|
610
|
+
return false;
|
|
611
|
+
}
|
|
612
|
+
case 'result.stored': {
|
|
613
|
+
const record = entry.result;
|
|
614
|
+
if (!record || typeof record.instance_key !== 'string') return false;
|
|
615
|
+
this.results.set(record.instance_key, { ...record });
|
|
616
|
+
const task = this.tasks.get(record.envelope?.task_id);
|
|
617
|
+
if (task && Number.isInteger(record.seq)) task.result_seqs.push(record.seq);
|
|
618
|
+
return false;
|
|
619
|
+
}
|
|
620
|
+
case 'task.cancelled': {
|
|
621
|
+
const task = this.tasks.get(entry.task_id);
|
|
622
|
+
if (task) {
|
|
623
|
+
task.cancelled = true;
|
|
624
|
+
task.cancel_reason = entry.reason ?? null;
|
|
625
|
+
}
|
|
626
|
+
return false;
|
|
627
|
+
}
|
|
628
|
+
default:
|
|
629
|
+
return false; // unknown entry type: forward compatible, ignored
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/* ---------------- events (§3) ---------------- */
|
|
634
|
+
|
|
635
|
+
nextSeq() { this.seq += 1; return this.seq; }
|
|
636
|
+
|
|
637
|
+
/** Reserve a seq WITHOUT buffering (used for the per-connection `ready` frame). */
|
|
638
|
+
reserveSeq() { return this.nextSeq(); }
|
|
639
|
+
|
|
640
|
+
onEvent(listener) {
|
|
641
|
+
this._listeners.add(listener);
|
|
642
|
+
return () => this._listeners.delete(listener);
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
/**
|
|
646
|
+
* Emit an event: allocates the next global seq, buffers it (ring of N) and
|
|
647
|
+
* notifies subscribers. `target` is the machine_id the event is addressed to
|
|
648
|
+
* (null = broadcast). The target is NOT serialized except as `machine_id`.
|
|
649
|
+
*/
|
|
650
|
+
emit(type, fields = {}, opts = {}) {
|
|
651
|
+
const target = opts.target ?? null;
|
|
652
|
+
const seq = this.nextSeq();
|
|
653
|
+
const event = {
|
|
654
|
+
type,
|
|
655
|
+
seq,
|
|
656
|
+
rabbit_time: this.nowIso(),
|
|
657
|
+
...(target ? { machine_id: target } : {}),
|
|
658
|
+
...fields,
|
|
659
|
+
};
|
|
660
|
+
const entry = { seq, event, target };
|
|
661
|
+
this.buffer.push(entry);
|
|
662
|
+
if (this.buffer.length > this.eventBufferSize) {
|
|
663
|
+
this.buffer.splice(0, this.buffer.length - this.eventBufferSize);
|
|
664
|
+
}
|
|
665
|
+
for (const listener of this._listeners) {
|
|
666
|
+
try { listener(entry); } catch { /* subscriber errors must never break state */ }
|
|
667
|
+
}
|
|
668
|
+
return event;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
oldestBufferedSeq() { return this.buffer.length ? this.buffer[0].seq : this.seq + 1; }
|
|
672
|
+
/**
|
|
673
|
+
* Highest seq present in the replay ring (0 when empty).
|
|
674
|
+
*
|
|
675
|
+
* Deliberately different from `lastSeq()`: `ready` frames allocate seq numbers
|
|
676
|
+
* without being buffered, so `lastSeq()` can run ahead of anything a client can
|
|
677
|
+
* actually replay. Diagnostics that describe the REPLAY WINDOW must use this one.
|
|
678
|
+
*/
|
|
679
|
+
lastBufferedSeq() { return this.buffer.length ? this.buffer[this.buffer.length - 1].seq : 0; }
|
|
680
|
+
lastSeq() { return this.seq; }
|
|
681
|
+
|
|
682
|
+
/** Replay entries with seq >= from (used by GET /v1/stream?seq=|Last-Event-ID). */
|
|
683
|
+
replayEntries(from) {
|
|
684
|
+
return this.buffer.filter((e) => e.seq >= from);
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
eventsSince(from) { return this.replayEntries(from).map((e) => e.event); }
|
|
688
|
+
|
|
689
|
+
/* ---------------- pairing (§2) ---------------- */
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* Mint a pairing code.
|
|
693
|
+
*
|
|
694
|
+
* **v0.7.7: the code carries the relay's fleet.** This is the link that makes the scheme work,
|
|
695
|
+
* and it is easy to leave out: `pair()` reads the fleet *from the code*, so a code with no
|
|
696
|
+
* fleet pairs a device into no fleet -- and a device in no fleet reaches nothing on a scoped
|
|
697
|
+
* relay. `--fleet X` would then have had no effect on anything, silently.
|
|
698
|
+
*
|
|
699
|
+
* Defaulting to `this.fleetId` rather than requiring the caller to pass it means there is no
|
|
700
|
+
* path that mints a fleetless code on a scoped relay. An explicit `fleetId` overrides, which
|
|
701
|
+
* is what a relay serving several fleets would need.
|
|
702
|
+
*/
|
|
703
|
+
createPairingCode({ ttlMs = this.pairingTtlMs, fleetId = this.fleetId } = {}) {
|
|
704
|
+
let code;
|
|
705
|
+
do { code = `PAIR-${this._randomChars(8)}`; } while (this.pairingCodes.has(code));
|
|
706
|
+
const now = this.nowMs();
|
|
707
|
+
this.pairingCodes.set(code, {
|
|
708
|
+
code,
|
|
709
|
+
created_at_ms: now,
|
|
710
|
+
expires_at_ms: now + ttlMs,
|
|
711
|
+
used: false,
|
|
712
|
+
// Present as an explicit key even when null, so a code dump shows whether a fleet was
|
|
713
|
+
// attached rather than leaving it to be inferred from an absent field.
|
|
714
|
+
fleet_id: typeof fleetId === 'string' && fleetId !== '' ? fleetId : null,
|
|
715
|
+
});
|
|
716
|
+
return code;
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
/** Revoke every outstanding code and mint a new one. */
|
|
720
|
+
rotatePairingCode(opts) {
|
|
721
|
+
for (const rec of this.pairingCodes.values()) rec.used = true;
|
|
722
|
+
return this.createPairingCode(opts);
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
/**
|
|
726
|
+
* Register an externally supplied code (`--pairing-code <c>`).
|
|
727
|
+
*
|
|
728
|
+
* v0.1.2 BUG-1: assigning the code to `relay.pairingCode` without registering it
|
|
729
|
+
* advertised a code that `/v1/pair` then rejected with PAIRING_INVALID, so every
|
|
730
|
+
* documented scripted-onboarding flow failed. "Which code do we use" and "is the
|
|
731
|
+
* code registered" are two different questions; this answers the second.
|
|
732
|
+
*/
|
|
733
|
+
registerPairingCode(code, { ttlMs = this.pairingTtlMs } = {}) {
|
|
734
|
+
if (typeof code !== 'string' || code.trim() === '') {
|
|
735
|
+
throw new ProtocolError('BAD_REQUEST', 'pairing code must be a non-empty string', { pairing_code: code });
|
|
736
|
+
}
|
|
737
|
+
const value = code.trim();
|
|
738
|
+
const now = this.nowMs();
|
|
739
|
+
this.pairingCodes.set(value, { code: value, created_at_ms: now, expires_at_ms: now + ttlMs, used: false });
|
|
740
|
+
return value;
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
/** §2.2 step 3: validate a one-time code. */
|
|
744
|
+
validatePairingCode(code) {
|
|
745
|
+
if (typeof code !== 'string' || code.length === 0) {
|
|
746
|
+
throw new ProtocolError('PAIRING_INVALID', 'pairing_code is required');
|
|
747
|
+
}
|
|
748
|
+
const rec = this.pairingCodes.get(code);
|
|
749
|
+
if (!rec) throw new ProtocolError('PAIRING_INVALID', 'unknown pairing code', { pairing_code: code });
|
|
750
|
+
if (rec.used) throw new ProtocolError('PAIRING_INVALID', 'pairing code already used', { pairing_code: code });
|
|
751
|
+
if (this.nowMs() >= rec.expires_at_ms) {
|
|
752
|
+
throw new ProtocolError('PAIRING_EXPIRED', 'pairing code expired', { pairing_code: code, expired_at: rfc3339(rec.expires_at_ms) });
|
|
753
|
+
}
|
|
754
|
+
return rec;
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
/** §2.2: exchange a one-time code for a device_token. §1: idempotent. */
|
|
758
|
+
pair(request = {}) {
|
|
759
|
+
/**
|
|
760
|
+
* v0.8.1: a pairing code **or** an account-derived fleet key.
|
|
761
|
+
*
|
|
762
|
+
* The code path is unchanged. The key path is what lets a second machine on one account join
|
|
763
|
+
* with no human step and no relay configuration — the gap that left a new PC invisible beside
|
|
764
|
+
* its own fleet.
|
|
765
|
+
*
|
|
766
|
+
* ## Why a key may stand in for a code
|
|
767
|
+
*
|
|
768
|
+
* They authorise different things. A **code** is the operator saying *"let this machine in"*,
|
|
769
|
+
* and it is the only way to place a device in a fleet the device could not compute for itself.
|
|
770
|
+
* A **key** is the account saying *"this machine is mine"*, and it can only ever place the
|
|
771
|
+
* device in the fleet derived from that key — so it cannot be used to reach a fleet its holder
|
|
772
|
+
* is not already entitled to.
|
|
773
|
+
*
|
|
774
|
+
* ## Why a malformed key refuses rather than falling back
|
|
775
|
+
*
|
|
776
|
+
* A request with a `fleet_key` is a request that *meant* to join by account. Quietly demanding a
|
|
777
|
+
* code instead would surface at the client as an unexplained `PAIRING_INVALID`, which is the
|
|
778
|
+
* silent-wrong-answer shape this codebase keeps producing; the reason is named instead.
|
|
779
|
+
*/
|
|
780
|
+
const hasKey = typeof request.fleet_key === 'string' && request.fleet_key !== '';
|
|
781
|
+
let rec = null;
|
|
782
|
+
if (hasKey) {
|
|
783
|
+
const join = planJoin({ fleetKeyEncoded: request.fleet_key });
|
|
784
|
+
if (!join.ok) {
|
|
785
|
+
throw new ProtocolError('PAIRING_INVALID', join.text, { reason: join.reason });
|
|
786
|
+
}
|
|
787
|
+
// A synthetic code record so the fleet binding below reads from one place. `fleet_id` is the
|
|
788
|
+
// DERIVED value, never anything the request claimed.
|
|
789
|
+
rec = { code: null, fleet_id: join.fleetId, from_fleet_key: true };
|
|
790
|
+
} else {
|
|
791
|
+
rec = this.validatePairingCode(request.pairing_code);
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
const machineId = request.machine_id;
|
|
795
|
+
if (typeof machineId !== 'string' || machineId.length === 0) {
|
|
796
|
+
throw new ProtocolError('BAD_REQUEST', 'machine_id is required');
|
|
797
|
+
}
|
|
798
|
+
const now = this.nowMs();
|
|
799
|
+
const token = this.newToken();
|
|
800
|
+
const existing = this.devices.get(machineId);
|
|
801
|
+
if (existing?.device_token) this.tokens.delete(existing.device_token);
|
|
802
|
+
|
|
803
|
+
const device = {
|
|
804
|
+
machine_id: machineId,
|
|
805
|
+
machine_name: typeof request.machine_name === 'string' && request.machine_name
|
|
806
|
+
? request.machine_name : machineId.slice(0, 12),
|
|
807
|
+
platform: request.platform ?? {},
|
|
808
|
+
caps: request.caps ?? {},
|
|
809
|
+
user_id: request.user_id ?? null,
|
|
810
|
+
// v0.7.7: which fleet this device belongs to, and where that fact came from.
|
|
811
|
+
//
|
|
812
|
+
// **The fleet comes from the pairing code, not from the request.** A device that could
|
|
813
|
+
// name its own fleet would join any fleet, and every signature would faithfully attest
|
|
814
|
+
// to it. `deviceFleetFields` is the single place that binding is made, and it records
|
|
815
|
+
// its provenance so a fleet that arrived from a request is visible as such in
|
|
816
|
+
// `devices.json` rather than looking identical to one that came from a code.
|
|
817
|
+
//
|
|
818
|
+
// v0.8.1 adds the third source: an account-derived `fleet_key`. That is what lets a second
|
|
819
|
+
// machine on one account join with **no human step and no relay configuration** -- the gap
|
|
820
|
+
// that left a new PC invisible beside its own fleet. The relay does not trust a claimed
|
|
821
|
+
// fleet id; it *derives* one from the key, so a device can only land in the fleet whose key
|
|
822
|
+
// it holds and anything else forms a separate, empty fleet.
|
|
823
|
+
...deviceFleetFields({
|
|
824
|
+
codeRecord: rec,
|
|
825
|
+
fleetKeyEncoded: typeof request.fleet_key === 'string' ? request.fleet_key : null,
|
|
826
|
+
requestedFleet: null,
|
|
827
|
+
}),
|
|
828
|
+
/**
|
|
829
|
+
* v0.7.10: the device's Ed25519 public key, if it offered one.
|
|
830
|
+
*
|
|
831
|
+
* A public key is not a secret, so storing it costs nothing and it is the whole basis of
|
|
832
|
+
* per-device verification: the relay can then check a signature only *this* machine could
|
|
833
|
+
* have made, rather than a shared secret every machine holds.
|
|
834
|
+
*
|
|
835
|
+
* Not validated here, deliberately. A malformed key fails at verification with a named
|
|
836
|
+
* reason, while rejecting a pairing over it would lock out every existing device the
|
|
837
|
+
* moment this field was added -- and devices that send no key keep working, with
|
|
838
|
+
* `public_key: null` meaning "signs with the shared secret or not at all".
|
|
839
|
+
*/
|
|
840
|
+
public_key: typeof request.public_key === 'string' && request.public_key !== '' ? request.public_key : null,
|
|
841
|
+
device_token: token,
|
|
842
|
+
paired_at_ms: now,
|
|
843
|
+
re_paired: Boolean(existing),
|
|
844
|
+
last_seen_at_ms: now,
|
|
845
|
+
online: true,
|
|
846
|
+
streams: existing?.streams ?? 0,
|
|
847
|
+
// v0.3.0 RTT: starts unknown. `null` is "never measured"; 0 would mean "instant".
|
|
848
|
+
rtt_ms: null,
|
|
849
|
+
rtt_at_ms: null,
|
|
850
|
+
last_heartbeat_at_ms: null,
|
|
851
|
+
stream_connects: existing?.stream_connects ?? 0,
|
|
852
|
+
};
|
|
853
|
+
this.devices.set(machineId, device);
|
|
854
|
+
this.tokens.set(token, machineId);
|
|
855
|
+
rec.used = true;
|
|
856
|
+
|
|
857
|
+
// A group has more than one machine, and a single-use code would leave the
|
|
858
|
+
// operator unable to pair the second one without restarting the relay. So
|
|
859
|
+
// consuming a code immediately mints the next one and announces it: the
|
|
860
|
+
// operator pairs machine A, sees a fresh code, and pairs machine B with it.
|
|
861
|
+
// `pairingCodeReusable` keeps one code alive for the whole window instead,
|
|
862
|
+
// for scripted onboarding where nobody is watching stdout.
|
|
863
|
+
let nextCode = null;
|
|
864
|
+
const reusable = this.pairingCodeReusable === true;
|
|
865
|
+
if (reusable) {
|
|
866
|
+
rec.used = false;
|
|
867
|
+
} else {
|
|
868
|
+
nextCode = this.rotatePairingCode();
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
this.emit('peer.hello', {
|
|
872
|
+
machine_id: machineId,
|
|
873
|
+
machine_name: device.machine_name,
|
|
874
|
+
platform: device.platform,
|
|
875
|
+
});
|
|
876
|
+
|
|
877
|
+
if (nextCode) {
|
|
878
|
+
this.emit('notice', {
|
|
879
|
+
level: 'info',
|
|
880
|
+
code: 'PAIRING_CODE_ROTATED',
|
|
881
|
+
message: 'pairing code consumed; a new one is available',
|
|
882
|
+
pairing_code: nextCode,
|
|
883
|
+
});
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
// §7: a device change is snapshotted immediately.
|
|
887
|
+
this._saveDevices();
|
|
888
|
+
this._ledger({
|
|
889
|
+
type: 'device.paired',
|
|
890
|
+
machine_id: machineId,
|
|
891
|
+
machine_name: device.machine_name,
|
|
892
|
+
re_paired: device.re_paired,
|
|
893
|
+
});
|
|
894
|
+
|
|
895
|
+
return {
|
|
896
|
+
device_token: token,
|
|
897
|
+
protocol_version: PROTOCOL_VERSION,
|
|
898
|
+
rabbit_time: rfc3339(now),
|
|
899
|
+
...(nextCode ? { next_pairing_code: nextCode } : {}),
|
|
900
|
+
};
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
/** @returns {object|null} device for a bearer token. */
|
|
904
|
+
authenticate(token) {
|
|
905
|
+
if (typeof token !== 'string' || token.length === 0) return null;
|
|
906
|
+
const machineId = this.tokens.get(token);
|
|
907
|
+
if (!machineId) return null;
|
|
908
|
+
return this.devices.get(machineId) ?? null;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
touchDevice(machineId, { online = true } = {}) {
|
|
912
|
+
const device = this.devices.get(machineId);
|
|
913
|
+
if (!device) return null;
|
|
914
|
+
device.last_seen_at_ms = this.nowMs();
|
|
915
|
+
if (online) device.online = true;
|
|
916
|
+
return device;
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
setOnline(machineId, online, reason = null) {
|
|
920
|
+
const device = this.devices.get(machineId);
|
|
921
|
+
if (!device) return null;
|
|
922
|
+
if (device.online === online) return device;
|
|
923
|
+
device.online = online;
|
|
924
|
+
if (online) {
|
|
925
|
+
this.touchDevice(machineId);
|
|
926
|
+
this.emit('peer.hello', {
|
|
927
|
+
machine_id: machineId,
|
|
928
|
+
machine_name: device.machine_name,
|
|
929
|
+
platform: device.platform,
|
|
930
|
+
});
|
|
931
|
+
} else {
|
|
932
|
+
this.emit('peer.bye', { machine_id: machineId, reason: reason ?? 'stream_closed' });
|
|
933
|
+
}
|
|
934
|
+
return device;
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
/* ---------------- RTT (v0.3.0 §R) ---------------- */
|
|
938
|
+
|
|
939
|
+
/**
|
|
940
|
+
* Validate an inbound `rtt_ms`.
|
|
941
|
+
*
|
|
942
|
+
* The contract is deliberately liberal: a MISSING or INVALID value must never
|
|
943
|
+
* reject the heartbeat and must never clobber a previously known good value.
|
|
944
|
+
* RTT is a diagnostic; it must not be able to break lease renewal.
|
|
945
|
+
*
|
|
946
|
+
* Sanity ceiling: anything above `MAX_RTT_MS` (24h) is treated as invalid. A
|
|
947
|
+
* single garbage value would otherwise poison the fleet aggregate.
|
|
948
|
+
*/
|
|
949
|
+
static normalizeRtt(value) {
|
|
950
|
+
if (value === undefined) return { provided: false, valid: false };
|
|
951
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
952
|
+
return { provided: true, valid: false };
|
|
953
|
+
}
|
|
954
|
+
if (value > MAX_RTT_MS) return { provided: true, valid: false };
|
|
955
|
+
return { provided: true, valid: true, value };
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
/**
|
|
959
|
+
* RTT is a VOLATILE diagnostic: it is intentionally NOT written to
|
|
960
|
+
* `devices.json` and NOT replayed from the ledger, so a restarted relay reports
|
|
961
|
+
* `rtt_ms: null` until a machine heartbeats again. A persisted value would be
|
|
962
|
+
* indistinguishable from a fresh one and would let the relay claim knowledge it
|
|
963
|
+
* lost when the process died.
|
|
964
|
+
*/
|
|
965
|
+
rttStaleAfterMs() { return this.rttStaleMs; }
|
|
966
|
+
|
|
967
|
+
/**
|
|
968
|
+
* v0.3.0: absorb the optional `rtt_ms` carried by a heartbeat.
|
|
969
|
+
*
|
|
970
|
+
* Deliberately liberal, because RTT is a diagnostic and must never be able to
|
|
971
|
+
* break lease renewal:
|
|
972
|
+
* - the field is optional; a v0.2.3 agent never sends it
|
|
973
|
+
* - a MISSING or INVALID value is IGNORED, keeping the previous value, and the
|
|
974
|
+
* heartbeat still succeeds
|
|
975
|
+
* - unknown extra fields are ignored entirely (forward compatibility)
|
|
976
|
+
*
|
|
977
|
+
* `last_heartbeat_at` advances on EVERY accepted heartbeat, with or without a
|
|
978
|
+
* usable RTT: it answers "is the machine alive", while `rtt_at` answers "when
|
|
979
|
+
* was this number measured". Keeping them separate is what lets a reader tell a
|
|
980
|
+
* genuinely slow machine from one that is simply gone.
|
|
981
|
+
*/
|
|
982
|
+
recordHeartbeatMetrics(machineId, input = {}, nowMs = this.nowMs()) {
|
|
983
|
+
const device = this.devices.get(machineId);
|
|
984
|
+
if (!device) return null;
|
|
985
|
+
device.last_heartbeat_at_ms = nowMs;
|
|
986
|
+
const rtt = RabbitState.normalizeRtt(input.rtt_ms);
|
|
987
|
+
if (rtt.valid) {
|
|
988
|
+
device.rtt_ms = rtt.value;
|
|
989
|
+
device.rtt_at_ms = nowMs;
|
|
990
|
+
}
|
|
991
|
+
return device;
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
/** True when there is no value, or the value is older than the staleness window. */
|
|
995
|
+
isRttStale(device, nowMs = this.nowMs()) {
|
|
996
|
+
if (!device || device.rtt_ms === null || device.rtt_ms === undefined) return true;
|
|
997
|
+
if (device.rtt_at_ms === null || device.rtt_at_ms === undefined) return true;
|
|
998
|
+
return nowMs - device.rtt_at_ms > this.rttStaleMs;
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
/** How old the RTT measurement is, in ms; null when there is none. */
|
|
1002
|
+
rttAgeMs(device, nowMs = this.nowMs()) {
|
|
1003
|
+
if (!device || device.rtt_at_ms === null || device.rtt_at_ms === undefined) return null;
|
|
1004
|
+
return Math.max(0, nowMs - device.rtt_at_ms);
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
/** Record that a machine attached a stream (a reconnect is any attach after the first). */
|
|
1008
|
+
noteStreamConnect(machineId) {
|
|
1009
|
+
const device = this.devices.get(machineId);
|
|
1010
|
+
if (!device) return null;
|
|
1011
|
+
device.stream_connects = (device.stream_connects ?? 0) + 1;
|
|
1012
|
+
return device.stream_connects;
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
/**
|
|
1016
|
+
* v0.3.0 aggregate for `/healthz`.
|
|
1017
|
+
*
|
|
1018
|
+
* Only FRESH measurements count. A machine that has been gone for a day must not
|
|
1019
|
+
* drag min/max/avg around, and with no fresh data every numeric field is null --
|
|
1020
|
+
* never 0, which would read as "instantaneous".
|
|
1021
|
+
*/
|
|
1022
|
+
rttSummary(nowMs = this.nowMs()) {
|
|
1023
|
+
const fresh = [];
|
|
1024
|
+
let stale = 0;
|
|
1025
|
+
let unknown = 0;
|
|
1026
|
+
for (const device of this.devices.values()) {
|
|
1027
|
+
if (device.rtt_ms === null || device.rtt_ms === undefined) { unknown += 1; continue; }
|
|
1028
|
+
if (this.isRttStale(device, nowMs)) { stale += 1; continue; }
|
|
1029
|
+
fresh.push(device.rtt_ms);
|
|
1030
|
+
}
|
|
1031
|
+
if (fresh.length === 0) {
|
|
1032
|
+
return {
|
|
1033
|
+
machines_reporting: 0,
|
|
1034
|
+
min_ms: null,
|
|
1035
|
+
max_ms: null,
|
|
1036
|
+
avg_ms: null,
|
|
1037
|
+
machines_stale: stale,
|
|
1038
|
+
machines_unknown: unknown,
|
|
1039
|
+
};
|
|
1040
|
+
}
|
|
1041
|
+
const sum = fresh.reduce((a, b) => a + b, 0);
|
|
1042
|
+
return {
|
|
1043
|
+
machines_reporting: fresh.length,
|
|
1044
|
+
min_ms: Math.min(...fresh),
|
|
1045
|
+
max_ms: Math.max(...fresh),
|
|
1046
|
+
avg_ms: Math.round((sum / fresh.length) * 10) / 10,
|
|
1047
|
+
machines_stale: stale,
|
|
1048
|
+
machines_unknown: unknown,
|
|
1049
|
+
};
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
/**
|
|
1053
|
+
* v0.3.0 single-machine status view.
|
|
1054
|
+
*
|
|
1055
|
+
* `last_heartbeat_at` and `rtt_at` answer different questions, and the gap
|
|
1056
|
+
* between them is the whole point: a machine can be heartbeating (alive) while
|
|
1057
|
+
* its RTT number is old (a v0.2.3 agent that never sends `rtt_ms`), and a
|
|
1058
|
+
* machine can be gone while a plausible-looking RTT is still on file.
|
|
1059
|
+
*/
|
|
1060
|
+
deviceStatus(machineId, nowMs = this.nowMs()) {
|
|
1061
|
+
const device = this.devices.get(machineId);
|
|
1062
|
+
if (!device) return null;
|
|
1063
|
+
const rttAt = device.rtt_at_ms ?? null;
|
|
1064
|
+
const lastHeartbeat = device.last_heartbeat_at_ms ?? null;
|
|
1065
|
+
return {
|
|
1066
|
+
machine_id: device.machine_id,
|
|
1067
|
+
machine_name: device.machine_name,
|
|
1068
|
+
relay_id: this.relayId,
|
|
1069
|
+
connected: (device.streams ?? 0) > 0,
|
|
1070
|
+
streams: device.streams ?? 0,
|
|
1071
|
+
online: device.online === true,
|
|
1072
|
+
rtt_ms: device.rtt_ms ?? null,
|
|
1073
|
+
rtt_at: rttAt === null ? null : rfc3339(rttAt),
|
|
1074
|
+
rtt_age_ms: this.rttAgeMs(device, nowMs),
|
|
1075
|
+
rtt_stale: this.isRttStale(device, nowMs),
|
|
1076
|
+
last_heartbeat_at: lastHeartbeat === null ? null : rfc3339(lastHeartbeat),
|
|
1077
|
+
last_seen_at: device.last_seen_at_ms === null || device.last_seen_at_ms === undefined
|
|
1078
|
+
? null : rfc3339(device.last_seen_at_ms),
|
|
1079
|
+
reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
|
|
1080
|
+
stream_connects: device.stream_connects ?? 0,
|
|
1081
|
+
paired_at: device.paired_at_ms === null || device.paired_at_ms === undefined
|
|
1082
|
+
? null : rfc3339(device.paired_at_ms),
|
|
1083
|
+
platform: device.platform,
|
|
1084
|
+
caps: device.caps,
|
|
1085
|
+
user_id: device.user_id ?? null,
|
|
1086
|
+
now: rfc3339(nowMs),
|
|
1087
|
+
};
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
/** Public device view — never leaks device_token. */
|
|
1091
|
+
publicDevice(device) {
|
|
1092
|
+
return {
|
|
1093
|
+
machine_id: device.machine_id,
|
|
1094
|
+
machine_name: device.machine_name,
|
|
1095
|
+
platform: device.platform,
|
|
1096
|
+
caps: device.caps,
|
|
1097
|
+
user_id: device.user_id,
|
|
1098
|
+
/**
|
|
1099
|
+
* v0.7.7: the fleet this device belongs to, and where the fact came from.
|
|
1100
|
+
*
|
|
1101
|
+
* **Carried in the public shape on purpose, and it has to be**, because `devices.json`
|
|
1102
|
+
* is written from this shape -- a fleet that lived only on the in-memory record would
|
|
1103
|
+
* vanish on the next restart, which is precisely when an operator is looking at it. A
|
|
1104
|
+
* first version omitted these and the live test caught the fleet disappearing from
|
|
1105
|
+
* `listDevices()`.
|
|
1106
|
+
*
|
|
1107
|
+
* Safe to expose: `fleet_id` is a truncated hash of the account id, not the account id,
|
|
1108
|
+
* and devices on one relay can already enumerate each other. Narrowing `/v1/devices` to
|
|
1109
|
+
* a device's own fleet is a further step, recorded as such rather than implied.
|
|
1110
|
+
*/
|
|
1111
|
+
fleet_id: device.fleet_id ?? null,
|
|
1112
|
+
fleet_source: device.fleet_source ?? null,
|
|
1113
|
+
// v0.7.10: a boolean rather than the key itself. The key is not a secret, but a device
|
|
1114
|
+
// list is read by every device, and there is no reason to hand every machine every other
|
|
1115
|
+
// machine's key when the verification happens relay-side.
|
|
1116
|
+
public_key_present: typeof device.public_key === 'string' && device.public_key !== '',
|
|
1117
|
+
online: device.online,
|
|
1118
|
+
streams: device.streams,
|
|
1119
|
+
paired_at: rfc3339(device.paired_at_ms),
|
|
1120
|
+
last_seen_at: rfc3339(device.last_seen_at_ms),
|
|
1121
|
+
// v0.3.0: `null` means "never measured", which is NOT the same as 0ms.
|
|
1122
|
+
rtt_ms: device.rtt_ms ?? null,
|
|
1123
|
+
rtt_at: device.rtt_at_ms === null || device.rtt_at_ms === undefined ? null : rfc3339(device.rtt_at_ms),
|
|
1124
|
+
rtt_age_ms: this.rttAgeMs(device),
|
|
1125
|
+
rtt_stale: this.isRttStale(device),
|
|
1126
|
+
last_heartbeat_at: device.last_heartbeat_at_ms === null || device.last_heartbeat_at_ms === undefined
|
|
1127
|
+
? null : rfc3339(device.last_heartbeat_at_ms),
|
|
1128
|
+
reconnect_attempts: Math.max(0, (device.stream_connects ?? 0) - 1),
|
|
1129
|
+
};
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
listDevices() { return [...this.devices.values()].map((d) => this.publicDevice(d)); }
|
|
1133
|
+
|
|
1134
|
+
/* ---------------- tasks (§4) ---------------- */
|
|
1135
|
+
|
|
1136
|
+
/**
|
|
1137
|
+
* v0.4.0: validate the routing metadata a `POST /v1/task` body may carry.
|
|
1138
|
+
*
|
|
1139
|
+
* A machine, not a mode, is `origin_machine_id`: where the task was typed (`device.json`), so an
|
|
1140
|
+
* offer can carry that fact to the machine that has to run it. `p2p.mode` is the dispatcher's
|
|
1141
|
+
* resolved mode for this particular task — the plugin may be configured `direct` while a
|
|
1142
|
+
* scheduled run is `relay`.
|
|
1143
|
+
*
|
|
1144
|
+
* Validation is deliberately shallow and strictly typed, in the house style of this file: a
|
|
1145
|
+
* missing value is the v0.3.9 shape and stays valid, while a present-but-wrong value is refused
|
|
1146
|
+
* by name. Accepting a non-string here would put a number into `origin_machine_id` and only fail
|
|
1147
|
+
* later, on a machine that cannot tell a bad field from a bad relay.
|
|
1148
|
+
*
|
|
1149
|
+
* @returns {{origin_machine_id:string|null, p2p:{mode:string}|null}}
|
|
1150
|
+
*/
|
|
1151
|
+
static validateTaskRouting(input = {}) {
|
|
1152
|
+
let originMachineId = null;
|
|
1153
|
+
if (input.origin_machine_id !== undefined && input.origin_machine_id !== null) {
|
|
1154
|
+
if (typeof input.origin_machine_id !== 'string'
|
|
1155
|
+
|| input.origin_machine_id.length === 0
|
|
1156
|
+
|| input.origin_machine_id.length > MAX_ORIGIN_MACHINE_ID_LENGTH) {
|
|
1157
|
+
throw new ProtocolError(
|
|
1158
|
+
'BAD_REQUEST',
|
|
1159
|
+
`origin_machine_id must be a non-empty string of at most ${MAX_ORIGIN_MACHINE_ID_LENGTH} characters`,
|
|
1160
|
+
{ origin_machine_id: input.origin_machine_id },
|
|
1161
|
+
);
|
|
1162
|
+
}
|
|
1163
|
+
originMachineId = input.origin_machine_id;
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
let p2p = null;
|
|
1167
|
+
if (input.p2p !== undefined && input.p2p !== null) {
|
|
1168
|
+
if (typeof input.p2p !== 'object' || Array.isArray(input.p2p)) {
|
|
1169
|
+
throw new ProtocolError('BAD_REQUEST', 'p2p must be an object with a `mode` field', { p2p: input.p2p });
|
|
1170
|
+
}
|
|
1171
|
+
if (!P2P_MODES.includes(input.p2p.mode)) {
|
|
1172
|
+
throw new ProtocolError(
|
|
1173
|
+
'BAD_REQUEST',
|
|
1174
|
+
`p2p.mode must be one of ${P2P_MODES.join('|')}`,
|
|
1175
|
+
{ 'p2p.mode': input.p2p.mode ?? null },
|
|
1176
|
+
);
|
|
1177
|
+
}
|
|
1178
|
+
// Copied rather than referenced: the recorded task must not be mutable through the request
|
|
1179
|
+
// body the caller still holds.
|
|
1180
|
+
p2p = { ...input.p2p };
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
return { origin_machine_id: originMachineId, p2p };
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
/**
|
|
1187
|
+
* v0.4.0 §5.2: validate the optional transport fields of a result envelope.
|
|
1188
|
+
*
|
|
1189
|
+
* Same reasoning as `validateTaskRouting`: `transport` is an enum and `p2p` is an opaque object,
|
|
1190
|
+
* so anything else is refused by name at the door. Both fields stay OUT of
|
|
1191
|
+
* `REQUIRED_ENVELOPE_FIELDS` (a missing one is a v0.3.9 envelope, not an incomplete one) and out
|
|
1192
|
+
* of `COMPARABLE_FIELDS` (a path is not a result).
|
|
1193
|
+
*/
|
|
1194
|
+
static validateTransportFields(envelope = {}) {
|
|
1195
|
+
if (envelope.transport !== undefined && envelope.transport !== null) {
|
|
1196
|
+
if (!TRANSPORT_VALUES.includes(envelope.transport)) {
|
|
1197
|
+
throw new ProtocolError(
|
|
1198
|
+
'BAD_REQUEST',
|
|
1199
|
+
`transport must be one of ${TRANSPORT_VALUES.join('|')}`,
|
|
1200
|
+
{ transport: envelope.transport },
|
|
1201
|
+
);
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
if (envelope.p2p !== undefined && envelope.p2p !== null
|
|
1205
|
+
&& (typeof envelope.p2p !== 'object' || Array.isArray(envelope.p2p))) {
|
|
1206
|
+
throw new ProtocolError('BAD_REQUEST', 'p2p must be an object', { p2p: envelope.p2p });
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
/**
|
|
1211
|
+
* §4.1 POST /v1/task.
|
|
1212
|
+
* Machine selection is not specified by the PROTOCOL; we implement:
|
|
1213
|
+
* - `target_machines` (optional explicit list), else every paired device
|
|
1214
|
+
* - capability gate §6.1 per machine (failures become `refused` leases)
|
|
1215
|
+
* - index assignment: replicate -> 0, split -> round-robin modulo index_total
|
|
1216
|
+
*/
|
|
1217
|
+
createTask(input = {}) {
|
|
1218
|
+
// v0.4.0: refused BEFORE the task literal is built and before any offer is emitted, so a bad
|
|
1219
|
+
// value cannot leave a half-created task behind. `POST /v1/task` is all-or-nothing.
|
|
1220
|
+
const routing = RabbitState.validateTaskRouting(input);
|
|
1221
|
+
|
|
1222
|
+
const mode = input.mode ?? 'replicate';
|
|
1223
|
+
if (mode !== 'replicate' && mode !== 'split' && mode !== 'broadcast' && mode !== 'pipeline' && mode !== 'compose') {
|
|
1224
|
+
throw new ProtocolError(
|
|
1225
|
+
'BAD_REQUEST',
|
|
1226
|
+
'mode must be replicate|split|broadcast|pipeline|compose',
|
|
1227
|
+
{ mode },
|
|
1228
|
+
);
|
|
1229
|
+
}
|
|
1230
|
+
/**
|
|
1231
|
+
* v0.3.3 `pipeline`: the whole chain runs on each machine, stage after stage.
|
|
1232
|
+
*
|
|
1233
|
+
* The semantics were chosen against the alternative (fan out stage 1, converge its outputs into
|
|
1234
|
+
* stage 2) because that one has no well-defined input: on a fleet, stage 1 produces n different
|
|
1235
|
+
* outputs, and "feed them all to stage 2" does not say in what order, or whether stage 2 then
|
|
1236
|
+
* runs once or n times. Worse, two machines finishing in a different order would produce
|
|
1237
|
+
* different stage-2 inputs, so one task_id would no longer describe one reproducible execution --
|
|
1238
|
+
* which is the property the rest of this system is built on. See docs/PIPELINE-DESIGN.md.
|
|
1239
|
+
*
|
|
1240
|
+
* So: a pipeline is `replicate` applied to a sequence. Every machine runs the whole chain, there
|
|
1241
|
+
* is no cross-machine data flow, and nothing new can be silently dropped in transit.
|
|
1242
|
+
*/
|
|
1243
|
+
const rawStages = Array.isArray(input.stages) ? input.stages : null;
|
|
1244
|
+
const isPipeline = mode === 'pipeline';
|
|
1245
|
+
if (isPipeline) {
|
|
1246
|
+
if (rawStages === null || rawStages.length === 0) {
|
|
1247
|
+
throw new ProtocolError('BAD_REQUEST', 'mode=pipeline requires a non-empty `stages` array', {
|
|
1248
|
+
stages: rawStages === null ? null : rawStages.length,
|
|
1249
|
+
});
|
|
1250
|
+
}
|
|
1251
|
+
if (rawStages.length > MAX_PIPELINE_STAGES) {
|
|
1252
|
+
throw new ProtocolError('BAD_REQUEST', `stages must have at most ${MAX_PIPELINE_STAGES} entries`, {
|
|
1253
|
+
stages: rawStages.length,
|
|
1254
|
+
});
|
|
1255
|
+
}
|
|
1256
|
+
rawStages.forEach((stage, i) => {
|
|
1257
|
+
const argv = stage?.command_argv;
|
|
1258
|
+
if (!Array.isArray(argv) || argv.length === 0 || argv.some((a) => typeof a !== 'string')) {
|
|
1259
|
+
throw new ProtocolError('BAD_REQUEST', `stages[${i}].command_argv must be a non-empty string[]`, { index: i });
|
|
1260
|
+
}
|
|
1261
|
+
});
|
|
1262
|
+
// A chain is one execution on one machine; sharding it would mean each machine ran a chain of
|
|
1263
|
+
// its own slice, which is a different feature and not what the caller asked for.
|
|
1264
|
+
if (Number.isInteger(input.index_total) && input.index_total !== 1) {
|
|
1265
|
+
throw new ProtocolError('BAD_REQUEST', 'mode=pipeline runs the whole chain per machine, so index_total must be 1', {
|
|
1266
|
+
index_total: input.index_total,
|
|
1267
|
+
});
|
|
1268
|
+
}
|
|
1269
|
+
} else if (rawStages !== null) {
|
|
1270
|
+
// Refused rather than ignored: the caller believes they described a sequence, and running only
|
|
1271
|
+
// `command_argv` would silently drop every stage after the first.
|
|
1272
|
+
throw new ProtocolError('BAD_REQUEST', '`stages` is only valid with mode=pipeline', { mode, stages: rawStages.length });
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
/**
|
|
1276
|
+
* Normalized stage list. For every mode there is at least one stage, so downstream code has a
|
|
1277
|
+
* single shape to reason about and `pipeline` is not a special case at the execution layer.
|
|
1278
|
+
*/
|
|
1279
|
+
const stages = isPipeline
|
|
1280
|
+
? rawStages.map((s, i) => ({
|
|
1281
|
+
index: i,
|
|
1282
|
+
command_argv: [...s.command_argv],
|
|
1283
|
+
cwd_rel: typeof s.cwd_rel === 'string' && s.cwd_rel ? s.cwd_rel : (typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.'),
|
|
1284
|
+
timeout_ms: Number.isInteger(s.timeout_ms) ? s.timeout_ms : null,
|
|
1285
|
+
continue_on_failure: s.continue_on_failure === true,
|
|
1286
|
+
}))
|
|
1287
|
+
: null;
|
|
1288
|
+
|
|
1289
|
+
const commandArgv = isPipeline ? [...rawStages[0].command_argv] : input.command_argv;
|
|
1290
|
+
if (!Array.isArray(commandArgv) || commandArgv.length === 0 || commandArgv.some((a) => typeof a !== 'string')) {
|
|
1291
|
+
throw new ProtocolError('BAD_REQUEST', 'command_argv must be a non-empty string[]');
|
|
1292
|
+
}
|
|
1293
|
+
const indexTotal = Number.isInteger(input.index_total) ? input.index_total : 1;
|
|
1294
|
+
if (indexTotal < 1) throw new ProtocolError('BAD_REQUEST', 'index_total must be >= 1');
|
|
1295
|
+
|
|
1296
|
+
const halt = input.halt ?? 'never';
|
|
1297
|
+
if (halt !== 'never' && halt !== 'now') throw new ProtocolError('BAD_REQUEST', 'halt must be never|now', { halt });
|
|
1298
|
+
|
|
1299
|
+
const taskId = input.task_id ?? this.nextId();
|
|
1300
|
+
if (this.tasks.has(taskId)) {
|
|
1301
|
+
throw new ProtocolError('TASK_EXISTS', 'task_id already exists', { task_id: taskId });
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
const now = this.nowMs();
|
|
1305
|
+
const cwdRel = typeof input.cwd_rel === 'string' && input.cwd_rel ? input.cwd_rel : '.';
|
|
1306
|
+
const baseTree = input.base_tree ?? null;
|
|
1307
|
+
const commandHash = input.command_hash ?? (isPipeline
|
|
1308
|
+
? computePipelineCommandHash(stages, SHELL_ID_DIRECT)
|
|
1309
|
+
: computeCommandHash(commandArgv, SHELL_ID_DIRECT, cwdRel));
|
|
1310
|
+
const timeoutMs = Number.isInteger(input.timeout_ms) ? input.timeout_ms : 300_000;
|
|
1311
|
+
|
|
1312
|
+
// v0.3.3 `broadcast`: one machine runs the command, every other machine is told the outcome.
|
|
1313
|
+
//
|
|
1314
|
+
// The executor choice is deterministic rather than random. A random leader makes a task
|
|
1315
|
+
// impossible to reproduce from its own record -- the same task id would run somewhere else on a
|
|
1316
|
+
// retry -- and this project's whole value rests on being able to say exactly where something ran.
|
|
1317
|
+
// The first target in the relay's own device order is stable for a given device table.
|
|
1318
|
+
//
|
|
1319
|
+
// Observers get a lease so they are tracked and notified, but their lease is `observing`, which
|
|
1320
|
+
// every aggregation step treats as "did not run". That distinction is the point: a machine that
|
|
1321
|
+
// never executed anything agreeing with a result would be a fabrication, and it is exactly the
|
|
1322
|
+
// fabrication this mode could most easily produce.
|
|
1323
|
+
//
|
|
1324
|
+
// Resolved before the task literal because the literal records it. (`targets` is computed below,
|
|
1325
|
+
// so the executor is re-validated against it after that.)
|
|
1326
|
+
const targetIds = Array.isArray(input.target_machines) && input.target_machines.length > 0
|
|
1327
|
+
? input.target_machines
|
|
1328
|
+
: [...this.devices.keys()];
|
|
1329
|
+
const executorMachineId = mode === 'broadcast'
|
|
1330
|
+
? (typeof input.executor_machine_id === 'string' && input.executor_machine_id !== ''
|
|
1331
|
+
? input.executor_machine_id
|
|
1332
|
+
: targetIds[0] ?? null)
|
|
1333
|
+
: null;
|
|
1334
|
+
|
|
1335
|
+
const task = {
|
|
1336
|
+
task_id: taskId,
|
|
1337
|
+
mode,
|
|
1338
|
+
command_argv: commandArgv,
|
|
1339
|
+
cwd_rel: cwdRel,
|
|
1340
|
+
index_total: indexTotal,
|
|
1341
|
+
timeout_ms: timeoutMs,
|
|
1342
|
+
write: input.write === true,
|
|
1343
|
+
write_scope: input.write_scope ?? [],
|
|
1344
|
+
require_exclusive_write: input.require_exclusive_write === true,
|
|
1345
|
+
base_commit: input.base_commit ?? null,
|
|
1346
|
+
base_tree: baseTree,
|
|
1347
|
+
requirements: input.requirements ?? {},
|
|
1348
|
+
compare_policy: input.compare_policy ?? {},
|
|
1349
|
+
halt,
|
|
1350
|
+
created_by: input.created_by ?? null,
|
|
1351
|
+
created_at_ms: now,
|
|
1352
|
+
command_hash: commandHash,
|
|
1353
|
+
attempt: 1,
|
|
1354
|
+
leases: new Map(),
|
|
1355
|
+
cancelled: false,
|
|
1356
|
+
cancel_reason: null,
|
|
1357
|
+
degraded: null,
|
|
1358
|
+
deadline_ms: now + timeoutMs,
|
|
1359
|
+
result_seqs: [],
|
|
1360
|
+
/** v0.3.3: which machine actually executes under `broadcast`; null for the other modes. */
|
|
1361
|
+
executor_machine_id: executorMachineId,
|
|
1362
|
+
/**
|
|
1363
|
+
* v0.3.3: the normalized stage list for `pipeline`, or null.
|
|
1364
|
+
*
|
|
1365
|
+
* Sent to the machine in the offer so it can run the whole chain. Kept on the task as well so
|
|
1366
|
+
* a report can say how many stages were supposed to run even when the machine crashed before
|
|
1367
|
+
* reporting any of them.
|
|
1368
|
+
*/
|
|
1369
|
+
stages,
|
|
1370
|
+
/**
|
|
1371
|
+
* v0.4.0: where the task came from and under which P2P mode it was dispatched, or null.
|
|
1372
|
+
*
|
|
1373
|
+
* Kept on the task (not just passed to `emitOffer`) because an offer is emitted more than
|
|
1374
|
+
* once: `redeliverPendingOffers` re-states unclaimed work on every reconnect and
|
|
1375
|
+
* `_takeover` re-offers a freed index. A machine that reconnects must see the same routing
|
|
1376
|
+
* facts as the machine that was connected at dispatch time, or the direct-push half of the
|
|
1377
|
+
* feature would work exactly once and then quietly stop.
|
|
1378
|
+
*/
|
|
1379
|
+
origin_machine_id: routing.origin_machine_id,
|
|
1380
|
+
p2p: routing.p2p,
|
|
1381
|
+
};
|
|
1382
|
+
|
|
1383
|
+
/**
|
|
1384
|
+
* Fleet scoping (v0.7.7).
|
|
1385
|
+
*
|
|
1386
|
+
* This replaces `targets = [...this.devices.values()]`, which reached **every device on the
|
|
1387
|
+
* relay** with no filter — the finding that accounts were not isolated. The rule now lives in
|
|
1388
|
+
* `fleet-wiring.mjs` so it is one function with its own exhaustive tests rather than three
|
|
1389
|
+
* lines inside a two-thousand-line object.
|
|
1390
|
+
*
|
|
1391
|
+
* ## The migration is deliberate, and so is its limit
|
|
1392
|
+
*
|
|
1393
|
+
* The shared relay holds four devices that paired before fleets existed, and none of them has
|
|
1394
|
+
* a `fleet_id`. `targetsFor` therefore has two modes: a relay with no fleet configured keeps
|
|
1395
|
+
* exactly the old behaviour, and a relay that *is* scoped refuses a fleetless requester rather
|
|
1396
|
+
* than handing it the run of the fleet. So deploying this cannot be an outage, and scoping it
|
|
1397
|
+
* cannot be a silent grant.
|
|
1398
|
+
*
|
|
1399
|
+
* ## What this does NOT do, stated here rather than discovered later
|
|
1400
|
+
*
|
|
1401
|
+
* `POST /v1/task` authenticates with the **operator token**, and `authorizeTaskDispatch`
|
|
1402
|
+
* returns `{ role: 'operator', machine_id: null }` for it — there is no device, so there is no
|
|
1403
|
+
* fleet to scope by. Every device that authenticates with a **device token** gets full
|
|
1404
|
+
* isolation from this change; an operator-token dispatch remains relay-wide, because the
|
|
1405
|
+
* operator token *is* the relay-wide credential by design.
|
|
1406
|
+
*
|
|
1407
|
+
* `input.created_by` is the one device identity a task carries, and it is used when present:
|
|
1408
|
+
* a task submitted on behalf of a machine is scoped to that machine's fleet.
|
|
1409
|
+
*/
|
|
1410
|
+
const requester =
|
|
1411
|
+
(typeof input.created_by === 'string' && this.devices.get(input.created_by)) ||
|
|
1412
|
+
(typeof input.requester_machine_id === 'string' && this.devices.get(input.requester_machine_id)) ||
|
|
1413
|
+
null;
|
|
1414
|
+
|
|
1415
|
+
/**
|
|
1416
|
+
* An operator-token dispatch, on a relay that has been scoped, acts **within that scope**.
|
|
1417
|
+
*
|
|
1418
|
+
* A first version passed `requester: null` and let it be refused -- which broke every
|
|
1419
|
+
* operator dispatch on a scoped relay, and broke it in the least useful way: the operator had
|
|
1420
|
+
* just configured the fleet, and configuring it made them unable to dispatch at all.
|
|
1421
|
+
*
|
|
1422
|
+
* The reasoning for the fix: the relay's own `fleetId` *is* an admin's statement about which
|
|
1423
|
+
* fleet this relay serves. An operator dispatch therefore has an unambiguous scope, and
|
|
1424
|
+
* refusing it would be refusing the case that was explicitly asked for.
|
|
1425
|
+
*
|
|
1426
|
+
* What stays genuinely ambiguous is an operator dispatch on a relay holding **several** fleets
|
|
1427
|
+
* with no fleet of its own -- there is no way to tell which was meant, so `targetsFor` refuses
|
|
1428
|
+
* that rather than guessing.
|
|
1429
|
+
*/
|
|
1430
|
+
const scopedRequester =
|
|
1431
|
+
requester ??
|
|
1432
|
+
(this.fleetId === null ? null : { machine_id: null, fleet_id: this.fleetId, fleet_source: 'relay_scope' });
|
|
1433
|
+
|
|
1434
|
+
const scoped = targetsFor({
|
|
1435
|
+
devices: [...this.devices.values()],
|
|
1436
|
+
requester: scopedRequester,
|
|
1437
|
+
targetMachines: input.target_machines,
|
|
1438
|
+
relayFleetId: this.fleetId ?? null,
|
|
1439
|
+
// The existing code did not filter offline devices at selection time -- it offered to them and
|
|
1440
|
+
// let the lease expire. Keeping that here avoids changing two behaviours at once, and the
|
|
1441
|
+
// lease expiry is what `handoff.mjs` is for.
|
|
1442
|
+
requireOnline: false,
|
|
1443
|
+
});
|
|
1444
|
+
if (!scoped.ok) {
|
|
1445
|
+
/**
|
|
1446
|
+
* `NO_ONLINE_DEVICE`, not a new code.
|
|
1447
|
+
*
|
|
1448
|
+
* A first version invented `NO_ELIGIBLE_DEVICE`, which broke the one relay test that pins
|
|
1449
|
+
* this contract — and worse, it would have broken **callers**: `chatops/fleet.mjs` reads
|
|
1450
|
+
* `NO_ONLINE_DEVICE` to produce *"start an agent on each machine"*, and a code it has never
|
|
1451
|
+
* seen falls through to a generic refusal. The protocol's code table is part of the
|
|
1452
|
+
* protocol, and adding a member to it is a decision for a protocol revision, not for a
|
|
1453
|
+
* wiring change.
|
|
1454
|
+
*
|
|
1455
|
+
* What *is* new is why nothing was eligible, and that belongs in `detail`, where a caller
|
|
1456
|
+
* can read it without a new code entering the table.
|
|
1457
|
+
*/
|
|
1458
|
+
throw new ProtocolError('NO_ONLINE_DEVICE', scoped.text, {
|
|
1459
|
+
reason: scoped.reason,
|
|
1460
|
+
legacy: scoped.legacy,
|
|
1461
|
+
fleet_scoped: scoped.legacy === false,
|
|
1462
|
+
hint:
|
|
1463
|
+
scoped.reason === 'requester_has_no_fleet'
|
|
1464
|
+
? 'this relay is bound to a fleet and the requesting device is in none; pair it with a code issued for that fleet'
|
|
1465
|
+
: scoped.reason === 'target_outside_fleet'
|
|
1466
|
+
? 'the named target is not in this fleet'
|
|
1467
|
+
: 'check that at least one device in this fleet is paired and streaming',
|
|
1468
|
+
});
|
|
1469
|
+
}
|
|
1470
|
+
const targets = scoped.targets;
|
|
1471
|
+
if (scoped.legacy) {
|
|
1472
|
+
// Logged at debug so an operator reading logs during a migration can see which rule applied.
|
|
1473
|
+
// "N targets" alone does not say whether isolation is on, and that is the question being asked.
|
|
1474
|
+
this.log?.('debug', 'task targets selected without a fleet (legacy path)', {
|
|
1475
|
+
task_candidates: targets.length,
|
|
1476
|
+
relay_fleet_id: this.fleetId ?? null,
|
|
1477
|
+
});
|
|
1478
|
+
}
|
|
1479
|
+
|
|
1480
|
+
// v0.3.3 `broadcast`: validated here, where `targets` is known. The executor id itself was
|
|
1481
|
+
// resolved before the task literal because the literal records it.
|
|
1482
|
+
if (mode === 'broadcast' && executorMachineId !== null && !targets.some((d) => d.machine_id === executorMachineId)) {
|
|
1483
|
+
throw new ProtocolError('BAD_REQUEST', 'executor_machine_id is not among the target machines', {
|
|
1484
|
+
executor_machine_id: executorMachineId,
|
|
1485
|
+
targets: targets.map((d) => d.machine_id),
|
|
1486
|
+
});
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1489
|
+
if (targets.length === 0) {
|
|
1490
|
+
throw new ProtocolError('NO_ONLINE_DEVICE', 'no paired device available for this task', {
|
|
1491
|
+
known_devices: all.length,
|
|
1492
|
+
});
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
const dedupeKeyBase = computeDedupeKey(taskId, 0, commandHash, baseTree);
|
|
1496
|
+
if (typeof dedupeKeyBase !== 'string' || dedupeKeyBase.length !== 64) {
|
|
1497
|
+
throw new ProtocolError('INTERNAL', 'failed to compute dedupe_key');
|
|
1498
|
+
}
|
|
1499
|
+
let lastSeq = this.seq;
|
|
1500
|
+
targets.forEach((device, i) => {
|
|
1501
|
+
const index = mode === 'replicate' ? 0 : i % indexTotal;
|
|
1502
|
+
const observing = mode === 'broadcast' && device.machine_id !== executorMachineId;
|
|
1503
|
+
const lease = {
|
|
1504
|
+
machine_id: device.machine_id,
|
|
1505
|
+
machine_name: device.machine_name,
|
|
1506
|
+
index,
|
|
1507
|
+
attempt: 1,
|
|
1508
|
+
// `observing` is a lease state, not an outcome: the machine holds a place in this task and
|
|
1509
|
+
// will be told what happened, but it was never handed the command. `gateMachine` below is
|
|
1510
|
+
// skipped for it because refusing to do work you were never asked to do is not a refusal.
|
|
1511
|
+
state: observing ? 'observing' : 'queued',
|
|
1512
|
+
refusal_reason: null,
|
|
1513
|
+
refusal_detail: null,
|
|
1514
|
+
dedupe_key: computeDedupeKey(taskId, index, commandHash, baseTree),
|
|
1515
|
+
last_heartbeat_ms: now,
|
|
1516
|
+
claimed_at_ms: now,
|
|
1517
|
+
lease_until_ms: now + this.leaseWindowMs(),
|
|
1518
|
+
phase: null,
|
|
1519
|
+
progress: null,
|
|
1520
|
+
offered_seq: null,
|
|
1521
|
+
expired_at_ms: null,
|
|
1522
|
+
finished_at_ms: null,
|
|
1523
|
+
heartbeat_count: 0,
|
|
1524
|
+
};
|
|
1525
|
+
if (observing) {
|
|
1526
|
+
// No capability gate and no offer: the machine is not being asked to run anything, so there
|
|
1527
|
+
// is nothing to refuse and nothing to hand out. It is recorded so the task can tell it the
|
|
1528
|
+
// outcome afterwards.
|
|
1529
|
+
task.leases.set(device.machine_id, lease);
|
|
1530
|
+
return;
|
|
1531
|
+
}
|
|
1532
|
+
const gate = gateMachine(task, device);
|
|
1533
|
+
if (gate) {
|
|
1534
|
+
lease.state = 'refused';
|
|
1535
|
+
lease.refusal_reason = gate.refusal_reason;
|
|
1536
|
+
lease.refusal_detail = gate.detail;
|
|
1537
|
+
task.leases.set(device.machine_id, lease);
|
|
1538
|
+
this.emit('notice', {
|
|
1539
|
+
level: 'warn',
|
|
1540
|
+
code: 'CAPABILITY_REFUSED',
|
|
1541
|
+
message: `${device.machine_id} refused: ${gate.refusal_reason}`,
|
|
1542
|
+
task_id: taskId,
|
|
1543
|
+
});
|
|
1544
|
+
return;
|
|
1545
|
+
}
|
|
1546
|
+
task.leases.set(device.machine_id, lease);
|
|
1547
|
+
const grace = this._p2pOfferGraceMs(task);
|
|
1548
|
+
if (grace > 0) {
|
|
1549
|
+
// v0.4.0: the dispatcher is pushing this offer over the direct path right now. Hold the
|
|
1550
|
+
// relay's copy for `grace` ms so the direct one can win; see `p2pOfferGraceMs`.
|
|
1551
|
+
this._scheduleOffer(task, lease, grace);
|
|
1552
|
+
} else {
|
|
1553
|
+
const event = this.emitOffer(task, lease);
|
|
1554
|
+
lastSeq = event.seq;
|
|
1555
|
+
}
|
|
1556
|
+
});
|
|
1557
|
+
|
|
1558
|
+
this.tasks.set(taskId, task);
|
|
1559
|
+
|
|
1560
|
+
this._ledger({
|
|
1561
|
+
type: 'task.created',
|
|
1562
|
+
task: {
|
|
1563
|
+
task_id: task.task_id,
|
|
1564
|
+
mode: task.mode,
|
|
1565
|
+
command_argv: task.command_argv,
|
|
1566
|
+
cwd_rel: task.cwd_rel,
|
|
1567
|
+
index_total: task.index_total,
|
|
1568
|
+
timeout_ms: task.timeout_ms,
|
|
1569
|
+
write: task.write,
|
|
1570
|
+
write_scope: task.write_scope,
|
|
1571
|
+
require_exclusive_write: task.require_exclusive_write,
|
|
1572
|
+
base_commit: task.base_commit,
|
|
1573
|
+
base_tree: task.base_tree,
|
|
1574
|
+
requirements: task.requirements,
|
|
1575
|
+
compare_policy: task.compare_policy,
|
|
1576
|
+
halt: task.halt,
|
|
1577
|
+
created_by: task.created_by,
|
|
1578
|
+
created_at_ms: task.created_at_ms,
|
|
1579
|
+
command_hash: task.command_hash,
|
|
1580
|
+
attempt: task.attempt,
|
|
1581
|
+
deadline_ms: task.deadline_ms,
|
|
1582
|
+
// v0.4.0: routing metadata, so a restarted relay re-offers with the same facts.
|
|
1583
|
+
origin_machine_id: task.origin_machine_id ?? null,
|
|
1584
|
+
p2p: task.p2p ?? null,
|
|
1585
|
+
},
|
|
1586
|
+
leases: [...task.leases.values()].map((l) => ({ ...l })),
|
|
1587
|
+
});
|
|
1588
|
+
|
|
1589
|
+
const refusedOnly = [...task.leases.values()].every((l) => l.state === 'refused');
|
|
1590
|
+
if (refusedOnly) {
|
|
1591
|
+
// §6.3 Step 0 territory: a task nobody may execute is still a valid, reportable task.
|
|
1592
|
+
this.emit('notice', {
|
|
1593
|
+
level: 'warn',
|
|
1594
|
+
code: 'NO_ONLINE_DEVICE',
|
|
1595
|
+
message: 'every candidate machine refused the capability gate',
|
|
1596
|
+
task_id: taskId,
|
|
1597
|
+
});
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
return {
|
|
1601
|
+
task_id: taskId,
|
|
1602
|
+
leases: [...task.leases.values()].map((l) => ({
|
|
1603
|
+
machine_id: l.machine_id,
|
|
1604
|
+
index: l.index,
|
|
1605
|
+
state: l.state,
|
|
1606
|
+
// v0.4.0: the dispatcher pushes an offer over the direct path, and the executor dedupes it
|
|
1607
|
+
// against the relay's copy by (task_id, attempt, dedupe_key). Without this field the
|
|
1608
|
+
// dispatcher has to reproduce the §4.4 formula locally, which works for a single-command
|
|
1609
|
+
// task and cannot work for a pipeline one -- the key covers the relay's normalised stage
|
|
1610
|
+
// list. Publishing it makes the pushed copy recognisably the same work item, which is what
|
|
1611
|
+
// keeps the direct path from turning exactly-once into at-least-twice.
|
|
1612
|
+
dedupe_key: l.dedupe_key ?? null,
|
|
1613
|
+
attempt: l.attempt,
|
|
1614
|
+
...(l.refusal_reason ? { refusal_reason: l.refusal_reason } : {}),
|
|
1615
|
+
})),
|
|
1616
|
+
seq: lastSeq,
|
|
1617
|
+
};
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
/**
|
|
1621
|
+
* v0.4.0: should the relay hold its offer back for this task?
|
|
1622
|
+
*
|
|
1623
|
+
* Only when the dispatcher said two things: that it is a machine (`origin_machine_id`), and that
|
|
1624
|
+
* the direct path is in play (`p2p.mode` of `auto` or `direct`). A `relay` mode or a v0.3.9
|
|
1625
|
+
* dispatcher gets the old behaviour exactly, which is why this predicate is the only gate on the
|
|
1626
|
+
* change and every pre-existing test still sees an immediate offer.
|
|
1627
|
+
*
|
|
1628
|
+
* @param {object} task
|
|
1629
|
+
* @returns {number} milliseconds to hold, or 0 for "offer now"
|
|
1630
|
+
*/
|
|
1631
|
+
_p2pOfferGraceMs(task) {
|
|
1632
|
+
if (this.p2pOfferGraceMs <= 0) return 0;
|
|
1633
|
+
const origin = task?.origin_machine_id;
|
|
1634
|
+
if (typeof origin !== 'string' || origin === '') return 0;
|
|
1635
|
+
const mode = task?.p2p?.mode;
|
|
1636
|
+
if (mode !== 'auto' && mode !== 'direct') return 0;
|
|
1637
|
+
return this.p2pOfferGraceMs;
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
/**
|
|
1641
|
+
* v0.4.0: emit `lease`'s offer after `delayMs`, unless the direct path got there first.
|
|
1642
|
+
*
|
|
1643
|
+
* "Got there first" is one fact and it is already tracked: the executor renews the lease
|
|
1644
|
+
* (`POST /v1/heartbeat`) as its first act on any offer it accepts, and a renewal moves the lease
|
|
1645
|
+
* out of `queued`. So the check is the lease's own state — no new message, no new endpoint, and
|
|
1646
|
+
* nothing the executor has to know about.
|
|
1647
|
+
*
|
|
1648
|
+
* The timer is `unref()`ed: a relay with nothing else to do must still be able to exit, and a
|
|
1649
|
+
* pending offer is not a reason to keep a process alive.
|
|
1650
|
+
*/
|
|
1651
|
+
_scheduleOffer(task, lease, delayMs) {
|
|
1652
|
+
const key = `${task.task_id}:${lease.machine_id}`;
|
|
1653
|
+
const timer = setTimeout(() => {
|
|
1654
|
+
this._pendingOffers.delete(key);
|
|
1655
|
+
// A lease that was re-created (takeover) or swept is not this one any more.
|
|
1656
|
+
if (task.leases.get(lease.machine_id) !== lease) return;
|
|
1657
|
+
if (lease.state !== 'queued') return; // the direct copy won; nothing to fall back to
|
|
1658
|
+
if (this.isLeaseExpired(lease, this.nowMs())) return; // the sweep owns it now
|
|
1659
|
+
this.emitOffer(task, lease);
|
|
1660
|
+
}, delayMs);
|
|
1661
|
+
timer.unref?.();
|
|
1662
|
+
this._pendingOffers.set(key, timer);
|
|
1663
|
+
}
|
|
1664
|
+
|
|
1665
|
+
/** Drop every deferred offer. Called by the relay's own teardown path. */
|
|
1666
|
+
clearPendingOffers() {
|
|
1667
|
+
for (const timer of this._pendingOffers.values()) clearTimeout(timer);
|
|
1668
|
+
this._pendingOffers.clear();
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
emitOffer(task, lease) {
|
|
1672
|
+
const event = this.emit('task.offer', {
|
|
1673
|
+
task_id: task.task_id,
|
|
1674
|
+
attempt: lease.attempt,
|
|
1675
|
+
mode: task.mode,
|
|
1676
|
+
index: lease.index,
|
|
1677
|
+
index_total: task.index_total,
|
|
1678
|
+
command_argv: task.command_argv,
|
|
1679
|
+
// v0.3.3: present only for `pipeline`, and only then does the machine run a chain. Omitted
|
|
1680
|
+
// entirely otherwise, so an offer for a single-command task is byte-identical to before --
|
|
1681
|
+
// which is what keeps a v0.2.3 agent working against this relay.
|
|
1682
|
+
...(task.stages ? { stages: task.stages } : {}),
|
|
1683
|
+
/**
|
|
1684
|
+
* v0.4.0: always present, `null` when the caller supplied nothing.
|
|
1685
|
+
*
|
|
1686
|
+
* Deliberately NOT conditional (unlike `stages` above, which is omitted so a single-command
|
|
1687
|
+
* offer stays byte-identical for an old agent): the contract fixes the shape of these two as
|
|
1688
|
+
* `string|null` and `{mode}|null`, and `null` is the honest value for "the dispatcher did not
|
|
1689
|
+
* say" — which is exactly the v0.3.9 case. They are additive keys in an event whose consumers
|
|
1690
|
+
* already ignore unknown fields.
|
|
1691
|
+
*/
|
|
1692
|
+
origin_machine_id: task.origin_machine_id ?? null,
|
|
1693
|
+
p2p: task.p2p ?? null,
|
|
1694
|
+
cwd_rel: task.cwd_rel,
|
|
1695
|
+
write: task.write,
|
|
1696
|
+
timeout_ms: task.timeout_ms,
|
|
1697
|
+
base_commit: task.base_commit,
|
|
1698
|
+
base_tree: task.base_tree,
|
|
1699
|
+
requirements: task.requirements,
|
|
1700
|
+
compare_policy: task.compare_policy,
|
|
1701
|
+
dedupe_key: lease.dedupe_key,
|
|
1702
|
+
deadline: rfc3339(task.deadline_ms),
|
|
1703
|
+
}, { target: lease.machine_id });
|
|
1704
|
+
lease.state = 'offered';
|
|
1705
|
+
lease.offered_seq = event.seq;
|
|
1706
|
+
lease.last_heartbeat_ms = this.nowMs();
|
|
1707
|
+
lease.lease_until_ms = lease.last_heartbeat_ms + this.leaseWindowMs();
|
|
1708
|
+
return event;
|
|
1709
|
+
}
|
|
1710
|
+
|
|
1711
|
+
getTask(taskId) { return this.tasks.get(taskId) ?? null; }
|
|
1712
|
+
|
|
1713
|
+
requireTask(taskId) {
|
|
1714
|
+
const task = this.tasks.get(taskId);
|
|
1715
|
+
if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
|
|
1716
|
+
return task;
|
|
1717
|
+
}
|
|
1718
|
+
|
|
1719
|
+
/**
|
|
1720
|
+
* Re-offer the work a machine was handed but has not yet started (v0.1.2 §8.6).
|
|
1721
|
+
*
|
|
1722
|
+
* WHY THIS EXISTS
|
|
1723
|
+
*
|
|
1724
|
+
* An offer is a point-in-time event on a stream. `createTask` publishes
|
|
1725
|
+
* `task.offer` once, and if the target machine's stream happens to be down at
|
|
1726
|
+
* that instant -- the ordinary situation after a relay restart, a tunnel drop,
|
|
1727
|
+
* or any cross-network interruption -- the event is simply gone. The lease sits
|
|
1728
|
+
* in `offered` until the sweep expires it, the task never runs, and the
|
|
1729
|
+
* operator sees a timeout with no explanation. Measured: the machine is back
|
|
1730
|
+
* online with a live stream within milliseconds, and still receives nothing.
|
|
1731
|
+
*
|
|
1732
|
+
* So the relay re-states unclaimed work when a machine connects. This is safe
|
|
1733
|
+
* because the state it acts on is durable: the lease the offer refers to is
|
|
1734
|
+
* still in the ledger.
|
|
1735
|
+
*
|
|
1736
|
+
* WHY ONLY `offered` AND `queued`
|
|
1737
|
+
*
|
|
1738
|
+
* `running` is deliberately excluded, and that exclusion is the whole safety
|
|
1739
|
+
* argument. A running lease means a machine has already STARTED the command --
|
|
1740
|
+
* it acknowledged the offer and is heartbeating. Re-offering it would make that
|
|
1741
|
+
* machine execute the work a second time, so the fix for a lost offer would
|
|
1742
|
+
* introduce the one failure this project is built to avoid: two machines, or
|
|
1743
|
+
* one machine twice, running the same side-effecting command.
|
|
1744
|
+
*
|
|
1745
|
+
* A machine that dies mid-run therefore does not get its work back here; that
|
|
1746
|
+
* path stays with the sweep, which expires the lease and hands the index to
|
|
1747
|
+
* another machine with `attempt + 1`.
|
|
1748
|
+
*
|
|
1749
|
+
* Re-emitted offers are new events and are replayed like any other, so a client
|
|
1750
|
+
* that reconnects repeatedly can see the same offer more than once. That is
|
|
1751
|
+
* intentional: the agent already de-duplicates by `task_id` (it ignores an
|
|
1752
|
+
* offer whose task is queued or running), and an at-least-once offer with a
|
|
1753
|
+
* client-side dedupe is strictly better than an at-most-once offer that is
|
|
1754
|
+
* silently dropped.
|
|
1755
|
+
*
|
|
1756
|
+
* @param {string} machineId
|
|
1757
|
+
* @returns {{count: number, task_ids: string[]}}
|
|
1758
|
+
*/
|
|
1759
|
+
redeliverPendingOffers(machineId) {
|
|
1760
|
+
const taskIds = [];
|
|
1761
|
+
const now = this.nowMs();
|
|
1762
|
+
|
|
1763
|
+
for (const task of this.tasks.values()) {
|
|
1764
|
+
if (task.cancelled === true) continue;
|
|
1765
|
+
const lease = task.leases.get(machineId);
|
|
1766
|
+
if (!lease) continue;
|
|
1767
|
+
// Only work the machine has not begun. See the note above: `running` is
|
|
1768
|
+
// excluded on purpose, not by omission.
|
|
1769
|
+
if (lease.state !== 'offered' && lease.state !== 'queued') continue;
|
|
1770
|
+
// Do not resurrect a lease the sweep has already written off; the takeover
|
|
1771
|
+
// path owns that decision.
|
|
1772
|
+
if (this.isLeaseExpired(lease, now)) continue;
|
|
1773
|
+
|
|
1774
|
+
this.emitOffer(task, lease);
|
|
1775
|
+
taskIds.push(task.task_id);
|
|
1776
|
+
}
|
|
1777
|
+
|
|
1778
|
+
return { count: taskIds.length, task_ids: taskIds };
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
listTasks({ limit = 50 } = {}) {
|
|
1782
|
+
const n = Number.isInteger(limit) && limit > 0 ? limit : 50;
|
|
1783
|
+
return [...this.tasks.values()]
|
|
1784
|
+
.sort((a, b) => b.created_at_ms - a.created_at_ms)
|
|
1785
|
+
.slice(0, n)
|
|
1786
|
+
.map((t) => this.taskSummary(t));
|
|
1787
|
+
}
|
|
1788
|
+
|
|
1789
|
+
taskSummary(task) {
|
|
1790
|
+
return {
|
|
1791
|
+
task_id: task.task_id,
|
|
1792
|
+
mode: task.mode,
|
|
1793
|
+
created_at: rfc3339(task.created_at_ms),
|
|
1794
|
+
created_by: task.created_by,
|
|
1795
|
+
index_total: task.index_total,
|
|
1796
|
+
cancelled: task.cancelled,
|
|
1797
|
+
degraded: task.degraded,
|
|
1798
|
+
leases: [...task.leases.values()].map((l) => l.machine_id),
|
|
1799
|
+
lease_states: Object.fromEntries([...task.leases.values()].map((l) => [l.machine_id, l.state])),
|
|
1800
|
+
};
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
cancelTask(taskId, reason = 'cancelled_by_operator') {
|
|
1804
|
+
const task = this.requireTask(taskId);
|
|
1805
|
+
if (task.cancelled) return task;
|
|
1806
|
+
task.cancelled = true;
|
|
1807
|
+
task.cancel_reason = reason;
|
|
1808
|
+
for (const lease of task.leases.values()) {
|
|
1809
|
+
if (ACTIVE_LEASE_STATES.includes(lease.state)) lease.state = 'cancelled';
|
|
1810
|
+
}
|
|
1811
|
+
this.emit('task.cancel', { task_id: taskId, reason });
|
|
1812
|
+
this._ledger({ type: 'task.cancelled', task_id: taskId, reason });
|
|
1813
|
+
return task;
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
/* ---------------- leases / heartbeat (§4.3) ---------------- */
|
|
1817
|
+
|
|
1818
|
+
/** Shared `phase`/`progress` validation for both heartbeat modes. */
|
|
1819
|
+
_validatePhaseProgress(input) {
|
|
1820
|
+
if (input.phase !== undefined && input.phase !== null && !LEASE_PHASES.includes(input.phase)) {
|
|
1821
|
+
throw new ProtocolError('BAD_REQUEST', 'phase must be preparing|running|finalizing', { phase: input.phase });
|
|
1822
|
+
}
|
|
1823
|
+
if (input.progress !== undefined && input.progress !== null
|
|
1824
|
+
&& (typeof input.progress !== 'number' || input.progress < 0 || input.progress > 1)) {
|
|
1825
|
+
throw new ProtocolError('BAD_REQUEST', 'progress must be a number in [0,1]', { progress: input.progress });
|
|
1826
|
+
}
|
|
1827
|
+
}
|
|
1828
|
+
|
|
1829
|
+
/**
|
|
1830
|
+
* v0.3.0 IDLE (diagnostic) heartbeat — an agent with no lease keeps its latency
|
|
1831
|
+
* and liveness visible to other machines.
|
|
1832
|
+
*
|
|
1833
|
+
* This path deliberately NEVER looks at `this.tasks`. That is the whole safety
|
|
1834
|
+
* argument, and it is structural rather than a matter of remembering a rule: an
|
|
1835
|
+
* idle heartbeat cannot renew, resurrect or otherwise touch a lease because it
|
|
1836
|
+
* never obtains one. Without that, an idle ping from a machine whose lease had
|
|
1837
|
+
* expired would flip `expired` back to `running` and manufacture a live task out
|
|
1838
|
+
* of nothing -- a fake "the work is still being done" signal, which is the most
|
|
1839
|
+
* dangerous lie this relay can tell.
|
|
1840
|
+
*/
|
|
1841
|
+
diagnosticHeartbeat(input = {}) {
|
|
1842
|
+
const machineId = typeof input.machine_id === 'string' && input.machine_id !== ''
|
|
1843
|
+
? input.machine_id : null;
|
|
1844
|
+
if (machineId === null) {
|
|
1845
|
+
throw new ProtocolError('BAD_REQUEST', 'machine_id is required', {});
|
|
1846
|
+
}
|
|
1847
|
+
if (!this.devices.has(machineId)) {
|
|
1848
|
+
throw new ProtocolError('NOT_FOUND', 'unknown machine_id', { machine_id: machineId });
|
|
1849
|
+
}
|
|
1850
|
+
this._validatePhaseProgress(input);
|
|
1851
|
+
|
|
1852
|
+
const device = this.touchDevice(machineId);
|
|
1853
|
+
this.recordHeartbeatMetrics(machineId, input);
|
|
1854
|
+
return {
|
|
1855
|
+
// No lease is involved, and the response says so instead of inventing one.
|
|
1856
|
+
lease_until: null,
|
|
1857
|
+
cancel: false,
|
|
1858
|
+
diagnostic: true,
|
|
1859
|
+
machine_id: machineId,
|
|
1860
|
+
rtt_ms: device?.rtt_ms ?? null,
|
|
1861
|
+
};
|
|
1862
|
+
}
|
|
1863
|
+
|
|
1864
|
+
/**
|
|
1865
|
+
* §4.3 POST /v1/heartbeat — renews the lease using SERVER time only.
|
|
1866
|
+
*
|
|
1867
|
+
* Two modes since v0.3.0:
|
|
1868
|
+
* - `task_id` present -> LEASE heartbeat (unchanged §4.3 behaviour)
|
|
1869
|
+
* - `diagnostic: true`, no task -> IDLE heartbeat (diagnostics only, no lease)
|
|
1870
|
+
* - neither -> 400. A heartbeat that silently renewed
|
|
1871
|
+
* nothing would let an agent whose `task_id` went missing stop renewing while
|
|
1872
|
+
* believing it was fine, ending in a falsely `expired` task. §0 forbids
|
|
1873
|
+
* silent degradation, so this is loud.
|
|
1874
|
+
*
|
|
1875
|
+
* @returns {{lease_until:string|null, cancel:boolean, phase?:string, progress?:number, diagnostic?:boolean}}
|
|
1876
|
+
*/
|
|
1877
|
+
heartbeat(input = {}) {
|
|
1878
|
+
const hasTask = typeof input.task_id === 'string' && input.task_id !== '';
|
|
1879
|
+
if (!hasTask) {
|
|
1880
|
+
if (input.diagnostic !== true) {
|
|
1881
|
+
throw new ProtocolError('BAD_REQUEST',
|
|
1882
|
+
'a heartbeat must carry task_id (lease renewal) or diagnostic:true (idle diagnostics); '
|
|
1883
|
+
+ 'refusing to treat a missing task_id as a no-op renewal', {
|
|
1884
|
+
received_keys: Object.keys(input).sort(),
|
|
1885
|
+
});
|
|
1886
|
+
}
|
|
1887
|
+
return this.diagnosticHeartbeat(input);
|
|
1888
|
+
}
|
|
1889
|
+
if (input.diagnostic === true) {
|
|
1890
|
+
// task_id wins: a heartbeat that names a task IS a lease heartbeat.
|
|
1891
|
+
// (documented; the flag is simply redundant here)
|
|
1892
|
+
}
|
|
1893
|
+
return this.leaseHeartbeat(input);
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
leaseHeartbeat(input = {}) {
|
|
1897
|
+
const task = this.requireTask(input.task_id);
|
|
1898
|
+
const lease = task.leases.get(input.machine_id);
|
|
1899
|
+
if (!lease) {
|
|
1900
|
+
throw new ProtocolError('NOT_FOUND', 'no lease for this machine on this task', {
|
|
1901
|
+
task_id: input.task_id, machine_id: input.machine_id,
|
|
1902
|
+
});
|
|
1903
|
+
}
|
|
1904
|
+
this._validatePhaseProgress(input);
|
|
1905
|
+
|
|
1906
|
+
const now = this.nowMs();
|
|
1907
|
+
this.touchDevice(input.machine_id);
|
|
1908
|
+
this.recordHeartbeatMetrics(input.machine_id, input, now);
|
|
1909
|
+
|
|
1910
|
+
if (task.cancelled) {
|
|
1911
|
+
return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: task.cancel_reason };
|
|
1912
|
+
}
|
|
1913
|
+
|
|
1914
|
+
// §4.3: once halt=now has handed this index to another attempt, the old owner
|
|
1915
|
+
// must stop — a late heartbeat from the expired machine is not a valid re-claim.
|
|
1916
|
+
if (lease.state === 'expired' && task.attempt > lease.attempt) {
|
|
1917
|
+
return { lease_until: rfc3339(lease.lease_until_ms), cancel: true, reason: 'LEASE_EXPIRED_TAKEN_OVER' };
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
const stateBefore = lease.state;
|
|
1921
|
+
lease.last_heartbeat_ms = now;
|
|
1922
|
+
lease.lease_until_ms = now + this.leaseWindowMs();
|
|
1923
|
+
lease.heartbeat_count += 1;
|
|
1924
|
+
if (input.phase !== undefined && input.phase !== null) lease.phase = input.phase;
|
|
1925
|
+
if (input.progress !== undefined && input.progress !== null) lease.progress = input.progress;
|
|
1926
|
+
if (Number.isInteger(input.attempt) && input.attempt >= 1) lease.attempt = input.attempt;
|
|
1927
|
+
if (lease.state === 'queued' || lease.state === 'offered') lease.state = 'running';
|
|
1928
|
+
// A live heartbeat is stronger evidence than a timer: with halt=never nobody took
|
|
1929
|
+
// the work away, so the late-but-alive agent re-claims its own lease.
|
|
1930
|
+
if (lease.state === 'expired') { lease.state = 'running'; lease.expired_at_ms = null; }
|
|
1931
|
+
|
|
1932
|
+
// §7 ledger records lease STATE changes, not every 10s heartbeat.
|
|
1933
|
+
if (lease.state !== stateBefore) this._ledgerLease(task, lease);
|
|
1934
|
+
|
|
1935
|
+
return { lease_until: rfc3339(lease.lease_until_ms), cancel: false, phase: lease.phase, progress: lease.progress };
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1938
|
+
/** §4.3: expiry is decided by Rabbit's clock only. */
|
|
1939
|
+
isLeaseExpired(lease, nowMs = this.nowMs()) {
|
|
1940
|
+
if (!ACTIVE_LEASE_STATES.includes(lease.state)) return false;
|
|
1941
|
+
return nowMs >= lease.lease_until_ms;
|
|
1942
|
+
}
|
|
1943
|
+
|
|
1944
|
+
/**
|
|
1945
|
+
* §4.3 expiry sweep.
|
|
1946
|
+
* - halt=never → book-keeping only, task is marked `partial`
|
|
1947
|
+
* - halt=now → notify remaining machines immediately that they may take over
|
|
1948
|
+
*/
|
|
1949
|
+
sweepExpired(nowMs = this.nowMs()) {
|
|
1950
|
+
const expired = [];
|
|
1951
|
+
for (const task of this.tasks.values()) {
|
|
1952
|
+
if (task.cancelled) continue;
|
|
1953
|
+
let changed = false;
|
|
1954
|
+
for (const lease of task.leases.values()) {
|
|
1955
|
+
if (!this.isLeaseExpired(lease, nowMs)) continue;
|
|
1956
|
+
lease.state = 'expired';
|
|
1957
|
+
lease.expired_at_ms = nowMs;
|
|
1958
|
+
changed = true;
|
|
1959
|
+
expired.push({ task, lease });
|
|
1960
|
+
this._ledgerLease(task, lease);
|
|
1961
|
+
this.emit('notice', {
|
|
1962
|
+
level: 'warn',
|
|
1963
|
+
code: 'LEASE_EXPIRED',
|
|
1964
|
+
message: `lease expired: machine=${lease.machine_id} task=${task.task_id} attempt=${lease.attempt}`,
|
|
1965
|
+
task_id: task.task_id,
|
|
1966
|
+
machine_id: lease.machine_id,
|
|
1967
|
+
});
|
|
1968
|
+
if (task.halt === 'now') {
|
|
1969
|
+
task.attempt += 1;
|
|
1970
|
+
this._takeover(task, lease, nowMs);
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1973
|
+
if (changed) task.degraded = 'partial';
|
|
1974
|
+
}
|
|
1975
|
+
return expired;
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
/**
|
|
1979
|
+
* §4.3 `halt=now`: "立即通知其他机器可接管".
|
|
1980
|
+
*
|
|
1981
|
+
* v0.0.1's PROTOCOL defines no per-index lease reassignment, so takeover is
|
|
1982
|
+
* implemented as: re-offer the freed `index` (attempt+1) to every live machine
|
|
1983
|
+
* that is idle — no lease yet, or a lease still queued/offered — and broadcast a
|
|
1984
|
+
* notice. Machines already `running` are never re-offered, to avoid double
|
|
1985
|
+
* execution of the same index.
|
|
1986
|
+
*/
|
|
1987
|
+
_takeover(task, expiredLease, nowMs = this.nowMs()) {
|
|
1988
|
+
/**
|
|
1989
|
+
* v0.7.8: the two safety guards this path was missing.
|
|
1990
|
+
*
|
|
1991
|
+
* `_takeover` already existed and is invoked on `halt=now`, so hand-off was not absent — but it
|
|
1992
|
+
* would re-offer a dead machine's index to **every idle device the relay holds**, with no fleet
|
|
1993
|
+
* filter and no consideration of whether the task writes. Both are reachable defects:
|
|
1994
|
+
*
|
|
1995
|
+
* 1. **A device in another account could be handed the work.** On a relay serving several
|
|
1996
|
+
* fleets, `this.devices` includes all of them, so a takeover crossed accounts — which is
|
|
1997
|
+
* the isolation this release series is about, broken in the one path that is easy to miss
|
|
1998
|
+
* because it only runs when something has already gone wrong.
|
|
1999
|
+
* 2. **A writing task could be run a second time.** Hand-off makes execution at-least-once,
|
|
2000
|
+
* which is harmless for `node --test` and is a duplicated write for anything else. The
|
|
2001
|
+
* design forbids automatic hand-off for writing tasks until a fencing token exists.
|
|
2002
|
+
*
|
|
2003
|
+
* The re-offer shape itself is unchanged: PROTOCOL v0.0.1 defines no per-index lease
|
|
2004
|
+
* reassignment, so takeover is expressed as a re-offer to idle machines. Narrowing that to a
|
|
2005
|
+
* single successor is a protocol revision, not a wiring change, and is not made here.
|
|
2006
|
+
*/
|
|
2007
|
+
if (task.write === true) {
|
|
2008
|
+
this.emit('notice', {
|
|
2009
|
+
level: 'warn',
|
|
2010
|
+
code: 'LEASE_TAKEOVER_REFUSED',
|
|
2011
|
+
message:
|
|
2012
|
+
`task ${task.task_id} index ${expiredLease.index} is a writing task, so no machine takes it over: `
|
|
2013
|
+
+ 'a second run could apply the same change twice, and the caller cannot tell from the verdict',
|
|
2014
|
+
task_id: task.task_id,
|
|
2015
|
+
reason: 'write_not_idempotent',
|
|
2016
|
+
});
|
|
2017
|
+
return [];
|
|
2018
|
+
}
|
|
2019
|
+
|
|
2020
|
+
const requesterFleet = this.fleetId ?? null;
|
|
2021
|
+
|
|
2022
|
+
const candidates = [];
|
|
2023
|
+
for (const device of this.devices.values()) {
|
|
2024
|
+
if (device.machine_id === expiredLease.machine_id) continue;
|
|
2025
|
+
/**
|
|
2026
|
+
* Fleet membership, when the relay is scoped.
|
|
2027
|
+
*
|
|
2028
|
+
* A device with its own `fleet_id` must match. A device with none cannot take over on a
|
|
2029
|
+
* scoped relay, because there is no fleet to match it against — the same rule `targetsFor`
|
|
2030
|
+
* applies to dispatch, kept identical here so the two paths cannot disagree.
|
|
2031
|
+
*/
|
|
2032
|
+
if (requesterFleet !== null) {
|
|
2033
|
+
const deviceFleet = typeof device.fleet_id === 'string' && device.fleet_id !== '' ? device.fleet_id : null;
|
|
2034
|
+
if (deviceFleet !== requesterFleet) {
|
|
2035
|
+
this.log?.('debug', 'takeover candidate skipped: not in this fleet', {
|
|
2036
|
+
task_id: task.task_id,
|
|
2037
|
+
machine_id: device.machine_id,
|
|
2038
|
+
device_fleet: deviceFleet,
|
|
2039
|
+
relay_fleet: requesterFleet,
|
|
2040
|
+
});
|
|
2041
|
+
continue;
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
const lease = task.leases.get(device.machine_id);
|
|
2045
|
+
if (lease && !['queued', 'offered'].includes(lease.state)) continue;
|
|
2046
|
+
if (!lease && gateMachine(task, device)) continue;
|
|
2047
|
+
candidates.push({ device, lease: lease ?? null });
|
|
2048
|
+
}
|
|
2049
|
+
|
|
2050
|
+
for (const { device, lease } of candidates) {
|
|
2051
|
+
const target = lease ?? {
|
|
2052
|
+
machine_id: device.machine_id,
|
|
2053
|
+
machine_name: device.machine_name,
|
|
2054
|
+
index: expiredLease.index,
|
|
2055
|
+
attempt: task.attempt,
|
|
2056
|
+
state: 'queued',
|
|
2057
|
+
refusal_reason: null,
|
|
2058
|
+
refusal_detail: null,
|
|
2059
|
+
dedupe_key: null,
|
|
2060
|
+
last_heartbeat_ms: nowMs,
|
|
2061
|
+
claimed_at_ms: nowMs,
|
|
2062
|
+
lease_until_ms: nowMs + this.leaseWindowMs(),
|
|
2063
|
+
phase: null,
|
|
2064
|
+
progress: null,
|
|
2065
|
+
offered_seq: null,
|
|
2066
|
+
expired_at_ms: null,
|
|
2067
|
+
finished_at_ms: null,
|
|
2068
|
+
heartbeat_count: 0,
|
|
2069
|
+
};
|
|
2070
|
+
target.index = expiredLease.index;
|
|
2071
|
+
target.attempt = task.attempt;
|
|
2072
|
+
target.dedupe_key = computeDedupeKey(task.task_id, target.index, task.command_hash, task.base_tree);
|
|
2073
|
+
task.leases.set(device.machine_id, target);
|
|
2074
|
+
this.emitOffer(task, target);
|
|
2075
|
+
this._ledgerLease(task, target);
|
|
2076
|
+
}
|
|
2077
|
+
|
|
2078
|
+
this.emit('notice', {
|
|
2079
|
+
level: 'info',
|
|
2080
|
+
code: candidates.length > 0 ? 'LEASE_TAKEOVER' : 'LEASE_TAKEOVER_UNAVAILABLE',
|
|
2081
|
+
message: candidates.length > 0
|
|
2082
|
+
? `task ${task.task_id} index ${expiredLease.index} re-offered at attempt ${task.attempt} to: `
|
|
2083
|
+
+ candidates.map((c) => c.device.machine_id).join(',')
|
|
2084
|
+
: `task ${task.task_id} index ${expiredLease.index} has no idle machine to take over`,
|
|
2085
|
+
task_id: task.task_id,
|
|
2086
|
+
});
|
|
2087
|
+
return candidates.map((c) => c.device.machine_id);
|
|
2088
|
+
}
|
|
2089
|
+
|
|
2090
|
+
/* ---------------- results (§5) ---------------- */
|
|
2091
|
+
|
|
2092
|
+
static instanceKey(machineId, dedupeKey, attempt) {
|
|
2093
|
+
return `${machineId}${INSTANCE_SEP}${dedupeKey}${INSTANCE_SEP}${attempt}`;
|
|
2094
|
+
}
|
|
2095
|
+
|
|
2096
|
+
/**
|
|
2097
|
+
* §5 result envelope validation.
|
|
2098
|
+
* Missing §5.1 required fields do NOT reject the frame — they make the result
|
|
2099
|
+
* `unverifiable` (§5.1 "缺一即 unverifiable").
|
|
2100
|
+
*/
|
|
2101
|
+
static validateEnvelope(envelope) {
|
|
2102
|
+
const missing = [];
|
|
2103
|
+
if (!envelope || typeof envelope !== 'object' || Array.isArray(envelope)) {
|
|
2104
|
+
return { missing: [...REQUIRED_ENVELOPE_FIELDS], malformed: true };
|
|
2105
|
+
}
|
|
2106
|
+
for (const field of REQUIRED_ENVELOPE_FIELDS) {
|
|
2107
|
+
if (!(field in envelope) || envelope[field] === undefined) missing.push(field);
|
|
2108
|
+
}
|
|
2109
|
+
const problems = [];
|
|
2110
|
+
if ('status' in envelope && !RESULT_STATUSES.includes(envelope.status)) {
|
|
2111
|
+
problems.push(`status must be one of ${RESULT_STATUSES.join('|')}`);
|
|
2112
|
+
}
|
|
2113
|
+
if (envelope.status === 'refused' && !envelope.refusal_reason) {
|
|
2114
|
+
problems.push('refusal_reason is required when status=refused');
|
|
2115
|
+
}
|
|
2116
|
+
if ('warnings' in envelope && !Array.isArray(envelope.warnings)) problems.push('warnings must be string[]');
|
|
2117
|
+
return { missing, problems };
|
|
2118
|
+
}
|
|
2119
|
+
|
|
2120
|
+
/**
|
|
2121
|
+
* §1 idempotent POST /v1/result.
|
|
2122
|
+
*
|
|
2123
|
+
* NOTE (contract ambiguity, see report): §4.4 `dedupe_key` deliberately excludes
|
|
2124
|
+
* `machine_id`/`attempt`, but §6.3 aggregates results from *several* machines in
|
|
2125
|
+
* replicate mode — they all share one dedupe_key. Deduping purely on dedupe_key
|
|
2126
|
+
* would silently drop every machine's result but the first. We therefore key the
|
|
2127
|
+
* idempotency check on (machine_id, dedupe_key, attempt): an identical redelivery
|
|
2128
|
+
* is `deduped:true`, distinct machines/attempts are all recorded.
|
|
2129
|
+
*/
|
|
2130
|
+
submitResult(envelope = {}) {
|
|
2131
|
+
const { missing, problems, malformed } = RabbitState.validateEnvelope(envelope);
|
|
2132
|
+
if (malformed) throw new ProtocolError('BAD_REQUEST', 'result must be a JSON object');
|
|
2133
|
+
|
|
2134
|
+
/**
|
|
2135
|
+
* v0.4.0: `transport` / `p2p` are OPTIONAL envelope fields, and they are validated as values
|
|
2136
|
+
* rather than added to the §5.1 required set. A v0.3.9 agent sends neither and must keep
|
|
2137
|
+
* working unchanged; an agent that sends `transport: 'satellite'` is refused by name instead of
|
|
2138
|
+
* storing a value the report would then show as a path that does not exist.
|
|
2139
|
+
*/
|
|
2140
|
+
RabbitState.validateTransportFields(envelope);
|
|
2141
|
+
|
|
2142
|
+
const taskId = envelope.task_id;
|
|
2143
|
+
if (typeof taskId !== 'string' || taskId.length === 0) {
|
|
2144
|
+
throw new ProtocolError('BAD_REQUEST', 'task_id is required');
|
|
2145
|
+
}
|
|
2146
|
+
const task = this.tasks.get(taskId);
|
|
2147
|
+
if (!task) throw new ProtocolError('NOT_FOUND', 'unknown task_id', { task_id: taskId });
|
|
2148
|
+
|
|
2149
|
+
const machineId = typeof envelope.machine_id === 'string' ? envelope.machine_id : null;
|
|
2150
|
+
const dedupeKey = typeof envelope.dedupe_key === 'string' && envelope.dedupe_key
|
|
2151
|
+
? envelope.dedupe_key : null;
|
|
2152
|
+
const attempt = Number.isInteger(envelope.attempt) && envelope.attempt >= 1 ? envelope.attempt : 1;
|
|
2153
|
+
|
|
2154
|
+
if (dedupeKey === null) {
|
|
2155
|
+
// Cannot dedupe without a key; still record it, as unverifiable.
|
|
2156
|
+
missing.push('dedupe_key');
|
|
2157
|
+
}
|
|
2158
|
+
const instanceKey = RabbitState.instanceKey(machineId ?? '<unknown>', dedupeKey ?? '<none>', attempt);
|
|
2159
|
+
|
|
2160
|
+
// §4.4 sanity check against the lease this machine actually holds. Booked as a
|
|
2161
|
+
// problem instead of a hard rejection so a buggy client cannot lose its result.
|
|
2162
|
+
const leaseForMachine = machineId ? task.leases.get(machineId) : null;
|
|
2163
|
+
if (leaseForMachine?.dedupe_key && dedupeKey && leaseForMachine.dedupe_key !== dedupeKey) {
|
|
2164
|
+
problems.push('dedupe_key does not match the §4.4 formula for this lease');
|
|
2165
|
+
}
|
|
2166
|
+
|
|
2167
|
+
if (this.results.has(instanceKey)) {
|
|
2168
|
+
const prior = this.results.get(instanceKey);
|
|
2169
|
+
return { deduped: true, task_id: taskId, dedupe_key: dedupeKey, seq: prior.seq };
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
const now = this.nowMs();
|
|
2173
|
+
let envelopeShaOk = null;
|
|
2174
|
+
if (typeof envelope.envelope_sha256 === 'string') {
|
|
2175
|
+
const { envelope_sha256: _drop, ...rest } = envelope;
|
|
2176
|
+
envelopeShaOk = sha256Hex(jcs(rest)) === envelope.envelope_sha256;
|
|
2177
|
+
}
|
|
2178
|
+
|
|
2179
|
+
const record = {
|
|
2180
|
+
envelope,
|
|
2181
|
+
machine_id: machineId,
|
|
2182
|
+
dedupe_key: dedupeKey,
|
|
2183
|
+
attempt,
|
|
2184
|
+
instance_key: instanceKey,
|
|
2185
|
+
received_at_ms: now,
|
|
2186
|
+
validation: {
|
|
2187
|
+
missing: [...new Set(missing)],
|
|
2188
|
+
problems,
|
|
2189
|
+
envelope_sha256_ok: envelopeShaOk,
|
|
2190
|
+
},
|
|
2191
|
+
};
|
|
2192
|
+
|
|
2193
|
+
const effectiveStatus = record.validation.missing.length > 0 ? 'unverifiable' : envelope.status;
|
|
2194
|
+
|
|
2195
|
+
const seq = this.nextSeq();
|
|
2196
|
+
record.seq = seq;
|
|
2197
|
+
record.effective_status = effectiveStatus;
|
|
2198
|
+
this.results.set(instanceKey, record);
|
|
2199
|
+
|
|
2200
|
+
if (machineId) {
|
|
2201
|
+
const lease = task.leases.get(machineId);
|
|
2202
|
+
if (lease) {
|
|
2203
|
+
lease.state = effectiveStatus === 'refused' ? 'refused' : 'done';
|
|
2204
|
+
if (effectiveStatus === 'refused' && !lease.refusal_reason) {
|
|
2205
|
+
lease.refusal_reason = envelope.refusal_reason ?? 'REFUSED';
|
|
2206
|
+
}
|
|
2207
|
+
lease.finished_at_ms = now;
|
|
2208
|
+
lease.attempt = attempt;
|
|
2209
|
+
}
|
|
2210
|
+
this.touchDevice(machineId);
|
|
2211
|
+
}
|
|
2212
|
+
task.result_seqs.push(seq);
|
|
2213
|
+
|
|
2214
|
+
this.emit('task.result', {
|
|
2215
|
+
task_id: taskId,
|
|
2216
|
+
dedupe_key: dedupeKey,
|
|
2217
|
+
attempt,
|
|
2218
|
+
index: envelope.index ?? null,
|
|
2219
|
+
status: effectiveStatus,
|
|
2220
|
+
machine_id: machineId,
|
|
2221
|
+
deduped: false,
|
|
2222
|
+
validation_errors: record.validation.missing,
|
|
2223
|
+
});
|
|
2224
|
+
|
|
2225
|
+
// §7: results go into the ledger so the dedupe index survives a restart.
|
|
2226
|
+
this._ledger({ type: 'result.stored', task_id: taskId, machine_id: machineId, result: record });
|
|
2227
|
+
if (machineId) {
|
|
2228
|
+
const changed = task.leases.get(machineId);
|
|
2229
|
+
if (changed) this._ledgerLease(task, changed);
|
|
2230
|
+
}
|
|
2231
|
+
|
|
2232
|
+
return { deduped: false, task_id: taskId, dedupe_key: dedupeKey, seq, _instanceKey: instanceKey };
|
|
2233
|
+
}
|
|
2234
|
+
|
|
2235
|
+
/** All recorded results for a task, keyed by machine_id (later attempts win). */
|
|
2236
|
+
taskResults(taskId) {
|
|
2237
|
+
const task = this.tasks.get(taskId);
|
|
2238
|
+
if (!task) return new Map();
|
|
2239
|
+
const out = new Map();
|
|
2240
|
+
for (const rec of this.results.values()) {
|
|
2241
|
+
if (rec.envelope.task_id !== taskId) continue;
|
|
2242
|
+
const prev = out.get(rec.machine_id);
|
|
2243
|
+
if (!prev || rec.received_at_ms >= prev.received_at_ms) out.set(rec.machine_id, rec);
|
|
2244
|
+
}
|
|
2245
|
+
return out;
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2248
|
+
/** Global dedupe book-keeping: dedupe_key -> instances seen. */
|
|
2249
|
+
dedupeReport() {
|
|
2250
|
+
const out = {};
|
|
2251
|
+
for (const rec of this.results.values()) {
|
|
2252
|
+
if (!rec.dedupe_key) continue;
|
|
2253
|
+
out[rec.dedupe_key] = out[rec.dedupe_key] ?? [];
|
|
2254
|
+
out[rec.dedupe_key].push({
|
|
2255
|
+
machine_id: rec.machine_id,
|
|
2256
|
+
attempt: rec.attempt,
|
|
2257
|
+
received_at: rfc3339(rec.received_at_ms),
|
|
2258
|
+
});
|
|
2259
|
+
}
|
|
2260
|
+
return out;
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
/* ---------------- serialization ---------------- */
|
|
2264
|
+
|
|
2265
|
+
stats() {
|
|
2266
|
+
return {
|
|
2267
|
+
protocol_version: PROTOCOL_VERSION,
|
|
2268
|
+
seq: this.seq,
|
|
2269
|
+
buffered_events: this.buffer.length,
|
|
2270
|
+
devices: this.devices.size,
|
|
2271
|
+
tasks: this.tasks.size,
|
|
2272
|
+
results: this.results.size,
|
|
2273
|
+
pairing_codes: [...this.pairingCodes.values()].filter((c) => !c.used && this.nowMs() < c.expires_at_ms).length,
|
|
2274
|
+
uptime_ms: this.nowMs() - this.startedAtMs,
|
|
2275
|
+
};
|
|
2276
|
+
}
|
|
2277
|
+
}
|
|
2278
|
+
|
|
2279
|
+
export default RabbitState;
|