@twinsearth/w2m-dsh-plugin 0.0.0-stage → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1288 -0
- package/LICENSE +21 -0
- package/PROTOCOL-v0.1.2.md +165 -0
- package/PROTOCOL-v0.3.0.md +454 -0
- package/PROTOCOL-v0.3.9.md +207 -0
- package/PROTOCOL-v0.4.0.md +255 -0
- package/PROTOCOL.md +339 -0
- package/README.md +665 -2
- package/bin/w2m-localside.mjs +303 -0
- package/bin/w2m-rabbit.mjs +396 -0
- package/bin/w2m-stun.mjs +231 -0
- package/cordis.patch.yml +94 -0
- package/deploy/caddy/Caddyfile.example +118 -0
- package/deploy/docker/Dockerfile +144 -0
- package/deploy/docker/README.md +338 -0
- package/deploy/docker/docker-compose.yml +179 -0
- package/deploy/nginx/w2m.conf.example +196 -0
- package/deploy/shared-server/README.md +118 -0
- package/deploy/shared-server/install.sh +255 -0
- package/deploy/systemd/w2m-rabbit.service +150 -0
- package/deploy/systemd/w2m-stun.service +69 -0
- package/deploy/windows/README.md +115 -0
- package/deploy/windows/install-service.ps1 +195 -0
- package/deploy/windows/w2m-agent-supervisor.mjs +205 -0
- package/deploy/windows/winsw/w2m-localside.xml +50 -0
- package/docs/DEPLOY.md +902 -0
- package/docs/LINT-AUDIT.md +492 -0
- package/docs/PIPELINE-DESIGN.md +95 -0
- package/docs/TROUBLESHOOTING.md +560 -0
- package/docs/UI-CARDS.md +81 -0
- package/lib/index.js +13 -0
- package/lib/tools.js +11 -0
- package/package.json +133 -4
- package/src/agent/agent.mjs +3317 -0
- package/src/agent/allowed-commands.mjs +116 -0
- package/src/agent/caps.mjs +437 -0
- package/src/agent/exec.mjs +356 -0
- package/src/agent/git.mjs +616 -0
- package/src/agent/identity.mjs +253 -0
- package/src/agent/p2p-node.mjs +1243 -0
- package/src/agent/p2p.mjs +605 -0
- package/src/agent/resolve.mjs +148 -0
- package/src/agent/spool.mjs +252 -0
- package/src/agent/stun.mjs +397 -0
- package/src/agent/url.mjs +127 -0
- package/src/plugin/auto-update.mjs +574 -0
- package/src/plugin/schedule.mjs +452 -0
- package/src/plugin/shared-config.mjs +264 -0
- package/src/plugin/tools.mjs +3891 -0
- package/src/plugin/update-install.mjs +764 -0
- package/src/plugin/update-source.mjs +686 -0
- package/src/relay/metrics.mjs +284 -0
- package/src/relay/peers.mjs +180 -0
- package/src/relay/persistence.mjs +207 -0
- package/src/relay/report.mjs +591 -0
- package/src/relay/server.mjs +1187 -0
- package/src/relay/state.mjs +2004 -0
- package/src/relay/stun-server.mjs +733 -0
- package/src/signing.mjs +359 -0
- package/src/util/cli.mjs +70 -0
|
@@ -0,0 +1,3317 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Localside agent: everything between "Rabbit hands us an offer" and
|
|
3
|
+
* "Rabbit has our envelope".
|
|
4
|
+
*
|
|
5
|
+
* Flow, in the order the contract requires it:
|
|
6
|
+
*
|
|
7
|
+
* pair -> SSE long-poll -> offer -> **spool first** -> capability gate ->
|
|
8
|
+
* anchors -> execute -> anchors -> envelope -> spool -> POST -> ack -> delete
|
|
9
|
+
*
|
|
10
|
+
* Notable decisions, all traceable to the protocol:
|
|
11
|
+
*
|
|
12
|
+
* * `shell_id` is always `direct-exec` (§5.1); a shell is never used.
|
|
13
|
+
* * The command allow-list is **default-deny**: an empty list refuses
|
|
14
|
+
* everything, and an offer may only run an argv that *starts with* one of
|
|
15
|
+
* the configured prefixes.
|
|
16
|
+
* * Capability mismatches are `refused` with a reason code, never a silent
|
|
17
|
+
* downgrade (§6.1).
|
|
18
|
+
* * The lease is renewed by heartbeat only; the agent never decides on its
|
|
19
|
+
* own that a lease expired (§4.3) -- Rabbit's clock is authoritative.
|
|
20
|
+
* * Reconnect backoff is exactly 500ms -> 1s -> 2s -> 4s -> 8s -> 10s with
|
|
21
|
+
* 50%-100% jitter, matching DSH's own behaviour so it is predictable.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { Readable } from 'node:stream';
|
|
25
|
+
import { access, constants as fsConstants } from 'node:fs/promises';
|
|
26
|
+
import { mkdirSync, renameSync, writeFileSync } from 'node:fs';
|
|
27
|
+
import { join } from 'node:path';
|
|
28
|
+
import { randomUUID } from 'node:crypto';
|
|
29
|
+
|
|
30
|
+
import {
|
|
31
|
+
EMPTY_SHA256,
|
|
32
|
+
classifyExit,
|
|
33
|
+
normalizeOutput,
|
|
34
|
+
runArgv,
|
|
35
|
+
sha256Hex,
|
|
36
|
+
} from './exec.mjs';
|
|
37
|
+
import {
|
|
38
|
+
FINGERPRINT_ALGO,
|
|
39
|
+
WARN_DIRTY_WORKTREE,
|
|
40
|
+
WARN_FINGERPRINT_UNAVAILABLE,
|
|
41
|
+
collectPostState,
|
|
42
|
+
collectPreAnchors,
|
|
43
|
+
commandHash,
|
|
44
|
+
envelopeSha256,
|
|
45
|
+
gitVersion,
|
|
46
|
+
normalizeRelPath,
|
|
47
|
+
pipelineCommandHash,
|
|
48
|
+
resolveProjectCwd,
|
|
49
|
+
} from './git.mjs';
|
|
50
|
+
import { createSpool } from './spool.mjs';
|
|
51
|
+
import { matchAllowedCommand, parseAllowedCommands } from './allowed-commands.mjs';
|
|
52
|
+
import { DEFAULT_P2P_MODE, P2PNode, normalizeP2PMode, normalizeP2PPort } from './p2p-node.mjs';
|
|
53
|
+
import { createSigner } from '../signing.mjs';
|
|
54
|
+
import { joinUrl, resolveBaseUrl } from './url.mjs';
|
|
55
|
+
|
|
56
|
+
/** Wire protocol version (§1). */
|
|
57
|
+
export const PROTOCOL_VERSION = 1;
|
|
58
|
+
|
|
59
|
+
/** Envelope schema version (§5.1). */
|
|
60
|
+
export const ENVELOPE_VERSION = '1.0';
|
|
61
|
+
|
|
62
|
+
/** v0.0.1 only ever executes directly (§5.1). */
|
|
63
|
+
export const SHELL_ID = 'direct-exec';
|
|
64
|
+
|
|
65
|
+
/** Reconnect backoff ladder, exactly as specified in task-8 / §3. */
|
|
66
|
+
export const BACKOFF_MS = [500, 1000, 2000, 4000, 8000, 10000];
|
|
67
|
+
|
|
68
|
+
/** Lease renewal cadence (§4.3). */
|
|
69
|
+
export const HEARTBEAT_INTERVAL_MS = 10_000;
|
|
70
|
+
|
|
71
|
+
/** Default task timeout when the offer does not carry one. */
|
|
72
|
+
export const DEFAULT_TIMEOUT_MS = 300_000;
|
|
73
|
+
|
|
74
|
+
/** Rolling window of heartbeat round-trip times kept for `w2m_status` (§8.2). */
|
|
75
|
+
export const RTT_WINDOW = 5;
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Largest round-trip time the agent will report to the relay (v0.3.0).
|
|
79
|
+
*
|
|
80
|
+
* A minute to answer a `POST /v1/heartbeat` is not a latency measurement any
|
|
81
|
+
* more; it is a symptom (a stalled socket, a suspended laptop, a clock jump).
|
|
82
|
+
* An out-of-range sample is **omitted rather than clamped**, because clamping
|
|
83
|
+
* would publish a number nobody measured -- the relay's aggregate can tolerate
|
|
84
|
+
* a missing sample, but not a fabricated one.
|
|
85
|
+
*/
|
|
86
|
+
export const MAX_HEARTBEAT_RTT_MS = 60_000;
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Idle diagnostic heartbeat cadence (v0.3.0).
|
|
90
|
+
*
|
|
91
|
+
* A machine with nothing to do is exactly the one an operator looks at -- "is
|
|
92
|
+
* that Mac still up, and how far away is it?" -- while a lease heartbeat only
|
|
93
|
+
* exists while a task does. Without this the relay's view of an idle machine
|
|
94
|
+
* freezes at its last task, and a `rtt_stale` flag that is permanently true
|
|
95
|
+
* reads like "the machine is there, just slow", which is worse than no value
|
|
96
|
+
* at all.
|
|
97
|
+
*
|
|
98
|
+
* The request is explicitly marked `diagnostic: true`. That flag is what tells
|
|
99
|
+
* the relay "touch no lease": a heartbeat that merely *lacks* `task_id` is
|
|
100
|
+
* indistinguishable from one whose field was lost, and §0 forbids trading a
|
|
101
|
+
* loud error for a quiet false success.
|
|
102
|
+
*/
|
|
103
|
+
export const IDLE_HEARTBEAT_INTERVAL_MS = 60_000;
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* A stream that stayed up at least this long is considered healthy, so the
|
|
107
|
+
* next reconnect starts from the bottom of the backoff ladder again.
|
|
108
|
+
*/
|
|
109
|
+
export const STABLE_STREAM_MS = 1000;
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Delay before reconnecting after a *healthy* stream ended (relay restart,
|
|
113
|
+
* proxy idle timeout). Small but non-zero: reconnecting in a hot loop would
|
|
114
|
+
* hammer a relay that is merely restarting, which is exactly the situation
|
|
115
|
+
* v0.1.2 is about.
|
|
116
|
+
*/
|
|
117
|
+
export const RECONNECT_DELAY_AFTER_STABLE_MS = 500;
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Diagnostics file the agent publishes for *other processes* (task-16).
|
|
121
|
+
*
|
|
122
|
+
* §8.2 puts RTT on the agent's in-memory state, which only helps a plugin that
|
|
123
|
+
* started the agent inside its own process. In a cross-region deployment the
|
|
124
|
+
* agent is its own process, so the same numbers are mirrored to
|
|
125
|
+
* `<stateDir>/agent-state.json`, which is what the plugin's `w2m_status` reads.
|
|
126
|
+
*/
|
|
127
|
+
export const AGENT_STATE_FILE = 'agent-state.json';
|
|
128
|
+
|
|
129
|
+
/** Schema version of {@link AGENT_STATE_FILE}. */
|
|
130
|
+
export const AGENT_STATE_SCHEMA_VERSION = 1;
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* How often the diagnostics file is refreshed while the agent sits idle.
|
|
134
|
+
*
|
|
135
|
+
* Without this, `updated_at` only moves when a heartbeat or a connection change
|
|
136
|
+
* happens -- so an idle-but-healthy agent would look frozen, and a reader could
|
|
137
|
+
* not tell "running, nothing to do" from "killed an hour ago". Cheap: one small
|
|
138
|
+
* atomic write per interval.
|
|
139
|
+
*/
|
|
140
|
+
export const AGENT_STATE_PUBLISH_INTERVAL_MS = 30_000;
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* How many times a relay identity change may trigger an *immediate* cursor-less
|
|
144
|
+
* re-attach before the agent falls back to the normal backoff ladder. A healthy
|
|
145
|
+
* relay identifies itself once per stream, so more than a couple in a row means
|
|
146
|
+
* something is wrong and hammering it would not help.
|
|
147
|
+
*/
|
|
148
|
+
export const MAX_IMMEDIATE_RESTART_RECONNECTS = 3;
|
|
149
|
+
|
|
150
|
+
/** Refusal reason codes this agent can produce. */
|
|
151
|
+
export const REFUSAL = {
|
|
152
|
+
COMMAND_NOT_ALLOWED: 'COMMAND_NOT_ALLOWED',
|
|
153
|
+
PLATFORM_MISMATCH: 'PLATFORM_MISMATCH',
|
|
154
|
+
READ_ONLY_MACHINE: 'READ_ONLY_MACHINE',
|
|
155
|
+
CWD_OUTSIDE_PROJECT: 'CWD_OUTSIDE_PROJECT',
|
|
156
|
+
INVALID_OFFER: 'INVALID_OFFER',
|
|
157
|
+
/**
|
|
158
|
+
* v0.4.0: an offer arrived over the relay while this agent is configured `direct`.
|
|
159
|
+
*
|
|
160
|
+
* See `p2pRefusal()` for why this is the one refusal the *mode* decides rather than the offer:
|
|
161
|
+
* `direct` exists to make a silent fallback impossible, so accepting the relay's copy would
|
|
162
|
+
* quietly turn the mode back into `auto` on the machine whose configuration said otherwise.
|
|
163
|
+
*/
|
|
164
|
+
P2P_UNAVAILABLE: 'P2P_UNAVAILABLE',
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* How long the agent waits for the dispatcher's `result.ack` on a direct channel (v0.4.0).
|
|
169
|
+
*
|
|
170
|
+
* This bound is the whole reason the direct copy is safe to attempt at all. The relay copy is
|
|
171
|
+
* posted only once this wait is over, because `result_path` has to be inside the envelope when
|
|
172
|
+
* `envelope_sha256` is computed -- attaching it afterwards would leave a hash that does not cover
|
|
173
|
+
* the bytes it claims to cover. So the wait is a real delay on the relay copy, and a *bounded* one
|
|
174
|
+
* is the difference between "the fast path is usually faster" and "a wedged dispatcher stalls the
|
|
175
|
+
* ledger". Two seconds is far above any plausible loopback or cross-network ack and far below the
|
|
176
|
+
* relay's lease window, so the relay copy still lands long before anyone would call it late.
|
|
177
|
+
*/
|
|
178
|
+
export const P2P_RESULT_ACK_TIMEOUT_MS = 2_000;
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* How long a task's direct channel outlives the attempt that used it.
|
|
182
|
+
*
|
|
183
|
+
* Long enough to answer a duplicate of a finished attempt over the same channel (the relay replaying
|
|
184
|
+
* an offer, a dispatcher retrying a push), short enough that a long-lived agent does not accumulate
|
|
185
|
+
* one live session per task it has ever run. Injectable as `p2pChannelLingerMs`, because a test that
|
|
186
|
+
* asserts "the channel is released" should be able to make that fact observable in milliseconds
|
|
187
|
+
* rather than describe a 30-second wait.
|
|
188
|
+
*/
|
|
189
|
+
export const P2P_CHANNEL_LINGER_MS = 30_000;
|
|
190
|
+
/**
|
|
191
|
+
* Upper bound on the stages a single offer may carry (v0.3.3 `pipeline`).
|
|
192
|
+
*
|
|
193
|
+
* Mirrors the relay's `MAX_PIPELINE_STAGES`. Kept as a separate literal rather than imported so the
|
|
194
|
+
* agent's validation does not depend on the relay module: the agent must refuse an oversized chain
|
|
195
|
+
* on its own authority, and a shared import would make "the relay said so" the only reason it does.
|
|
196
|
+
*/
|
|
197
|
+
const MAX_STAGES = 16;
|
|
198
|
+
|
|
199
|
+
/** Required envelope fields (§5.1). Missing any one makes a result unverifiable. */
|
|
200
|
+
export const REQUIRED_ENVELOPE_FIELDS = [ 'envelope_version',
|
|
201
|
+
'task_id',
|
|
202
|
+
'attempt',
|
|
203
|
+
'dedupe_key',
|
|
204
|
+
'machine_id',
|
|
205
|
+
'machine_name',
|
|
206
|
+
'platform',
|
|
207
|
+
'caps',
|
|
208
|
+
'index',
|
|
209
|
+
'index_total',
|
|
210
|
+
'mode',
|
|
211
|
+
'cwd_rel',
|
|
212
|
+
'base_commit',
|
|
213
|
+
'base_tree',
|
|
214
|
+
'pre_tree_fingerprint',
|
|
215
|
+
'post_tree_fingerprint',
|
|
216
|
+
'fingerprint_algo',
|
|
217
|
+
'fingerprint_error',
|
|
218
|
+
'head_commit',
|
|
219
|
+
'dirty_before',
|
|
220
|
+
'command_argv',
|
|
221
|
+
'command_hash',
|
|
222
|
+
'shell_id',
|
|
223
|
+
'started_at',
|
|
224
|
+
'ended_at',
|
|
225
|
+
'duration_ms',
|
|
226
|
+
'exit_code',
|
|
227
|
+
'status',
|
|
228
|
+
'refusal_reason',
|
|
229
|
+
'stdout_sha256',
|
|
230
|
+
'stdout_bytes',
|
|
231
|
+
'stderr_sha256',
|
|
232
|
+
'stderr_bytes',
|
|
233
|
+
'warnings',
|
|
234
|
+
'envelope_sha256',
|
|
235
|
+
];
|
|
236
|
+
|
|
237
|
+
// ---------------------------------------------------------------------------
|
|
238
|
+
// Small utilities
|
|
239
|
+
// ---------------------------------------------------------------------------
|
|
240
|
+
|
|
241
|
+
/** @param {number} ms @param {AbortSignal} [signal] */
|
|
242
|
+
export function sleep(ms, signal) {
|
|
243
|
+
return new Promise((resolve) => {
|
|
244
|
+
if (signal?.aborted) {
|
|
245
|
+
resolve();
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
const timer = setTimeout(resolve, ms);
|
|
249
|
+
signal?.addEventListener(
|
|
250
|
+
'abort',
|
|
251
|
+
() => {
|
|
252
|
+
clearTimeout(timer);
|
|
253
|
+
resolve();
|
|
254
|
+
},
|
|
255
|
+
{ once: true },
|
|
256
|
+
);
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Reconnect delay: the ladder position for `attempt`, jittered to 50%-100%.
|
|
262
|
+
*
|
|
263
|
+
* Jitter never exceeds the ladder value, so the sequence stays recognisable
|
|
264
|
+
* (500 / 1000 / 2000 / 4000 / 8000 / 10000) while still de-synchronising
|
|
265
|
+
* many clients that dropped at the same moment.
|
|
266
|
+
*
|
|
267
|
+
* @param {number} attempt 0-based reconnect attempt.
|
|
268
|
+
* @param {() => number} [rand]
|
|
269
|
+
* @returns {number} milliseconds
|
|
270
|
+
*/
|
|
271
|
+
export function backoffDelay(attempt, rand = Math.random) {
|
|
272
|
+
const index = Math.max(0, Math.min(Math.trunc(attempt) || 0, BACKOFF_MS.length - 1));
|
|
273
|
+
const base = BACKOFF_MS[index];
|
|
274
|
+
const factor = 0.5 + Math.min(Math.max(rand(), 0), 1) * 0.5;
|
|
275
|
+
return Math.round(base * factor);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Summarise heartbeat round-trip samples for `state.rttMs` (§8.2).
|
|
280
|
+
*
|
|
281
|
+
* Exported so the shape the plugin reads through `w2m_status` is testable
|
|
282
|
+
* without a live relay.
|
|
283
|
+
*
|
|
284
|
+
* @param {number[]} samples Round-trip times in milliseconds, oldest first.
|
|
285
|
+
* @param {number} [window]
|
|
286
|
+
* @returns {{last: number|null, avg: number|null, samples: number[]}}
|
|
287
|
+
*/
|
|
288
|
+
export function summarizeRtt(samples, window = RTT_WINDOW) {
|
|
289
|
+
const kept = (Array.isArray(samples) ? samples : [])
|
|
290
|
+
.filter((value) => Number.isFinite(value))
|
|
291
|
+
.slice(-window)
|
|
292
|
+
.map((value) => Math.round(value));
|
|
293
|
+
if (kept.length === 0) return { last: null, avg: null, samples: [] };
|
|
294
|
+
const avg = Math.round(kept.reduce((sum, value) => sum + value, 0) / kept.length);
|
|
295
|
+
return { last: kept[kept.length - 1], avg, samples: kept };
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* Decide whether the stored event cursor may be sent when attaching.
|
|
300
|
+
*
|
|
301
|
+
* A cursor is only meaningful to the relay process that issued it. `seq` is
|
|
302
|
+
* per-process (v0.1.2 §8.3), so after a restart the new process numbers events
|
|
303
|
+
* from 1 again: replaying `Last-Event-ID: 42` tells it we are already past
|
|
304
|
+
* events 1..42 -- including the `task.offer` that landed at seq 1, which is
|
|
305
|
+
* exactly the event the agent then waits forever for. The cursor therefore
|
|
306
|
+
* travels with the `relay_id` that issued it, and is withheld whenever that
|
|
307
|
+
* differs from the relay we are attaching to.
|
|
308
|
+
*
|
|
309
|
+
* Two cases must NOT be over-corrected:
|
|
310
|
+
* * a pre-v0.1.2 relay sends no `relay_id`, so there is nothing to compare
|
|
311
|
+
* against and the v1 behaviour (resume) is kept;
|
|
312
|
+
* * an unchanged `relay_id` must still resume, or every reconnect becomes a
|
|
313
|
+
* full replay.
|
|
314
|
+
*
|
|
315
|
+
* @param {{seq?: number, cursorRelayId?: string|null, relayId?: string|null}} cursor
|
|
316
|
+
* @returns {boolean}
|
|
317
|
+
*/
|
|
318
|
+
export function cursorUsableFor({ seq, cursorRelayId = null, relayId = null } = {}) {
|
|
319
|
+
if (!Number.isInteger(seq) || seq <= 0) return false;
|
|
320
|
+
// No relay identity to compare against: keep the v1 resume behaviour.
|
|
321
|
+
if (relayId === null || relayId === undefined) return true;
|
|
322
|
+
// The relay identifies itself, so only a cursor it issued can be replayed.
|
|
323
|
+
return cursorRelayId === relayId;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* Extra fields the heartbeat carries so a *remote* reader can see this
|
|
328
|
+
* machine's link quality (v0.3.0): the relay stores them and exposes them
|
|
329
|
+
* through its per-machine status endpoint, which is the only way a plugin on
|
|
330
|
+
* another host can see the latency.
|
|
331
|
+
*
|
|
332
|
+
* Rules, all of them about not inventing data:
|
|
333
|
+
*
|
|
334
|
+
* * `rtt_ms` is present **only** when a round trip was actually measured.
|
|
335
|
+
* Absence means "unknown"; `null` would be an extra shape for the relay to
|
|
336
|
+
* handle, and `0` would be read as "extremely fast", turning ignorance into
|
|
337
|
+
* a good score.
|
|
338
|
+
* * The value is a non-negative integer (`last` is already rounded), and a
|
|
339
|
+
* sample above {@link MAX_HEARTBEAT_RTT_MS} is dropped rather than clamped.
|
|
340
|
+
* * `unstable_reconnects` is always sent: it is a counter the agent owns, and
|
|
341
|
+
* its absence would be indistinguishable from zero.
|
|
342
|
+
*
|
|
343
|
+
* `unstable_reconnects` deliberately does **not** reuse the name
|
|
344
|
+
* `reconnect_attempts`: the relay publishes its own `reconnect_attempts`
|
|
345
|
+
* (attaches - 1, see `src/relay/state.mjs`), and two different quantities under
|
|
346
|
+
* one name would make "this link is flapping right now" and "this machine has
|
|
347
|
+
* reconnected N times in total" look like the same fact. The relay's count is
|
|
348
|
+
* authoritative during a crash (it keeps counting); this one is the only thing
|
|
349
|
+
* that can say the link is unstable *at the moment*.
|
|
350
|
+
*
|
|
351
|
+
* The local file channel keeps working in parallel -- same numbers, different
|
|
352
|
+
* failure modes: it is faster on the same host and still correct when the relay
|
|
353
|
+
* is unreachable, while this one is the only channel that crosses machines.
|
|
354
|
+
*
|
|
355
|
+
* @param {{rttMs?: {last: number|null}|null, reconnectAttempts?: number|null}} snapshot
|
|
356
|
+
* `reconnectAttempts` is the in-process counter (also exposed as
|
|
357
|
+
* `state.unstableReconnects`); it is published as `unstable_reconnects`.
|
|
358
|
+
* @param {{maxRttMs?: number}} [options]
|
|
359
|
+
* @returns {Record<string, number>}
|
|
360
|
+
*/
|
|
361
|
+
export function heartbeatDiagnostics(snapshot = {}, options = {}) {
|
|
362
|
+
const maxRttMs = options.maxRttMs ?? MAX_HEARTBEAT_RTT_MS;
|
|
363
|
+
/** @type {Record<string, number>} */
|
|
364
|
+
const extra = {};
|
|
365
|
+
|
|
366
|
+
const last = snapshot?.rttMs?.last;
|
|
367
|
+
if (typeof last === 'number' && Number.isFinite(last) && last >= 0 && last <= maxRttMs) {
|
|
368
|
+
extra.rtt_ms = Math.round(last);
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
const attempts = snapshot?.reconnectAttempts;
|
|
372
|
+
if (typeof attempts === 'number' && Number.isFinite(attempts) && attempts >= 0) {
|
|
373
|
+
extra.unstable_reconnects = Math.trunc(attempts);
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
return extra;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// ---------------------------------------------------------------------------
|
|
380
|
+
// P2P transport facts (v0.4.0)
|
|
381
|
+
// ---------------------------------------------------------------------------
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* The two transports an offer or a result can travel over.
|
|
385
|
+
*
|
|
386
|
+
* Duplicated from `src/relay/state.mjs`'s `TRANSPORT_VALUES` on purpose, for the same reason
|
|
387
|
+
* `P2PNode` duplicates the relay's IPv4 rule: they are wire values that two independently
|
|
388
|
+
* deployed ends must agree on, and a shared import would make one side's edit silently change
|
|
389
|
+
* what the other accepts.
|
|
390
|
+
*/
|
|
391
|
+
export const TRANSPORT_VALUES = Object.freeze(['p2p', 'relay']);
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* A JSON *frame* arriving on a P2P channel.
|
|
395
|
+
*
|
|
396
|
+
* The channel carries opaque bytes (`P2PNode` deliberately does not parse them) and the W2M
|
|
397
|
+
* convention is that the bytes are the relay's own JSON frames. Parsing happens here, in the
|
|
398
|
+
* agent, because "what a frame means for a task" is a task-layer question; from this point on
|
|
399
|
+
* the payload takes `handleFrame`'s path with `transport: 'p2p'`, which is what makes a direct
|
|
400
|
+
* offer and an SSE offer the *same* code path rather than two that have to be kept in step.
|
|
401
|
+
*
|
|
402
|
+
* @param {unknown} payload The channel's message body.
|
|
403
|
+
* @returns {object|null} The payload object, or null when it is not a JSON object with a `type`.
|
|
404
|
+
*/
|
|
405
|
+
export function parseP2PFrame(payload) {
|
|
406
|
+
const text = Buffer.isBuffer(payload) ? payload.toString('utf8') : payload;
|
|
407
|
+
let parsed;
|
|
408
|
+
try {
|
|
409
|
+
parsed = JSON.parse(typeof text === 'string' ? text : String(text));
|
|
410
|
+
} catch {
|
|
411
|
+
return null;
|
|
412
|
+
}
|
|
413
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return null;
|
|
414
|
+
if (typeof parsed.type !== 'string' || parsed.type === '') return null;
|
|
415
|
+
return parsed;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Which path an offer arrived on, and -- when it arrived over the relay -- why the direct path
|
|
420
|
+
* was not the answer.
|
|
421
|
+
*
|
|
422
|
+
* The transport value is taken from `offer.p2p_transport`, which the agent itself stamps on a
|
|
423
|
+
* channel-delivered offer and only there. Nothing on the wire can set it: a relay frame that
|
|
424
|
+
* carried the field would still be stamped `'relay'` by the SSE path, so a dispatcher cannot
|
|
425
|
+
* claim a direct path it never opened.
|
|
426
|
+
*
|
|
427
|
+
* The reason is named only when the relay really was the answer. `P2P_NO_CANDIDATES` is not a
|
|
428
|
+
* guess about the dispatcher's punch -- it is the agent's own finding for *this* task: a relay
|
|
429
|
+
* offer means no direct channel exists for this `task_id`, and the reason says exactly that.
|
|
430
|
+
* `P2P_DISABLED` covers the mechanical case (`mode: 'relay'`, where no direct path exists at all);
|
|
431
|
+
* `P2P_UNREACHABLE` covers the rare ordering where the offer beat the node's own startup.
|
|
432
|
+
*
|
|
433
|
+
* @param {object|null} offer
|
|
434
|
+
* @param {{mode: string, running: boolean}} p2p
|
|
435
|
+
* @returns {{offer_path: 'p2p'|'relay', reason: string|null}}
|
|
436
|
+
*/
|
|
437
|
+
export function offerTransportOf(offer, p2p) {
|
|
438
|
+
if (offer?.p2p_transport === 'p2p') return { offer_path: 'p2p', reason: null };
|
|
439
|
+
if (p2p.mode === 'relay') {
|
|
440
|
+
return {
|
|
441
|
+
offer_path: 'relay',
|
|
442
|
+
reason: 'P2P_DISABLED: p2pMode is "relay", so no direct path is attempted',
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
if (p2p.running !== true) {
|
|
446
|
+
return {
|
|
447
|
+
offer_path: 'relay',
|
|
448
|
+
reason: 'P2P_UNREACHABLE: the P2P node is not running, so no direct path was available',
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
return {
|
|
452
|
+
offer_path: 'relay',
|
|
453
|
+
reason: 'P2P_NO_CANDIDATES: no direct channel was open for this task, so the relay carried the offer',
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/**
|
|
458
|
+
* The result-delivery mark: how the *envelope* is being carried, as opposed to how the offer
|
|
459
|
+
* arrived.
|
|
460
|
+
*
|
|
461
|
+
* Kept as a plain object so it can be handed through `finish()`/`makeEnvelope()` unchanged, and
|
|
462
|
+
* `asked` distinguishes "nothing to acknowledge over" from "asked, and nobody answered" -- both
|
|
463
|
+
* end up as `result_path: null`, but only the second one is worth a metric.
|
|
464
|
+
*
|
|
465
|
+
* @param {{path: 'p2p'|null, asked: boolean, rtt_ms: number|null, error: string|null}} value
|
|
466
|
+
* @returns {{path: 'p2p'|null, asked: boolean, rtt_ms: number|null, error: string|null}}
|
|
467
|
+
*/
|
|
468
|
+
export function p2pDeliveryMark(value = {}) {
|
|
469
|
+
return {
|
|
470
|
+
path: value.path === 'p2p' ? 'p2p' : null,
|
|
471
|
+
asked: value.asked === true,
|
|
472
|
+
rtt_ms: typeof value.rtt_ms === 'number' && Number.isFinite(value.rtt_ms) ? value.rtt_ms : null,
|
|
473
|
+
error: typeof value.error === 'string' && value.error !== '' ? value.error : null,
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
// ---------------------------------------------------------------------------
|
|
478
|
+
// Command allow-list (default-deny)
|
|
479
|
+
// ---------------------------------------------------------------------------
|
|
480
|
+
|
|
481
|
+
// The matcher itself lives in `./allowed-commands.mjs`, because the plugin's dispatch-time
|
|
482
|
+
// pre-flight must apply the *same* rule (see that file's header). It is imported above for this
|
|
483
|
+
// module's own use and re-exported here so the exports of this module are unchanged: the CLI and
|
|
484
|
+
// `test/agent.test.mjs` have always imported these two names from the agent.
|
|
485
|
+
export { parseAllowedCommands, matchAllowedCommand } from './allowed-commands.mjs';
|
|
486
|
+
|
|
487
|
+
// ---------------------------------------------------------------------------
|
|
488
|
+
// Capability gate (§6.1)
|
|
489
|
+
// ---------------------------------------------------------------------------
|
|
490
|
+
|
|
491
|
+
/** @param {string} value @returns {number[]} */
|
|
492
|
+
function versionParts(value) {
|
|
493
|
+
return String(value)
|
|
494
|
+
.trim()
|
|
495
|
+
.replace(/^v/i, '')
|
|
496
|
+
.split(/[.\-+]/)
|
|
497
|
+
.map((part) => Number.parseInt(part, 10))
|
|
498
|
+
.map((part) => (Number.isFinite(part) ? part : 0));
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/**
|
|
502
|
+
* Compare two version strings numerically.
|
|
503
|
+
*
|
|
504
|
+
* @param {string} a
|
|
505
|
+
* @param {string} b
|
|
506
|
+
* @returns {number} -1, 0 or 1
|
|
507
|
+
*/
|
|
508
|
+
export function compareVersions(a, b) {
|
|
509
|
+
const left = versionParts(a);
|
|
510
|
+
const right = versionParts(b);
|
|
511
|
+
const length = Math.max(left.length, right.length);
|
|
512
|
+
for (let i = 0; i < length; i += 1) {
|
|
513
|
+
const l = left[i] ?? 0;
|
|
514
|
+
const r = right[i] ?? 0;
|
|
515
|
+
if (l < r) return -1;
|
|
516
|
+
if (l > r) return 1;
|
|
517
|
+
}
|
|
518
|
+
return 0;
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
/** @param {number[]} a @param {number[]} b @param {number} length */
|
|
522
|
+
function samePrefix(a, b, length) {
|
|
523
|
+
for (let i = 0; i < length; i += 1) {
|
|
524
|
+
if ((a[i] ?? 0) !== (b[i] ?? 0)) return false;
|
|
525
|
+
}
|
|
526
|
+
return true;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/**
|
|
530
|
+
* Semver-ish range check, deliberately small: `>=20`, `>=20 <22`, `^20.1`,
|
|
531
|
+
* `~20`, `20`, `=20.1.0`, with `||` alternatives.
|
|
532
|
+
*
|
|
533
|
+
* @param {string} version
|
|
534
|
+
* @param {string} range
|
|
535
|
+
* @returns {boolean}
|
|
536
|
+
*/
|
|
537
|
+
export function satisfiesVersion(version, range) {
|
|
538
|
+
if (typeof range !== 'string' || range.trim() === '') return true;
|
|
539
|
+
const current = versionParts(version);
|
|
540
|
+
return range.split('||').some((alternative) => {
|
|
541
|
+
const tokens = alternative.trim().split(/[\s,]+/).filter((token) => token !== '');
|
|
542
|
+
if (tokens.length === 0) return true;
|
|
543
|
+
return tokens.every((token) => {
|
|
544
|
+
const match = token.match(/^(\^|~|>=|<=|==|=|>|<)?\s*(.+)$/);
|
|
545
|
+
if (!match) return false;
|
|
546
|
+
const op = match[1] ?? '';
|
|
547
|
+
const target = versionParts(match[2]);
|
|
548
|
+
switch (op) {
|
|
549
|
+
case '>=':
|
|
550
|
+
return compareVersions(version, match[2]) >= 0;
|
|
551
|
+
case '<=':
|
|
552
|
+
return compareVersions(version, match[2]) <= 0;
|
|
553
|
+
case '>':
|
|
554
|
+
return compareVersions(version, match[2]) > 0;
|
|
555
|
+
case '<':
|
|
556
|
+
return compareVersions(version, match[2]) < 0;
|
|
557
|
+
case '^': {
|
|
558
|
+
// `^20.1.0` means >=20.1.0 <21.0.0: the upper bound comes from the
|
|
559
|
+
// *target's* major, not from the version being tested.
|
|
560
|
+
const major = target[0] ?? 0;
|
|
561
|
+
const upper = major > 0 ? [major + 1, 0, 0] : [0, (target[1] ?? 0) + 1, 0];
|
|
562
|
+
return compareVersions(version, match[2]) >= 0 && compareVersions(version, upper.join('.')) < 0;
|
|
563
|
+
}
|
|
564
|
+
case '~': {
|
|
565
|
+
const upper = [target[0] ?? 0, (target[1] ?? 0) + 1, 0];
|
|
566
|
+
return compareVersions(version, match[2]) >= 0 && compareVersions(version, upper.join('.')) < 0;
|
|
567
|
+
}
|
|
568
|
+
default:
|
|
569
|
+
// Bare version: prefix match (`20` matches 20.9.1, not 21.0.0).
|
|
570
|
+
return samePrefix(current, target, target.length);
|
|
571
|
+
}
|
|
572
|
+
});
|
|
573
|
+
});
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* Evaluate §6.1 for one machine.
|
|
578
|
+
*
|
|
579
|
+
* Toolchain lookups consult `caps` first and fall back to `toolchain`, because
|
|
580
|
+
* the contract's `caps` object only carries `python`/`npm`/`node` -- a task
|
|
581
|
+
* requiring e.g. `toolchain: {git: ">=2"}` would otherwise be refused as
|
|
582
|
+
* `MISSING_GIT` on a machine that has git installed.
|
|
583
|
+
*
|
|
584
|
+
* @param {object} offer
|
|
585
|
+
* @param {object} context
|
|
586
|
+
* @param {Record<string, string|null>} context.caps
|
|
587
|
+
* @param {{os: string}} context.platform
|
|
588
|
+
* @param {boolean} context.writable False when the machine cannot write.
|
|
589
|
+
* @param {Record<string, string|null>} [context.toolchain] Extra detected tools.
|
|
590
|
+
* @returns {{ok: boolean, refusal_reason: string|null, detail: string|null}}
|
|
591
|
+
*/
|
|
592
|
+
export function evaluateGate(offer, context) {
|
|
593
|
+
const requirements = offer?.requirements ?? {};
|
|
594
|
+
const toolchain = requirements.toolchain ?? {};
|
|
595
|
+
for (const [tool, range] of Object.entries(toolchain)) {
|
|
596
|
+
const have = context.caps?.[tool] ?? context.toolchain?.[tool] ?? null;
|
|
597
|
+
if (have === null) {
|
|
598
|
+
return { ok: false, refusal_reason: `MISSING_${tool.toUpperCase()}`, detail: `${tool} not installed` };
|
|
599
|
+
}
|
|
600
|
+
if (typeof range !== 'string') {
|
|
601
|
+
return {
|
|
602
|
+
ok: false,
|
|
603
|
+
refusal_reason: `MISSING_${tool.toUpperCase()}`,
|
|
604
|
+
detail: `unusable toolchain constraint for ${tool}`,
|
|
605
|
+
};
|
|
606
|
+
}
|
|
607
|
+
if (!satisfiesVersion(have, range)) {
|
|
608
|
+
return {
|
|
609
|
+
ok: false,
|
|
610
|
+
refusal_reason: `MISSING_${tool.toUpperCase()}`,
|
|
611
|
+
detail: `${tool} ${have} does not satisfy ${range}`,
|
|
612
|
+
};
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
const platforms = requirements.platform;
|
|
616
|
+
if (Array.isArray(platforms) && platforms.length > 0 && !platforms.includes(context.platform?.os)) {
|
|
617
|
+
return {
|
|
618
|
+
ok: false,
|
|
619
|
+
refusal_reason: REFUSAL.PLATFORM_MISMATCH,
|
|
620
|
+
detail: `this machine is ${context.platform?.os}`,
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
if (offer?.write === true && context.writable === false) {
|
|
624
|
+
return { ok: false, refusal_reason: REFUSAL.READ_ONLY_MACHINE, detail: 'project is not writable' };
|
|
625
|
+
}
|
|
626
|
+
return { ok: true, refusal_reason: null, detail: null };
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
// ---------------------------------------------------------------------------
|
|
630
|
+
// SSE parsing
|
|
631
|
+
// ---------------------------------------------------------------------------
|
|
632
|
+
|
|
633
|
+
/**
|
|
634
|
+
* Incremental SSE frame parser (§3.1).
|
|
635
|
+
*
|
|
636
|
+
* Handles frames split across chunks, CRLF, multi-line `data:`, and comment
|
|
637
|
+
* keepalives (`: keepalive`), which must not be mistaken for events.
|
|
638
|
+
*/
|
|
639
|
+
export class SseParser {
|
|
640
|
+
constructor() {
|
|
641
|
+
this.buffer = '';
|
|
642
|
+
this.event = '';
|
|
643
|
+
this.data = [];
|
|
644
|
+
this.id = null;
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/**
|
|
648
|
+
* Feed decoded text, get whatever complete events it produced.
|
|
649
|
+
*
|
|
650
|
+
* @param {string} chunk
|
|
651
|
+
* @returns {Array<{event: string, data: string, id: string|null}>}
|
|
652
|
+
*/
|
|
653
|
+
push(chunk) {
|
|
654
|
+
this.buffer += chunk;
|
|
655
|
+
const events = [];
|
|
656
|
+
let index = this.buffer.indexOf('\n');
|
|
657
|
+
while (index !== -1) {
|
|
658
|
+
let line = this.buffer.slice(0, index);
|
|
659
|
+
this.buffer = this.buffer.slice(index + 1);
|
|
660
|
+
if (line.endsWith('\r')) line = line.slice(0, -1);
|
|
661
|
+
|
|
662
|
+
if (line === '') {
|
|
663
|
+
if (this.data.length > 0) {
|
|
664
|
+
events.push({ event: this.event || 'message', data: this.data.join('\n'), id: this.id });
|
|
665
|
+
}
|
|
666
|
+
this.event = '';
|
|
667
|
+
this.data = [];
|
|
668
|
+
this.id = null;
|
|
669
|
+
} else if (line.startsWith(':')) {
|
|
670
|
+
// keepalive / comment
|
|
671
|
+
} else {
|
|
672
|
+
const colon = line.indexOf(':');
|
|
673
|
+
const field = colon === -1 ? line : line.slice(0, colon);
|
|
674
|
+
let value = colon === -1 ? '' : line.slice(colon + 1);
|
|
675
|
+
if (value.startsWith(' ')) value = value.slice(1);
|
|
676
|
+
if (field === 'event') this.event = value;
|
|
677
|
+
else if (field === 'data') this.data.push(value);
|
|
678
|
+
else if (field === 'id') this.id = value;
|
|
679
|
+
}
|
|
680
|
+
index = this.buffer.indexOf('\n');
|
|
681
|
+
}
|
|
682
|
+
return events;
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
// ---------------------------------------------------------------------------
|
|
687
|
+
// Envelope
|
|
688
|
+
// ---------------------------------------------------------------------------
|
|
689
|
+
|
|
690
|
+
/**
|
|
691
|
+
* Attach `envelope_sha256` (§5.1: JCS hash of the envelope without it).
|
|
692
|
+
*
|
|
693
|
+
* @param {object} envelope
|
|
694
|
+
* @returns {object}
|
|
695
|
+
*/
|
|
696
|
+
export function buildEnvelope(envelope) {
|
|
697
|
+
for (const field of REQUIRED_ENVELOPE_FIELDS) {
|
|
698
|
+
if (field === 'envelope_sha256') continue;
|
|
699
|
+
if (!(field in envelope)) {
|
|
700
|
+
throw new TypeError(`envelope is missing required field: ${field}`);
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
return { ...envelope, envelope_sha256: envelopeSha256(envelope) };
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
/**
|
|
707
|
+
* Verify a received envelope's `envelope_sha256`.
|
|
708
|
+
*
|
|
709
|
+
* @param {object} envelope
|
|
710
|
+
* @returns {boolean}
|
|
711
|
+
*/
|
|
712
|
+
export function verifyEnvelope(envelope) {
|
|
713
|
+
if (!envelope || typeof envelope.envelope_sha256 !== 'string') return false;
|
|
714
|
+
return envelopeSha256(envelope) === envelope.envelope_sha256;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
// ---------------------------------------------------------------------------
|
|
718
|
+
// Heartbeat
|
|
719
|
+
// ---------------------------------------------------------------------------
|
|
720
|
+
|
|
721
|
+
/**
|
|
722
|
+
* Lease heartbeat (§4.3). `phase` moves preparing -> running -> finalizing and
|
|
723
|
+
* the response's `cancel` flag is honoured by aborting the running command.
|
|
724
|
+
*/
|
|
725
|
+
export class Heartbeat {
|
|
726
|
+
/**
|
|
727
|
+
* @param {object} options
|
|
728
|
+
* @param {(body: object) => Promise<object|null>} options.send
|
|
729
|
+
* @param {string} options.taskId
|
|
730
|
+
* @param {string} options.machineId
|
|
731
|
+
* @param {number} options.attempt
|
|
732
|
+
* @param {number} [options.intervalMs]
|
|
733
|
+
* @param {(error: unknown) => void} [options.onError]
|
|
734
|
+
* @param {() => void} [options.onCancel]
|
|
735
|
+
* @param {(ms: number) => void} [options.onRtt] Called with each successful round trip.
|
|
736
|
+
* @param {() => object} [options.diagnostics] Extra body fields per heartbeat (v0.3.0).
|
|
737
|
+
*/
|
|
738
|
+
constructor(options) {
|
|
739
|
+
this.send = options.send;
|
|
740
|
+
this.taskId = options.taskId;
|
|
741
|
+
this.machineId = options.machineId;
|
|
742
|
+
this.attempt = options.attempt;
|
|
743
|
+
this.intervalMs = options.intervalMs ?? HEARTBEAT_INTERVAL_MS;
|
|
744
|
+
this.onError = options.onError ?? (() => {});
|
|
745
|
+
this.onCancel = options.onCancel ?? (() => {});
|
|
746
|
+
this.onRtt = options.onRtt ?? (() => {});
|
|
747
|
+
this.diagnostics = options.diagnostics ?? null;
|
|
748
|
+
this.phase = 'preparing';
|
|
749
|
+
this.progress = 0;
|
|
750
|
+
this.ticks = 0;
|
|
751
|
+
this.timer = null;
|
|
752
|
+
this.cancelled = false;
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
/** @param {'preparing'|'running'|'finalizing'} phase @param {number} [progress] */
|
|
756
|
+
setPhase(phase, progress) {
|
|
757
|
+
this.phase = phase;
|
|
758
|
+
if (typeof progress === 'number') this.progress = progress;
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
/**
|
|
762
|
+
* The body of one heartbeat, including whatever diagnostics the caller
|
|
763
|
+
* supplies.
|
|
764
|
+
*
|
|
765
|
+
* @returns {object}
|
|
766
|
+
*/
|
|
767
|
+
heartbeatBody() {
|
|
768
|
+
const body = {
|
|
769
|
+
task_id: this.taskId,
|
|
770
|
+
machine_id: this.machineId,
|
|
771
|
+
attempt: this.attempt,
|
|
772
|
+
phase: this.phase,
|
|
773
|
+
progress: this.progress,
|
|
774
|
+
};
|
|
775
|
+
if (this.diagnostics) {
|
|
776
|
+
try {
|
|
777
|
+
Object.assign(body, this.diagnostics() ?? {});
|
|
778
|
+
} catch {
|
|
779
|
+
// A broken diagnostics provider must never cost us the lease renewal:
|
|
780
|
+
// the heartbeat's job is to keep the task alive, and the extra fields
|
|
781
|
+
// are exactly that -- extra.
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
return body;
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
/** Send one heartbeat; returns the decoded response. */
|
|
788
|
+
async tick() {
|
|
789
|
+
this.ticks += 1;
|
|
790
|
+
const startedHr = process.hrtime.bigint();
|
|
791
|
+
try {
|
|
792
|
+
const response = await this.send(this.heartbeatBody());
|
|
793
|
+
// Measured on success only: a timeout is not a round-trip time, and
|
|
794
|
+
// folding it in would make "latency to the relay" look catastrophic.
|
|
795
|
+
this.onRtt(Number(process.hrtime.bigint() - startedHr) / 1e6);
|
|
796
|
+
if (response && response.cancel === true) {
|
|
797
|
+
this.cancelled = true;
|
|
798
|
+
this.onCancel();
|
|
799
|
+
}
|
|
800
|
+
return response;
|
|
801
|
+
} catch (error) {
|
|
802
|
+
this.onError(error);
|
|
803
|
+
return null;
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
start() {
|
|
808
|
+
if (this.timer) return this;
|
|
809
|
+
// Fire once immediately so Rabbit learns about the claim without waiting a
|
|
810
|
+
// full interval, then keep the lease warm.
|
|
811
|
+
void this.tick();
|
|
812
|
+
this.timer = setInterval(() => void this.tick(), this.intervalMs);
|
|
813
|
+
return this;
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
stop() {
|
|
817
|
+
if (this.timer) clearInterval(this.timer);
|
|
818
|
+
this.timer = null;
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
// ---------------------------------------------------------------------------
|
|
823
|
+
// The agent
|
|
824
|
+
// ---------------------------------------------------------------------------
|
|
825
|
+
|
|
826
|
+
/**
|
|
827
|
+
* @typedef {object} AgentOptions
|
|
828
|
+
* @property {string} rabbitUrl Base address; a sub-path is allowed
|
|
829
|
+
* (`https://host/w2m`), a query string or fragment is rejected. See ./url.mjs.
|
|
830
|
+
* @property {string} project Absolute project root.
|
|
831
|
+
* @property {string} stateDir Directory for spool/state (never the project).
|
|
832
|
+
* @property {object} identity `{machine_id, machine_name, device_token, rabbit_url}`.
|
|
833
|
+
* @property {Record<string, unknown>} caps
|
|
834
|
+
* @property {{os: string, os_version: string, arch: string, shell: string, shell_version: string|null}} platform
|
|
835
|
+
* @property {string[][]} [allowedCommands]
|
|
836
|
+
* @property {boolean} [once]
|
|
837
|
+
* @property {(level: string, message: string, extra?: object) => void} [log]
|
|
838
|
+
* @property {typeof fetch} [fetchImpl]
|
|
839
|
+
* @property {number} [heartbeatIntervalMs]
|
|
840
|
+
* @property {number} [maxOutputBytes]
|
|
841
|
+
* @property {'temp'|'project'} [objectMode]
|
|
842
|
+
* @property {string} [tmpDir]
|
|
843
|
+
* @property {number} [onceIdleMs]
|
|
844
|
+
* @property {number} [resultRetries]
|
|
845
|
+
* @property {number} [statePublishIntervalMs] Idle refresh cadence for the
|
|
846
|
+
* diagnostics file; 0 disables it (task-16).
|
|
847
|
+
* @property {string|null} [operatorToken] Only needed when this host submits
|
|
848
|
+
* tasks itself (v0.1.2 §5); never sent by the Localside agent.
|
|
849
|
+
* @property {string} [p2pMode] `auto` (default), `direct` or `relay` (v0.4.0). Anything
|
|
850
|
+
* else is a `TypeError` naming {@link normalizeP2PMode}'s reason: a mode is a wire value,
|
|
851
|
+
* so a typo must fail at the call site rather than degrade into a silent default.
|
|
852
|
+
* @property {string[]|string} [stunServers] Complete STUN list, in query order, for the
|
|
853
|
+
* agent's own node. See {@link stunServersWithShared}.
|
|
854
|
+
* @property {number|null} [p2pPort] Local UDP port the punch socket binds, 1..65535; `null`
|
|
855
|
+
* or omitted means an ephemeral port, which is the default. A machine that must **accept**
|
|
856
|
+
* a punch needs a stable port: an ephemeral one changes on every restart, so a firewall
|
|
857
|
+
* rule has to be re-pointed after each start and every dispatch falls back to the relay in
|
|
858
|
+
* the meantime (`P2P_PUNCH_TIMEOUT` on the dispatcher, `offer_path: relay` in the ledger).
|
|
859
|
+
* Anything that is not a valid port is reported as a `P2P_PORT_INVALID` start failure, never
|
|
860
|
+
* thrown -- see {@link normalizeP2PPort} and `startP2P()`, which treats it like every other
|
|
861
|
+
* P2P error: recorded, and never fatal to the relay path.
|
|
862
|
+
* @property {P2PNode|((options: object) => P2PNode)} [p2pNode] The node to use. An
|
|
863
|
+
* instance is adopted; a function is called with the derived options. Injectable
|
|
864
|
+
* because a test needs a real socket and a deterministic discovery answer, and
|
|
865
|
+
* because `test/outbound-only.test.mjs` counts binds through it.
|
|
866
|
+
* @property {object} [p2pNodeOptions] Extra `P2PNode` constructor options, merged last.
|
|
867
|
+
* @property {number} [p2pAckTimeoutMs] Bounded wait for `result.ack` on the direct
|
|
868
|
+
* channel; see {@link P2P_RESULT_ACK_TIMEOUT_MS}.
|
|
869
|
+
* @property {(status: object) => void} [onP2PStatus] Called with a snapshot of the agent's
|
|
870
|
+
* P2P facts whenever they change (the node starting, an announcement, an error). Push,
|
|
871
|
+
* not poll: the plugin's `w2m_status` must not have to ask repeatedly.
|
|
872
|
+
*/
|
|
873
|
+
|
|
874
|
+
/**
|
|
875
|
+
* Build an agent instance.
|
|
876
|
+
*
|
|
877
|
+
* @param {AgentOptions} options
|
|
878
|
+
*/
|
|
879
|
+
export function createAgent(options) {
|
|
880
|
+
const {
|
|
881
|
+
rabbitUrl,
|
|
882
|
+
project,
|
|
883
|
+
stateDir,
|
|
884
|
+
identity,
|
|
885
|
+
caps,
|
|
886
|
+
platform,
|
|
887
|
+
once = false,
|
|
888
|
+
fetchImpl = globalThis.fetch,
|
|
889
|
+
heartbeatIntervalMs = HEARTBEAT_INTERVAL_MS,
|
|
890
|
+
maxOutputBytes,
|
|
891
|
+
objectMode = 'temp',
|
|
892
|
+
tmpDir,
|
|
893
|
+
onceIdleMs = 0,
|
|
894
|
+
resultRetries = 4,
|
|
895
|
+
operatorToken = null,
|
|
896
|
+
statePublishIntervalMs = AGENT_STATE_PUBLISH_INTERVAL_MS,
|
|
897
|
+
idleHeartbeatIntervalMs = IDLE_HEARTBEAT_INTERVAL_MS,
|
|
898
|
+
onP2PStatus = () => {},
|
|
899
|
+
p2pAckTimeoutMs = P2P_RESULT_ACK_TIMEOUT_MS,
|
|
900
|
+
p2pChannelLingerMs = P2P_CHANNEL_LINGER_MS,
|
|
901
|
+
} = options;
|
|
902
|
+
|
|
903
|
+
// Normalise here so every caller gets the same contract: the documented
|
|
904
|
+
// `string | string[][]` form is accepted, and a bare string list such as
|
|
905
|
+
// `['node --test']` cannot reach `.map((prefix) => prefix.join(' '))`.
|
|
906
|
+
const allowedCommands = parseAllowedCommands(options.allowedCommands ?? []);
|
|
907
|
+
|
|
908
|
+
if (!rabbitUrl) throw new TypeError('createAgent: rabbitUrl is required');
|
|
909
|
+
if (!project) throw new TypeError('createAgent: project is required');
|
|
910
|
+
if (!stateDir) throw new TypeError('createAgent: stateDir is required');
|
|
911
|
+
if (!identity?.machine_id) throw new TypeError('createAgent: identity.machine_id is required');
|
|
912
|
+
if (typeof fetchImpl !== 'function') throw new TypeError('createAgent: no fetch implementation available');
|
|
913
|
+
|
|
914
|
+
// v0.4.0: the mode is validated at construction, with the same loud helper the node and the
|
|
915
|
+
// CLI use. `normalizeP2PMode` rejects `'AUTO'`, `' auto '` and friends by name; doing that here
|
|
916
|
+
// means an unusable mode can never reach `start()` as a half-started agent.
|
|
917
|
+
const p2pModeInput = options.p2pMode === undefined ? DEFAULT_P2P_MODE : options.p2pMode;
|
|
918
|
+
const normalizedMode = normalizeP2PMode(p2pModeInput);
|
|
919
|
+
if (!normalizedMode.ok) throw new TypeError(`createAgent: ${normalizedMode.reason}`);
|
|
920
|
+
const p2pMode = normalizedMode.mode;
|
|
921
|
+
|
|
922
|
+
// The punch port is validated at construction with the same named helper the CLI uses, but an
|
|
923
|
+
// unusable value is *not* thrown: a machine whose P2P settings are wrong must still work over the
|
|
924
|
+
// relay, so it becomes a start failure (`P2P_PORT_INVALID`) reported by `startP2P()` exactly like
|
|
925
|
+
// `P2P_NODE_INVALID` or `P2P_BIND_FAILED`. The error is recorded here and logged where the node
|
|
926
|
+
// would have been built.
|
|
927
|
+
const p2pPortResult = normalizeP2PPort(options.p2pPort);
|
|
928
|
+
const p2pPort = p2pPortResult.ok ? p2pPortResult.port : null;
|
|
929
|
+
const p2pPortError = p2pPortResult.ok ? null : `P2P_PORT_INVALID: ${p2pPortResult.reason}`;
|
|
930
|
+
|
|
931
|
+
if (typeof onP2PStatus !== 'function') throw new TypeError('createAgent: onP2PStatus must be a function');
|
|
932
|
+
if (typeof p2pAckTimeoutMs !== 'number' || !Number.isFinite(p2pAckTimeoutMs) || p2pAckTimeoutMs < 0) {
|
|
933
|
+
throw new TypeError('createAgent: p2pAckTimeoutMs must be a non-negative number of milliseconds');
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
// Validate + normalise once, at construction: a `rabbitUrl` carrying a query
|
|
937
|
+
// string, a fragment or a non-http scheme must fail loudly here rather than
|
|
938
|
+
// turn into mysterious 404s at request time. Sub-paths are preserved.
|
|
939
|
+
const baseUrl = resolveBaseUrl(rabbitUrl);
|
|
940
|
+
|
|
941
|
+
// ---- v0.3.0 request signing ------------------------------------------
|
|
942
|
+
// Absent/`null`/`''` means "signing is not configured", and in that case **not
|
|
943
|
+
// one signing header is added anywhere** -- the wire stays byte-identical to
|
|
944
|
+
// v0.2.3, which is the backward-compatibility gate the tests pin down. The
|
|
945
|
+
// empty string is the explicit "off" spelling (same convention as the relay's
|
|
946
|
+
// `--require-signature` settings), so it disables rather than throws.
|
|
947
|
+
//
|
|
948
|
+
// Any *other* unusable value is an error instead of a silent downgrade: a
|
|
949
|
+
// caller who passes a number, an object or `true` meant to configure signing,
|
|
950
|
+
// and quietly giving them an unsigned link is exactly the silent degradation
|
|
951
|
+
// §0 forbids. `signing.mjs`'s `BAD_SECRET` exists for this distinction.
|
|
952
|
+
const signingSecret = options.signingSecret ?? null;
|
|
953
|
+
if (signingSecret !== null && typeof signingSecret !== 'string') {
|
|
954
|
+
throw new TypeError(
|
|
955
|
+
`createAgent: signingSecret must be a string (got ${typeof signingSecret}); ` +
|
|
956
|
+
"pass '' or omit it to disable signing",
|
|
957
|
+
);
|
|
958
|
+
}
|
|
959
|
+
const signer = signingSecret === null || signingSecret === ''
|
|
960
|
+
? null
|
|
961
|
+
: createSigner({ secret: signingSecret, ...(options.signerOptions ?? {}) });
|
|
962
|
+
|
|
963
|
+
const log = options.log ?? (() => {});
|
|
964
|
+
const spool = createSpool(stateDir);
|
|
965
|
+
|
|
966
|
+
// -- P2P transport (v0.4.0) ----------------------------------------------
|
|
967
|
+
//
|
|
968
|
+
// Everything the direct path owns lives in these few bindings, and the shape is deliberate:
|
|
969
|
+
// the agent does not *have* a P2P stack, it *borrows* one node and treats every failure as a
|
|
970
|
+
// reportable fact. `p2pMode: 'relay'` leaves the whole block inert -- `configureP2P()` never
|
|
971
|
+
// constructs a node, so no UDP socket exists and no STUN server is ever queried, which is what
|
|
972
|
+
// makes the setting a mechanical statement ("this machine is exactly v0.3.9") rather than a
|
|
973
|
+
// promise about behaviour.
|
|
974
|
+
|
|
975
|
+
/** @type {P2PNode|null} The node, once one exists. Null in `relay` mode, forever. */
|
|
976
|
+
let p2pNode = null;
|
|
977
|
+
/** @type {Promise<object>|null} In-flight start, so `stop()` can await it. */
|
|
978
|
+
let p2pStarting = null;
|
|
979
|
+
/** @type {object|null} Start outcome, so the status can report *why* the node is not running. */
|
|
980
|
+
let p2pStartResult = null;
|
|
981
|
+
/** @type {object|null} Last snapshot handed to `onP2PStatus`, so a no-op change is not pushed. */
|
|
982
|
+
let lastP2PStatusJson = null;
|
|
983
|
+
|
|
984
|
+
/**
|
|
985
|
+
* Result envelopes awaiting a `result.ack` over a direct channel.
|
|
986
|
+
*
|
|
987
|
+
* Keyed by `task_id` (one result per task and attempt is the whole point of the dedupe work), so
|
|
988
|
+
* a late ack for a re-delivered attempt resolves the live waiter rather than a stale one.
|
|
989
|
+
*
|
|
990
|
+
* @type {Map<string, {settle: (acked: boolean, error: string|null) => void}>}
|
|
991
|
+
*/
|
|
992
|
+
const p2pResultWaiters = new Map();
|
|
993
|
+
|
|
994
|
+
/**
|
|
995
|
+
* Live channels per `task_id`, for the direct result copy *and* for knowing which tasks the
|
|
996
|
+
* dispatcher can already reach us about. Populated only from a `task.offer` that arrived on a
|
|
997
|
+
* channel: a channel that was opened but never used for an offer has no task to be the result of.
|
|
998
|
+
*
|
|
999
|
+
* `onClose` is the listener attached to the channel, kept here so a channel replaced for the
|
|
1000
|
+
* same task can have its old listener removed instead of leaking one per re-punch.
|
|
1001
|
+
*
|
|
1002
|
+
* @type {Map<string, {channel: object, onClose: () => void}>}
|
|
1003
|
+
*/
|
|
1004
|
+
const p2pChannels = new Map();
|
|
1005
|
+
|
|
1006
|
+
/** The agent's P2P facts, always a fresh plain object. */
|
|
1007
|
+
function p2pStatusSnapshot() {
|
|
1008
|
+
const nodeStatus = p2pNode?.status ?? null;
|
|
1009
|
+
const startError = p2pStartResult?.ok === false ? p2pStartResult.error ?? null : null;
|
|
1010
|
+
return {
|
|
1011
|
+
mode: p2pMode,
|
|
1012
|
+
enabled: p2pMode !== 'relay',
|
|
1013
|
+
running: nodeStatus?.running === true,
|
|
1014
|
+
machine_id: identity.machine_id,
|
|
1015
|
+
rabbit_url: baseUrl,
|
|
1016
|
+
local: nodeStatus?.local ?? null,
|
|
1017
|
+
reflexive: nodeStatus?.reflexive ?? null,
|
|
1018
|
+
mapping: nodeStatus?.mapping ?? null,
|
|
1019
|
+
candidates: nodeStatus?.candidates ?? [],
|
|
1020
|
+
announced_at: nodeStatus?.announced_at ?? null,
|
|
1021
|
+
announce_ok: nodeStatus?.announce_ok === true,
|
|
1022
|
+
announce_failures: nodeStatus?.announce_failures ?? 0,
|
|
1023
|
+
last_announce_error: nodeStatus?.last_announce_error ?? null,
|
|
1024
|
+
punches_out: nodeStatus?.punches_out ?? 0,
|
|
1025
|
+
punches_in: nodeStatus?.punches_in ?? 0,
|
|
1026
|
+
dial_failures: nodeStatus?.dial_failures ?? 0,
|
|
1027
|
+
channels: p2pChannels.size,
|
|
1028
|
+
start_error: startError,
|
|
1029
|
+
last_error: startError ?? nodeStatus?.last_error ?? null,
|
|
1030
|
+
};
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
/**
|
|
1034
|
+
* Push the current facts to `onP2PStatus`, but only when they actually changed.
|
|
1035
|
+
*
|
|
1036
|
+
* Deduplicated on the serialised snapshot: `publishState()` runs on every heartbeat and an
|
|
1037
|
+
* unconditional push would turn a 10s cadence into a stream of identical callbacks. The
|
|
1038
|
+
* comparison is on JSON, not on object identity, because every read builds a new object by
|
|
1039
|
+
* design (no caller may hold a reference into the agent's state).
|
|
1040
|
+
*/
|
|
1041
|
+
function refreshP2PStatus() {
|
|
1042
|
+
const snapshot = p2pStatusSnapshot();
|
|
1043
|
+
const json = JSON.stringify(snapshot);
|
|
1044
|
+
if (json === lastP2PStatusJson) return snapshot;
|
|
1045
|
+
lastP2PStatusJson = json;
|
|
1046
|
+
try {
|
|
1047
|
+
onP2PStatus(snapshot);
|
|
1048
|
+
} catch (error) {
|
|
1049
|
+
// A diagnostic consumer that throws must not cost the agent anything: the callback belongs
|
|
1050
|
+
// to a *reader* (the plugin), and "the reporter is broken" is not a transport failure.
|
|
1051
|
+
log('warn', 'onP2PStatus threw; P2P status is still in state.p2p', {
|
|
1052
|
+
error: error?.message ?? String(error),
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
return snapshot;
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
/**
|
|
1059
|
+
* Build (but do not start) the node, unless the mode says there is no direct path.
|
|
1060
|
+
*
|
|
1061
|
+
* A node the caller injected is adopted as-is -- including its mode, which is why an injected
|
|
1062
|
+
* node and `p2pMode` disagreeing is logged rather than silently resolved: the injected node is
|
|
1063
|
+
* the one that would actually bind a socket, so it is the one that decides, and a caller
|
|
1064
|
+
* comparing the two configuration surfaces deserves to know they are not the same thing.
|
|
1065
|
+
*/
|
|
1066
|
+
function configureP2P() {
|
|
1067
|
+
if (p2pMode === 'relay') return;
|
|
1068
|
+
if (p2pNode !== null) return;
|
|
1069
|
+
if (p2pPortError !== null) {
|
|
1070
|
+
// Loud, and before anything is built. `relay` mode above is deliberately checked first: an
|
|
1071
|
+
// operator who asked for no direct path at all should not be told about a port that would
|
|
1072
|
+
// never be bound.
|
|
1073
|
+
p2pStartResult = { ok: false, enabled: true, error: p2pPortError };
|
|
1074
|
+
log('error', 'refusing to build the P2P node with an unusable p2pPort; the relay path is unaffected', {
|
|
1075
|
+
error: p2pPortError,
|
|
1076
|
+
});
|
|
1077
|
+
return;
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
const injected = options.p2pNode ?? null;
|
|
1081
|
+
const extra = options.p2pNodeOptions ?? {};
|
|
1082
|
+
const derived = {
|
|
1083
|
+
mode: p2pMode,
|
|
1084
|
+
rabbitUrl: baseUrl,
|
|
1085
|
+
machineId: identity.machine_id,
|
|
1086
|
+
postJson: (path, body) => postJson(path, body),
|
|
1087
|
+
getJson: (path) => getJson(path),
|
|
1088
|
+
...(options.stunServers === undefined ? {} : { stunServers: options.stunServers }),
|
|
1089
|
+
log: (line) => log('info', line, {}),
|
|
1090
|
+
...extra,
|
|
1091
|
+
// After `extra`, so the *validated* named option is the one that decides: `p2pNodeOptions` is
|
|
1092
|
+
// an escape hatch for the node's other constructor options, and letting an unvalidated
|
|
1093
|
+
// `bindPort` there silently win would route around the check above.
|
|
1094
|
+
...(p2pPort === null ? {} : { bindPort: p2pPort }),
|
|
1095
|
+
};
|
|
1096
|
+
|
|
1097
|
+
try {
|
|
1098
|
+
if (typeof injected === 'function') {
|
|
1099
|
+
p2pNode = injected(derived);
|
|
1100
|
+
} else if (injected !== null && typeof injected === 'object') {
|
|
1101
|
+
p2pNode = injected;
|
|
1102
|
+
if (p2pNode.mode !== p2pMode) {
|
|
1103
|
+
log('warn', 'injected p2pNode mode differs from p2pMode; the node decides', {
|
|
1104
|
+
p2pMode,
|
|
1105
|
+
node_mode: p2pNode.mode,
|
|
1106
|
+
});
|
|
1107
|
+
}
|
|
1108
|
+
} else {
|
|
1109
|
+
p2pNode = new P2PNode(derived);
|
|
1110
|
+
}
|
|
1111
|
+
} catch (error) {
|
|
1112
|
+
// Never fatal, for the same reason as a failed start: a machine whose P2P configuration is
|
|
1113
|
+
// wrong must still work over the relay.
|
|
1114
|
+
p2pStartResult = {
|
|
1115
|
+
ok: false,
|
|
1116
|
+
enabled: p2pMode !== 'relay',
|
|
1117
|
+
error: `P2P_NODE_INVALID: ${error?.message ?? String(error)}`,
|
|
1118
|
+
};
|
|
1119
|
+
log('error', 'could not build the P2P node; the relay path is unaffected', {
|
|
1120
|
+
error: p2pStartResult.error,
|
|
1121
|
+
});
|
|
1122
|
+
return;
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
p2pNode.on('message', (event) => handleP2PMessage(event));
|
|
1126
|
+
// `'announce'` is the only event that changes the facts a reader cares about between
|
|
1127
|
+
// heartbeats, so it is what refreshes the published status.
|
|
1128
|
+
p2pNode.on('announce', () => refreshP2PStatus());
|
|
1129
|
+
// Deliberately after `configureP2P`: a node so broken it binds twice is the *only* thing this
|
|
1130
|
+
// catches, and it is a bug in the node rather than anything the agent can recover from.
|
|
1131
|
+
p2pNode.on('error', (error) => {
|
|
1132
|
+
log('warn', `p2p: ${error?.message ?? String(error)}`, {});
|
|
1133
|
+
});
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
/**
|
|
1137
|
+
* Bring the direct path up. Failures are recorded, never thrown.
|
|
1138
|
+
*
|
|
1139
|
+
* The guard is the whole point: a machine that cannot punch (no UDP egress, a symmetric NAT, an
|
|
1140
|
+
* unreachable relay route from `POST /v1/peer/announce`) is still a working W2M machine, because
|
|
1141
|
+
* the relay is always still there. An agent that refused to start because a *fast path* was
|
|
1142
|
+
* unavailable would have turned an optimisation into a new failure mode.
|
|
1143
|
+
*
|
|
1144
|
+
* @returns {Promise<{ok: boolean, enabled: boolean, error: string|null}>}
|
|
1145
|
+
*/
|
|
1146
|
+
async function startP2P() {
|
|
1147
|
+
if (p2pMode === 'relay') return { ok: true, enabled: false, error: null };
|
|
1148
|
+
configureP2P();
|
|
1149
|
+
if (p2pNode === null) return p2pStartResult ?? { ok: false, enabled: true, error: 'P2P_NODE_INVALID' };
|
|
1150
|
+
if (p2pStarting !== null) return p2pStarting;
|
|
1151
|
+
|
|
1152
|
+
const started = (async () => {
|
|
1153
|
+
let result;
|
|
1154
|
+
try {
|
|
1155
|
+
result = await p2pNode.start();
|
|
1156
|
+
} catch (error) {
|
|
1157
|
+
result = { ok: false, enabled: true, error: `P2P_START_THREW: ${error?.message ?? String(error)}` };
|
|
1158
|
+
}
|
|
1159
|
+
p2pStartResult = result;
|
|
1160
|
+
log(result.ok ? 'info' : 'warn', `p2p ${p2pMode}: ${result.ok ? 'node started' : 'node did not start'}`, {
|
|
1161
|
+
error: result.error ?? null,
|
|
1162
|
+
});
|
|
1163
|
+
refreshP2PStatus();
|
|
1164
|
+
return result;
|
|
1165
|
+
})();
|
|
1166
|
+
p2pStarting = started;
|
|
1167
|
+
|
|
1168
|
+
try {
|
|
1169
|
+
return await started;
|
|
1170
|
+
} finally {
|
|
1171
|
+
// Only the caller that owns the in-flight start clears it. Clearing unconditionally would be
|
|
1172
|
+
// safe today (the promise is shared and idempotent) but would break the moment a second start
|
|
1173
|
+
// could begin while the first was still running.
|
|
1174
|
+
if (p2pStarting === started) p2pStarting = null;
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
/** Stop the node and let go of every channel. Idempotent and awaitable. */
|
|
1179
|
+
async function closeP2P() {
|
|
1180
|
+
// Await a start that is still in flight first: closing a node that is halfway through binding
|
|
1181
|
+
// would race the bind and could leave a socket nobody references.
|
|
1182
|
+
if (p2pStarting !== null) await p2pStarting.catch(() => null);
|
|
1183
|
+
const node = p2pNode;
|
|
1184
|
+
if (node === null) return;
|
|
1185
|
+
try {
|
|
1186
|
+
await node.close();
|
|
1187
|
+
} catch (error) {
|
|
1188
|
+
log('warn', 'p2p: closing the node failed', { error: error?.message ?? String(error) });
|
|
1189
|
+
}
|
|
1190
|
+
p2pChannels.clear();
|
|
1191
|
+
refreshP2PStatus();
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
/** Remember which channel a task's result should travel back over, and release it on close. */
|
|
1195
|
+
function rememberChannel(taskId, channel) {
|
|
1196
|
+
const previous = p2pChannels.get(taskId);
|
|
1197
|
+
if (previous?.channel === channel) return;
|
|
1198
|
+
if (previous) previous.channel.removeListener('close', previous.onClose);
|
|
1199
|
+
const onClose = () => {
|
|
1200
|
+
if (p2pChannels.get(taskId)?.channel === channel) p2pChannels.delete(taskId);
|
|
1201
|
+
};
|
|
1202
|
+
p2pChannels.set(taskId, { channel, onClose });
|
|
1203
|
+
channel.on('close', onClose);
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
/** The channel a task's result can be acknowledged over, or null when there is none. */
|
|
1207
|
+
const channelFor = (taskId) => p2pChannels.get(taskId)?.channel ?? null;
|
|
1208
|
+
|
|
1209
|
+
/**
|
|
1210
|
+
* Release the channel a task used, once the attempt is over.
|
|
1211
|
+
*
|
|
1212
|
+
* A channel exists for one attempt: the offer arrives on it and the result notice is acknowledged
|
|
1213
|
+
* on it, and nothing else is expected. Without this, an agent accumulated one live session per task
|
|
1214
|
+
* it ever handled -- and a session whose peer has gone away is invisible in every status surface
|
|
1215
|
+
* except the channel count, which is exactly how a leak becomes a mystery months later. Closing is
|
|
1216
|
+
* cheap: the mapping is re-punchable, and a duplicate offer that arrives afterwards simply has no
|
|
1217
|
+
* channel to answer on (the relay copy is the durable one).
|
|
1218
|
+
*/
|
|
1219
|
+
function releaseChannel(taskId) {
|
|
1220
|
+
const entry = p2pChannels.get(taskId);
|
|
1221
|
+
if (!entry) return;
|
|
1222
|
+
p2pChannels.delete(taskId);
|
|
1223
|
+
entry.channel.removeListener('close', entry.onClose);
|
|
1224
|
+
clearTimeout(entry.lingerTimer);
|
|
1225
|
+
try {
|
|
1226
|
+
entry.channel.close('attempt-complete');
|
|
1227
|
+
} catch {
|
|
1228
|
+
/* the path is already gone; that is the state this function produces */
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
/**
|
|
1233
|
+
* Release a task's channel after a bounded linger.
|
|
1234
|
+
*
|
|
1235
|
+
* The linger is the difference between "no leak" and "a duplicate cannot be answered": a
|
|
1236
|
+
* re-delivery of a finished attempt (the relay replaying an offer, or a dispatcher retrying a push)
|
|
1237
|
+
* is answered over the channel, and that is only possible while the channel exists. 30 seconds is
|
|
1238
|
+
* long enough to cover a retry and short enough that a long-lived agent does not accumulate one
|
|
1239
|
+
* session per task it has ever run.
|
|
1240
|
+
*/
|
|
1241
|
+
function scheduleChannelRelease(taskId) {
|
|
1242
|
+
const entry = p2pChannels.get(taskId);
|
|
1243
|
+
if (!entry) return;
|
|
1244
|
+
clearTimeout(entry.lingerTimer);
|
|
1245
|
+
const timer = setTimeout(() => releaseChannel(taskId), p2pChannelLingerMs);
|
|
1246
|
+
timer.unref?.();
|
|
1247
|
+
entry.lingerTimer = timer;
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
/** The STUN list the node is actually querying, for a status surface that can be quoted. */
|
|
1251
|
+
const p2pStunServers = () => p2pNode?.stunServers ?? null;
|
|
1252
|
+
|
|
1253
|
+
/**
|
|
1254
|
+
* Absolute path of the cross-process diagnostics file (task-16).
|
|
1255
|
+
*
|
|
1256
|
+
* Deliberately under `stateDir` (the `--state` directory), not next to
|
|
1257
|
+
* `device.json`: the spool, the state and this file belong together, and a
|
|
1258
|
+
* read-only project must never be written to.
|
|
1259
|
+
*/
|
|
1260
|
+
const stateFilePath = join(stateDir, AGENT_STATE_FILE);
|
|
1261
|
+
let statePublishWarned = false;
|
|
1262
|
+
|
|
1263
|
+
const state = {
|
|
1264
|
+
seq: 0,
|
|
1265
|
+
connected: false,
|
|
1266
|
+
handled: 0,
|
|
1267
|
+
stopped: false,
|
|
1268
|
+
/** `relay_id` of the relay we are currently talking to (v0.1.2 §8.3). */
|
|
1269
|
+
relayId: null,
|
|
1270
|
+
/**
|
|
1271
|
+
* `relay_id` that issued `state.seq`, or null when the cursor cannot be
|
|
1272
|
+
* attributed to a relay process (pre-v0.1.2 relay, or none yet).
|
|
1273
|
+
*/
|
|
1274
|
+
cursorRelayId: null,
|
|
1275
|
+
/**
|
|
1276
|
+
* Set when a relay restart makes the *current* stream unusable: it was
|
|
1277
|
+
* opened with the previous process's cursor, so it has already skipped the
|
|
1278
|
+
* new process's early events and must be replaced.
|
|
1279
|
+
*/
|
|
1280
|
+
streamRestartRequested: false,
|
|
1281
|
+
/** How many times a relay restart was detected. */
|
|
1282
|
+
relayIdChanges: 0,
|
|
1283
|
+
/** Last `REPLAY_TRUNCATED` notice we had to act on (v0.1.2 §8.4). */
|
|
1284
|
+
replayTruncated: null,
|
|
1285
|
+
/** Rolling heartbeat round-trip times, read by `w2m_status` (§8.2). */
|
|
1286
|
+
rttMs: summarizeRtt([]),
|
|
1287
|
+
/**
|
|
1288
|
+
* Consecutive unstable reconnects: 0 once a stream has stayed up for
|
|
1289
|
+
* {@link STABLE_STREAM_MS}, incremented for each reconnect before that.
|
|
1290
|
+
*
|
|
1291
|
+
* Published as `unstable_reconnects` in both channels (the heartbeat and
|
|
1292
|
+
* `agent-state.json`) so they agree by construction. The JavaScript property
|
|
1293
|
+
* keeps the shorter historical name because `scripts/probe-restart.mjs`
|
|
1294
|
+
* reads it; `unstableReconnects` is the same value under the published name.
|
|
1295
|
+
*/
|
|
1296
|
+
get reconnectAttempts() {
|
|
1297
|
+
return reconnectAttempt;
|
|
1298
|
+
},
|
|
1299
|
+
/** Alias of {@link reconnectAttempts}, spelled like the published field. */
|
|
1300
|
+
get unstableReconnects() {
|
|
1301
|
+
return reconnectAttempt;
|
|
1302
|
+
},
|
|
1303
|
+
/** Path of the published diagnostics file. */
|
|
1304
|
+
stateFile: stateFilePath,
|
|
1305
|
+
/** `updated_at` of the last successful publish, or null. */
|
|
1306
|
+
statePublishedAt: null,
|
|
1307
|
+
/**
|
|
1308
|
+
* Last non-2xx response from the relay, with its own error code preserved
|
|
1309
|
+
* (v0.3.0). On a signed link this is what distinguishes "your clock drifted
|
|
1310
|
+
* outside the window" from "your secret is wrong".
|
|
1311
|
+
*/
|
|
1312
|
+
lastRelayError: null,
|
|
1313
|
+
/**
|
|
1314
|
+
* The agent's P2P facts (v0.4.0): mode, whether the node is running, what it announced, how
|
|
1315
|
+
* many punches went each way, and the start error when there is one.
|
|
1316
|
+
*
|
|
1317
|
+
* A getter rather than a field because every read must be a *fresh* plain object: this is
|
|
1318
|
+
* published to another process (the plugin's `w2m_status`), and a shared object would let a
|
|
1319
|
+
* reader mutate the agent's own view of its transport. The full node status -- including the
|
|
1320
|
+
* reflexive address and candidate list -- is available from `agent.p2pNode`.
|
|
1321
|
+
*/
|
|
1322
|
+
get p2p() {
|
|
1323
|
+
return p2pStatusSnapshot();
|
|
1324
|
+
},
|
|
1325
|
+
/** @type {object|null} */
|
|
1326
|
+
current: null,
|
|
1327
|
+
};
|
|
1328
|
+
|
|
1329
|
+
/** @type {object[]} */
|
|
1330
|
+
const queue = [];
|
|
1331
|
+
/**
|
|
1332
|
+
* Offers whose attempt is already queued or running, keyed by {@link identityKeyFor}.
|
|
1333
|
+
*
|
|
1334
|
+
* This is v0.4.0's exactly-once guard, and it is deliberately *synchronous*: `enqueue` checks it
|
|
1335
|
+
* and writes to it without an `await` anywhere between, so two copies of one offer -- one over
|
|
1336
|
+
* the relay, one over a channel, in either order -- cannot both reach the queue. `completed` and
|
|
1337
|
+
* the spool cannot do this job on their own: neither knows anything until an envelope exists,
|
|
1338
|
+
* which is exactly the window a duplicate arrives in.
|
|
1339
|
+
*
|
|
1340
|
+
* Released in `pump`'s `finally`, once the attempt has delivered (or failed to). It is not
|
|
1341
|
+
* released in `handleOffer` so an offer that throws is still retryable.
|
|
1342
|
+
*
|
|
1343
|
+
* @type {Set<string>}
|
|
1344
|
+
*/
|
|
1345
|
+
const claimed = new Set();
|
|
1346
|
+
/**
|
|
1347
|
+
* Recently delivered envelopes, keyed by `dedupe_key#attempt`.
|
|
1348
|
+
*
|
|
1349
|
+
* Rabbit's idempotency triple is `(machine_id, dedupe_key, attempt)`, so:
|
|
1350
|
+
*
|
|
1351
|
+
* * a **transport** re-delivery of the same attempt is answered from this
|
|
1352
|
+
* cache (or the spool) instead of being executed twice;
|
|
1353
|
+
* * a **new attempt** for the same `dedupe_key` is a genuine retry and must
|
|
1354
|
+
* run again -- §4.4 keeps `attempt` out of `dedupe_key` for exactly that
|
|
1355
|
+
* reason, and replaying the old envelope would leave the new lease
|
|
1356
|
+
* without a result of its own.
|
|
1357
|
+
*
|
|
1358
|
+
* Bounded and memory-only: after a restart Rabbit's own accounting is the
|
|
1359
|
+
* backstop.
|
|
1360
|
+
*
|
|
1361
|
+
* @type {Map<string, object>}
|
|
1362
|
+
*/
|
|
1363
|
+
const completed = new Map();
|
|
1364
|
+
const COMPLETED_CACHE_LIMIT = 64;
|
|
1365
|
+
let pumping = false;
|
|
1366
|
+
let streamAbort = null;
|
|
1367
|
+
let runAbort = null;
|
|
1368
|
+
/** @type {Heartbeat|null} */
|
|
1369
|
+
let heartbeat = null;
|
|
1370
|
+
/** Consecutive unstable reconnects; drives the backoff ladder (§8.1). */
|
|
1371
|
+
let reconnectAttempt = 0;
|
|
1372
|
+
/**
|
|
1373
|
+
* Consecutive immediate re-attaches caused by a relay identity change. Capped
|
|
1374
|
+
* so a relay that cannot keep a stable identity degrades to plain backoff
|
|
1375
|
+
* instead of a hot loop.
|
|
1376
|
+
*/
|
|
1377
|
+
let restartReconnects = 0;
|
|
1378
|
+
/**
|
|
1379
|
+
* What the current stream was opened with, so `ready` can tell whether the
|
|
1380
|
+
* events on it are complete. `seq: null` means no cursor was sent, i.e. this
|
|
1381
|
+
* stream starts from the relay's own beginning and is never "tainted".
|
|
1382
|
+
*
|
|
1383
|
+
* @type {{seq: number|null, attributionRelayId: string|null}}
|
|
1384
|
+
*/
|
|
1385
|
+
let attachInfo = { seq: null, attributionRelayId: null };
|
|
1386
|
+
/** @type {NodeJS.Timeout|null} Idle refresh of the diagnostics file. */
|
|
1387
|
+
let statePublishTimer = null;
|
|
1388
|
+
/** @type {NodeJS.Timeout|null} Idle diagnostic heartbeat (v0.3.0). */
|
|
1389
|
+
let idleHeartbeatTimer = null;
|
|
1390
|
+
|
|
1391
|
+
/**
|
|
1392
|
+
* Remember a delivered envelope, bounded.
|
|
1393
|
+
*
|
|
1394
|
+
* The key is {@link identityKeyFor}'s, not a second spelling of the same idea: a cache keyed
|
|
1395
|
+
* differently from the dedupe check would be a cache that never hits.
|
|
1396
|
+
*
|
|
1397
|
+
* @param {string|null} key
|
|
1398
|
+
* @param {object} envelope
|
|
1399
|
+
*/
|
|
1400
|
+
function rememberCompleted(key, envelope) {
|
|
1401
|
+
if (!key) return;
|
|
1402
|
+
completed.delete(key);
|
|
1403
|
+
completed.set(key, envelope);
|
|
1404
|
+
while (completed.size > COMPLETED_CACHE_LIMIT) {
|
|
1405
|
+
const oldest = completed.keys().next().value;
|
|
1406
|
+
completed.delete(oldest);
|
|
1407
|
+
}
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
// -- Cross-process diagnostics file (task-16) ----------------------------
|
|
1411
|
+
|
|
1412
|
+
/**
|
|
1413
|
+
* Snapshot published to `<stateDir>/agent-state.json`.
|
|
1414
|
+
*
|
|
1415
|
+
* An allow-list, not a spread of internal state: the plugin reads this
|
|
1416
|
+
* cross-process, so the only safe rule is that a field must be named here to
|
|
1417
|
+
* exist. No token -- device or operator -- can ever appear, because neither
|
|
1418
|
+
* is referenced.
|
|
1419
|
+
*
|
|
1420
|
+
* @returns {object}
|
|
1421
|
+
*/
|
|
1422
|
+
function agentStateSnapshot() {
|
|
1423
|
+
return {
|
|
1424
|
+
schema_version: AGENT_STATE_SCHEMA_VERSION,
|
|
1425
|
+
machine_id: identity.machine_id,
|
|
1426
|
+
machine_name: identity.machine_name ?? null,
|
|
1427
|
+
updated_at: new Date().toISOString(),
|
|
1428
|
+
connected: state.connected === true,
|
|
1429
|
+
relay_id: state.relayId ?? null,
|
|
1430
|
+
rttMs: {
|
|
1431
|
+
last: state.rttMs.last,
|
|
1432
|
+
avg: state.rttMs.avg,
|
|
1433
|
+
samples: [...state.rttMs.samples],
|
|
1434
|
+
},
|
|
1435
|
+
// Named to match the heartbeat field (and to stay distinct from the
|
|
1436
|
+
// relay's own `reconnect_attempts`, which answers a different question).
|
|
1437
|
+
unstable_reconnects: reconnectAttempt,
|
|
1438
|
+
replay_truncated: state.replayTruncated !== null,
|
|
1439
|
+
// v0.4.0: the direct path's facts, so a plugin on another host can answer "which path is this
|
|
1440
|
+
// machine actually using?" without asking the agent process. A summarised shape on purpose --
|
|
1441
|
+
// the candidate list and the reflexive address are available in-process from `agent.p2p`.
|
|
1442
|
+
p2p: p2pSummary(),
|
|
1443
|
+
};
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
/**
|
|
1447
|
+
* The published (cross-process) subset of the P2P facts.
|
|
1448
|
+
*
|
|
1449
|
+
* Deliberately not the whole {@link p2pStatusSnapshot}: this is written to disk and read by a
|
|
1450
|
+
* plugin in another process, so it names a fixed set of fields rather than mirroring whatever
|
|
1451
|
+
* the node happens to expose.
|
|
1452
|
+
*
|
|
1453
|
+
* @returns {object}
|
|
1454
|
+
*/
|
|
1455
|
+
function p2pSummary() {
|
|
1456
|
+
const snapshot = p2pStatusSnapshot();
|
|
1457
|
+
return {
|
|
1458
|
+
mode: snapshot.mode,
|
|
1459
|
+
enabled: snapshot.enabled,
|
|
1460
|
+
running: snapshot.running,
|
|
1461
|
+
mapping: snapshot.mapping,
|
|
1462
|
+
reflexive: snapshot.reflexive,
|
|
1463
|
+
candidates: snapshot.candidates.length,
|
|
1464
|
+
announced_at: snapshot.announced_at,
|
|
1465
|
+
punches_out: snapshot.punches_out,
|
|
1466
|
+
punches_in: snapshot.punches_in,
|
|
1467
|
+
dial_failures: snapshot.dial_failures,
|
|
1468
|
+
channels: snapshot.channels,
|
|
1469
|
+
last_error: snapshot.last_error,
|
|
1470
|
+
};
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
/**
|
|
1474
|
+
* Atomically publish the snapshot: temp file in the same directory, then
|
|
1475
|
+
* rename, so a reader never sees half a JSON document.
|
|
1476
|
+
*
|
|
1477
|
+
* Never throws. A full disk or a state directory that vanished must not take
|
|
1478
|
+
* the agent (or a running task) down -- the file is a diagnostic, not a
|
|
1479
|
+
* delivery guarantee -- so failure is logged once and otherwise ignored.
|
|
1480
|
+
*
|
|
1481
|
+
* @returns {object|null} The snapshot written, or null when the write failed.
|
|
1482
|
+
*/
|
|
1483
|
+
function publishState() {
|
|
1484
|
+
const snapshot = agentStateSnapshot();
|
|
1485
|
+
try {
|
|
1486
|
+
mkdirSync(stateDir, { recursive: true, mode: 0o700 });
|
|
1487
|
+
const tmp = `${stateFilePath}.tmp-${process.pid}-${randomUUID().slice(0, 8)}`;
|
|
1488
|
+
writeFileSync(tmp, `${JSON.stringify(snapshot, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
|
|
1489
|
+
renameSync(tmp, stateFilePath);
|
|
1490
|
+
state.statePublishedAt = snapshot.updated_at;
|
|
1491
|
+
statePublishWarned = false;
|
|
1492
|
+
return snapshot;
|
|
1493
|
+
} catch (error) {
|
|
1494
|
+
if (!statePublishWarned) {
|
|
1495
|
+
statePublishWarned = true;
|
|
1496
|
+
log('warn', `could not publish ${AGENT_STATE_FILE} (diagnostics only; continuing)`, {
|
|
1497
|
+
path: stateFilePath,
|
|
1498
|
+
error: error?.message ?? String(error),
|
|
1499
|
+
});
|
|
1500
|
+
}
|
|
1501
|
+
return null;
|
|
1502
|
+
}
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
const token = () => identity.device_token;
|
|
1506
|
+
|
|
1507
|
+
/**
|
|
1508
|
+
* Fold one heartbeat round trip into the rolling window (§8.2).
|
|
1509
|
+
*
|
|
1510
|
+
* Exposed as `state.rttMs` for `w2m_status`; deliberately *not* added to the
|
|
1511
|
+
* result envelope, which must stay comparable across machines.
|
|
1512
|
+
*
|
|
1513
|
+
* @param {number} ms
|
|
1514
|
+
*/
|
|
1515
|
+
function recordRtt(ms) {
|
|
1516
|
+
state.rttMs = summarizeRtt([...state.rttMs.samples, ms]);
|
|
1517
|
+
// Publish on every successful heartbeat: this is the moment the numbers a
|
|
1518
|
+
// remote operator cares about actually changed (task-16).
|
|
1519
|
+
publishState();
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
/** Detected tool versions, resolved once per process (git needs a spawn). */
|
|
1523
|
+
let toolchainCache = null;
|
|
1524
|
+
async function toolchainInfo() {
|
|
1525
|
+
if (toolchainCache) return toolchainCache;
|
|
1526
|
+
let git = null;
|
|
1527
|
+
try {
|
|
1528
|
+
git = await gitVersion({ cwd: project });
|
|
1529
|
+
} catch {
|
|
1530
|
+
git = null;
|
|
1531
|
+
}
|
|
1532
|
+
// Guarded above by `if (toolchainCache) return`, and two concurrent callers would derive the same
|
|
1533
|
+
// value from immutable project facts -- the worst case is doing the work twice, not caching
|
|
1534
|
+
// something wrong.
|
|
1535
|
+
// eslint-disable-next-line require-atomic-updates -- guarded memo; value derived from immutables
|
|
1536
|
+
toolchainCache = {
|
|
1537
|
+
node: caps.node ?? null,
|
|
1538
|
+
python: caps.python ?? null,
|
|
1539
|
+
npm: caps.npm ?? null,
|
|
1540
|
+
git,
|
|
1541
|
+
};
|
|
1542
|
+
return toolchainCache;
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
// -- HTTP helpers --------------------------------------------------------
|
|
1546
|
+
|
|
1547
|
+
/**
|
|
1548
|
+
* @param {string} path
|
|
1549
|
+
* @returns {string}
|
|
1550
|
+
*/
|
|
1551
|
+
const url = (path) => joinUrl(baseUrl, path);
|
|
1552
|
+
|
|
1553
|
+
/**
|
|
1554
|
+
* Signing headers for one request, or nothing at all.
|
|
1555
|
+
*
|
|
1556
|
+
* The signed `path` is the **API-relative** path plus its query
|
|
1557
|
+
* (`/v1/stream?machine_id=…&seq=5`), never the URL as sent.
|
|
1558
|
+
*
|
|
1559
|
+
* That looks backwards -- surely the signature should cover the bytes on the
|
|
1560
|
+
* wire? -- so here is the case that settles it. With `rabbitUrl =
|
|
1561
|
+
* https://host/w2m` there are two proxy shapes, and the client cannot tell
|
|
1562
|
+
* them apart:
|
|
1563
|
+
*
|
|
1564
|
+
* * the proxy forwards `/w2m/v1/heartbeat` and the relay strips its
|
|
1565
|
+
* `--base-path` itself;
|
|
1566
|
+
* * the proxy strips `/w2m` and the relay never sees the prefix at all.
|
|
1567
|
+
*
|
|
1568
|
+
* In the second shape the relay *cannot* reconstruct `/w2m/v1/heartbeat` --
|
|
1569
|
+
* those bytes never reached it -- so a signature over the wire path can never
|
|
1570
|
+
* verify there, and every request 401s. The one value both sides always agree
|
|
1571
|
+
* on is the path *after* routing: `/v1/heartbeat`. That is what is signed.
|
|
1572
|
+
* (The verifier's side of the same argument is PROTOCOL-v0.3.0 §9.4.)
|
|
1573
|
+
*
|
|
1574
|
+
* The gate is deliberately absolute: with no secret configured this returns an
|
|
1575
|
+
* empty object, so the wire is byte-identical to v0.2.3 (a hard
|
|
1576
|
+
* backward-compatibility requirement, asserted in the tests). A relay that
|
|
1577
|
+
* *requires* signatures then answers 401 with its own code, which the caller
|
|
1578
|
+
* surfaces rather than masking.
|
|
1579
|
+
*
|
|
1580
|
+
* @param {'GET'|'POST'} method
|
|
1581
|
+
* @param {string} apiPath API-relative path, query included when there is one.
|
|
1582
|
+
* @param {string|Buffer} [bodyText] Exact body bytes that will be sent.
|
|
1583
|
+
* @returns {Record<string, string>}
|
|
1584
|
+
*/
|
|
1585
|
+
function signingHeaders(method, apiPath, bodyText = '') {
|
|
1586
|
+
if (!signer) return {};
|
|
1587
|
+
return signer.headers({ method, path: apiPath, body: bodyText });
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1590
|
+
/**
|
|
1591
|
+
* The relay's machine-readable error code from a response body.
|
|
1592
|
+
*
|
|
1593
|
+
* Kept separate so a 401 says `SIGNATURE_REQUIRED` (or whatever the relay
|
|
1594
|
+
* decided) instead of a generic "unauthorized": on a signed link the code is
|
|
1595
|
+
* the difference between "your clock is off" and "your secret is wrong".
|
|
1596
|
+
*
|
|
1597
|
+
* @param {{json?: any}} response
|
|
1598
|
+
* @returns {string|null}
|
|
1599
|
+
*/
|
|
1600
|
+
function relayErrorCode(response) {
|
|
1601
|
+
const code = response?.json?.error?.code;
|
|
1602
|
+
return typeof code === 'string' && code !== '' ? code : null;
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
/**
|
|
1606
|
+
* POST JSON and decode the response body.
|
|
1607
|
+
*
|
|
1608
|
+
* @param {string} path
|
|
1609
|
+
* @param {object} body
|
|
1610
|
+
* @param {{auth?: boolean}} [opts]
|
|
1611
|
+
*/
|
|
1612
|
+
async function postJson(path, body, opts = {}) {
|
|
1613
|
+
const headers = { 'content-type': 'application/json', accept: 'application/json' };
|
|
1614
|
+
if (opts.auth !== false && token()) headers.authorization = `Bearer ${token()}`;
|
|
1615
|
+
const target = url(path);
|
|
1616
|
+
// Stringify once: the signature must cover the exact bytes that go on the
|
|
1617
|
+
// wire, so it can never be computed from a second, differently-ordered copy.
|
|
1618
|
+
const bodyText = JSON.stringify(body);
|
|
1619
|
+
Object.assign(headers, signingHeaders('POST', path, bodyText));
|
|
1620
|
+
const response = await fetchImpl(target, {
|
|
1621
|
+
method: 'POST',
|
|
1622
|
+
headers,
|
|
1623
|
+
body: bodyText,
|
|
1624
|
+
});
|
|
1625
|
+
const text = await response.text();
|
|
1626
|
+
let json = null;
|
|
1627
|
+
try {
|
|
1628
|
+
json = text === '' ? null : JSON.parse(text);
|
|
1629
|
+
} catch {
|
|
1630
|
+
json = null;
|
|
1631
|
+
}
|
|
1632
|
+
const result = { status: response.status, ok: response.ok, json, text };
|
|
1633
|
+
if (!response.ok) {
|
|
1634
|
+
// Remember the last refusal so diagnostics can name it (and so a 401 is
|
|
1635
|
+
// distinguishable from a 500 at a glance). One request at a time per agent, so there is no
|
|
1636
|
+
// concurrent writer here and no disable is needed.
|
|
1637
|
+
state.lastRelayError = {
|
|
1638
|
+
at: new Date().toISOString(),
|
|
1639
|
+
http_status: response.status,
|
|
1640
|
+
code: relayErrorCode(result) ?? `HTTP_${response.status}`,
|
|
1641
|
+
path,
|
|
1642
|
+
};
|
|
1643
|
+
}
|
|
1644
|
+
return result;
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
/** @param {string} path @param {object} body @param {number} [retries] */
|
|
1648
|
+
async function postJsonWithRetry(path, body, retries = resultRetries) {
|
|
1649
|
+
let lastError = null;
|
|
1650
|
+
for (let attempt = 0; attempt <= retries; attempt += 1) {
|
|
1651
|
+
if (state.stopped) break;
|
|
1652
|
+
try {
|
|
1653
|
+
const response = await postJson(path, body);
|
|
1654
|
+
if (response.ok) return response;
|
|
1655
|
+
// A 4xx is permanent: retrying a rejected signature or a bad request
|
|
1656
|
+
// only burns the lease window.
|
|
1657
|
+
if (response.status >= 400 && response.status < 500) return response;
|
|
1658
|
+
lastError = new Error(`HTTP ${response.status}`);
|
|
1659
|
+
} catch (error) {
|
|
1660
|
+
lastError = error;
|
|
1661
|
+
}
|
|
1662
|
+
if (attempt < retries) await sleep(backoffDelay(attempt));
|
|
1663
|
+
}
|
|
1664
|
+
if (lastError) log('warn', `${path} failed`, { error: lastError.message });
|
|
1665
|
+
return null;
|
|
1666
|
+
}
|
|
1667
|
+
|
|
1668
|
+
/**
|
|
1669
|
+
* GET JSON and decode the response body (v0.4.0).
|
|
1670
|
+
*
|
|
1671
|
+
* The mirror of {@link postJson} for the one read the direct path needs: a peer's announced
|
|
1672
|
+
* candidates (`GET /v1/peer/{machineId}`). Signed exactly like every other request, so the
|
|
1673
|
+
* P2P signalling routes cannot be the one place a signed link silently stops being signed.
|
|
1674
|
+
*
|
|
1675
|
+
* It deliberately does **not** record `state.lastRelayError` the way `postJson` does. A 404 here
|
|
1676
|
+
* is the ordinary answer for "this machine has not announced", which is a normal state in
|
|
1677
|
+
* `auto` mode -- recording it as the last relay error would make a healthy link look broken
|
|
1678
|
+
* every time a relay-only peer was dialled.
|
|
1679
|
+
*
|
|
1680
|
+
* @param {string} path
|
|
1681
|
+
* @returns {Promise<{status: number, ok: boolean, json: any, text: string, error: string|null}>}
|
|
1682
|
+
*/
|
|
1683
|
+
async function getJson(path) {
|
|
1684
|
+
const headers = { accept: 'application/json' };
|
|
1685
|
+
if (token()) headers.authorization = `Bearer ${token()}`;
|
|
1686
|
+
Object.assign(headers, signingHeaders('GET', path));
|
|
1687
|
+
let response;
|
|
1688
|
+
try {
|
|
1689
|
+
response = await fetchImpl(url(path), { method: 'GET', headers });
|
|
1690
|
+
} catch (error) {
|
|
1691
|
+
// Returned rather than thrown: `P2PNode.dial()` names a lookup failure with its own code, and
|
|
1692
|
+
// an exception crossing that boundary would be reported as a punch failure instead.
|
|
1693
|
+
return { status: 0, ok: false, json: null, text: '', error: error?.message ?? String(error) };
|
|
1694
|
+
}
|
|
1695
|
+
const text = await response.text();
|
|
1696
|
+
let json = null;
|
|
1697
|
+
try {
|
|
1698
|
+
json = text === '' ? null : JSON.parse(text);
|
|
1699
|
+
} catch {
|
|
1700
|
+
json = null;
|
|
1701
|
+
}
|
|
1702
|
+
return { status: response.status, ok: response.ok, json, text, error: null };
|
|
1703
|
+
}
|
|
1704
|
+
|
|
1705
|
+
// -- Pairing -------------------------------------------------------------
|
|
1706
|
+
|
|
1707
|
+
/**
|
|
1708
|
+
* Exchange a one-time pairing code for a device token (§2.2).
|
|
1709
|
+
*
|
|
1710
|
+
* @param {string} pairingCode
|
|
1711
|
+
* @param {string} [userId]
|
|
1712
|
+
*/
|
|
1713
|
+
async function pair(pairingCode, userId) {
|
|
1714
|
+
const body = {
|
|
1715
|
+
pairing_code: pairingCode,
|
|
1716
|
+
machine_id: identity.machine_id,
|
|
1717
|
+
machine_name: identity.machine_name,
|
|
1718
|
+
platform,
|
|
1719
|
+
caps,
|
|
1720
|
+
};
|
|
1721
|
+
if (userId) body.user_id = userId;
|
|
1722
|
+
const response = await postJson('/v1/pair', body, { auth: false });
|
|
1723
|
+
if (!response.ok) {
|
|
1724
|
+
const code = response.json?.error?.code ?? `HTTP_${response.status}`;
|
|
1725
|
+
const error = new Error(`pairing failed: ${code} ${response.json?.error?.message ?? ''}`.trim());
|
|
1726
|
+
error.code = code;
|
|
1727
|
+
throw error;
|
|
1728
|
+
}
|
|
1729
|
+
const version = response.json?.protocol_version;
|
|
1730
|
+
if (version !== undefined && version !== PROTOCOL_VERSION) {
|
|
1731
|
+
const error = new Error(
|
|
1732
|
+
`protocol mismatch: Rabbit speaks ${version}, this agent speaks ${PROTOCOL_VERSION}`,
|
|
1733
|
+
);
|
|
1734
|
+
error.code = 'PROTOCOL_MISMATCH';
|
|
1735
|
+
throw error;
|
|
1736
|
+
}
|
|
1737
|
+
if (typeof response.json?.device_token !== 'string' || response.json.device_token === '') {
|
|
1738
|
+
const error = new Error('pairing response carried no device_token');
|
|
1739
|
+
error.code = 'PAIRING_RESPONSE_INVALID';
|
|
1740
|
+
throw error;
|
|
1741
|
+
}
|
|
1742
|
+
// Pairing runs once, from the CLI, before any loop starts, so there is no interleaving here.
|
|
1743
|
+
// eslint-disable-next-line require-atomic-updates -- single call site, no concurrency
|
|
1744
|
+
identity.device_token = response.json.device_token;
|
|
1745
|
+
// eslint-disable-next-line require-atomic-updates -- single call site, no concurrency
|
|
1746
|
+
identity.rabbit_url = baseUrl;
|
|
1747
|
+
log('info', 'paired', { machine_id: identity.machine_id });
|
|
1748
|
+
return response.json;
|
|
1749
|
+
}
|
|
1750
|
+
|
|
1751
|
+
// -- Spool flushing ------------------------------------------------------
|
|
1752
|
+
|
|
1753
|
+
/** Resend envelopes that were written but never acknowledged. */
|
|
1754
|
+
async function flushSpool() {
|
|
1755
|
+
const pending = spool.pendingResults();
|
|
1756
|
+
if (pending.length === 0) return 0;
|
|
1757
|
+
log('info', `resending ${pending.length} spooled result(s)`);
|
|
1758
|
+
let sent = 0;
|
|
1759
|
+
for (const record of pending) {
|
|
1760
|
+
if (state.stopped) break;
|
|
1761
|
+
const response = await postJson('/v1/result', record.envelope);
|
|
1762
|
+
if (response.ok) {
|
|
1763
|
+
spool.ack({ task_id: record.task_id, attempt: record.attempt });
|
|
1764
|
+
sent += 1;
|
|
1765
|
+
} else {
|
|
1766
|
+
log('warn', `spooled result ${record.task_id} still undelivered`, {
|
|
1767
|
+
status: response.status,
|
|
1768
|
+
// The relay's own code, not a paraphrase: SIGNATURE_EXPIRED and
|
|
1769
|
+
// SIGNATURE_REQUIRED need different fixes.
|
|
1770
|
+
code: relayErrorCode(response),
|
|
1771
|
+
});
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
return sent;
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1777
|
+
/** Warn about entries that were claimed but never finished (a crash). */
|
|
1778
|
+
function reportInterrupted() {
|
|
1779
|
+
const interrupted = spool.interruptedTasks();
|
|
1780
|
+
for (const record of interrupted) {
|
|
1781
|
+
log('warn', `interrupted task left in spool (not replayed automatically)`, {
|
|
1782
|
+
task_id: record.task_id,
|
|
1783
|
+
attempt: record.attempt,
|
|
1784
|
+
});
|
|
1785
|
+
}
|
|
1786
|
+
return interrupted.length;
|
|
1787
|
+
}
|
|
1788
|
+
|
|
1789
|
+
// -- Envelope assembly ---------------------------------------------------
|
|
1790
|
+
|
|
1791
|
+
/**
|
|
1792
|
+
* The `p2p` block of a result envelope (v0.4.0 §6).
|
|
1793
|
+
*
|
|
1794
|
+
* It is built **before** `envelope_sha256` is computed, because the hash covers the whole
|
|
1795
|
+
* envelope: attaching these fields afterwards would leave a hash that does not cover the bytes
|
|
1796
|
+
* it claims to cover, and `verifyEnvelope` would fail on an envelope nobody tampered with. That
|
|
1797
|
+
* is also why the direct copy uses the very same object -- one envelope, two deliveries.
|
|
1798
|
+
*
|
|
1799
|
+
* Every field answers a question a reader of a *stored* result actually asks:
|
|
1800
|
+
*
|
|
1801
|
+
* * `mode` -- this agent's configured mode, or the dispatcher's when it stated one. The
|
|
1802
|
+
* dispatcher's value is used when present because a task dispatched by an
|
|
1803
|
+
* `auto` machine may be executed by a `direct` one, and "why did this path
|
|
1804
|
+
* win" is a question about the dispatcher's intent.
|
|
1805
|
+
* * `offer_path`-- how the offer got here, as this agent observed it (never as claimed on the
|
|
1806
|
+
* wire: only the channel path can set it).
|
|
1807
|
+
* * `result_path`-- `'p2p'` **only** when the envelope was acknowledged over the direct
|
|
1808
|
+
* channel. Absent means "no direct result", which is not the same fact as
|
|
1809
|
+
* `'relay'` and must not be spelled as one.
|
|
1810
|
+
* * `reason` -- why the direct path was not the answer, or null when it was.
|
|
1811
|
+
* * `rtt_ms` -- the punch's measured round trip, or null. Null rather than 0: a round trip
|
|
1812
|
+
* that was never measured is not an instantaneous one.
|
|
1813
|
+
* * `peer` -- the peer's address as seen by this agent, or null for an inbound channel
|
|
1814
|
+
* (a HELLO carries a session, not an identity).
|
|
1815
|
+
* * `session` -- the punch's session id, which is what correlates two agents' logs.
|
|
1816
|
+
* * `mapping` -- the NAT mapping this agent's announcement measured.
|
|
1817
|
+
*
|
|
1818
|
+
* @param {object} context
|
|
1819
|
+
* @returns {object}
|
|
1820
|
+
*/
|
|
1821
|
+
function envelopeP2P(context) {
|
|
1822
|
+
const offer = context.offer ?? {};
|
|
1823
|
+
const transport = context.p2pTransport ?? offerTransportOf(offer, p2pStatusSnapshot());
|
|
1824
|
+
const delivery = context.p2pDelivery ?? p2pDeliveryMark();
|
|
1825
|
+
const record = context.p2pRecord ?? null;
|
|
1826
|
+
const dispatcherMode = offer?.p2p?.mode;
|
|
1827
|
+
return {
|
|
1828
|
+
mode: typeof dispatcherMode === 'string' && dispatcherMode !== '' ? dispatcherMode : p2pMode,
|
|
1829
|
+
offer_path: transport.offer_path,
|
|
1830
|
+
...(delivery.path === null ? {} : { result_path: delivery.path }),
|
|
1831
|
+
// The key is OMITTED when there is nothing to explain, and that is not the same as setting it
|
|
1832
|
+
// to `undefined`: `JSON.stringify` would drop that, but the envelope hash is computed with JCS,
|
|
1833
|
+
// whose implementation (rightly) refuses a non-JSON value. Measured -- `reason: undefined`
|
|
1834
|
+
// here made every direct-path result fail to build its envelope with
|
|
1835
|
+
// `jcs: property reason is undefined`, so the task executed and then vanished from the ledger.
|
|
1836
|
+
...(typeof transport.reason === 'string' && transport.reason !== '' ? { reason: transport.reason } : {}),
|
|
1837
|
+
rtt_ms: delivery.rtt_ms ?? record?.rtt_ms ?? null,
|
|
1838
|
+
peer: record?.peer ?? null,
|
|
1839
|
+
session: record?.session ?? null,
|
|
1840
|
+
mapping: p2pNode?.status.mapping ?? null,
|
|
1841
|
+
};
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1844
|
+
/**
|
|
1845
|
+
* Turn a finished (or refused) run into a complete §5.1 envelope.
|
|
1846
|
+
*
|
|
1847
|
+
* @param {object} context
|
|
1848
|
+
*/
|
|
1849
|
+
function makeEnvelope(context) {
|
|
1850
|
+
const {
|
|
1851
|
+
offer,
|
|
1852
|
+
cwdRel,
|
|
1853
|
+
commandArgv,
|
|
1854
|
+
anchorsBefore,
|
|
1855
|
+
anchorsAfter,
|
|
1856
|
+
execution,
|
|
1857
|
+
status,
|
|
1858
|
+
refusalReason,
|
|
1859
|
+
warnings,
|
|
1860
|
+
startedAt,
|
|
1861
|
+
endedAt,
|
|
1862
|
+
toolchain,
|
|
1863
|
+
comparePolicy,
|
|
1864
|
+
pipeline,
|
|
1865
|
+
protocolViolations,
|
|
1866
|
+
} = context;
|
|
1867
|
+
|
|
1868
|
+
const stdout = execution?.stdout ?? Buffer.alloc(0);
|
|
1869
|
+
const normalized = normalizeOutput(stdout, comparePolicy ?? {});
|
|
1870
|
+
|
|
1871
|
+
/**
|
|
1872
|
+
* The stage descriptors the pipeline hash is taken over, or null for a single command.
|
|
1873
|
+
*
|
|
1874
|
+
* Normalized exactly as the relay normalizes them (`cwd_rel` defaults to the task cwd,
|
|
1875
|
+
* `continue_on_failure` to false), because the two sides must hash the same shape or every
|
|
1876
|
+
* pipeline anchors as `unverifiable`.
|
|
1877
|
+
*/
|
|
1878
|
+
const pipelineHashMaterial = pipeline && Array.isArray(pipeline.stages) && pipeline.total_stages > 1
|
|
1879
|
+
? pipeline.stages.map((s) => ({
|
|
1880
|
+
command_argv: s.command_argv,
|
|
1881
|
+
cwd_rel: s.cwd_rel ?? cwdRel,
|
|
1882
|
+
continue_on_failure: s.continue_on_failure === true,
|
|
1883
|
+
}))
|
|
1884
|
+
: null;
|
|
1885
|
+
|
|
1886
|
+
const envelope = {
|
|
1887
|
+
envelope_version: ENVELOPE_VERSION,
|
|
1888
|
+
task_id: offer.task_id,
|
|
1889
|
+
attempt: Number(offer.attempt) || 1,
|
|
1890
|
+
dedupe_key: offer.dedupe_key ?? '',
|
|
1891
|
+
machine_id: identity.machine_id,
|
|
1892
|
+
machine_name: identity.machine_name,
|
|
1893
|
+
platform,
|
|
1894
|
+
caps,
|
|
1895
|
+
index: Number.isInteger(offer.index) ? offer.index : 0,
|
|
1896
|
+
index_total: Number.isInteger(offer.index_total) ? offer.index_total : 1,
|
|
1897
|
+
mode: offer.mode ?? 'replicate',
|
|
1898
|
+
cwd_rel: normalizeRelPath(cwdRel),
|
|
1899
|
+
base_commit: offer.base_commit ?? anchorsBefore?.head_commit ?? '',
|
|
1900
|
+
base_tree: offer.base_tree ?? null,
|
|
1901
|
+
pre_tree_fingerprint: anchorsBefore?.pre_tree_fingerprint ?? null,
|
|
1902
|
+
post_tree_fingerprint: anchorsAfter?.post_tree_fingerprint ?? null,
|
|
1903
|
+
fingerprint_algo: FINGERPRINT_ALGO,
|
|
1904
|
+
fingerprint_error: anchorsBefore?.fingerprint_error ?? null,
|
|
1905
|
+
head_commit: anchorsAfter?.head_commit ?? anchorsBefore?.head_commit ?? null,
|
|
1906
|
+
dirty_before: anchorsBefore?.dirty_before === true,
|
|
1907
|
+
command_argv: commandArgv,
|
|
1908
|
+
// A pipeline's hash covers the whole chain, matching the relay's `computePipelineCommandHash`.
|
|
1909
|
+
// Hashing only stage 0 would make every pipeline `unverifiable` at the anchor -- the machine ran
|
|
1910
|
+
// exactly what it was asked and the relay still could not confirm it.
|
|
1911
|
+
command_hash: pipelineHashMaterial === null
|
|
1912
|
+
? commandHash(commandArgv, SHELL_ID, cwdRel)
|
|
1913
|
+
: pipelineCommandHash(pipelineHashMaterial, SHELL_ID),
|
|
1914
|
+
// The material the pipeline hash was taken over. Kept in the envelope so a mismatch can be
|
|
1915
|
+
// diagnosed from the result itself rather than by re-running with instrumentation -- which is
|
|
1916
|
+
// what was needed to find the relay/agent disagreement this field now prevents. Present only for
|
|
1917
|
+
// a chain, so a single-command envelope is byte-unchanged.
|
|
1918
|
+
...(pipelineHashMaterial === null ? {} : { pipeline_hash_material: pipelineHashMaterial }),
|
|
1919
|
+
shell_id: SHELL_ID,
|
|
1920
|
+
started_at: startedAt,
|
|
1921
|
+
ended_at: endedAt,
|
|
1922
|
+
duration_ms: execution?.duration_ms ?? 0,
|
|
1923
|
+
exit_code: execution ? execution.exit_code : null,
|
|
1924
|
+
status,
|
|
1925
|
+
refusal_reason: refusalReason ?? null,
|
|
1926
|
+
stdout_sha256: execution ? execution.stdout_sha256 : EMPTY_SHA256,
|
|
1927
|
+
stdout_bytes: execution ? execution.stdout_bytes : 0,
|
|
1928
|
+
stderr_sha256: execution ? execution.stderr_sha256 : EMPTY_SHA256,
|
|
1929
|
+
stderr_bytes: execution ? execution.stderr_bytes : 0,
|
|
1930
|
+
warnings,
|
|
1931
|
+
// Optional fields (§5.2): present on both machines for the same task, so
|
|
1932
|
+
// they stay comparable; head/tail are informational only.
|
|
1933
|
+
stdout_normalized_sha256: sha256Hex(normalized),
|
|
1934
|
+
stdout_head: execution?.stdout_head ?? '',
|
|
1935
|
+
stdout_tail: execution?.stdout_tail ?? '',
|
|
1936
|
+
stderr_head: execution?.stderr_head ?? '',
|
|
1937
|
+
stderr_tail: execution?.stderr_tail ?? '',
|
|
1938
|
+
untracked: anchorsAfter?.untracked ?? anchorsBefore?.untracked ?? [],
|
|
1939
|
+
diff_numstat: anchorsAfter?.diff_numstat ?? [],
|
|
1940
|
+
toolchain: toolchain ?? null,
|
|
1941
|
+
};
|
|
1942
|
+
if (execution?.signal) envelope.signal = execution.signal;
|
|
1943
|
+
if (offer.write === true) envelope.write = true;
|
|
1944
|
+
|
|
1945
|
+
// v0.3.3 `pipeline` (§5.2 optional). Present only for a chain, so a single-command envelope is
|
|
1946
|
+
// byte-identical to before -- which is what keeps a v0.2.3 relay able to verify this one.
|
|
1947
|
+
//
|
|
1948
|
+
// The per-stage record is the point: a final exit code cannot distinguish "stage 3 failed" from
|
|
1949
|
+
// "stages 3..5 never ran", and those need different responses from a reader.
|
|
1950
|
+
if (pipeline && typeof pipeline === 'object') {
|
|
1951
|
+
envelope.pipeline = pipeline;
|
|
1952
|
+
}
|
|
1953
|
+
if (Array.isArray(protocolViolations) && protocolViolations.length > 0) {
|
|
1954
|
+
envelope.protocol_violations = protocolViolations;
|
|
1955
|
+
}
|
|
1956
|
+
|
|
1957
|
+
// v0.4.0 §6. Always present, like the relay's own offer fields and for the same reason: a
|
|
1958
|
+
// v0.3.9 machine and an `auto` one are then distinguishable from the stored envelope alone,
|
|
1959
|
+
// instead of `transport` being absent both when the direct path was never tried and when it
|
|
1960
|
+
// was tried and lost. `transport` is how the *offer* arrived; `p2p.result_path` is how the
|
|
1961
|
+
// *result* left. They are different facts and are allowed to differ -- a relay offer can be
|
|
1962
|
+
// answered over a channel the agent opened for that task.
|
|
1963
|
+
const transport = context.p2pTransport ?? offerTransportOf(offer, p2pStatusSnapshot());
|
|
1964
|
+
envelope.transport = transport.offer_path;
|
|
1965
|
+
envelope.p2p = envelopeP2P({ ...context, p2pTransport: transport });
|
|
1966
|
+
return buildEnvelope(envelope);
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
// -- Task handling -------------------------------------------------------
|
|
1970
|
+
|
|
1971
|
+
/** @param {object} offer */
|
|
1972
|
+
function validateOffer(offer) {
|
|
1973
|
+
if (!offer || typeof offer !== 'object') return 'NOT_AN_OBJECT';
|
|
1974
|
+
if (typeof offer.task_id !== 'string' || offer.task_id === '') return 'MISSING_TASK_ID';
|
|
1975
|
+
if (!Array.isArray(offer.command_argv) || offer.command_argv.length === 0) return 'MISSING_COMMAND_ARGV';
|
|
1976
|
+
if (!offer.command_argv.every((part) => typeof part === 'string')) return 'COMMAND_ARGV_NOT_STRINGS';
|
|
1977
|
+
if (offer.index !== undefined && !Number.isInteger(offer.index)) return 'INDEX_NOT_INTEGER';
|
|
1978
|
+
// v0.3.3: a pipeline carries its chain in `stages`. Validated here rather than trusted, because
|
|
1979
|
+
// a malformed chain would otherwise run partially and report a result for work it never did.
|
|
1980
|
+
if (offer.stages !== undefined) {
|
|
1981
|
+
if (offer.mode !== 'pipeline') return 'STAGES_WITHOUT_PIPELINE_MODE';
|
|
1982
|
+
if (!Array.isArray(offer.stages) || offer.stages.length === 0) return 'STAGES_NOT_NON_EMPTY_ARRAY';
|
|
1983
|
+
if (offer.stages.length > MAX_STAGES) return 'TOO_MANY_STAGES';
|
|
1984
|
+
for (const stage of offer.stages) {
|
|
1985
|
+
if (!Array.isArray(stage?.command_argv) || stage.command_argv.length === 0) return 'STAGE_COMMAND_ARGV_MISSING';
|
|
1986
|
+
if (!stage.command_argv.every((part) => typeof part === 'string')) return 'STAGE_COMMAND_ARGV_NOT_STRINGS';
|
|
1987
|
+
}
|
|
1988
|
+
}
|
|
1989
|
+
return null;
|
|
1990
|
+
}
|
|
1991
|
+
|
|
1992
|
+
/**
|
|
1993
|
+
* Does this agent's mode allow the offer to run at all (v0.4.0 §1)?
|
|
1994
|
+
*
|
|
1995
|
+
* `direct` is the one mode where the *transport* is a gate rather than a preference, and it is
|
|
1996
|
+
* deliberately total: any offer that did not arrive over a channel is refused, including one
|
|
1997
|
+
* that arrived over the relay while a punch for the same task was still being attempted. The
|
|
1998
|
+
* alternative -- accepting the relay's copy when a channel happens to exist -- would make the
|
|
1999
|
+
* mode's meaning depend on a race, which is exactly the ambiguity `direct` exists to remove
|
|
2000
|
+
* ("a failed punch is not hidden").
|
|
2001
|
+
*
|
|
2002
|
+
* @param {object} offer
|
|
2003
|
+
* @returns {{reason: string, detail: string}|null}
|
|
2004
|
+
*/
|
|
2005
|
+
function p2pRefusal(offer) {
|
|
2006
|
+
if (p2pMode !== 'direct') return null;
|
|
2007
|
+
if (offer.p2p_transport === 'p2p') return null;
|
|
2008
|
+
return {
|
|
2009
|
+
reason: REFUSAL.P2P_UNAVAILABLE,
|
|
2010
|
+
detail: 'p2pMode is "direct" and this offer arrived over the relay',
|
|
2011
|
+
};
|
|
2012
|
+
}
|
|
2013
|
+
|
|
2014
|
+
/**
|
|
2015
|
+
* The full lifecycle of one offer.
|
|
2016
|
+
*
|
|
2017
|
+
* @param {object} offer
|
|
2018
|
+
*/
|
|
2019
|
+
async function handleOffer(offer) {
|
|
2020
|
+
const problem = validateOffer(offer);
|
|
2021
|
+
if (problem) {
|
|
2022
|
+
log('error', `ignoring malformed offer: ${problem}`, { offer });
|
|
2023
|
+
state.handled += 1;
|
|
2024
|
+
return;
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
const commandArgv = [...offer.command_argv];
|
|
2028
|
+
const cwdOutcome = resolveProjectCwd(project, offer.cwd_rel ?? '.');
|
|
2029
|
+
const cwdRel = cwdOutcome.ok ? cwdOutcome.cwd_rel : normalizeRelPath(offer.cwd_rel ?? '.');
|
|
2030
|
+
const controller = new AbortController();
|
|
2031
|
+
runAbort = controller;
|
|
2032
|
+
state.current = {
|
|
2033
|
+
task_id: offer.task_id,
|
|
2034
|
+
attempt: Number(offer.attempt) || 1,
|
|
2035
|
+
controller,
|
|
2036
|
+
key: identityKeyFor(offer),
|
|
2037
|
+
};
|
|
2038
|
+
|
|
2039
|
+
// ---- 0. mode gate (v0.4.0 §1) ----------------------------------------
|
|
2040
|
+
// Checked before the spool: a refusal is not a run. Spooling it would leave a `claimed` entry
|
|
2041
|
+
// whose envelope is written by `deliver`, which would in turn claim a task that never executed
|
|
2042
|
+
// -- so the refusal is answered and the attempt is done.
|
|
2043
|
+
const refusal = p2pRefusal(offer);
|
|
2044
|
+
if (refusal) {
|
|
2045
|
+
log('warn', `refused ${offer.task_id}: ${refusal.reason}`, { detail: refusal.detail });
|
|
2046
|
+
runAbort = null;
|
|
2047
|
+
state.current = null;
|
|
2048
|
+
state.handled += 1;
|
|
2049
|
+
await finish({
|
|
2050
|
+
offer,
|
|
2051
|
+
cwdRel,
|
|
2052
|
+
commandArgv,
|
|
2053
|
+
anchorsBefore: null,
|
|
2054
|
+
anchorsAfter: null,
|
|
2055
|
+
execution: null,
|
|
2056
|
+
status: 'refused',
|
|
2057
|
+
refusalReason: refusal.reason,
|
|
2058
|
+
warnings: [`P2P_UNAVAILABLE: ${refusal.detail}`],
|
|
2059
|
+
startedAt: new Date().toISOString(),
|
|
2060
|
+
endedAt: new Date().toISOString(),
|
|
2061
|
+
});
|
|
2062
|
+
return;
|
|
2063
|
+
}
|
|
2064
|
+
|
|
2065
|
+
// ---- 1. spool before anything else (§0) ------------------------------
|
|
2066
|
+
// Fail closed: if the durable copy cannot be written, running the command
|
|
2067
|
+
// would produce a result we could never replay, so the offer is dropped
|
|
2068
|
+
// with an explicit reason instead of executing unspooled. This also keeps
|
|
2069
|
+
// `state.current` from being left pointing at a task that never started.
|
|
2070
|
+
try {
|
|
2071
|
+
spool.saveTask({
|
|
2072
|
+
task_id: offer.task_id,
|
|
2073
|
+
attempt: offer.attempt,
|
|
2074
|
+
dedupe_key: offer.dedupe_key,
|
|
2075
|
+
offer,
|
|
2076
|
+
});
|
|
2077
|
+
} catch (error) {
|
|
2078
|
+
log('error', `could not spool ${offer.task_id}; not running it (spool-first, §0)`, {
|
|
2079
|
+
error: error?.message ?? String(error),
|
|
2080
|
+
});
|
|
2081
|
+
runAbort = null;
|
|
2082
|
+
state.current = null;
|
|
2083
|
+
state.handled += 1;
|
|
2084
|
+
return;
|
|
2085
|
+
}
|
|
2086
|
+
log('info', `offer ${offer.task_id} attempt ${offer.attempt ?? 1}`, {
|
|
2087
|
+
argv: commandArgv,
|
|
2088
|
+
cwd_rel: cwdRel,
|
|
2089
|
+
write: offer.write === true,
|
|
2090
|
+
});
|
|
2091
|
+
|
|
2092
|
+
// ---- 2. lease heartbeat ---------------------------------------------
|
|
2093
|
+
heartbeat = new Heartbeat({
|
|
2094
|
+
send: (body) => postJson('/v1/heartbeat', body).then((response) => response.json),
|
|
2095
|
+
taskId: offer.task_id,
|
|
2096
|
+
machineId: identity.machine_id,
|
|
2097
|
+
attempt: Number(offer.attempt) || 1,
|
|
2098
|
+
intervalMs: heartbeatIntervalMs,
|
|
2099
|
+
onError: (error) => log('warn', 'heartbeat failed', { error: error.message }),
|
|
2100
|
+
onRtt: recordRtt,
|
|
2101
|
+
// v0.3.0: the same numbers the local file carries, sent to the relay so a
|
|
2102
|
+
// plugin on another machine can see them.
|
|
2103
|
+
diagnostics: () =>
|
|
2104
|
+
heartbeatDiagnostics({
|
|
2105
|
+
rttMs: state.rttMs,
|
|
2106
|
+
reconnectAttempts: state.reconnectAttempts,
|
|
2107
|
+
}),
|
|
2108
|
+
onCancel: () => {
|
|
2109
|
+
log('warn', `Rabbit cancelled ${offer.task_id} via heartbeat`);
|
|
2110
|
+
controller.abort();
|
|
2111
|
+
},
|
|
2112
|
+
}).start();
|
|
2113
|
+
|
|
2114
|
+
const startedAt = new Date().toISOString();
|
|
2115
|
+
const startedHr = process.hrtime.bigint();
|
|
2116
|
+
/** @type {string[]} */
|
|
2117
|
+
const warnings = [];
|
|
2118
|
+
|
|
2119
|
+
try {
|
|
2120
|
+
// ---- 3. capability gate (§6.1) -------------------------------------
|
|
2121
|
+
const writable = offer.write === true ? await isWritableProject(project) : true;
|
|
2122
|
+
const gate = evaluateGate(offer, { caps, platform, writable, toolchain: await toolchainInfo() });
|
|
2123
|
+
|
|
2124
|
+
// ---- 4. anchors ------------------------------------------------------
|
|
2125
|
+
const anchorsBefore = await safeAnchors(warnings);
|
|
2126
|
+
|
|
2127
|
+
if (!gate.ok) {
|
|
2128
|
+
log('warn', `refused ${offer.task_id}: ${gate.refusal_reason}`, { detail: gate.detail });
|
|
2129
|
+
await finish({
|
|
2130
|
+
offer,
|
|
2131
|
+
cwdRel,
|
|
2132
|
+
commandArgv,
|
|
2133
|
+
anchorsBefore,
|
|
2134
|
+
anchorsAfter: null,
|
|
2135
|
+
execution: null,
|
|
2136
|
+
status: 'refused',
|
|
2137
|
+
refusalReason: gate.refusal_reason,
|
|
2138
|
+
warnings,
|
|
2139
|
+
startedAt,
|
|
2140
|
+
endedAt: new Date().toISOString(),
|
|
2141
|
+
});
|
|
2142
|
+
return;
|
|
2143
|
+
}
|
|
2144
|
+
|
|
2145
|
+
// ---- 5. allow-list (default-deny) -----------------------------------
|
|
2146
|
+
const allowed = matchAllowedCommand(commandArgv, allowedCommands);
|
|
2147
|
+
if (!allowed.allowed) {
|
|
2148
|
+
log('warn', `refused ${offer.task_id}: COMMAND_NOT_ALLOWED`, { argv: commandArgv });
|
|
2149
|
+
await finish({
|
|
2150
|
+
offer,
|
|
2151
|
+
cwdRel,
|
|
2152
|
+
commandArgv,
|
|
2153
|
+
anchorsBefore,
|
|
2154
|
+
anchorsAfter: null,
|
|
2155
|
+
execution: null,
|
|
2156
|
+
status: 'refused',
|
|
2157
|
+
refusalReason: REFUSAL.COMMAND_NOT_ALLOWED,
|
|
2158
|
+
warnings,
|
|
2159
|
+
startedAt,
|
|
2160
|
+
endedAt: new Date().toISOString(),
|
|
2161
|
+
});
|
|
2162
|
+
return;
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
if (!cwdOutcome.ok) {
|
|
2166
|
+
log('warn', `refused ${offer.task_id}: ${cwdOutcome.reason}`, { cwd_rel: offer.cwd_rel });
|
|
2167
|
+
await finish({
|
|
2168
|
+
offer,
|
|
2169
|
+
cwdRel,
|
|
2170
|
+
commandArgv,
|
|
2171
|
+
anchorsBefore,
|
|
2172
|
+
anchorsAfter: null,
|
|
2173
|
+
execution: null,
|
|
2174
|
+
status: 'refused',
|
|
2175
|
+
refusalReason: cwdOutcome.reason,
|
|
2176
|
+
warnings,
|
|
2177
|
+
startedAt,
|
|
2178
|
+
endedAt: new Date().toISOString(),
|
|
2179
|
+
});
|
|
2180
|
+
return;
|
|
2181
|
+
}
|
|
2182
|
+
|
|
2183
|
+
// ---- 6. execute ------------------------------------------------------
|
|
2184
|
+
//
|
|
2185
|
+
// Every offer has at least one stage: a single-command task is the one-stage case (the relay
|
|
2186
|
+
// normalizes it that way), so there is no separate branch for the chain. `pipeline` adds
|
|
2187
|
+
// ordering and stop-on-failure, not a second execution path.
|
|
2188
|
+
const isPipeline = Array.isArray(offer.stages) && offer.stages.length > 0;
|
|
2189
|
+
const plannedStages = isPipeline
|
|
2190
|
+
? offer.stages.map((s, i) => ({
|
|
2191
|
+
index: i,
|
|
2192
|
+
command_argv: [...s.command_argv],
|
|
2193
|
+
cwd_rel: typeof s.cwd_rel === 'string' && s.cwd_rel ? s.cwd_rel : (offer.cwd_rel ?? '.'),
|
|
2194
|
+
continue_on_failure: s.continue_on_failure === true,
|
|
2195
|
+
}))
|
|
2196
|
+
: [{ index: 0, command_argv: [...offer.command_argv], cwd_rel: offer.cwd_rel ?? '.', continue_on_failure: false }];
|
|
2197
|
+
|
|
2198
|
+
// The task's timeout is a budget for the whole chain, not per stage. Giving each stage the full
|
|
2199
|
+
// timeout would let a 16-stage pipeline run 16x longer than the caller asked for, and the relay
|
|
2200
|
+
// would have given up long before.
|
|
2201
|
+
const totalBudgetMs = Number.isFinite(offer.timeout_ms) ? offer.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
2202
|
+
const chainStartedAt = Date.now();
|
|
2203
|
+
|
|
2204
|
+
const stageReports = [];
|
|
2205
|
+
/** The last stage's execution, used for the envelope's top-level exit code and output hashes. */
|
|
2206
|
+
let execution = null;
|
|
2207
|
+
let stoppedAt = null;
|
|
2208
|
+
const protocolViolations = [];
|
|
2209
|
+
|
|
2210
|
+
for (const stage of plannedStages) {
|
|
2211
|
+
heartbeat.setPhase('running', 50);
|
|
2212
|
+
|
|
2213
|
+
const stageCwdOutcome = resolveProjectCwd(project, stage.cwd_rel);
|
|
2214
|
+
if (!stageCwdOutcome.ok) {
|
|
2215
|
+
// A stage whose cwd is outside the project is a refusal of the whole chain: continuing
|
|
2216
|
+
// would run later stages against a working directory the caller never asked for.
|
|
2217
|
+
protocolViolations.push(`stage ${stage.index}: ${stageCwdOutcome.reason}`);
|
|
2218
|
+
stageReports.push({
|
|
2219
|
+
index: stage.index,
|
|
2220
|
+
command_argv: stage.command_argv,
|
|
2221
|
+
cwd_rel: stage.cwd_rel,
|
|
2222
|
+
status: 'refused',
|
|
2223
|
+
refusal_reason: stageCwdOutcome.reason,
|
|
2224
|
+
exit_code: null,
|
|
2225
|
+
duration_ms: 0,
|
|
2226
|
+
});
|
|
2227
|
+
stoppedAt = stage.index;
|
|
2228
|
+
break;
|
|
2229
|
+
}
|
|
2230
|
+
|
|
2231
|
+
const elapsedMs = Date.now() - chainStartedAt;
|
|
2232
|
+
const remainingMs = totalBudgetMs - elapsedMs;
|
|
2233
|
+
if (remainingMs <= 0) {
|
|
2234
|
+
warnings.push('PIPELINE_BUDGET_EXHAUSTED');
|
|
2235
|
+
stageReports.push({
|
|
2236
|
+
index: stage.index,
|
|
2237
|
+
command_argv: stage.command_argv,
|
|
2238
|
+
cwd_rel: stage.cwd_rel,
|
|
2239
|
+
status: 'timeout',
|
|
2240
|
+
exit_code: null,
|
|
2241
|
+
duration_ms: 0,
|
|
2242
|
+
});
|
|
2243
|
+
stoppedAt = stage.index;
|
|
2244
|
+
break;
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
const ran = await runArgv(stage.command_argv, {
|
|
2248
|
+
cwd: stageCwdOutcome.cwd,
|
|
2249
|
+
timeoutMs: remainingMs,
|
|
2250
|
+
maxOutputBytes,
|
|
2251
|
+
signal: controller.signal,
|
|
2252
|
+
});
|
|
2253
|
+
execution = ran;
|
|
2254
|
+
for (const warning of ran.warnings) if (!warnings.includes(warning)) warnings.push(warning);
|
|
2255
|
+
|
|
2256
|
+
const stageStatus = classifyExit(ran);
|
|
2257
|
+
stageReports.push({
|
|
2258
|
+
index: stage.index,
|
|
2259
|
+
command_argv: stage.command_argv,
|
|
2260
|
+
cwd_rel: stageCwdOutcome.cwd_rel ?? stage.cwd_rel,
|
|
2261
|
+
// Carried so the agent's pipeline hash covers the same shape the relay hashes. Omitting it
|
|
2262
|
+
// would make the two disagree the moment a caller sets `continue_on_failure: true`.
|
|
2263
|
+
continue_on_failure: stage.continue_on_failure,
|
|
2264
|
+
status: stageStatus,
|
|
2265
|
+
exit_code: ran.exit_code,
|
|
2266
|
+
duration_ms: ran.duration_ms,
|
|
2267
|
+
stdout_sha256: ran.stdout_sha256,
|
|
2268
|
+
stderr_sha256: ran.stderr_sha256,
|
|
2269
|
+
});
|
|
2270
|
+
|
|
2271
|
+
if (ran.timed_out) log('warn', `${offer.task_id} stage ${stage.index} timed out`);
|
|
2272
|
+
if (ran.cancelled) log('warn', `${offer.task_id} stage ${stage.index} was cancelled`);
|
|
2273
|
+
|
|
2274
|
+
if (stageStatus !== 'ok' && !stage.continue_on_failure) {
|
|
2275
|
+
stoppedAt = stage.index;
|
|
2276
|
+
break;
|
|
2277
|
+
}
|
|
2278
|
+
}
|
|
2279
|
+
|
|
2280
|
+
heartbeat.setPhase('finalizing', 90);
|
|
2281
|
+
if (execution === null) {
|
|
2282
|
+
// Nothing ran at all (a refused first stage, or a budget already spent). `execution` stays
|
|
2283
|
+
// null, which `finish` renders as an empty-output envelope -- the honest shape for "no
|
|
2284
|
+
// command produced output".
|
|
2285
|
+
log('warn', `${offer.task_id} ran no pipeline stage`);
|
|
2286
|
+
}
|
|
2287
|
+
if (execution?.timed_out) log('warn', `${offer.task_id} timed out after ${offer.timeout_ms}ms`);
|
|
2288
|
+
if (execution?.cancelled) log('warn', `${offer.task_id} was cancelled`);
|
|
2289
|
+
|
|
2290
|
+
const anchorsAfter = await safeAnchors(warnings);
|
|
2291
|
+
const status = execution ? classifyExit(execution) : (protocolViolations.length > 0 ? 'refused' : 'unverifiable');
|
|
2292
|
+
|
|
2293
|
+
await finish({
|
|
2294
|
+
offer,
|
|
2295
|
+
cwdRel,
|
|
2296
|
+
commandArgv,
|
|
2297
|
+
anchorsBefore,
|
|
2298
|
+
anchorsAfter,
|
|
2299
|
+
execution,
|
|
2300
|
+
status,
|
|
2301
|
+
refusalReason: null,
|
|
2302
|
+
warnings,
|
|
2303
|
+
startedAt,
|
|
2304
|
+
endedAt: new Date().toISOString(),
|
|
2305
|
+
// v0.3.3: the per-stage record travels with the envelope so a reader can see which stage
|
|
2306
|
+
// failed and which never ran, instead of only a final exit code.
|
|
2307
|
+
pipeline: isPipeline
|
|
2308
|
+
? {
|
|
2309
|
+
total_stages: plannedStages.length,
|
|
2310
|
+
completed_stages: stageReports.length,
|
|
2311
|
+
stopped_at: stoppedAt,
|
|
2312
|
+
stages: stageReports,
|
|
2313
|
+
}
|
|
2314
|
+
: null,
|
|
2315
|
+
protocolViolations,
|
|
2316
|
+
});
|
|
2317
|
+
} catch (error) {
|
|
2318
|
+
// Never leave a claimed task unspooled: emit a crashed envelope so the
|
|
2319
|
+
// relay can account for it instead of waiting for a lease to expire.
|
|
2320
|
+
log('error', `error while handling ${offer.task_id}`, { error: error?.message ?? String(error) });
|
|
2321
|
+
try {
|
|
2322
|
+
const envelope = makeEnvelope({
|
|
2323
|
+
offer,
|
|
2324
|
+
cwdRel,
|
|
2325
|
+
commandArgv,
|
|
2326
|
+
anchorsBefore: null,
|
|
2327
|
+
anchorsAfter: null,
|
|
2328
|
+
execution: null,
|
|
2329
|
+
status: 'crashed',
|
|
2330
|
+
refusalReason: null,
|
|
2331
|
+
// The error text travels with the envelope. Without it, a crashed pipeline is visible only as
|
|
2332
|
+
// `status: crashed` on the wire and the cause has to be reproduced to be seen -- which is
|
|
2333
|
+
// exactly the situation this field was added to end.
|
|
2334
|
+
warnings: [...warnings, 'AGENT_ERROR', `AGENT_ERROR_DETAIL: ${error?.message ?? String(error)}`],
|
|
2335
|
+
startedAt,
|
|
2336
|
+
endedAt: new Date().toISOString(),
|
|
2337
|
+
});
|
|
2338
|
+
await deliver(envelope);
|
|
2339
|
+
} catch (nested) {
|
|
2340
|
+
log('error', 'could not even build a crashed envelope', {
|
|
2341
|
+
error: nested?.message ?? String(nested),
|
|
2342
|
+
});
|
|
2343
|
+
}
|
|
2344
|
+
} finally {
|
|
2345
|
+
heartbeat?.stop();
|
|
2346
|
+
heartbeat = null;
|
|
2347
|
+
runAbort = null;
|
|
2348
|
+
state.current = null;
|
|
2349
|
+
state.handled += 1;
|
|
2350
|
+
log('success', `task ${offer.task_id} complete`, {
|
|
2351
|
+
duration_ms: Math.round(Number(process.hrtime.bigint() - startedHr) / 1e6),
|
|
2352
|
+
});
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
|
|
2356
|
+
/** Collect anchors, converting failures into warnings instead of throws. */
|
|
2357
|
+
async function safeAnchors(warnings) {
|
|
2358
|
+
try {
|
|
2359
|
+
const anchors = await collectPreAnchors({
|
|
2360
|
+
cwd: project,
|
|
2361
|
+
tmpDir,
|
|
2362
|
+
objectMode,
|
|
2363
|
+
});
|
|
2364
|
+
if (anchors.fingerprint_error && !warnings.includes(WARN_FINGERPRINT_UNAVAILABLE)) {
|
|
2365
|
+
warnings.push(WARN_FINGERPRINT_UNAVAILABLE);
|
|
2366
|
+
}
|
|
2367
|
+
if (anchors.dirty_before === true && !warnings.includes(WARN_DIRTY_WORKTREE)) {
|
|
2368
|
+
warnings.push(WARN_DIRTY_WORKTREE);
|
|
2369
|
+
}
|
|
2370
|
+
return anchors;
|
|
2371
|
+
} catch (error) {
|
|
2372
|
+
warnings.push(WARN_FINGERPRINT_UNAVAILABLE);
|
|
2373
|
+
log('warn', 'could not collect anchors', { error: error?.message ?? String(error) });
|
|
2374
|
+
return {
|
|
2375
|
+
head_commit: null,
|
|
2376
|
+
head_tree: null,
|
|
2377
|
+
pre_tree_fingerprint: null,
|
|
2378
|
+
fingerprint_error: 'ANCHOR_COLLECTION_FAILED',
|
|
2379
|
+
fingerprint_error_detail: error?.message ?? String(error),
|
|
2380
|
+
dirty_before: null,
|
|
2381
|
+
untracked: [],
|
|
2382
|
+
};
|
|
2383
|
+
}
|
|
2384
|
+
}
|
|
2385
|
+
|
|
2386
|
+
/** Post-run state, converted the same way. */
|
|
2387
|
+
async function safePostState(offer, warnings) {
|
|
2388
|
+
try {
|
|
2389
|
+
return await collectPostState({
|
|
2390
|
+
cwd: project,
|
|
2391
|
+
base: offer.base_commit ?? undefined,
|
|
2392
|
+
tmpDir,
|
|
2393
|
+
objectMode,
|
|
2394
|
+
});
|
|
2395
|
+
} catch (error) {
|
|
2396
|
+
warnings.push(WARN_FINGERPRINT_UNAVAILABLE);
|
|
2397
|
+
log('warn', 'could not collect post-state', { error: error?.message ?? String(error) });
|
|
2398
|
+
return {
|
|
2399
|
+
head_commit: null,
|
|
2400
|
+
post_tree_fingerprint: null,
|
|
2401
|
+
fingerprint_error: 'ANCHOR_COLLECTION_FAILED',
|
|
2402
|
+
untracked: [],
|
|
2403
|
+
diff_numstat: [],
|
|
2404
|
+
};
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
|
|
2408
|
+
/**
|
|
2409
|
+
* Try the direct path for one attempt, before the envelope exists.
|
|
2410
|
+
*
|
|
2411
|
+
* WHY THIS RUNS FIRST, AND WHY THE FRAME IS NOT THE ENVELOPE
|
|
2412
|
+
*
|
|
2413
|
+
* `result_path` is `'p2p'` only when the envelope was *actually acknowledged* over the channel,
|
|
2414
|
+
* and `envelope_sha256` covers the whole envelope -- so the fact has to be known before the
|
|
2415
|
+
* envelope is built, or the hash would not cover the bytes it claims to cover. The obvious way
|
|
2416
|
+
* out would be to send the finished envelope and stamp `result_path` afterwards, which is
|
|
2417
|
+
* precisely the mutation that would break `verifyEnvelope`. So the order is inverted instead:
|
|
2418
|
+
* announce the result on the channel (`{type:'task.result', task_id}`), wait for the dispatcher's
|
|
2419
|
+
* `{type:'result.ack', task_id}`, and only then build the one envelope that describes what
|
|
2420
|
+
* happened.
|
|
2421
|
+
*
|
|
2422
|
+
* The direct frame therefore carries the frame *type* and the task identity rather than the whole
|
|
2423
|
+
* envelope. That is a deliberate narrowing, and it is worth stating plainly: the fast path
|
|
2424
|
+
* currently saves the dispatcher the wait for the relay's copy, not the bytes. Carrying the full
|
|
2425
|
+
* envelope on the channel would need the envelope before the acknowledgement that decides one of
|
|
2426
|
+
* its fields -- see the note in the v0.4.0 report and `PROTOCOL-v0.4.0.md`'s open question.
|
|
2427
|
+
*
|
|
2428
|
+
* @param {object} offer
|
|
2429
|
+
* @returns {Promise<ReturnType<typeof p2pDeliveryMark>>}
|
|
2430
|
+
*/
|
|
2431
|
+
async function planDirectDelivery(offer) {
|
|
2432
|
+
if (channelFor(offer?.task_id) === null) return p2pDeliveryMark();
|
|
2433
|
+
const direct = await sendResultDirect(offer.task_id);
|
|
2434
|
+
if (direct.delivered) {
|
|
2435
|
+
log('info', `result acknowledged over the direct channel for ${offer.task_id}`, {
|
|
2436
|
+
rtt_ms: direct.rtt_ms,
|
|
2437
|
+
});
|
|
2438
|
+
return p2pDeliveryMark({ path: 'p2p', asked: true, rtt_ms: direct.rtt_ms });
|
|
2439
|
+
}
|
|
2440
|
+
// A timeout is a reported failure; "no channel at all" is not, and `asked` keeps the two
|
|
2441
|
+
// apart for anything reading the agent's own diagnostics.
|
|
2442
|
+
log('warn', `direct result delivery failed for ${offer.task_id}; the relay copy is unaffected`, {
|
|
2443
|
+
error: direct.error,
|
|
2444
|
+
});
|
|
2445
|
+
return p2pDeliveryMark({ asked: true, error: direct.error });
|
|
2446
|
+
}
|
|
2447
|
+
|
|
2448
|
+
/**
|
|
2449
|
+
* Announce one result on its direct channel and wait for the acknowledgement, bounded.
|
|
2450
|
+
*
|
|
2451
|
+
* This is the fast path, and it is deliberately *only* a fast path: the relay copy is posted by
|
|
2452
|
+
* the caller whatever happens here, so nothing in this function can lose a result. The wait is
|
|
2453
|
+
* bounded by {@link P2P_ACK_TIMEOUT_MS} for the same reason -- a dispatcher that stops answering
|
|
2454
|
+
* must cost the ledger a bounded delay, never a hang.
|
|
2455
|
+
*
|
|
2456
|
+
* It is deliberately **not** retried. `P2PChannel.send` already retransmits every fragment until
|
|
2457
|
+
* the peer acknowledges it; a retry on top would be a second retransmission protocol layered on
|
|
2458
|
+
* the first, and "the dispatcher is gone" is not a condition that improves by asking again.
|
|
2459
|
+
*
|
|
2460
|
+
* @param {string} taskId
|
|
2461
|
+
* @returns {Promise<{delivered: boolean, rtt_ms: number|null, error: string|null}>}
|
|
2462
|
+
*/
|
|
2463
|
+
async function sendResultDirect(taskId) {
|
|
2464
|
+
const channel = channelFor(taskId);
|
|
2465
|
+
if (channel === null) return { delivered: false, rtt_ms: null, error: null };
|
|
2466
|
+
|
|
2467
|
+
/** Why the wait ended without an ack, when the channel itself said so. */
|
|
2468
|
+
let lastAckError = null;
|
|
2469
|
+
let registration = null;
|
|
2470
|
+
const acked = new Promise((resolve) => {
|
|
2471
|
+
const timer = setTimeout(() => {
|
|
2472
|
+
p2pResultWaiters.delete(taskId);
|
|
2473
|
+
resolve(false);
|
|
2474
|
+
}, p2pAckTimeoutMs);
|
|
2475
|
+
// Never keep the process alive for an acknowledgement.
|
|
2476
|
+
timer.unref?.();
|
|
2477
|
+
registration = {
|
|
2478
|
+
settle: (value, reason) => {
|
|
2479
|
+
clearTimeout(timer);
|
|
2480
|
+
if (p2pResultWaiters.get(taskId) === registration) p2pResultWaiters.delete(taskId);
|
|
2481
|
+
if (value === false && typeof reason === 'string' && reason !== '') lastAckError = reason;
|
|
2482
|
+
resolve(value);
|
|
2483
|
+
},
|
|
2484
|
+
};
|
|
2485
|
+
p2pResultWaiters.set(taskId, registration);
|
|
2486
|
+
});
|
|
2487
|
+
|
|
2488
|
+
const startedAt = process.hrtime.bigint();
|
|
2489
|
+
// `machine_id` travels with the notice because the dispatcher has to be able to say *which*
|
|
2490
|
+
// machine finished without waiting for the relay's copy — the plugin names its direct-result
|
|
2491
|
+
// inbox file after it. The frame still carries no envelope: `result_path` is decided by the
|
|
2492
|
+
// acknowledgement this frame is waiting for, and `envelope_sha256` covers `result_path`, so
|
|
2493
|
+
// sending the envelope first and stamping it afterwards would break the hash it is signed with.
|
|
2494
|
+
const frame = JSON.stringify({ type: 'task.result', task_id: taskId, machine_id: identity.machine_id });
|
|
2495
|
+
try {
|
|
2496
|
+
await channel.send(frame);
|
|
2497
|
+
} catch (error) {
|
|
2498
|
+
registration.settle(false, `P2P_SEND_FAILED: ${error?.message ?? String(error)}`);
|
|
2499
|
+
}
|
|
2500
|
+
|
|
2501
|
+
const delivered = await acked;
|
|
2502
|
+
const elapsed = Number(process.hrtime.bigint() - startedAt) / 1e6;
|
|
2503
|
+
if (delivered) return { delivered: true, rtt_ms: Math.round(elapsed * 100) / 100, error: null };
|
|
2504
|
+
return {
|
|
2505
|
+
delivered: false,
|
|
2506
|
+
rtt_ms: null,
|
|
2507
|
+
error: lastAckError ?? `P2P_ACK_TIMEOUT: no result.ack within ${p2pAckTimeoutMs}ms`,
|
|
2508
|
+
};
|
|
2509
|
+
}
|
|
2510
|
+
|
|
2511
|
+
/** Build, spool, send and acknowledge one envelope. */
|
|
2512
|
+
async function finish(context) {
|
|
2513
|
+
const anchorsAfter = context.anchorsAfter ?? (await safePostState(context.offer, context.warnings));
|
|
2514
|
+
const toolchain = await toolchainInfo();
|
|
2515
|
+
const envelope = makeEnvelope({
|
|
2516
|
+
...context,
|
|
2517
|
+
anchorsAfter,
|
|
2518
|
+
toolchain,
|
|
2519
|
+
comparePolicy: context.offer.compare_policy,
|
|
2520
|
+
// The bounded wait happens *before* the envelope is built, because `result_path` has to be
|
|
2521
|
+
// inside the envelope when `envelope_sha256` is taken; see `sendResultDirect`.
|
|
2522
|
+
p2pDelivery: context.p2pDelivery ?? (await planDirectDelivery(context.offer)),
|
|
2523
|
+
});
|
|
2524
|
+
await deliver(envelope);
|
|
2525
|
+
// The attempt is over. The channel lingers (bounded) so a duplicate of this attempt can still be
|
|
2526
|
+
// answered on it, and is released afterwards.
|
|
2527
|
+
scheduleChannelRelease(envelope.task_id);
|
|
2528
|
+
return envelope;
|
|
2529
|
+
}
|
|
2530
|
+
|
|
2531
|
+
/**
|
|
2532
|
+
* Spool -> POST to the relay -> ack.
|
|
2533
|
+
*
|
|
2534
|
+
* The spool is written first, so a crash anywhere later leaves a replayable envelope; the relay
|
|
2535
|
+
* copy is posted **unconditionally**, because the relay is the ledger and the only writer allowed
|
|
2536
|
+
* to store a machine's result. The direct copy has already been attempted and boundedly awaited
|
|
2537
|
+
* by the time this runs (`planDirectDelivery`), and a failure there changes nothing here -- which
|
|
2538
|
+
* is the property `test/p2p-agent.test.mjs` pins down with a dispatcher that stops answering.
|
|
2539
|
+
*/
|
|
2540
|
+
async function deliver(envelope) {
|
|
2541
|
+
spool.saveEnvelope({
|
|
2542
|
+
task_id: envelope.task_id,
|
|
2543
|
+
attempt: envelope.attempt,
|
|
2544
|
+
dedupe_key: envelope.dedupe_key,
|
|
2545
|
+
envelope,
|
|
2546
|
+
});
|
|
2547
|
+
const response = await postJsonWithRetry('/v1/result', envelope);
|
|
2548
|
+
if (response?.ok) {
|
|
2549
|
+
spool.ack({ task_id: envelope.task_id, attempt: envelope.attempt });
|
|
2550
|
+
// Keep the delivered envelope around (bounded) so a transport re-delivery
|
|
2551
|
+
// of this same attempt is answered instead of executed again.
|
|
2552
|
+
rememberCompleted(identityKeyFor(envelope), envelope);
|
|
2553
|
+
log('info', `result delivered for ${envelope.task_id}`, {
|
|
2554
|
+
status: envelope.status,
|
|
2555
|
+
transport: envelope.transport,
|
|
2556
|
+
result_path: envelope.p2p?.result_path ?? 'relay',
|
|
2557
|
+
});
|
|
2558
|
+
return true;
|
|
2559
|
+
}
|
|
2560
|
+
log('warn', `result kept in spool for ${envelope.task_id}`, {
|
|
2561
|
+
status: response?.status ?? null,
|
|
2562
|
+
code: relayErrorCode(response),
|
|
2563
|
+
});
|
|
2564
|
+
return false;
|
|
2565
|
+
}
|
|
2566
|
+
|
|
2567
|
+
// -- SSE -----------------------------------------------------------------
|
|
2568
|
+
|
|
2569
|
+
/** Open the event stream and dispatch until it ends or we stop. */
|
|
2570
|
+
async function streamOnce() {
|
|
2571
|
+
const headers = { accept: 'text/event-stream' };
|
|
2572
|
+
if (token()) headers.authorization = `Bearer ${token()}`;
|
|
2573
|
+
// Decide *before* connecting whether the cursor may be replayed: sending a
|
|
2574
|
+
// stale one is what makes a restarted relay believe we are already caught
|
|
2575
|
+
// up (task-18). Built by concatenation so a relay mounted under a sub-path
|
|
2576
|
+
// (https://host/w2m) keeps its prefix -- see ./url.mjs.
|
|
2577
|
+
const resume = cursorUsableFor({
|
|
2578
|
+
seq: state.seq,
|
|
2579
|
+
cursorRelayId: state.cursorRelayId,
|
|
2580
|
+
relayId: state.relayId,
|
|
2581
|
+
});
|
|
2582
|
+
const query = new URLSearchParams();
|
|
2583
|
+
query.set('machine_id', identity.machine_id);
|
|
2584
|
+
if (resume) {
|
|
2585
|
+
headers['last-event-id'] = String(state.seq);
|
|
2586
|
+
query.set('seq', String(state.seq + 1));
|
|
2587
|
+
}
|
|
2588
|
+
// Remembered so `ready` can decide whether this stream is trustworthy.
|
|
2589
|
+
attachInfo = { seq: resume ? state.seq : null, attributionRelayId: resume ? state.cursorRelayId : null };
|
|
2590
|
+
log('info', 'attaching to event stream', {
|
|
2591
|
+
from_seq: resume ? state.seq + 1 : null,
|
|
2592
|
+
cursor: resume ? state.seq : null,
|
|
2593
|
+
cursor_relay_id: resume ? state.cursorRelayId : null,
|
|
2594
|
+
relay_id: state.relayId,
|
|
2595
|
+
withheld_cursor: !resume && state.seq > 0,
|
|
2596
|
+
});
|
|
2597
|
+
const streamUrl = joinUrl(baseUrl, `/v1/stream?${query.toString()}`);
|
|
2598
|
+
// Signed once, at attach time, over the *API* path including its query: a
|
|
2599
|
+
// long-lived GET has no body, so the query (machine_id, and the resume seq
|
|
2600
|
+
// when we have a cursor) is the part of the request that must be bound.
|
|
2601
|
+
// Signing only the pathname is the classic mistake -- the tests prove such a
|
|
2602
|
+
// signature does not verify -- and signing the base-prefixed wire path would
|
|
2603
|
+
// break the "proxy strips the prefix" deployment (see signingHeaders).
|
|
2604
|
+
Object.assign(headers, signingHeaders('GET', `/v1/stream?${query.toString()}`));
|
|
2605
|
+
|
|
2606
|
+
const response = await fetchImpl(streamUrl, { headers, signal: streamAbort.signal });
|
|
2607
|
+
if (!response.ok) {
|
|
2608
|
+
const body = await response.text().catch(() => '');
|
|
2609
|
+
let code = null;
|
|
2610
|
+
try {
|
|
2611
|
+
code = JSON.parse(body)?.error?.code ?? null;
|
|
2612
|
+
} catch {
|
|
2613
|
+
code = null;
|
|
2614
|
+
}
|
|
2615
|
+
const error = new Error(`stream HTTP ${response.status} ${body.slice(0, 200)}`.trim());
|
|
2616
|
+
// Preserve the relay's own code (SIGNATURE_REQUIRED, SIGNATURE_EXPIRED, …)
|
|
2617
|
+
// so a signed link can be diagnosed without guessing.
|
|
2618
|
+
error.code = code ?? `HTTP_${response.status}`;
|
|
2619
|
+
state.lastRelayError = {
|
|
2620
|
+
at: new Date().toISOString(),
|
|
2621
|
+
http_status: response.status,
|
|
2622
|
+
code: error.code,
|
|
2623
|
+
path: '/v1/stream',
|
|
2624
|
+
};
|
|
2625
|
+
throw error; }
|
|
2626
|
+
if (!response.body) throw new Error('stream response had no body');
|
|
2627
|
+
|
|
2628
|
+
// Both assignments mark the same stream attempt as no longer live: one before the read loop, one
|
|
2629
|
+
// after it ends. Only one stream runs at a time, so there is nothing to interleave with.
|
|
2630
|
+
// eslint-disable-next-line require-atomic-updates -- one stream attempt at a time
|
|
2631
|
+
state.connected = false;
|
|
2632
|
+
const decoder = new TextDecoder('utf-8');
|
|
2633
|
+
const parser = new SseParser();
|
|
2634
|
+
const source = Readable.fromWeb(response.body);
|
|
2635
|
+
|
|
2636
|
+
for await (const chunk of source) {
|
|
2637
|
+
if (state.stopped) break;
|
|
2638
|
+
for (const frame of parser.push(decoder.decode(chunk, { stream: true }))) {
|
|
2639
|
+
handleFrame(frame);
|
|
2640
|
+
}
|
|
2641
|
+
}
|
|
2642
|
+
state.connected = false;
|
|
2643
|
+
publishState();
|
|
2644
|
+
}
|
|
2645
|
+
|
|
2646
|
+
/**
|
|
2647
|
+
* Decode one SSE frame and dispatch it.
|
|
2648
|
+
*
|
|
2649
|
+
* A thin wrapper on purpose (v0.4.0): the dispatch itself is `dispatchFrame`, so an SSE frame and
|
|
2650
|
+
* a JSON frame off a P2P channel reach *the same* handler. The alternative -- a second switch for
|
|
2651
|
+
* the direct path -- is how "the offer runs over either transport" quietly becomes "the offer
|
|
2652
|
+
* runs over the relay, and something almost like it runs over the channel".
|
|
2653
|
+
*
|
|
2654
|
+
* @param {{event: string, data: string, id: string|null}} frame
|
|
2655
|
+
*/
|
|
2656
|
+
function handleFrame(frame) {
|
|
2657
|
+
let payload;
|
|
2658
|
+
try {
|
|
2659
|
+
payload = JSON.parse(frame.data);
|
|
2660
|
+
} catch {
|
|
2661
|
+
log('warn', 'ignoring unparseable SSE frame', { event: frame.event, data: frame.data.slice(0, 120) });
|
|
2662
|
+
return;
|
|
2663
|
+
}
|
|
2664
|
+
dispatchFrame(payload, { event: frame.event, id: frame.id ?? null, transport: 'relay' });
|
|
2665
|
+
}
|
|
2666
|
+
|
|
2667
|
+
/**
|
|
2668
|
+
* One message arrived on a P2P channel.
|
|
2669
|
+
*
|
|
2670
|
+
* The body is a frame, not an offer: the W2M convention is the relay's own frame shape
|
|
2671
|
+
* (`{type:'task.offer', ...}`), so it is parsed with {@link parseP2PFrame} and then dispatched
|
|
2672
|
+
* exactly like an SSE frame would be.
|
|
2673
|
+
*
|
|
2674
|
+
* @param {{from: string|null, channel: object, peer: object, session: number, payload: unknown}} event
|
|
2675
|
+
*/
|
|
2676
|
+
function handleP2PMessage(event) {
|
|
2677
|
+
const payload = parseP2PFrame(event.payload);
|
|
2678
|
+
if (payload === null) {
|
|
2679
|
+
log('warn', 'ignoring non-JSON frame on a P2P channel', {
|
|
2680
|
+
peer: event.peer ?? null,
|
|
2681
|
+
session: event.session ?? null,
|
|
2682
|
+
bytes: Buffer.isBuffer(event.payload) ? event.payload.length : null,
|
|
2683
|
+
});
|
|
2684
|
+
return;
|
|
2685
|
+
}
|
|
2686
|
+
dispatchFrame(payload, { event: payload.type, id: null, transport: 'p2p', channel: event.channel });
|
|
2687
|
+
}
|
|
2688
|
+
|
|
2689
|
+
/**
|
|
2690
|
+
* The one dispatcher both transports feed.
|
|
2691
|
+
*
|
|
2692
|
+
* `transport` is the only difference between an offer from the relay and an offer from a channel,
|
|
2693
|
+
* and it is stamped onto the offer itself (`p2p_transport`) rather than carried alongside: the
|
|
2694
|
+
* dedupe path hands offers to `handleOffer` by reference and drops duplicates before they get
|
|
2695
|
+
* there, so the fact has to travel *with* the offer or it would be lost exactly when two copies
|
|
2696
|
+
* race -- which is the case it exists for.
|
|
2697
|
+
*
|
|
2698
|
+
* `state.seq` is advanced for relay frames only. A channel frame carries no `seq` (it never went
|
|
2699
|
+
* through the relay's event ring), and letting one move the cursor would make the agent tell the
|
|
2700
|
+
* relay it had already seen events it had not.
|
|
2701
|
+
*
|
|
2702
|
+
* @param {object} payload
|
|
2703
|
+
* @param {{event: string, id: string|null, transport: 'p2p'|'relay', channel?: object}} source
|
|
2704
|
+
*/
|
|
2705
|
+
function dispatchFrame(payload, source) {
|
|
2706
|
+
const type = payload?.type ?? source.event;
|
|
2707
|
+
if (source.transport === 'relay') {
|
|
2708
|
+
if (Number.isInteger(payload?.seq)) state.seq = Math.max(state.seq, payload.seq);
|
|
2709
|
+
else if (source.id && /^\d+$/.test(source.id)) state.seq = Math.max(state.seq, Number(source.id));
|
|
2710
|
+
}
|
|
2711
|
+
|
|
2712
|
+
switch (type) {
|
|
2713
|
+
case 'ready': {
|
|
2714
|
+
// A channel frame with this type is not the relay's `ready`: it is a dispatcher pushing a
|
|
2715
|
+
// frame it had no business pushing, and treating it as "connected" would mark an agent with
|
|
2716
|
+
// no stream as connected.
|
|
2717
|
+
if (source.transport !== 'relay') {
|
|
2718
|
+
log('info', 'ignoring a `ready` frame that did not come from the relay');
|
|
2719
|
+
break;
|
|
2720
|
+
}
|
|
2721
|
+
// §3.1: nothing counts as connected before `ready` arrives.
|
|
2722
|
+
state.connected = true;
|
|
2723
|
+
const relayId =
|
|
2724
|
+
typeof payload.relay_id === 'string' && payload.relay_id !== '' ? payload.relay_id : null;
|
|
2725
|
+
const previousRelayId = state.relayId;
|
|
2726
|
+
// A stream is only "tainted" when it was opened *with* a cursor that
|
|
2727
|
+
// this relay process never issued. Such a stream has already skipped
|
|
2728
|
+
// everything the process emitted before `ready` -- including the
|
|
2729
|
+
// `task.offer` at seq 1 -- so it must be dropped and re-attached
|
|
2730
|
+
// cursor-less (task-18). A stream opened without a cursor has seen
|
|
2731
|
+
// everything from the start and is kept: reconnecting there would be
|
|
2732
|
+
// pure churn.
|
|
2733
|
+
const tainted =
|
|
2734
|
+
attachInfo.seq !== null && relayId !== null && attachInfo.attributionRelayId !== relayId;
|
|
2735
|
+
if (tainted) {
|
|
2736
|
+
state.relayIdChanges += previousRelayId !== null && previousRelayId !== relayId ? 1 : 0;
|
|
2737
|
+
state.seq = 0;
|
|
2738
|
+
state.cursorRelayId = null;
|
|
2739
|
+
state.relayId = relayId;
|
|
2740
|
+
state.streamRestartRequested = true;
|
|
2741
|
+
log('warn', `relay restarted (relay_id ${previousRelayId ?? '<unknown>'} -> ${relayId}); discarding seq cursor and re-attaching without one`, {
|
|
2742
|
+
previous_relay_id: previousRelayId,
|
|
2743
|
+
relay_id: relayId,
|
|
2744
|
+
discarded_cursor: attachInfo.seq,
|
|
2745
|
+
});
|
|
2746
|
+
publishState();
|
|
2747
|
+
streamAbort?.abort();
|
|
2748
|
+
break;
|
|
2749
|
+
}
|
|
2750
|
+
if (relayId) state.relayId = relayId;
|
|
2751
|
+
// The cursor we hold was issued by this relay, so attribute it: that is
|
|
2752
|
+
// what lets a later restart be told apart from a plain reconnect.
|
|
2753
|
+
state.cursorRelayId = relayId ?? state.cursorRelayId;
|
|
2754
|
+
log('info', 'stream ready', {
|
|
2755
|
+
seq: payload.seq,
|
|
2756
|
+
protocol_version: payload.protocol_version,
|
|
2757
|
+
machine_id: payload.machine_id,
|
|
2758
|
+
relay_id: relayId,
|
|
2759
|
+
});
|
|
2760
|
+
// Connection state changed: make it visible to other processes.
|
|
2761
|
+
publishState();
|
|
2762
|
+
break;
|
|
2763
|
+
}
|
|
2764
|
+
case 'task.offer': {
|
|
2765
|
+
// The transport travels *on* the offer (see this function's note). Any `p2p_transport` a
|
|
2766
|
+
// peer tried to put on the wire is overwritten, so a relay frame can never claim the
|
|
2767
|
+
// direct path -- the only thing that sets `'p2p'` here is the channel having delivered it.
|
|
2768
|
+
const offer = { ...payload, p2p_transport: source.transport };
|
|
2769
|
+
if (source.transport === 'p2p') {
|
|
2770
|
+
if (typeof offer.task_id !== 'string' || offer.task_id === '') {
|
|
2771
|
+
log('warn', 'ignoring a direct offer with no task_id');
|
|
2772
|
+
break;
|
|
2773
|
+
}
|
|
2774
|
+
// Remembered before the offer is queued: the result path must exist by the time the
|
|
2775
|
+
// result is produced, and a channel lost between the two would leave `result_path` null
|
|
2776
|
+
// with no way to tell why.
|
|
2777
|
+
rememberChannel(offer.task_id, source.channel);
|
|
2778
|
+
}
|
|
2779
|
+
enqueue(offer);
|
|
2780
|
+
break;
|
|
2781
|
+
}
|
|
2782
|
+
case 'task.cancel': {
|
|
2783
|
+
cancelTask(payload.task_id, payload.reason);
|
|
2784
|
+
break;
|
|
2785
|
+
}
|
|
2786
|
+
case 'result.ack': {
|
|
2787
|
+
// Only meaningful on a channel: an ack the relay relays is not evidence about the direct
|
|
2788
|
+
// path, and `result_path` must never be set from it.
|
|
2789
|
+
if (source.transport !== 'p2p') {
|
|
2790
|
+
log('info', `ignoring result.ack for ${payload.task_id} on the relay path`);
|
|
2791
|
+
break;
|
|
2792
|
+
}
|
|
2793
|
+
settleResultAck(payload.task_id);
|
|
2794
|
+
break;
|
|
2795
|
+
}
|
|
2796
|
+
case 'peer.hello':
|
|
2797
|
+
log('info', `peer online: ${payload.machine_name ?? payload.machine_id}`);
|
|
2798
|
+
break;
|
|
2799
|
+
case 'peer.bye':
|
|
2800
|
+
log('info', `peer offline: ${payload.machine_id} (${payload.reason ?? ''})`);
|
|
2801
|
+
break;
|
|
2802
|
+
case 'notice': {
|
|
2803
|
+
log(payload.level === 'error' ? 'error' : 'info', `notice ${payload.code ?? ''}: ${payload.message ?? ''}`);
|
|
2804
|
+
if (payload.code === 'REPLAY_TRUNCATED') {
|
|
2805
|
+
// v0.1.2 §8.4: the relay tells us the oldest seq it still holds, so we
|
|
2806
|
+
// can re-align instead of only learning that we missed events. The
|
|
2807
|
+
// cursor moves to `oldest_available_seq - 1` because the next connect
|
|
2808
|
+
// asks for `cursor + 1`.
|
|
2809
|
+
const oldest = payload.oldest_available_seq;
|
|
2810
|
+
if (Number.isInteger(oldest) && oldest >= 1) {
|
|
2811
|
+
state.replayTruncated = {
|
|
2812
|
+
at: new Date().toISOString(),
|
|
2813
|
+
oldest_available_seq: oldest,
|
|
2814
|
+
previous_seq: state.seq,
|
|
2815
|
+
};
|
|
2816
|
+
// A plain assignment, not a `max`: after a restart the new process
|
|
2817
|
+
// may hold a *smaller* window than the cursor we arrived with, and
|
|
2818
|
+
// aligning downward (even to 0) is the whole point. The aligned
|
|
2819
|
+
// cursor belongs to the relay that sent this notice.
|
|
2820
|
+
state.seq = oldest - 1;
|
|
2821
|
+
state.cursorRelayId = state.relayId ?? state.cursorRelayId;
|
|
2822
|
+
log('warn', `replay window truncated; seq cursor aligned to ${state.seq} (next connect asks for ${oldest})`, {
|
|
2823
|
+
oldest_available_seq: oldest,
|
|
2824
|
+
previous_seq: state.replayTruncated.previous_seq,
|
|
2825
|
+
});
|
|
2826
|
+
publishState();
|
|
2827
|
+
} else {
|
|
2828
|
+
log('warn', 'REPLAY_TRUNCATED without a usable oldest_available_seq; cursor left unchanged', {
|
|
2829
|
+
oldest_available_seq: oldest ?? null,
|
|
2830
|
+
});
|
|
2831
|
+
}
|
|
2832
|
+
}
|
|
2833
|
+
break;
|
|
2834
|
+
}
|
|
2835
|
+
case 'task.result':
|
|
2836
|
+
// Two different frames wear this type. From the relay it is the notice that a result
|
|
2837
|
+
// reached Rabbit. From a channel it is a *dispatcher* pushing a result at this agent
|
|
2838
|
+
// (machine A's result travelling to machine B directly) -- which this agent does not
|
|
2839
|
+
// consume in v0.4.0, and says so rather than pretending it was the relay's notice.
|
|
2840
|
+
if (source.transport === 'p2p') {
|
|
2841
|
+
log('info', `result for ${payload.task_id} arrived over a direct channel (not consumed by the executor)`, {
|
|
2842
|
+
status: payload.status,
|
|
2843
|
+
});
|
|
2844
|
+
break;
|
|
2845
|
+
}
|
|
2846
|
+
log('info', `result accepted for ${payload.task_id}`, {
|
|
2847
|
+
status: payload.status,
|
|
2848
|
+
deduped: payload.deduped === true,
|
|
2849
|
+
});
|
|
2850
|
+
break;
|
|
2851
|
+
default:
|
|
2852
|
+
log('info', `ignoring unknown event type ${type}`);
|
|
2853
|
+
}
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
/** Resolve the waiter for one result acknowledgement, if there is one. */
|
|
2857
|
+
function settleResultAck(taskId) {
|
|
2858
|
+
if (typeof taskId !== 'string' || taskId === '') return;
|
|
2859
|
+
const waiter = p2pResultWaiters.get(taskId);
|
|
2860
|
+
if (!waiter) {
|
|
2861
|
+
// An ack for an already-settled wait is ordinary: the direct copy is sent once, and a
|
|
2862
|
+
// dispatcher that re-acks a frame it processed twice must not resurrect anything.
|
|
2863
|
+
log('info', `result.ack for ${taskId} with no wait in flight`);
|
|
2864
|
+
return;
|
|
2865
|
+
}
|
|
2866
|
+
waiter.settle(true, null);
|
|
2867
|
+
}
|
|
2868
|
+
|
|
2869
|
+
/** Drop a queued task and abort a running one (§3.4). Shared by both transports. */
|
|
2870
|
+
function cancelTask(taskId, reason) {
|
|
2871
|
+
const index = queue.findIndex((item) => item.task_id === taskId);
|
|
2872
|
+
if (index !== -1) queue.splice(index, 1);
|
|
2873
|
+
if (state.current && state.current.task_id === taskId) {
|
|
2874
|
+
log('warn', `task.cancel for running ${taskId}: ${reason ?? ''}`);
|
|
2875
|
+
state.current.controller.abort();
|
|
2876
|
+
} else {
|
|
2877
|
+
log('info', `task.cancel for idle ${taskId}`);
|
|
2878
|
+
}
|
|
2879
|
+
}
|
|
2880
|
+
|
|
2881
|
+
/**
|
|
2882
|
+
* The dedupe identity of one offer (v0.4.0).
|
|
2883
|
+
*
|
|
2884
|
+
* Two facts drive this, and the second one is new:
|
|
2885
|
+
*
|
|
2886
|
+
* * Rabbit's idempotency triple is `(machine_id, dedupe_key, attempt)` (§4.4), so `attempt` is
|
|
2887
|
+
* part of the identity: a new attempt is a genuine retry and must run again.
|
|
2888
|
+
* * The same offer now arrives over **two transports** -- pushed over a channel by the
|
|
2889
|
+
* dispatcher and published over the relay's SSE stream -- and either may arrive first. They
|
|
2890
|
+
* carry the same `task_id` and the same `dedupe_key`, so the identity must be computed from
|
|
2891
|
+
* the lease's facts and never from the path the copy took. A P2P copy that deduped differently
|
|
2892
|
+
* from the relay copy would execute a `write: true` command twice, which is the failure this
|
|
2893
|
+
* whole mechanism exists to prevent.
|
|
2894
|
+
*
|
|
2895
|
+
* `task_id#attempt` is the fallback for an offer with no `dedupe_key`. It is intentionally
|
|
2896
|
+
* *weaker* than the documented triple and only reached by a dispatcher that omitted the field, so
|
|
2897
|
+
* it widens dedupe rather than narrowing it -- for the two-transport case the alternative is not
|
|
2898
|
+
* "no dedupe", it is "no dedupe at all", which is strictly worse.
|
|
2899
|
+
*
|
|
2900
|
+
* @param {{task_id?: string, dedupe_key?: string|null, attempt?: number}|null} offer
|
|
2901
|
+
* @returns {string|null}
|
|
2902
|
+
*/
|
|
2903
|
+
function identityKeyFor(offer) {
|
|
2904
|
+
const taskId = typeof offer?.task_id === 'string' && offer.task_id !== '' ? offer.task_id : null;
|
|
2905
|
+
if (taskId === null) return null;
|
|
2906
|
+
const attempt = Number(offer?.attempt) || 1;
|
|
2907
|
+
const dedupeKey = typeof offer?.dedupe_key === 'string' && offer.dedupe_key !== '' ? offer.dedupe_key : null;
|
|
2908
|
+
return dedupeKey === null ? `${taskId}#${attempt}` : `${dedupeKey}#${attempt}`;
|
|
2909
|
+
}
|
|
2910
|
+
|
|
2911
|
+
/**
|
|
2912
|
+
* Queue an offer, unless we already have it -- over either transport.
|
|
2913
|
+
*
|
|
2914
|
+
* Dedupe by the lease's identity (§4.4 plus v0.4.0's two-transport rule): a re-delivery of a task
|
|
2915
|
+
* we already finished is answered from the cache or the spool instead of being executed a second
|
|
2916
|
+
* time, which matters for `write: true` offers.
|
|
2917
|
+
*
|
|
2918
|
+
* The three checks are in the order that keeps the promise above true:
|
|
2919
|
+
*
|
|
2920
|
+
* 1. **already claimed** -- queued, running, or already executed. Answered without touching the
|
|
2921
|
+
* queue. This is the branch the direct-path duplicate hits while the first copy is still
|
|
2922
|
+
* running: the second copy is recorded (its channel, if it came over one) and dropped.
|
|
2923
|
+
* 2. **cached envelope** -- the attempt finished and the relay accepted it. The envelope is
|
|
2924
|
+
* re-delivered, which is v0.3.9's behaviour and is safe because the relay dedupes it.
|
|
2925
|
+
* 3. **spooled envelope** -- the attempt finished but the relay has not confirmed it yet.
|
|
2926
|
+
*
|
|
2927
|
+
* A copy that arrives *while* the first is running therefore never re-executes and never produces
|
|
2928
|
+
* a second result. `pump` is single-entry and `claimed` is written synchronously, so there is no
|
|
2929
|
+
* interleaving between the check and the write.
|
|
2930
|
+
*
|
|
2931
|
+
* @param {object} offer
|
|
2932
|
+
*/
|
|
2933
|
+
function enqueue(offer) {
|
|
2934
|
+
const attempt = Number(offer?.attempt) || 1;
|
|
2935
|
+
const key = identityKeyFor(offer);
|
|
2936
|
+
|
|
2937
|
+
if (key !== null && claimed.has(key)) {
|
|
2938
|
+
// The claim covers exactly the window in which a second copy must not start a second run:
|
|
2939
|
+
// from here until `handleOffer` has delivered its envelope. The channel (if this copy came
|
|
2940
|
+
// over one) was already remembered by `dispatchFrame`, so the result will reach it.
|
|
2941
|
+
log('info', `ignoring duplicate offer ${offer.task_id} attempt ${attempt} (already claimed, arrived over ${offer.p2p_transport ?? 'relay'})`);
|
|
2942
|
+
return;
|
|
2943
|
+
}
|
|
2944
|
+
|
|
2945
|
+
if (key !== null) {
|
|
2946
|
+
const cached = completed.get(key);
|
|
2947
|
+
if (cached) {
|
|
2948
|
+
log('info', `dedupe hit: resending cached result for ${offer.task_id} attempt ${attempt}`);
|
|
2949
|
+
void deliver(cached);
|
|
2950
|
+
// If this copy arrived over a channel, the dispatcher is waiting for an answer on it. The
|
|
2951
|
+
// relay copy above is the durable one; this notice is what stops the dispatcher's bounded
|
|
2952
|
+
// wait from expiring on a task that is already finished. Fire-and-forget: the answer is a
|
|
2953
|
+
// courtesy, and failing to give it must never affect the spooled envelope.
|
|
2954
|
+
if (offer.p2p_transport === 'p2p' && channelFor(offer.task_id) !== null) {
|
|
2955
|
+
void sendResultDirect(offer.task_id).catch(() => {});
|
|
2956
|
+
}
|
|
2957
|
+
return;
|
|
2958
|
+
}
|
|
2959
|
+
const spooled = spool.pendingResults().find((record) => {
|
|
2960
|
+
if (!record.envelope) return false;
|
|
2961
|
+
return identityKeyFor({
|
|
2962
|
+
task_id: record.task_id,
|
|
2963
|
+
dedupe_key: record.dedupe_key ?? record.envelope.dedupe_key,
|
|
2964
|
+
attempt: record.attempt,
|
|
2965
|
+
}) === key;
|
|
2966
|
+
});
|
|
2967
|
+
if (spooled) {
|
|
2968
|
+
log('info', `dedupe hit: resending spooled result for ${offer.task_id} attempt ${attempt}`);
|
|
2969
|
+
void deliver(spooled.envelope);
|
|
2970
|
+
return;
|
|
2971
|
+
}
|
|
2972
|
+
claimed.add(key);
|
|
2973
|
+
}
|
|
2974
|
+
queue.push({ offer, key });
|
|
2975
|
+
void pump();
|
|
2976
|
+
}
|
|
2977
|
+
|
|
2978
|
+
/** Serial worker: one task at a time, in arrival order. */
|
|
2979
|
+
async function pump() {
|
|
2980
|
+
if (pumping) return;
|
|
2981
|
+
pumping = true;
|
|
2982
|
+
try {
|
|
2983
|
+
while (queue.length > 0 && !state.stopped) {
|
|
2984
|
+
const item = queue.shift();
|
|
2985
|
+
const offer = item.offer;
|
|
2986
|
+
try {
|
|
2987
|
+
await handleOffer(offer);
|
|
2988
|
+
} catch (error) {
|
|
2989
|
+
// A throw here must not kill the stream loop or wedge the queue.
|
|
2990
|
+
log('error', `unhandled failure in ${offer?.task_id ?? 'offer'}`, {
|
|
2991
|
+
error: error?.message ?? String(error),
|
|
2992
|
+
});
|
|
2993
|
+
} finally {
|
|
2994
|
+
// Released here, not in `handleOffer`: an offer that threw before delivering must still
|
|
2995
|
+
// become runnable again, or a bug would turn into a task that can never be retried.
|
|
2996
|
+
if (item.key !== null) claimed.delete(item.key);
|
|
2997
|
+
}
|
|
2998
|
+
if (once) {
|
|
2999
|
+
stop();
|
|
3000
|
+
break;
|
|
3001
|
+
}
|
|
3002
|
+
}
|
|
3003
|
+
} finally {
|
|
3004
|
+
// The pump is single-entry: `pumping` is set before the first await and cleared here, so a second
|
|
3005
|
+
// pump cannot be running to observe a stale value.
|
|
3006
|
+
// eslint-disable-next-line require-atomic-updates -- single-entry pump
|
|
3007
|
+
pumping = false;
|
|
3008
|
+
}
|
|
3009
|
+
}
|
|
3010
|
+
|
|
3011
|
+
// -- Lifecycle -----------------------------------------------------------
|
|
3012
|
+
|
|
3013
|
+
/**
|
|
3014
|
+
* The idle diagnostic heartbeat (v0.3.0): `POST /v1/heartbeat` with
|
|
3015
|
+
* `diagnostic: true` and **no** `task_id`.
|
|
3016
|
+
*
|
|
3017
|
+
* Three properties matter, and each of them is a way this could go wrong:
|
|
3018
|
+
*
|
|
3019
|
+
* 1. It holds no lease, so nothing in the response may change local state.
|
|
3020
|
+
* The body is deliberately not inspected -- not even `cancel`, which on a
|
|
3021
|
+
* lease heartbeat means "stop the task". (A relay answering `cancel:
|
|
3022
|
+
* true` here must not stop an idle agent.)
|
|
3023
|
+
* 2. It is best effort. A pre-v0.3.0 relay answers 404 because it demands a
|
|
3024
|
+
* `task_id`; that is expected, is logged at debug level, and must never
|
|
3025
|
+
* be retried in a storm, affect a task, or end the process.
|
|
3026
|
+
* 3. It is skipped while a task holds the lease: the 10s lease heartbeat is
|
|
3027
|
+
* already reporting, and two cadences reporting the same numbers would
|
|
3028
|
+
* just be noise.
|
|
3029
|
+
*
|
|
3030
|
+
* A successful round trip *is* a real measurement, so it feeds the same
|
|
3031
|
+
* rolling window the lease heartbeat uses -- that is what keeps the relay's
|
|
3032
|
+
* view fresh while the machine has nothing to do.
|
|
3033
|
+
*
|
|
3034
|
+
* @returns {Promise<{status: number, ok: boolean}|null>}
|
|
3035
|
+
*/
|
|
3036
|
+
async function sendIdleHeartbeat() {
|
|
3037
|
+
if (state.stopped || state.current) return null;
|
|
3038
|
+
if (!token()) return null; // nothing to authenticate with; not worth a 401
|
|
3039
|
+
const startedHr = process.hrtime.bigint();
|
|
3040
|
+
try {
|
|
3041
|
+
const response = await postJson('/v1/heartbeat', {
|
|
3042
|
+
diagnostic: true,
|
|
3043
|
+
machine_id: identity.machine_id,
|
|
3044
|
+
...heartbeatDiagnostics({
|
|
3045
|
+
rttMs: state.rttMs,
|
|
3046
|
+
reconnectAttempts: state.reconnectAttempts,
|
|
3047
|
+
}),
|
|
3048
|
+
});
|
|
3049
|
+
if (response.ok) {
|
|
3050
|
+
// Measured on success only, exactly like the lease heartbeat.
|
|
3051
|
+
recordRtt(Number(process.hrtime.bigint() - startedHr) / 1e6);
|
|
3052
|
+
} else {
|
|
3053
|
+
log('debug', `idle heartbeat rejected (HTTP ${response.status}); diagnostics only, continuing`, {
|
|
3054
|
+
body: response.json?.error?.code ?? null,
|
|
3055
|
+
});
|
|
3056
|
+
}
|
|
3057
|
+
return response;
|
|
3058
|
+
} catch (error) {
|
|
3059
|
+
log('debug', `idle heartbeat failed: ${error?.message ?? String(error)}; diagnostics only, continuing`, {});
|
|
3060
|
+
return null;
|
|
3061
|
+
}
|
|
3062
|
+
}
|
|
3063
|
+
|
|
3064
|
+
/**
|
|
3065
|
+
* Start the idle diagnostic cadence. Deliberately does **not** fire
|
|
3066
|
+
* immediately: the first beat lands one interval in, so booting an agent
|
|
3067
|
+
* stays quiet and a short-lived run produces no diagnostic traffic at all.
|
|
3068
|
+
*/
|
|
3069
|
+
function startIdleHeartbeat() {
|
|
3070
|
+
if (idleHeartbeatIntervalMs <= 0 || idleHeartbeatTimer) return;
|
|
3071
|
+
idleHeartbeatTimer = setInterval(() => {
|
|
3072
|
+
void sendIdleHeartbeat();
|
|
3073
|
+
}, idleHeartbeatIntervalMs);
|
|
3074
|
+
// A diagnostic must never be the reason the process stays alive.
|
|
3075
|
+
if (typeof idleHeartbeatTimer.unref === 'function') idleHeartbeatTimer.unref();
|
|
3076
|
+
}
|
|
3077
|
+
|
|
3078
|
+
/**
|
|
3079
|
+
* Stop the stream, the running command, the heartbeat and the direct path.
|
|
3080
|
+
*
|
|
3081
|
+
* Returns a promise so a caller can wait for the node's socket to be gone -- `close()` on the node
|
|
3082
|
+
* is asynchronous (it waits for the accept loop to end). Callers that do not care still get the
|
|
3083
|
+
* old fire-and-forget behaviour by ignoring the value; `w2m-localside` does not, because "the
|
|
3084
|
+
* process exited" and "the port was released" are different facts on Windows.
|
|
3085
|
+
*
|
|
3086
|
+
* @returns {Promise<void>}
|
|
3087
|
+
*/
|
|
3088
|
+
function stop() {
|
|
3089
|
+
if (state.stopped) return Promise.resolve();
|
|
3090
|
+
state.stopped = true;
|
|
3091
|
+
if (statePublishTimer) clearInterval(statePublishTimer);
|
|
3092
|
+
statePublishTimer = null;
|
|
3093
|
+
if (idleHeartbeatTimer) clearInterval(idleHeartbeatTimer);
|
|
3094
|
+
idleHeartbeatTimer = null;
|
|
3095
|
+
heartbeat?.stop();
|
|
3096
|
+
runAbort?.abort();
|
|
3097
|
+
streamAbort?.abort();
|
|
3098
|
+
// Final snapshot: a reader must be able to tell "stopped" from "crashed
|
|
3099
|
+
// mid-write", and from "still running but not connected".
|
|
3100
|
+
state.connected = false;
|
|
3101
|
+
publishState();
|
|
3102
|
+
// The node is closed last: it is the only part of `stop()` that is asynchronous, and everything
|
|
3103
|
+
// above must already be winding down before a channel can deliver a payload into a stopped agent.
|
|
3104
|
+
return closeP2P();
|
|
3105
|
+
}
|
|
3106
|
+
|
|
3107
|
+
/**
|
|
3108
|
+
* Run forever (or until `--once` has handled one task).
|
|
3109
|
+
*
|
|
3110
|
+
* @returns {Promise<void>}
|
|
3111
|
+
*/
|
|
3112
|
+
async function start() {
|
|
3113
|
+
log('info', `localside ${identity.machine_name} (${identity.machine_id})`, {
|
|
3114
|
+
rabbit: baseUrl,
|
|
3115
|
+
project,
|
|
3116
|
+
state: stateDir,
|
|
3117
|
+
allowed_commands: allowedCommands.map((prefix) => prefix.join(' ')),
|
|
3118
|
+
p2p_mode: p2pMode,
|
|
3119
|
+
});
|
|
3120
|
+
if (allowedCommands.length === 0) {
|
|
3121
|
+
log('warn', 'no --allowed-commands configured: every offer will be refused (default-deny)');
|
|
3122
|
+
}
|
|
3123
|
+
|
|
3124
|
+
// v0.4.0: bring the direct path up first, so a channel exists before the first offer can arrive
|
|
3125
|
+
// on one. Failures are recorded, never fatal -- see `startP2P`.
|
|
3126
|
+
await startP2P();
|
|
3127
|
+
|
|
3128
|
+
// Publish immediately so the file appears with the agent, not only after
|
|
3129
|
+
// the first heartbeat: "is an agent running here?" is the first question a
|
|
3130
|
+
// remote operator asks (task-16).
|
|
3131
|
+
publishState();
|
|
3132
|
+
if (statePublishIntervalMs > 0) {
|
|
3133
|
+
statePublishTimer = setInterval(() => publishState(), statePublishIntervalMs);
|
|
3134
|
+
// Never keep the process alive just to refresh a diagnostics file.
|
|
3135
|
+
if (typeof statePublishTimer.unref === 'function') statePublishTimer.unref();
|
|
3136
|
+
}
|
|
3137
|
+
// v0.3.0: keep the relay's view of *this* machine fresh even when it has
|
|
3138
|
+
// nothing to do. Best effort by construction -- see sendIdleHeartbeat().
|
|
3139
|
+
startIdleHeartbeat();
|
|
3140
|
+
|
|
3141
|
+
if (token()) {
|
|
3142
|
+
await flushSpool();
|
|
3143
|
+
reportInterrupted();
|
|
3144
|
+
} else {
|
|
3145
|
+
log('warn', 'no device_token yet: run with --pair <CODE> to pair');
|
|
3146
|
+
}
|
|
3147
|
+
|
|
3148
|
+
let idleTimer = null;
|
|
3149
|
+
if (once && onceIdleMs > 0) {
|
|
3150
|
+
idleTimer = setTimeout(() => {
|
|
3151
|
+
if (state.handled === 0) {
|
|
3152
|
+
log('warn', `--once: no offer within ${onceIdleMs}ms, exiting`);
|
|
3153
|
+
stop();
|
|
3154
|
+
}
|
|
3155
|
+
}, onceIdleMs);
|
|
3156
|
+
if (typeof idleTimer.unref === 'function') idleTimer.unref();
|
|
3157
|
+
}
|
|
3158
|
+
|
|
3159
|
+
try {
|
|
3160
|
+
while (!state.stopped) {
|
|
3161
|
+
streamAbort = new AbortController();
|
|
3162
|
+
state.streamRestartRequested = false;
|
|
3163
|
+
const openedAt = Date.now();
|
|
3164
|
+
/** @type {Error|null} */
|
|
3165
|
+
let failure = null;
|
|
3166
|
+
try {
|
|
3167
|
+
await streamOnce();
|
|
3168
|
+
} catch (error) {
|
|
3169
|
+
failure = error instanceof Error ? error : new Error(String(error));
|
|
3170
|
+
}
|
|
3171
|
+
if (state.stopped) break;
|
|
3172
|
+
state.connected = false;
|
|
3173
|
+
publishState();
|
|
3174
|
+
|
|
3175
|
+
const livedMs = Date.now() - openedAt;
|
|
3176
|
+
if (state.streamRestartRequested) {
|
|
3177
|
+
// The relay changed identity, so the cursor this stream was opened
|
|
3178
|
+
// with is void and the stream itself has already skipped the new
|
|
3179
|
+
// process's early events. Re-attach at once, without a cursor: any
|
|
3180
|
+
// delay here would widen exactly the window this fix closes.
|
|
3181
|
+
state.streamRestartRequested = false;
|
|
3182
|
+
restartReconnects += 1;
|
|
3183
|
+
if (restartReconnects <= MAX_IMMEDIATE_RESTART_RECONNECTS) {
|
|
3184
|
+
reconnectAttempt = 0;
|
|
3185
|
+
log('warn', `re-attaching after relay change without a cursor (attempt ${restartReconnects})`, {
|
|
3186
|
+
lived_ms: livedMs,
|
|
3187
|
+
});
|
|
3188
|
+
continue;
|
|
3189
|
+
}
|
|
3190
|
+
// A relay that keeps changing identity would otherwise spin us; fall
|
|
3191
|
+
// through to the normal backoff from here on.
|
|
3192
|
+
log('warn', 'relay identity keeps changing; falling back to backoff between attaches', {
|
|
3193
|
+
restarts: restartReconnects,
|
|
3194
|
+
});
|
|
3195
|
+
}
|
|
3196
|
+
|
|
3197
|
+
const stable = livedMs >= STABLE_STREAM_MS;
|
|
3198
|
+
if (stable) {
|
|
3199
|
+
reconnectAttempt = 0;
|
|
3200
|
+
restartReconnects = 0;
|
|
3201
|
+
}
|
|
3202
|
+
// A stream that ends immediately is treated as a failure even when the
|
|
3203
|
+
// socket closed cleanly; otherwise a flapping tunnel becomes a hot loop.
|
|
3204
|
+
const delay = stable ? RECONNECT_DELAY_AFTER_STABLE_MS : backoffDelay(reconnectAttempt);
|
|
3205
|
+
reconnectAttempt += 1;
|
|
3206
|
+
const reason = failure
|
|
3207
|
+
? `stream error: ${failure.message}`
|
|
3208
|
+
: `event stream closed by server after ${livedMs}ms`;
|
|
3209
|
+
// Cross-region debugging needs to answer "is it retrying, how often, and
|
|
3210
|
+
// how long until the next try" without attaching a debugger (§8.1).
|
|
3211
|
+
log('warn', `${reason}; reconnect #${reconnectAttempt} in ${delay}ms`, {
|
|
3212
|
+
attempt: reconnectAttempt,
|
|
3213
|
+
delay_ms: delay,
|
|
3214
|
+
lived_ms: livedMs,
|
|
3215
|
+
stable,
|
|
3216
|
+
});
|
|
3217
|
+
await sleep(delay, undefined);
|
|
3218
|
+
}
|
|
3219
|
+
} finally {
|
|
3220
|
+
if (idleTimer) clearTimeout(idleTimer);
|
|
3221
|
+
stop();
|
|
3222
|
+
log('info', 'localside stopped', { handled: state.handled });
|
|
3223
|
+
}
|
|
3224
|
+
}
|
|
3225
|
+
|
|
3226
|
+
return {
|
|
3227
|
+
start,
|
|
3228
|
+
stop,
|
|
3229
|
+
pair,
|
|
3230
|
+
flushSpool,
|
|
3231
|
+
reportInterrupted,
|
|
3232
|
+
handleOffer,
|
|
3233
|
+
get state() {
|
|
3234
|
+
return state;
|
|
3235
|
+
},
|
|
3236
|
+
spool,
|
|
3237
|
+
identity,
|
|
3238
|
+
caps,
|
|
3239
|
+
platform,
|
|
3240
|
+
allowedCommands,
|
|
3241
|
+
/** Write the diagnostics file now (also done automatically; task-16). */
|
|
3242
|
+
publishState,
|
|
3243
|
+
/** Send one idle diagnostic heartbeat now (v0.3.0). */
|
|
3244
|
+
sendIdleHeartbeat,
|
|
3245
|
+
/**
|
|
3246
|
+
* The direct path, for a caller that needs more than the summary (v0.4.0).
|
|
3247
|
+
*
|
|
3248
|
+
* Null in `relay` mode forever, and null until `start()` has built one. Exposed rather than
|
|
3249
|
+
* hidden because "which port did this machine announce?" is a question an operator asks with
|
|
3250
|
+
* `netstat` open, and the answer belongs to the object that owns the socket.
|
|
3251
|
+
*/
|
|
3252
|
+
get p2pNode() {
|
|
3253
|
+
return p2pNode;
|
|
3254
|
+
},
|
|
3255
|
+
/** The mode this agent was built with, validated at construction. */
|
|
3256
|
+
p2pMode,
|
|
3257
|
+
/** The STUN list the node queries, in order; null when there is no node. */
|
|
3258
|
+
p2pStunServers,
|
|
3259
|
+
/** Bring the direct path up on its own (also done by `start()`); never fatal. */
|
|
3260
|
+
startP2P,
|
|
3261
|
+
/** Close the direct path on its own (also done by `stop()`). */
|
|
3262
|
+
closeP2P,
|
|
3263
|
+
/**
|
|
3264
|
+
* Feed one already-decoded frame into the dispatcher both transports use (v0.4.0).
|
|
3265
|
+
*
|
|
3266
|
+
* Exists because "an offer from a channel" and "an offer from the relay" differ in exactly one
|
|
3267
|
+
* value, and a caller that needs a specific arrival order -- a test, or a future tool that
|
|
3268
|
+
* injects a frame -- should not have to open a socket to express it. `transport` is the only
|
|
3269
|
+
* thing it controls; the dedupe, the queue and the result path are the same code either way,
|
|
3270
|
+
* which is what makes an injected frame a fair test of the real path.
|
|
3271
|
+
*
|
|
3272
|
+
* @param {object} payload A frame (`{type:'task.offer', ...}`).
|
|
3273
|
+
* @param {{transport?: 'p2p'|'relay', channel?: object}} [origin]
|
|
3274
|
+
*/
|
|
3275
|
+
ingest(payload, origin = {}) {
|
|
3276
|
+
dispatchFrame(payload, {
|
|
3277
|
+
event: typeof payload?.type === 'string' ? payload.type : 'message',
|
|
3278
|
+
id: null,
|
|
3279
|
+
transport: origin.transport === 'p2p' ? 'p2p' : 'relay',
|
|
3280
|
+
channel: origin.channel,
|
|
3281
|
+
});
|
|
3282
|
+
},
|
|
3283
|
+
/** Whether request signing is configured (v0.3.0). */
|
|
3284
|
+
signingEnabled: signer !== null,
|
|
3285
|
+
/** Absolute path of the published diagnostics file. */
|
|
3286
|
+
stateFile: stateFilePath,
|
|
3287
|
+
/** Normalised base address actually used for every request (v0.1.2 §2). */
|
|
3288
|
+
baseUrl,
|
|
3289
|
+
/**
|
|
3290
|
+
* Operator token, when this host also submits tasks (v0.1.2 §5).
|
|
3291
|
+
*
|
|
3292
|
+
* The Localside agent never POSTs `/v1/task`, so this is carried for local
|
|
3293
|
+
* tooling (`w2m_status`, host-side scripts) and is never put on the wire by
|
|
3294
|
+
* this process. It is deliberately not persisted to disk either.
|
|
3295
|
+
*/
|
|
3296
|
+
operatorToken: operatorToken ?? null,
|
|
3297
|
+
};
|
|
3298
|
+
}
|
|
3299
|
+
|
|
3300
|
+
/**
|
|
3301
|
+
* `true` when the project directory can be written to.
|
|
3302
|
+
*
|
|
3303
|
+
* `fs.access(..., W_OK)` is the honest cheap check available to us; it is not
|
|
3304
|
+
* a substitute for actually writing, but a read-only mount or a missing
|
|
3305
|
+
* directory both fail it, which is exactly the §6.1 `READ_ONLY_MACHINE` case.
|
|
3306
|
+
*
|
|
3307
|
+
* @param {string} dir
|
|
3308
|
+
* @returns {Promise<boolean>}
|
|
3309
|
+
*/
|
|
3310
|
+
export async function isWritableProject(dir) {
|
|
3311
|
+
try {
|
|
3312
|
+
await access(dir, fsConstants.W_OK);
|
|
3313
|
+
return true;
|
|
3314
|
+
} catch {
|
|
3315
|
+
return false;
|
|
3316
|
+
}
|
|
3317
|
+
}
|