@byok-sdk/server 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +10 -0
- package/dist/auth.d.ts +171 -0
- package/dist/blob-store.d.ts +77 -0
- package/dist/event-queue.d.ts +20 -0
- package/dist/heartbeat.d.ts +30 -0
- package/dist/http.d.ts +24 -0
- package/dist/hub.d.ts +861 -0
- package/dist/ids.d.ts +3 -0
- package/dist/index.d.ts +116 -0
- package/dist/index.js +2675 -0
- package/dist/index.js.map +1 -0
- package/dist/pairing.d.ts +58 -0
- package/dist/rate-limiter.d.ts +114 -0
- package/dist/sqlite-blob-store.d.ts +89 -0
- package/dist/sqlite-support.d.ts +84 -0
- package/dist/sqlite-task-store.d.ts +123 -0
- package/dist/task-store.d.ts +125 -0
- package/dist/types.d.ts +378 -0
- package/dist/ws-server.d.ts +20 -0
- package/package.json +56 -0
package/dist/hub.d.ts
ADDED
|
@@ -0,0 +1,861 @@
|
|
|
1
|
+
import type { WebSocket } from 'ws';
|
|
2
|
+
import { type Envelope, type RuntimeId, type RuntimeInfo, type TaskState } from '@byok-sdk/protocol';
|
|
3
|
+
import type { DeviceRegistry } from './auth';
|
|
4
|
+
import { RateLimiter } from './rate-limiter';
|
|
5
|
+
import type { TaskStore } from './task-store';
|
|
6
|
+
import type { ByokServerEvent, DispatchInput, HubStats, MachineInfo, TaskHandle, TaskSnapshot } from './types';
|
|
7
|
+
/**
|
|
8
|
+
* The connection hub: tracks each device's live transport (WS or long-poll —
|
|
9
|
+
* never both at once, see {@link takeOverAsLongPoll}), routes `dispatch()`'d
|
|
10
|
+
* tasks to a device, and processes inbound task.* envelopes from daemons.
|
|
11
|
+
*
|
|
12
|
+
* Routing (M1): every `task.*` envelope carries a *required* envelope
|
|
13
|
+
* `task_id` — the sole routing key, both directions. No payload duplicates
|
|
14
|
+
* it, and none of the handlers below need to guard against a missing one
|
|
15
|
+
* (the wire schema already rejects such an envelope before it reaches here).
|
|
16
|
+
*
|
|
17
|
+
* Inbound gate ({@link handleInbound}): the single choke point both
|
|
18
|
+
* transports (`ws-server.ts`'s WS message handler, `http.ts`'s
|
|
19
|
+
* `POST /byok/messages`) call instead of reaching into per-type handlers
|
|
20
|
+
* directly. Runs, in order: (1) type-allow — only `DAEMON_TO_SERVER_TYPES`
|
|
21
|
+
* may pass, a server -> daemon type arriving inbound is rejected (P2); (2)
|
|
22
|
+
* ownership (N2) — an envelope for a task already owned by a *different*
|
|
23
|
+
* device is dropped and logged, never force-failed (force-failing on an
|
|
24
|
+
* authz mismatch would let an attacker who merely guesses a `taskId` kill
|
|
25
|
+
* the real owner's task); (3) dedup (N3) — an envelope `id` already seen
|
|
26
|
+
* from this device is a no-op, making the at-least-once wire (§9)
|
|
27
|
+
* effectively at-most-once server-side; (4) dispatch to the per-type
|
|
28
|
+
* handler. Because ownership is enforced once, centrally, here, the
|
|
29
|
+
* handlers below no longer carry their own device-mismatch checks.
|
|
30
|
+
*
|
|
31
|
+
* Outbound delivery (M1, §1.2/§9): every server -> daemon envelope
|
|
32
|
+
* (`conn.ack`, `task.offer/approve/reject/cancel/steer`) gets a fresh
|
|
33
|
+
* per-device monotonic `seq` and is retained in a capped ring buffer
|
|
34
|
+
* ({@link OUTBOX_RING_CAPACITY} entries) so it can be redelivered — in `seq`
|
|
35
|
+
* order, skipping anything whose task has since reached a terminal state —
|
|
36
|
+
* on reconnect (`redeliverAfterReconnect`) or long-poll
|
|
37
|
+
* (`pollEvents`/`collectRelevant`). `conn.ack` has no task association, so
|
|
38
|
+
* it's retained (for seq-counting purposes only) but never redelivered.
|
|
39
|
+
* Exception (N1/F4): `task.cancel`/`task.reject` are exempt from the
|
|
40
|
+
* terminal-task skip (`OutboxEntry.redeliverThroughTerminal`) because both
|
|
41
|
+
* move their own task to a terminal state before being queued — without the
|
|
42
|
+
* exemption they could never qualify for redelivery even when the original
|
|
43
|
+
* send never reached the daemon.
|
|
44
|
+
*
|
|
45
|
+
* State-machine (M1): `task.claim` only claims (`Offered -> Claimed`); it no
|
|
46
|
+
* longer implies `Running`. The daemon reports `Claimed -> Running`
|
|
47
|
+
* explicitly via `task.started` once its runtime session actually starts
|
|
48
|
+
* (§3.1). `task.decline` reports a pre-claim fail-closed rejection
|
|
49
|
+
* (`Offered -> Failed`, §3.2). `task.cancelled` is dual-purpose: an
|
|
50
|
+
* idempotent ack when the server already cancelled the task itself, or the
|
|
51
|
+
* authoritative trigger when the daemon observed the cancellation first
|
|
52
|
+
* (§3.3). Per §9, `task.complete`/`task.fail`/`task.cancelled` arriving for
|
|
53
|
+
* an already-terminal task are silently dropped as stale/duplicate — not a
|
|
54
|
+
* warning; this is what naturally resolves the M0 gatekeeper's cancel-race
|
|
55
|
+
* `console.warn` finding (a late `task.fail`/`task.cancelled` racing a
|
|
56
|
+
* server-initiated cancel is exactly this case).
|
|
57
|
+
*
|
|
58
|
+
* Task lease (M2): a periodic sweep (started in the constructor) reaps a
|
|
59
|
+
* `Claimed`/`Running`/`AwaitApproval` task to
|
|
60
|
+
* `Failed(retryable: true, reason: 'lease-expired')` once its owning device
|
|
61
|
+
* has been dark (disconnected, or long-poll-silent) AND the task itself has
|
|
62
|
+
* had no inbound activity for `taskLeaseMs` — see the "task-lease reaper"
|
|
63
|
+
* section further down for the full design, including why this does not
|
|
64
|
+
* reintroduce the disconnect-alone-fails-the-task bug M1 removed above.
|
|
65
|
+
*/
|
|
66
|
+
/**
|
|
67
|
+
* M4 Phase 3: thrown by {@link ConnectionHub.approveTask}/{@link
|
|
68
|
+
* ConnectionHub.rejectTask} for a `taskId` this hub has no record of at all
|
|
69
|
+
* — mirrors `task-store.ts`'s `IllegalTaskTransitionError` (typed error
|
|
70
|
+
* class + `instanceof` dispatch is this codebase's own established idiom for
|
|
71
|
+
* mapping a domain error to the right status code). Distinguished from
|
|
72
|
+
* {@link TaskNotAwaitingApprovalError} so a caller CAN tell the two failure
|
|
73
|
+
* modes apart (e.g. 404 vs. 409) instead of only ever seeing a single
|
|
74
|
+
* generic `Error`. There is no bearer-authed HTTP route for this on
|
|
75
|
+
* `http.ts`'s own app (see that file's own closing comment for why) — the
|
|
76
|
+
* supported entry point is calling `approveTask`/`rejectTask` directly, or
|
|
77
|
+
* via `TaskHandle.approve()`/`reject()` (thin wrappers over the same two
|
|
78
|
+
* methods); an embedder builds its own operator-facing surface on top of
|
|
79
|
+
* that, exactly like `examples/basic/server.ts`'s own
|
|
80
|
+
* `/api/tasks/:taskId/approve`/`reject` routes do.
|
|
81
|
+
*/
|
|
82
|
+
export declare class UnknownTaskError extends Error {
|
|
83
|
+
readonly taskId: string;
|
|
84
|
+
constructor(taskId: string);
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Thrown by {@link ConnectionHub.approveTask}/{@link ConnectionHub.rejectTask}
|
|
88
|
+
* when the task exists but isn't currently `AwaitApproval` — see {@link
|
|
89
|
+
* UnknownTaskError}'s own doc comment for why this is a distinct class.
|
|
90
|
+
* `verb` keeps the exact pre-existing message wording per call site
|
|
91
|
+
* ("cannot approve ..." vs. "cannot reject ...") byte-for-byte unchanged —
|
|
92
|
+
* this message is user-visible today (e.g. `examples/basic`'s own
|
|
93
|
+
* `/api/tasks/:taskId/approve` surfaces `err.message` straight to the
|
|
94
|
+
* caller), so only the error's TYPE changes here, not its text.
|
|
95
|
+
*/
|
|
96
|
+
export declare class TaskNotAwaitingApprovalError extends Error {
|
|
97
|
+
readonly taskId: string;
|
|
98
|
+
readonly state: TaskState;
|
|
99
|
+
constructor(taskId: string, state: TaskState, verb: 'approve' | 'reject');
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* M5 (approval targeting): thrown by {@link ConnectionHub.approveTask}/
|
|
103
|
+
* {@link ConnectionHub.rejectTask} when an operator-supplied
|
|
104
|
+
* `opts.approvalId` is provided but does NOT match this hub's own
|
|
105
|
+
* last-recorded {@link TaskSnapshot.pendingApprovalId} for `taskId` — i.e.
|
|
106
|
+
* the caller is targeting a SPECIFIC approval that this server's record
|
|
107
|
+
* shows has already been superseded by a newer one (see `onAwaitApproval`'s
|
|
108
|
+
* own doc comment for how that happens). Distinct from
|
|
109
|
+
* {@link TaskNotAwaitingApprovalError} (the task isn't `AwaitApproval` at
|
|
110
|
+
* all right now — checked FIRST, so it still wins when both would apply):
|
|
111
|
+
* this error means the task genuinely IS awaiting approval, just not the one
|
|
112
|
+
* the caller thinks it is. Thrown before any state change and before any
|
|
113
|
+
* wire message is sent — a stale-id call has zero side effects, same as any
|
|
114
|
+
* other validation failure in this file.
|
|
115
|
+
*/
|
|
116
|
+
export declare class StaleApprovalError extends Error {
|
|
117
|
+
readonly taskId: string;
|
|
118
|
+
readonly requestedApprovalId: string;
|
|
119
|
+
readonly currentApprovalId: string | undefined;
|
|
120
|
+
constructor(taskId: string, requestedApprovalId: string, currentApprovalId: string | undefined);
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* S0 (GAP-002): why {@link ConnectionHub.steerTask} refused. Stable strings —
|
|
124
|
+
* a caller (an embedder's operator UI, an HTTP surface mapping this to a
|
|
125
|
+
* status code) switches on these rather than matching error text.
|
|
126
|
+
*
|
|
127
|
+
* - `task_terminal` — the task already reached `Complete`/`Failed`/
|
|
128
|
+
* `Cancelled`. Checked FIRST, so a terminal task that is also (obviously)
|
|
129
|
+
* not `Running` reports the more specific truth, and a steer racing a
|
|
130
|
+
* terminal transition always resolves terminal-first.
|
|
131
|
+
* - `task_not_running` — the task exists and is live, but is `Offered`/
|
|
132
|
+
* `Claimed`/`AwaitApproval`; there is no running turn to steer yet.
|
|
133
|
+
* - `steer_unsupported_runtime` — the runtime that CLAIMED this task cannot
|
|
134
|
+
* be steered, per the claim-time capability snapshot
|
|
135
|
+
* (`TaskSnapshot.claimedRuntimeCapabilities`, sourced from the claiming
|
|
136
|
+
* adapter's own `task.claim.capabilities`). Fail-closed: a MISSING snapshot
|
|
137
|
+
* rejects under this same code, because "unknown" is not "supported" (see
|
|
138
|
+
* {@link SteerRejectedError}'s own doc comment).
|
|
139
|
+
*/
|
|
140
|
+
export type SteerRejectionCode = 'steer_unsupported_runtime' | 'task_not_running' | 'task_terminal';
|
|
141
|
+
/**
|
|
142
|
+
* S0 (GAP-002): thrown by {@link ConnectionHub.steerTask} instead of the
|
|
143
|
+
* pre-S0 generic `Error`, so a caller can tell WHY a steer was refused
|
|
144
|
+
* without matching on message text — same typed-error idiom as
|
|
145
|
+
* {@link UnknownTaskError}/{@link TaskNotAwaitingApprovalError}/
|
|
146
|
+
* {@link StaleApprovalError} above.
|
|
147
|
+
*
|
|
148
|
+
* The gap this closes: pre-S0 `steerTask` sent `task.steer` to ANY `Running`
|
|
149
|
+
* task, but only pi's adapter implements steering — Claude's and Codex's
|
|
150
|
+
* throw on receipt (`claude-adapter.ts`, `codex-adapter.ts`), which stalls
|
|
151
|
+
* the client's redelivery cursor at that seq and loops forever. So the
|
|
152
|
+
* decision has to be made server-side, from per-runtime truth, BEFORE an
|
|
153
|
+
* envelope exists.
|
|
154
|
+
*
|
|
155
|
+
* Fail-closed on unknown, deliberately: `steer_unsupported_runtime` covers
|
|
156
|
+
* both "the claiming adapter reported `steer: false`" and "this server has no
|
|
157
|
+
* capability snapshot for this task at all" (a pre-D-4 daemon whose claim
|
|
158
|
+
* carried no `capabilities`, a task record predating S0). Refusing an unknown
|
|
159
|
+
* is a recoverable operator-visible error; guessing "supported" reintroduces
|
|
160
|
+
* the exact permanent cursor stall this gate exists to prevent.
|
|
161
|
+
*
|
|
162
|
+
* Thrown before any state change and before any wire message is built — a
|
|
163
|
+
* rejected steer has zero side effects, same as every other validation
|
|
164
|
+
* failure in this file.
|
|
165
|
+
*
|
|
166
|
+
* SINGLE SOURCE, deliberately: the gate reads the claim payload and NOTHING
|
|
167
|
+
* from the connection layer — not {@link getDeviceCapabilities} (the
|
|
168
|
+
* CONNECTION-level `conn.hello` flag list) and not `ConnectionState.runtimes`.
|
|
169
|
+
* Two reasons, either sufficient. First, scope: connection-level data is
|
|
170
|
+
* discovery describing a daemon BUILD, not the per-runtime, per-task,
|
|
171
|
+
* claim-time truth this gate needs — conflating the two is the original bug.
|
|
172
|
+
* Second, reach: `conn.hello` is transport-shaped. A long-poll-only daemon
|
|
173
|
+
* never sends one (sole sender: `ws-transport.ts:192`, `packages/client`), so
|
|
174
|
+
* a connection-sourced snapshot is permanently `undefined` for an entire
|
|
175
|
+
* transport and a fail-closed gate reading it disables steer across that whole
|
|
176
|
+
* deployment surface — a regression, not a safety property. The claim, by
|
|
177
|
+
* contrast, is the message that establishes the task↔runtime binding on every
|
|
178
|
+
* transport, so the gate's input now shares a lifecycle with the thing it
|
|
179
|
+
* judges. Adding a connection-level fallback here would restore both defects
|
|
180
|
+
* at once and is what this design exists to forbid.
|
|
181
|
+
*/
|
|
182
|
+
export declare class SteerRejectedError extends Error {
|
|
183
|
+
readonly taskId: string;
|
|
184
|
+
readonly code: SteerRejectionCode;
|
|
185
|
+
/** The task's state at the moment the steer was refused. */
|
|
186
|
+
readonly state: TaskState;
|
|
187
|
+
/** `TaskSnapshot.claimedRuntime` — `undefined` when nothing was ever recorded (which is itself a reason `steer_unsupported_runtime` can fire). */
|
|
188
|
+
readonly runtime: RuntimeId | undefined;
|
|
189
|
+
constructor(taskId: string, code: SteerRejectionCode,
|
|
190
|
+
/** The task's state at the moment the steer was refused. */
|
|
191
|
+
state: TaskState,
|
|
192
|
+
/** `TaskSnapshot.claimedRuntime` — `undefined` when nothing was ever recorded (which is itself a reason `steer_unsupported_runtime` can fire). */
|
|
193
|
+
runtime: RuntimeId | undefined);
|
|
194
|
+
}
|
|
195
|
+
export declare class ConnectionHub {
|
|
196
|
+
private readonly taskStore;
|
|
197
|
+
private readonly devices;
|
|
198
|
+
/** See {@link CreateByokServerOptions.taskLeaseMs} — already defaulted by `createByokServer` before reaching here. */
|
|
199
|
+
private readonly taskLeaseMs;
|
|
200
|
+
/**
|
|
201
|
+
* M4 Phase 4 (part A): per-device inbound-envelope token bucket — see
|
|
202
|
+
* {@link CreateByokServerOptions.rateLimit} (already defaulted by
|
|
203
|
+
* `createByokServer` before reaching here) and {@link handleInbound}'s
|
|
204
|
+
* own doc comment for where it's enforced. Defaults to a fresh
|
|
205
|
+
* default-configured `RateLimiter` so every existing direct-construction
|
|
206
|
+
* call site (this hub is constructed directly by several tests) keeps
|
|
207
|
+
* working unchanged.
|
|
208
|
+
*/
|
|
209
|
+
private readonly rateLimiter;
|
|
210
|
+
private readonly connections;
|
|
211
|
+
private readonly outboxes;
|
|
212
|
+
/** Idempotency window per device (N3) — recent inbound envelope ids, capped at {@link DEDUP_RING_CAPACITY}. */
|
|
213
|
+
private readonly dedupRings;
|
|
214
|
+
private readonly longPollWaiters;
|
|
215
|
+
private readonly runtimes;
|
|
216
|
+
private readonly serverEvents;
|
|
217
|
+
/**
|
|
218
|
+
* Per-task last-inbound-activity timestamp (epoch ms) — the task-lease
|
|
219
|
+
* reaper's condition (c), see the "task-lease reaper" section below. Reset
|
|
220
|
+
* on every accepted inbound `task.*` envelope ({@link recordTaskActivity},
|
|
221
|
+
* called from {@link dispatchToHandler}); cleared once the task reaches a
|
|
222
|
+
* terminal state ({@link onStateChange}), so this map only ever holds
|
|
223
|
+
* entries for currently non-terminal claimed tasks.
|
|
224
|
+
*/
|
|
225
|
+
private readonly taskActivity;
|
|
226
|
+
/** The task-lease reaper's own periodic sweep timer — see the constructor and `sweepLeases` below. */
|
|
227
|
+
private readonly leaseReaperTimer;
|
|
228
|
+
/** {@link ConnectionHub.stats}'s `uptimeMs` origin — this hub's own construction instant. */
|
|
229
|
+
private readonly startedAtMs;
|
|
230
|
+
/** {@link ConnectionHub.stats}'s `envelopesIn` — every {@link handleInbound} call, every outcome. */
|
|
231
|
+
private envelopesInCount;
|
|
232
|
+
/** {@link ConnectionHub.stats}'s `envelopesOut` — every envelope built via the single outbound choke point, {@link sendToDevice}. */
|
|
233
|
+
private envelopesOutCount;
|
|
234
|
+
/** {@link ConnectionHub.stats}'s `dedupDrops` (N3). */
|
|
235
|
+
private dedupDropCount;
|
|
236
|
+
/** {@link ConnectionHub.stats}'s `rateLimitEvents` — see {@link handleRateLimited}. */
|
|
237
|
+
private rateLimitEventCount;
|
|
238
|
+
/**
|
|
239
|
+
* M4 Phase 4 (gatekeeper LOW advisory): devices that have already had a
|
|
240
|
+
* `device.rate_limited` embedder event emitted for their CURRENT
|
|
241
|
+
* over-budget episode — see {@link handleRateLimited}'s own doc comment.
|
|
242
|
+
* Coalescing state only; {@link rateLimitEventCount} still counts every
|
|
243
|
+
* single hit regardless of what this suppresses.
|
|
244
|
+
*/
|
|
245
|
+
private readonly rateLimitEventEmittedFor;
|
|
246
|
+
constructor(taskStore: TaskStore, devices: DeviceRegistry,
|
|
247
|
+
/** See {@link CreateByokServerOptions.taskLeaseMs} — already defaulted by `createByokServer` before reaching here. */
|
|
248
|
+
taskLeaseMs: number,
|
|
249
|
+
/**
|
|
250
|
+
* M4 Phase 4 (part A): per-device inbound-envelope token bucket — see
|
|
251
|
+
* {@link CreateByokServerOptions.rateLimit} (already defaulted by
|
|
252
|
+
* `createByokServer` before reaching here) and {@link handleInbound}'s
|
|
253
|
+
* own doc comment for where it's enforced. Defaults to a fresh
|
|
254
|
+
* default-configured `RateLimiter` so every existing direct-construction
|
|
255
|
+
* call site (this hub is constructed directly by several tests) keeps
|
|
256
|
+
* working unchanged.
|
|
257
|
+
*/
|
|
258
|
+
rateLimiter?: RateLimiter);
|
|
259
|
+
/**
|
|
260
|
+
* Stop the task-lease reaper's sweep timer — called by `ByokServer.stop()`
|
|
261
|
+
* (`index.ts`) on shutdown. Idempotent: clearing an already-cleared
|
|
262
|
+
* interval is a safe no-op.
|
|
263
|
+
*/
|
|
264
|
+
stopLeaseReaper(): void;
|
|
265
|
+
/** The top-level `events` feed returned by `createByokServer` — see {@link ByokServerEvent}. */
|
|
266
|
+
subscribeServerEvents(): AsyncIterable<ByokServerEvent>;
|
|
267
|
+
/**
|
|
268
|
+
* A daemon completed the WS handshake (`conn.hello`). Does not itself send
|
|
269
|
+
* `conn.ack` or redeliver — see {@link sendConnAck}/{@link redeliverAfterReconnect}.
|
|
270
|
+
*
|
|
271
|
+
* `capabilities` (M5, hello-capability plumbing): the daemon's own
|
|
272
|
+
* `conn.hello.capabilities` — previously silently ignored end to end (a
|
|
273
|
+
* verified gap: `ws-server.ts` forwarded only `runtimes`). Optional so
|
|
274
|
+
* every pre-M5 direct-construction call site (several tests construct a
|
|
275
|
+
* `ConnectionHub` and call this directly) keeps working unchanged; a
|
|
276
|
+
* connection this hub never learns capabilities for simply reads back
|
|
277
|
+
* `undefined` from {@link getDeviceCapabilities}.
|
|
278
|
+
*/
|
|
279
|
+
registerConnection(deviceId: string, ws: WebSocket, runtimes: RuntimeInfo[] | undefined, capabilities?: readonly string[]): void;
|
|
280
|
+
sendConnAck(deviceId: string, capabilities: string[]): void;
|
|
281
|
+
/**
|
|
282
|
+
* Reconnection procedure step 3 (§9): redeliver, in `seq` order, every
|
|
283
|
+
* retained envelope with `seq > cursor` that still belongs to a
|
|
284
|
+
* non-terminal task. Called after `conn.ack` (step 2), per the spec.
|
|
285
|
+
*/
|
|
286
|
+
redeliverAfterReconnect(deviceId: string, cursor: number): void;
|
|
287
|
+
/**
|
|
288
|
+
* A device's WS socket closed. `ws` identifies *which* socket closed: if
|
|
289
|
+
* it's no longer the one this device's connection state points at (a
|
|
290
|
+
* newer WS reconnected, or long-poll took over — "last transport wins"),
|
|
291
|
+
* this close is for a stale/superseded socket and the device isn't
|
|
292
|
+
* actually gone, so the bookkeeping below is skipped entirely.
|
|
293
|
+
*
|
|
294
|
+
* M1 note: the M0 server force-failed/cancelled every in-flight task for a
|
|
295
|
+
* device the instant it disconnected, on the stated premise that "a task
|
|
296
|
+
* still in flight for a device that just disconnected can't be resumed, so
|
|
297
|
+
* it's terminated" — true only in the absence of a redelivery cursor. M1
|
|
298
|
+
* adds exactly that (§9): a task's in-flight state is retained
|
|
299
|
+
* independently of any one connection, specifically so it can survive a
|
|
300
|
+
* disconnect and resume via redelivery once the device reconnects. Failing
|
|
301
|
+
* tasks here would make that feature unreachable in practice (nothing
|
|
302
|
+
* would ever still be non-terminal by the time a reconnect happened), so
|
|
303
|
+
* this now only updates connection bookkeeping and leaves task state
|
|
304
|
+
* alone. A task left in-flight by a device that never reconnects stays
|
|
305
|
+
* that way until the SaaS embedder explicitly cancels it — no
|
|
306
|
+
* disconnect-timeout is specified by the protocol, so none is invented
|
|
307
|
+
* here (see the M1-2 report's contract-gap notes).
|
|
308
|
+
*/
|
|
309
|
+
handleDisconnect(deviceId: string, ws: WebSocket): void;
|
|
310
|
+
/**
|
|
311
|
+
* Resolve immediately if there are already-relevant events past `cursor`;
|
|
312
|
+
* otherwise hold for up to `holdMs` and resolve with an empty result if
|
|
313
|
+
* nothing arrives. A device may be connected via WS or long-poll, not
|
|
314
|
+
* both simultaneously — a poll here supersedes (closes) any live WS for
|
|
315
|
+
* this device ("last one wins", documented at the type level on
|
|
316
|
+
* {@link ConnectionState}).
|
|
317
|
+
*/
|
|
318
|
+
pollEvents(deviceId: string, cursor: number, holdMs: number): Promise<{
|
|
319
|
+
events: Envelope[];
|
|
320
|
+
cursor: number;
|
|
321
|
+
}>;
|
|
322
|
+
/** Make long-poll this device's active transport, closing any live WS ("last one wins", §8). */
|
|
323
|
+
private takeOverAsLongPoll;
|
|
324
|
+
/** Resolve (settle) any long-poll request currently held open for `deviceId`, if one exists. */
|
|
325
|
+
private settleLongPollWaiter;
|
|
326
|
+
/**
|
|
327
|
+
* Single inbound choke point for every daemon -> server envelope (N2/N3/
|
|
328
|
+
* P2) — called by both the WS path (`ws-server.ts`) and the long-poll send
|
|
329
|
+
* path (`POST /byok/messages`, `http.ts`) in place of reaching into
|
|
330
|
+
* per-type handlers directly. Runs a fixed gate, in order:
|
|
331
|
+
*
|
|
332
|
+
* 0. **rate limit (M4 Phase 4, part A)** — one token debited from this
|
|
333
|
+
* device's bucket ({@link rateLimiter}) for EVERY inbound envelope,
|
|
334
|
+
* before anything else runs (including the type-allow check below) —
|
|
335
|
+
* a flood of garbage-typed envelopes must cost the same budget as a
|
|
336
|
+
* flood of well-formed ones. Checked first specifically so an
|
|
337
|
+
* over-budget device is turned away as cheaply as possible, before any
|
|
338
|
+
* taskStore lookup or dedup bookkeeping. See {@link handleRateLimited}
|
|
339
|
+
* for what happens on exceed (never a silent drop).
|
|
340
|
+
* 1. **type-allow (P2)** — only {@link DAEMON_TO_SERVER_TYPES} may pass; a
|
|
341
|
+
* server -> daemon type (or anything unrecognized, e.g. a stale/future
|
|
342
|
+
* `conn.hello` outside the handshake) arriving inbound is rejected
|
|
343
|
+
* before it's dispatched or counted accepted.
|
|
344
|
+
* 2. **ownership (N2)** — an envelope for a task already owned by a
|
|
345
|
+
* *different* device is dropped (logged), never force-failed:
|
|
346
|
+
* force-failing on an authz mismatch would let an attacker who merely
|
|
347
|
+
* guesses a `taskId` kill the real owner's task (a DoS). A task with no
|
|
348
|
+
* owner yet, or that doesn't exist at all, is not rejected here — the
|
|
349
|
+
* per-type handler's own no-op-on-missing-record behavior covers the
|
|
350
|
+
* latter.
|
|
351
|
+
* 3. **dedup (N3)** — an envelope `id` already seen from this device is a
|
|
352
|
+
* no-op: the wire is at-least-once (§9), this makes server-side
|
|
353
|
+
* processing at-most-once. Check-and-record is synchronous (Node is
|
|
354
|
+
* single-threaded), so it's atomic with respect to any other envelope
|
|
355
|
+
* for this device.
|
|
356
|
+
* 4. **dispatch** — handed to the existing per-type `on*` handler.
|
|
357
|
+
*
|
|
358
|
+
* Returns which outcome applied. A duplicate still counts as `accepted` on
|
|
359
|
+
* the `POST /byok/messages` wire (§8.2) — an idempotent replay is a
|
|
360
|
+
* wire-level success even though no handler ran a second time; only
|
|
361
|
+
* `rejected`/`rate_limited` (gate steps 0-2) are excluded from that count.
|
|
362
|
+
*/
|
|
363
|
+
handleInbound(deviceId: string, envelope: Envelope): 'accepted' | 'duplicate' | 'rejected' | 'rate_limited';
|
|
364
|
+
/**
|
|
365
|
+
* M4 Phase 4 (part A): `deviceId` just exceeded its inbound-envelope rate
|
|
366
|
+
* limit. Never a silent drop: counts the occurrence
|
|
367
|
+
* ({@link rateLimitEventCount}, surfaced via {@link stats} — every single
|
|
368
|
+
* hit, unconditionally) and, the FIRST time in this over-budget episode
|
|
369
|
+
* only, emits an embedder-facing `device.rate_limited`
|
|
370
|
+
* {@link ByokServerEvent} — see that variant's own doc comment (`types.ts`)
|
|
371
|
+
* for the full per-transport enforcement shape.
|
|
372
|
+
*
|
|
373
|
+
* Gatekeeper LOW advisory (event amplification): a single flood can make
|
|
374
|
+
* `handleInbound` call this many times in a row — e.g. several WS frames
|
|
375
|
+
* already in flight before the close below actually lands, or a
|
|
376
|
+
* long-poll device retrying its `POST /byok/messages` before its bucket
|
|
377
|
+
* has refilled. Without coalescing, an embedder subscribed to
|
|
378
|
+
* `events.subscribe()` would see one `device.rate_limited` per hit, which
|
|
379
|
+
* is noisy for what is really ONE ongoing episode of one device
|
|
380
|
+
* flooding. `rateLimitEventEmittedFor` suppresses the repeats: this
|
|
381
|
+
* method only pushes the event the first time it sees a given `deviceId`
|
|
382
|
+
* since `handleInbound`'s own success path last cleared it (i.e. since
|
|
383
|
+
* this device was last confirmed back under budget) — the COUNTER above
|
|
384
|
+
* is entirely unaffected by this and still increments on every call,
|
|
385
|
+
* unconditionally.
|
|
386
|
+
*
|
|
387
|
+
* This method only handles the WS half of the enforcement shape (closing
|
|
388
|
+
* the live connection, if any, so the client's existing backoff+reconnect
|
|
389
|
+
* takes over — mirrors `takeOverAsLongPoll`'s own `ws.close`, the only
|
|
390
|
+
* other place this hub closes a device's socket directly); a long-poll
|
|
391
|
+
* device has no live `ws` to close here at all (`conn.ws` is `undefined`
|
|
392
|
+
* while long-polling — see {@link ConnectionState}), so `http.ts`'s
|
|
393
|
+
* `/byok/messages` handler maps this same `'rate_limited'` `handleInbound`
|
|
394
|
+
* outcome to an HTTP 429 for that transport instead.
|
|
395
|
+
*/
|
|
396
|
+
private handleRateLimited;
|
|
397
|
+
/**
|
|
398
|
+
* Idempotency check-and-record (N3): `true` (duplicate) if `id` was
|
|
399
|
+
* already seen for `deviceId`; otherwise records it and returns `false`.
|
|
400
|
+
* Bounded to {@link DEDUP_RING_CAPACITY} ids per device — a ring, not an
|
|
401
|
+
* unbounded set — evicting the oldest once full.
|
|
402
|
+
*/
|
|
403
|
+
private checkAndRecordDuplicate;
|
|
404
|
+
/**
|
|
405
|
+
* Route one already-gated envelope (see {@link handleInbound}) to its
|
|
406
|
+
* per-type handler. Type-allow/ownership/dedup have already run by the
|
|
407
|
+
* time this executes, so the handlers below no longer need their own
|
|
408
|
+
* device-mismatch checks — that authz decision now lives solely in
|
|
409
|
+
* `handleInbound` (N2).
|
|
410
|
+
*
|
|
411
|
+
* Also the task-lease reaper's activity checkpoint
|
|
412
|
+
* ({@link recordTaskActivity}): every envelope for a task that currently
|
|
413
|
+
* *exists and is non-terminal* counts as proof of life for `taskId`'s
|
|
414
|
+
* lease, regardless of what its per-type handler below ends up doing with
|
|
415
|
+
* it (including a no-op/stale drop) — see the "task-lease reaper" section
|
|
416
|
+
* further down for why. Deliberately gated on the record's existence and
|
|
417
|
+
* non-terminal state *here*, before dispatch: `taskActivity` must never
|
|
418
|
+
* gain an entry for a taskId that doesn't exist (a nonexistent/garbage id
|
|
419
|
+
* an authenticated-but-malicious daemon could send indefinitely — an
|
|
420
|
+
* unbounded-growth vector, since `taskId`s aren't deduped the way envelope
|
|
421
|
+
* `id`s are) or for one that's already terminal (a stale/late message for
|
|
422
|
+
* a finished task — `onStateChange` deletes the entry on the *real*
|
|
423
|
+
* terminal transition, but a stale message arriving *after* that would
|
|
424
|
+
* otherwise silently recreate it, since every per-type handler's own
|
|
425
|
+
* terminal/unknown-task guard runs — and early-returns — only *after*
|
|
426
|
+
* this would already have recorded activity).
|
|
427
|
+
*/
|
|
428
|
+
private dispatchToHandler;
|
|
429
|
+
/** Reset the task-lease reaper's per-task clock (condition (c) in the "task-lease reaper" section below). */
|
|
430
|
+
private recordTaskActivity;
|
|
431
|
+
/**
|
|
432
|
+
* Ownership (record.deviceId matching the connection's authenticated
|
|
433
|
+
* deviceId) is enforced centrally by {@link handleInbound} (N2) before this
|
|
434
|
+
* runs; only the idempotent-claim CAS and the first-claim device patch
|
|
435
|
+
* happen here.
|
|
436
|
+
*
|
|
437
|
+
* M5 (claimed runtime, docs/protocol.md §3.1): `payload.runtime` — the
|
|
438
|
+
* ACTUAL adapter the daemon selected (`TaskRunner.pickAdapter`,
|
|
439
|
+
* `packages/client`'s `task-runner.ts`) — is recorded into
|
|
440
|
+
* `TaskSnapshot.claimedRuntime` alongside the device patch, distinct from
|
|
441
|
+
* the pre-existing `TaskSnapshot.runtime` (the merely REQUESTED runtime,
|
|
442
|
+
* untouched here and set only once, at `dispatch()` time). Only ever
|
|
443
|
+
* written on the FIRST real claim: the idempotent-CAS early return above
|
|
444
|
+
* fires before this for a retried claim from a device that already owns
|
|
445
|
+
* the task, so a redelivered/retried `task.claim` can never overwrite an
|
|
446
|
+
* already-recorded `claimedRuntime` — including with a stale or absent
|
|
447
|
+
* value from an out-of-order retry.
|
|
448
|
+
*
|
|
449
|
+
* S0/D-4 (claim-time capability snapshot): `payload.capabilities` — the
|
|
450
|
+
* claiming adapter's OWN self-report, carried on this same `task.claim`
|
|
451
|
+
* (docs/protocol.md §2.4) — supplies
|
|
452
|
+
* `TaskSnapshot.claimedRuntimeCapabilities`, written in the same patch and
|
|
453
|
+
* therefore under the same write-exactly-once property as `claimedRuntime`.
|
|
454
|
+
*
|
|
455
|
+
* Taken from the payload and from nowhere else. This hub deliberately does
|
|
456
|
+
* NOT consult connection state (`conn.hello.runtimes[]`) for it — see
|
|
457
|
+
* {@link SteerRejectedError} for why that source is structurally wrong for a
|
|
458
|
+
* control decision, and that field's own doc comment (`types.ts`) for why
|
|
459
|
+
* this is snapshotted rather than read live at steer time. A claim that
|
|
460
|
+
* carries no `capabilities` (a pre-D-4 daemon) records `undefined`, which
|
|
461
|
+
* the gate reads as "unknown" and refuses.
|
|
462
|
+
*/
|
|
463
|
+
private onClaim;
|
|
464
|
+
/**
|
|
465
|
+
* `Claimed -> Running` (§3.1) — a daemon actually starting the runtime
|
|
466
|
+
* session, distinct from merely claiming. Ownership is already enforced
|
|
467
|
+
* by {@link handleInbound} (N2) before this runs.
|
|
468
|
+
*/
|
|
469
|
+
private onStarted;
|
|
470
|
+
/**
|
|
471
|
+
* `Offered -> Failed` (§3.2) — a fail-closed pre-claim rejection. Only
|
|
472
|
+
* ever legal from `Offered`; anything else is stale. Ownership is already
|
|
473
|
+
* enforced by {@link handleInbound} (N2) before this runs.
|
|
474
|
+
*/
|
|
475
|
+
private onDecline;
|
|
476
|
+
private onProgress;
|
|
477
|
+
private onArtifact;
|
|
478
|
+
private onAwaitApproval;
|
|
479
|
+
private onComplete;
|
|
480
|
+
private onFail;
|
|
481
|
+
/**
|
|
482
|
+
* Dual-purpose on receipt (§3.3): if the server already moved this task to
|
|
483
|
+
* `Cancelled` on its own action (the common case — `cancelTask()` is
|
|
484
|
+
* authoritative immediately, §4), this is a late idempotent ack — silent,
|
|
485
|
+
* not a warning (this is the other half of the M0 gatekeeper finding this
|
|
486
|
+
* change resolves). Otherwise it's the authoritative trigger for a
|
|
487
|
+
* cancellation the daemon observed that the server didn't initiate.
|
|
488
|
+
* Ownership is already enforced by {@link handleInbound} (N2) before this
|
|
489
|
+
* runs.
|
|
490
|
+
*/
|
|
491
|
+
private onCancelled;
|
|
492
|
+
/**
|
|
493
|
+
* M4 (additive-minor, `task.approval_resolved`): the EXPLICIT counterpart
|
|
494
|
+
* to {@link resumeIfImplicitlyApproved} — a daemon that resolved a pending
|
|
495
|
+
* approval entirely LOCALLY now reports it immediately, instead of the
|
|
496
|
+
* server only finding out after the fact once evidence (a later
|
|
497
|
+
* `task.progress`/`task.artifact`/`task.complete`) proves it.
|
|
498
|
+
*
|
|
499
|
+
* Relationship to the implicit path (both stay, permanently — this is not
|
|
500
|
+
* a replacement): {@link resumeIfImplicitlyApproved} remains completely
|
|
501
|
+
* untouched as the fallback for (a) an old daemon that predates this
|
|
502
|
+
* message, and (b) a daemon connected to an old server that never
|
|
503
|
+
* advertised the `approval_resolved` capability flag (`version.ts`) at
|
|
504
|
+
* handshake time — in either case the daemon never sends this message at
|
|
505
|
+
* all (see `packages/client`'s `task-runner.ts`), and the server keeps
|
|
506
|
+
* inferring the resolution from evidence exactly as it did before this
|
|
507
|
+
* message existed. When THIS message does arrive first, it already moves
|
|
508
|
+
* the record out of `AwaitApproval` (see below) — so by the time any
|
|
509
|
+
* following `task.progress`/etc. reaches `onProgress`/`onArtifact`/
|
|
510
|
+
* `onComplete`, `resumeIfImplicitlyApproved`'s own `record.state !==
|
|
511
|
+
* 'AwaitApproval'` guard is already true and it no-ops, never firing its
|
|
512
|
+
* own `task.approval_resolved_implicit` event a second time for the same
|
|
513
|
+
* resolution. The two mechanisms race harmlessly: whichever one the
|
|
514
|
+
* server processes first is the one that actually performs the
|
|
515
|
+
* transition; the other is naturally inert once it runs.
|
|
516
|
+
*
|
|
517
|
+
* Three outcomes, mirroring this file's existing per-type idempotency
|
|
518
|
+
* conventions:
|
|
519
|
+
* - `AwaitApproval` (the expected case): legal transition to `Running`
|
|
520
|
+
* (an existing `TASK_TRANSITIONS` edge, the same one `approveTask`
|
|
521
|
+
* itself uses) plus a `task.approval_resolved` {@link ByokServerEvent}
|
|
522
|
+
* carrying `approvalId`/`decision`/`resolvedBy` for an embedder to
|
|
523
|
+
* observe.
|
|
524
|
+
* - Already `Running` (evidence — or the implicit path — already beat
|
|
525
|
+
* this message to it): idempotent no-op, silent, mirroring
|
|
526
|
+
* `onStarted`'s own already-running guard.
|
|
527
|
+
* - Terminal, or a state that was never `AwaitApproval` in the first
|
|
528
|
+
* place (`Offered`/`Claimed` — a genuinely out-of-sequence report):
|
|
529
|
+
* stale no-op with a `console.warn`, matching this file's existing
|
|
530
|
+
* stale-message convention (`forceFailOrDrop`, `handleInbound`'s
|
|
531
|
+
* ownership-mismatch drop) — never force-failed, since a late/
|
|
532
|
+
* redelivered report about a task that has already moved on is not
|
|
533
|
+
* evidence of anything currently wrong with it.
|
|
534
|
+
*
|
|
535
|
+
* This is also the residual-race resolution the accompanying protocol/docs
|
|
536
|
+
* update documents: a SaaS decision (`approveTask`/`rejectTask`) already in
|
|
537
|
+
* flight when the local resolution happens can still land on the server
|
|
538
|
+
* FIRST and move the record to a terminal state before this message
|
|
539
|
+
* arrives — in that case this message hits the terminal branch above and
|
|
540
|
+
* is a stale no-op, exactly like any other late message for an
|
|
541
|
+
* already-terminal task. The window for that crossing is now
|
|
542
|
+
* network-latency-sized (how long this message takes to arrive), not
|
|
543
|
+
* "until the next progress message" the way the pre-existing implicit-only
|
|
544
|
+
* inference left it.
|
|
545
|
+
*/
|
|
546
|
+
private onApprovalResolved;
|
|
547
|
+
/**
|
|
548
|
+
* M5 (approval targeting): single low-level wrapper around
|
|
549
|
+
* `TaskStore.transition` that every ACTUAL state-changing write in this
|
|
550
|
+
* file goes through — {@link applyOrFail}'s legal-transition branch,
|
|
551
|
+
* {@link forceFailOrDrop}, and {@link resumeIfImplicitlyApproved} (the one
|
|
552
|
+
* caller that transitions WITHOUT going through `applyOrFail` at all).
|
|
553
|
+
* Two responsibilities, folded in here once rather than duplicated at
|
|
554
|
+
* each call site:
|
|
555
|
+
*
|
|
556
|
+
* 1. Clears `pendingApprovalId` whenever `record` is LEAVING
|
|
557
|
+
* `AwaitApproval` (`record.state === 'AwaitApproval' && to !==
|
|
558
|
+
* 'AwaitApproval'`) — the id this hub last recorded for a task's
|
|
559
|
+
* pending approval ({@link onAwaitApproval}) is meaningless the
|
|
560
|
+
* instant that task is no longer awaiting it. Clearing it here,
|
|
561
|
+
* centrally, is what guarantees a FUTURE `AwaitApproval` cycle for
|
|
562
|
+
* the SAME task always starts from a clean slate instead of silently
|
|
563
|
+
* inheriting a stale id from a previous cycle (which would make a
|
|
564
|
+
* stale-approval check against the NEW cycle's real pending id
|
|
565
|
+
* spuriously pass just because a leftover value happened to still be
|
|
566
|
+
* sitting in the record).
|
|
567
|
+
* 2. Calls {@link onStateChange} — every call site already did this
|
|
568
|
+
* immediately after its own `transition` call; folding it in here
|
|
569
|
+
* removes the duplication and the chance of a future call site
|
|
570
|
+
* forgetting it.
|
|
571
|
+
*/
|
|
572
|
+
private transitionTask;
|
|
573
|
+
/**
|
|
574
|
+
* Apply `taskId`'s state -> `target`. If that's illegal per
|
|
575
|
+
* `TASK_TRANSITIONS`, fall back to `Failed` (if reachable from the current
|
|
576
|
+
* state); this is the "illegal transition = error + task.fail path" rule.
|
|
577
|
+
*/
|
|
578
|
+
private applyOrFail;
|
|
579
|
+
/**
|
|
580
|
+
* M4 Phase 3 hardening (orchestrator-directed fix for the server-state-
|
|
581
|
+
* machine trace finding): a task can be resolved entirely OUT-OF-BAND, on
|
|
582
|
+
* the daemon side only (M4 Phase 3's local `approvals.resolve`
|
|
583
|
+
* control-socket path, `packages/client`) — the server never sees a wire
|
|
584
|
+
* `task.approve`/`task.reject` for it, so its own record sits in
|
|
585
|
+
* `AwaitApproval` even though the daemon already resumed and moved on.
|
|
586
|
+
*
|
|
587
|
+
* The daemon is the execution authority in this security model (the SaaS
|
|
588
|
+
* only ever *proposes* — see docs/spec.md); the daemon sending ANY further
|
|
589
|
+
* task.* traffic for a task the server still thinks is `AwaitApproval` is
|
|
590
|
+
* itself sufficient proof the approval was resolved locally, one way or
|
|
591
|
+
* another. Rather than force-failing/dropping that traffic (the pre-fix
|
|
592
|
+
* behavior — `onProgress`/`onArtifact`'s own `!== 'Running'` guard,
|
|
593
|
+
* `onComplete`'s illegal-transition fallback), this applies the exact same
|
|
594
|
+
* `AwaitApproval -> Running` edge `approveTask` already uses (a
|
|
595
|
+
* pre-existing legal `TASK_TRANSITIONS` edge, not a new one) through the
|
|
596
|
+
* normal transition path — `taskStore.transition` + `onStateChange`, same
|
|
597
|
+
* as `applyOrFail`'s own legal-transition branch — so every existing
|
|
598
|
+
* consumer of task state (§, `TaskHandle.events()`, the lease reaper's
|
|
599
|
+
* `taskActivity`) observes it exactly as it would a real wire
|
|
600
|
+
* `task.approve`. Then emits `task.approval_resolved_implicit` (a
|
|
601
|
+
* `ByokServerEvent`, NOT a wire message — see that type's own doc comment)
|
|
602
|
+
* so an embedder can distinguish this from an operator-driven approval.
|
|
603
|
+
*
|
|
604
|
+
* M4 (additive-minor, superseding this method's own former "deferred"
|
|
605
|
+
* framing): a first-class `task.approval_resolved` WIRE notification now
|
|
606
|
+
* exists (`onApprovalResolved`, below) — a daemon that supports it, talking
|
|
607
|
+
* to a server that advertised the `approval_resolved` capability flag
|
|
608
|
+
* (`version.ts`), reports a local resolution explicitly and immediately
|
|
609
|
+
* instead of leaving the server to infer it here. This method is
|
|
610
|
+
* UNTOUCHED and remains the permanent fallback for the N/N-1 cases where
|
|
611
|
+
* that explicit report never arrives (an old daemon, or an old server this
|
|
612
|
+
* daemon is talking to) — see `onApprovalResolved`'s own doc comment for
|
|
613
|
+
* the full relationship between the two paths, including why they can
|
|
614
|
+
* never both fire for the same resolution.
|
|
615
|
+
*
|
|
616
|
+
* No-op (returns `record` unchanged) for any state other than
|
|
617
|
+
* `AwaitApproval` — every other guard (terminal, pre-claim, already-
|
|
618
|
+
* Running) keeps exactly its current behavior. `onFail`/`onCancelled`
|
|
619
|
+
* never call this: `Failed`/`Cancelled` are already direct, legal edges
|
|
620
|
+
* from `AwaitApproval`, so they never hit the illegal-transition path this
|
|
621
|
+
* exists to avoid in the first place.
|
|
622
|
+
*/
|
|
623
|
+
private resumeIfImplicitlyApproved;
|
|
624
|
+
/**
|
|
625
|
+
* A daemon message didn't fit the task's current state (e.g. progress
|
|
626
|
+
* while AwaitApproval). Force the task to `Failed` if that's reachable;
|
|
627
|
+
* otherwise it's already terminal (or `Offered`, which has no Failed edge)
|
|
628
|
+
* and there's nothing safe to do but log + drop.
|
|
629
|
+
*/
|
|
630
|
+
private forceFailOrDrop;
|
|
631
|
+
private onStateChange;
|
|
632
|
+
/**
|
|
633
|
+
* Task lease: a backstop for a device that goes dark mid-task and never
|
|
634
|
+
* comes back — distinct from, and layered on top of, M1's redelivery
|
|
635
|
+
* (docs/protocol.md §9), which already handles "device reconnects within
|
|
636
|
+
* the window, nothing lost." Decision (user+design): reuse the existing
|
|
637
|
+
* `Failed` terminal state and its `retryable` flag —
|
|
638
|
+
* `Failed(retryable: true, reason: 'lease-expired')` — exactly like any
|
|
639
|
+
* other `task.fail`. The embedder is expected to treat this exactly like
|
|
640
|
+
* any other retryable failure: re-dispatch as a brand-new task.
|
|
641
|
+
*
|
|
642
|
+
* Implemented as a periodic sweep (see the constructor), not a per-task
|
|
643
|
+
* timer, so a device that goes dark *after* being idle-but-connected for a
|
|
644
|
+
* while is still caught on a later tick without needing extra bookkeeping
|
|
645
|
+
* at disconnect time. `sweepLeases` reaps a task only when ALL of the
|
|
646
|
+
* following hold, checked fresh on every tick (never cached):
|
|
647
|
+
*
|
|
648
|
+
* (a) the task is in a non-terminal *claimed* state — `Claimed`,
|
|
649
|
+
* `Running`, or `AwaitApproval` ({@link isClaimedState}). `Offered`
|
|
650
|
+
* is excluded: it has no owning device yet, so there's nothing to
|
|
651
|
+
* be "dark".
|
|
652
|
+
* (b) the owning device is dark right now ({@link deviceDarkSince}
|
|
653
|
+
* returns a timestamp rather than `undefined`) — disconnected
|
|
654
|
+
* outright, or (long-poll only) hasn't been seen since before the
|
|
655
|
+
* lease window. A live WS connection is never dark from the
|
|
656
|
+
* reaper's point of view: `heartbeat.ts` already independently
|
|
657
|
+
* proves liveness at the transport level and flips
|
|
658
|
+
* `connected: false` via `handleDisconnect` once it stops getting
|
|
659
|
+
* pongs — the reaper just reads that flag rather than re-deriving
|
|
660
|
+
* it. `deviceDarkSince` also returns *when* darkness started
|
|
661
|
+
* ({@link ConnectionState.darkSince}, set the instant
|
|
662
|
+
* `handleDisconnect` flips the connection dark) — that instant
|
|
663
|
+
* feeds condition (c), below.
|
|
664
|
+
* (c) a full `taskLeaseMs` has elapsed since the *later* of: the task's
|
|
665
|
+
* own last inbound-activity timestamp ({@link taskActivity}, reset
|
|
666
|
+
* in {@link dispatchToHandler} on every accepted envelope for a
|
|
667
|
+
* known, non-terminal task — claim, started, progress, artifact,
|
|
668
|
+
* await_approval, anything), and (b)'s dark-since instant. Taking
|
|
669
|
+
* the *later* of the two — not the activity timestamp alone — is
|
|
670
|
+
* what makes a device going dark start a fresh, full countdown
|
|
671
|
+
* instead of reusing whatever (possibly already-stale) activity
|
|
672
|
+
* timestamp the task happened to have: a task can be legitimately
|
|
673
|
+
* idle *while connected* for longer than `taskLeaseMs` (a long turn
|
|
674
|
+
* with no progress events, or just a quiet stretch) without being
|
|
675
|
+
* touched — see (b) — but the instant such a task's device
|
|
676
|
+
* disconnects, that stale activity timestamp must NOT immediately
|
|
677
|
+
* satisfy (c) on its own, or the task would get reaped within one
|
|
678
|
+
* sweep tick of disconnect instead of waiting the full window. That
|
|
679
|
+
* was a real bug (a disconnect-after-long-idle reap effectively
|
|
680
|
+
* indistinguishable from the M0 disconnect-alone-fails-the-task
|
|
681
|
+
* behavior M1 removed, below); anchoring (c) to
|
|
682
|
+
* `max(lastActivity, darkSince)` fixes it — idle time that elapsed
|
|
683
|
+
* *before* the device went dark no longer counts toward the lease,
|
|
684
|
+
* only silence *after* dark-start does.
|
|
685
|
+
*
|
|
686
|
+
* (b) and (c) are deliberately independent clocks, not one merged check.
|
|
687
|
+
* The property this most exists to protect: a *connected*, momentarily
|
|
688
|
+
* idle device mid-turn must never be reaped, no matter how long
|
|
689
|
+
* `taskLeaseMs` is — condition (b) alone blocks that regardless of (c).
|
|
690
|
+
* This is also what keeps this from reintroducing the M0 bug M1
|
|
691
|
+
* deliberately removed (see `handleDisconnect`'s own doc comment above) —
|
|
692
|
+
* M0 force-failed a task the instant its device disconnected; M1
|
|
693
|
+
* correctly stopped doing that so a task could survive a disconnect and
|
|
694
|
+
* resume via redelivery. This reaper does not revert that: disconnect
|
|
695
|
+
* ALONE still does nothing here either — (c) still has to independently
|
|
696
|
+
* hold, and per the `max(...)` above it only will once a full
|
|
697
|
+
* `taskLeaseMs` has genuinely elapsed *since the device went dark*, no
|
|
698
|
+
* matter how stale the task's own activity timestamp already was at that
|
|
699
|
+
* moment.
|
|
700
|
+
*
|
|
701
|
+
* Interaction with redelivery (§9): redelivery is what handles "the
|
|
702
|
+
* device came back within the window" — nothing to reap, normal traffic
|
|
703
|
+
* resumes. This reaper is what handles "it never came back." Idempotent
|
|
704
|
+
* claim (`onClaim`'s CAS) still protects server-side bookkeeping if a
|
|
705
|
+
* device wakes up *after* its task was already reaped and retries a stale
|
|
706
|
+
* claim/progress/etc. for it: every per-type handler's existing
|
|
707
|
+
* stale/terminal-task guard (§9) drops it as a no-op, same as any other
|
|
708
|
+
* late message for an already-terminal task — no new guard was needed for
|
|
709
|
+
* that here.
|
|
710
|
+
*
|
|
711
|
+
* Accepted residual (by design, not a bug): idempotent claim protects
|
|
712
|
+
* *server-side* state, not the device's own local side effects. A dark
|
|
713
|
+
* device that wakes up after its task has already been reaped may still
|
|
714
|
+
* be mid-way through running real local work (file writes, shell
|
|
715
|
+
* commands, whatever the runtime adapter was doing) for a task the server
|
|
716
|
+
* has since moved on from — and that the embedder may have already
|
|
717
|
+
* re-dispatched elsewhere. There is no way to remotely guarantee a
|
|
718
|
+
* truly-dark device stops running; the mitigation is entirely
|
|
719
|
+
* `taskLeaseMs` being set far larger than any realistic task duration, so
|
|
720
|
+
* this can only happen to a device that was genuinely gone for a very
|
|
721
|
+
* long time, not a normal slow turn.
|
|
722
|
+
*/
|
|
723
|
+
private sweepLeases;
|
|
724
|
+
/**
|
|
725
|
+
* Condition (b) above: `undefined` while `deviceId`'s connection counts as
|
|
726
|
+
* alive (never reapable, no matter how stale (c) is); otherwise the
|
|
727
|
+
* epoch-ms instant it began counting as "dark" for lease purposes.
|
|
728
|
+
* `sweepLeases` combines this with (c)'s own last-activity instant via
|
|
729
|
+
* `max(...)` so the full `taskLeaseMs` silence window is always measured
|
|
730
|
+
* from whichever of the two happened later.
|
|
731
|
+
*/
|
|
732
|
+
private deviceDarkSince;
|
|
733
|
+
/** Reap one lease-expired task through the exact same TaskStore/canTransition path — and terminal-event emission — as any other `task.fail` (see {@link applyOrFail}). */
|
|
734
|
+
private reapTask;
|
|
735
|
+
dispatch(input: DispatchInput): Promise<TaskHandle>;
|
|
736
|
+
private buildTaskHandle;
|
|
737
|
+
/** Idempotent: cancelling an already-terminal task is a no-op, not an error. */
|
|
738
|
+
private cancelTask;
|
|
739
|
+
/**
|
|
740
|
+
* M4 Phase 3: made public (was private through M3) so an embedder can call
|
|
741
|
+
* it directly from its own operator-facing surface — there is no
|
|
742
|
+
* bearer-authed HTTP route for this on `http.ts`'s own app (see
|
|
743
|
+
* `UnknownTaskError`'s own doc comment for why, and
|
|
744
|
+
* `examples/basic/server.ts`'s `/api/tasks/:taskId/approve` for the
|
|
745
|
+
* intended shape of that embedder-built surface). See this file's own
|
|
746
|
+
* `UnknownTaskError`/`TaskNotAwaitingApprovalError` doc comments for why
|
|
747
|
+
* the two failure modes are now distinct typed errors rather than a
|
|
748
|
+
* single generic `Error`. Every thrown message's TEXT is byte-for-byte
|
|
749
|
+
* unchanged from M2/M3 — only the error's type changed (this is still also
|
|
750
|
+
* reachable via `TaskHandle.approve()`, unaffected).
|
|
751
|
+
*/
|
|
752
|
+
/**
|
|
753
|
+
* M5 (approval targeting, docs/protocol.md §5.3): `opts.approvalId`
|
|
754
|
+
* targets a SPECIFIC pending approval rather than "whichever one is
|
|
755
|
+
* currently pending" (the pre-M5 default, unchanged when `opts` is
|
|
756
|
+
* omitted). Validated FIRST, before any state change or wire send: if
|
|
757
|
+
* `opts.approvalId` is supplied and this hub has a recorded
|
|
758
|
+
* `pendingApprovalId` for `taskId` that DIFFERS, throws
|
|
759
|
+
* {@link StaleApprovalError} — no transition, no `task.approve` sent. If
|
|
760
|
+
* this hub never recorded a `pendingApprovalId` (a legacy daemon that
|
|
761
|
+
* never reported one), the call proceeds untargeted exactly as before.
|
|
762
|
+
* The outgoing `task.approve` carries `approvalId`: the caller-supplied
|
|
763
|
+
* one if given, else this hub's own recorded one, else omitted entirely
|
|
764
|
+
* (legacy wire shape) — so the daemon can apply its own exact-match check
|
|
765
|
+
* whenever this server has an id to offer at all.
|
|
766
|
+
*/
|
|
767
|
+
approveTask(taskId: string, opts?: {
|
|
768
|
+
approvalId?: string;
|
|
769
|
+
}): Promise<void>;
|
|
770
|
+
/**
|
|
771
|
+
* M4 Phase 3: made public — see {@link ConnectionHub.approveTask}'s own
|
|
772
|
+
* doc comment for the full rationale (identical reasoning applies here).
|
|
773
|
+
* M5: same `opts.approvalId` targeting semantics as `approveTask` above —
|
|
774
|
+
* see that method's own doc comment.
|
|
775
|
+
*/
|
|
776
|
+
rejectTask(taskId: string, reason?: string, opts?: {
|
|
777
|
+
approvalId?: string;
|
|
778
|
+
}): Promise<void>;
|
|
779
|
+
/**
|
|
780
|
+
* S0 (GAP-002): a task-level gate, evaluated in full before any envelope is
|
|
781
|
+
* built — see {@link SteerRejectedError} for the gap this closes and why an
|
|
782
|
+
* unknown capability must refuse rather than proceed. Order matters:
|
|
783
|
+
*
|
|
784
|
+
* 1. unknown task — unchanged pre-S0 `Error` (this is not a steer-policy
|
|
785
|
+
* decision, and `TaskHandle.steer` can only be reached with a taskId
|
|
786
|
+
* this hub minted, so it's a programming error, not an operator one);
|
|
787
|
+
* 2. terminal (`Complete`/`Failed`/`Cancelled`) -> `task_terminal`,
|
|
788
|
+
* checked BEFORE the `Running` check so a steer racing a terminal
|
|
789
|
+
* transition always resolves terminal-first;
|
|
790
|
+
* 3. not `Running` (`Offered`/`Claimed`/`AwaitApproval`) ->
|
|
791
|
+
* `task_not_running`;
|
|
792
|
+
* 4. the claim-time snapshot does not positively say `steer: true` ->
|
|
793
|
+
* `steer_unsupported_runtime`, including when there is no snapshot at
|
|
794
|
+
* all (fail-closed);
|
|
795
|
+
* 5. only then, the pre-existing device-liveness check and the send.
|
|
796
|
+
*
|
|
797
|
+
* Step 4 reads `TaskSnapshot.claimedRuntimeCapabilities` — the per-runtime,
|
|
798
|
+
* per-task value frozen at claim time from the claiming adapter's own
|
|
799
|
+
* `task.claim.capabilities` — and reads NO connection state whatsoever:
|
|
800
|
+
* not {@link getDeviceCapabilities}, not `ConnectionState.runtimes`, and
|
|
801
|
+
* with no fallback to either when the snapshot is absent. See
|
|
802
|
+
* {@link SteerRejectedError} for why a connection-sourced input is wrong
|
|
803
|
+
* both in scope (describes a daemon build, not this task's runtime) and in
|
|
804
|
+
* reach (absent entirely on long-poll-only daemons).
|
|
805
|
+
*/
|
|
806
|
+
private steerTask;
|
|
807
|
+
private pickFirstConnectedDevice;
|
|
808
|
+
/**
|
|
809
|
+
* Build a server -> daemon envelope with a fresh per-device `seq`, retain
|
|
810
|
+
* it in that device's outbox ring buffer, and deliver it now if a live
|
|
811
|
+
* transport is available (WS send, or wake a pending long-poll).
|
|
812
|
+
*
|
|
813
|
+
* `opts`'s type mirrors `createEnvelope`'s own per-type conditional
|
|
814
|
+
* requiredness (finding F1) minus `seq` (computed fresh right here on
|
|
815
|
+
* every call, never caller-supplied) — so every one of this method's 6
|
|
816
|
+
* callers below must supply `taskId` for the 5 types that need it
|
|
817
|
+
* (everything except `conn.ack`), same as calling `createEnvelope`
|
|
818
|
+
* directly would require.
|
|
819
|
+
*/
|
|
820
|
+
private sendToDevice;
|
|
821
|
+
private deliverToDevice;
|
|
822
|
+
/**
|
|
823
|
+
* Retained envelopes for `deviceId` with `seq > cursor` that still belong
|
|
824
|
+
* to a non-terminal task — OR are explicitly exempted from that filter
|
|
825
|
+
* (`redeliverThroughTerminal`, N1/F4: `task.cancel`/`task.reject`) — in
|
|
826
|
+
* `seq` order. The `seq > cursor` bound is what naturally stops an
|
|
827
|
+
* exempted entry from redelivering forever: once the daemon acks it (its
|
|
828
|
+
* reported cursor advances past that `seq`), it no longer qualifies here
|
|
829
|
+
* on any future reconnect/poll.
|
|
830
|
+
*/
|
|
831
|
+
private collectRelevant;
|
|
832
|
+
private isTaskTerminal;
|
|
833
|
+
/** The highest `seq` assigned to `deviceId` so far — the redelivery cursor to hand back on a poll/reconnect. */
|
|
834
|
+
private currentCursor;
|
|
835
|
+
private getOrCreateOutbox;
|
|
836
|
+
listMachines(): MachineInfo[];
|
|
837
|
+
/**
|
|
838
|
+
* M5 (approval targeting, hello-capability plumbing): the capability flags
|
|
839
|
+
* `deviceId`'s CURRENT connection advertised in its `conn.hello` —
|
|
840
|
+
* `undefined` if this hub has no connection state for the device at all,
|
|
841
|
+
* or one that never had capabilities recorded (a pre-M5 daemon, or a
|
|
842
|
+
* device this hub only ever saw over long-poll with no prior WS hello —
|
|
843
|
+
* see `ConnectionState.capabilities`'s own doc comment). Read fresh from
|
|
844
|
+
* live connection state, mirroring `listMachines()`'s own convention; an
|
|
845
|
+
* embedder can use this to distinguish a targeting-capable device from a
|
|
846
|
+
* legacy one for its own observability/UI purposes (see `version.ts`'s
|
|
847
|
+
* `approval-targeting` flag doc comment for why this is informational
|
|
848
|
+
* only, never a correctness gate).
|
|
849
|
+
*/
|
|
850
|
+
getDeviceCapabilities(deviceId: string): readonly string[] | undefined;
|
|
851
|
+
getTask(taskId: string): TaskSnapshot | undefined;
|
|
852
|
+
listTasks(): TaskSnapshot[];
|
|
853
|
+
/**
|
|
854
|
+
* A plain, serializable snapshot of this hub's current state, derived from
|
|
855
|
+
* existing structures (`connections`, `taskStore`) plus the small counters
|
|
856
|
+
* this file already maintains for exactly this purpose — no new
|
|
857
|
+
* bookkeeping structures beyond those counters. See {@link HubStats}
|
|
858
|
+
* (`types.ts`) for the full field-by-field contract.
|
|
859
|
+
*/
|
|
860
|
+
stats(): HubStats;
|
|
861
|
+
}
|