@relaycast/engine 8.2.2 → 8.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/db/migrations/0043_node_machine_id_index.sql +10 -0
- package/dist/db/migrations/0044_node_proven_live_at.sql +16 -0
- package/dist/db/schema.d.ts +17 -0
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +9 -1
- package/dist/db/schema.js.map +1 -1
- package/dist/engine/action.d.ts.map +1 -1
- package/dist/engine/action.js +35 -8
- package/dist/engine/action.js.map +1 -1
- package/dist/engine/eventDispatch.d.ts +71 -0
- package/dist/engine/eventDispatch.d.ts.map +1 -0
- package/dist/engine/eventDispatch.js +115 -0
- package/dist/engine/eventDispatch.js.map +1 -0
- package/dist/engine/node.d.ts +99 -2
- package/dist/engine/node.d.ts.map +1 -1
- package/dist/engine/node.js +115 -4
- package/dist/engine/node.js.map +1 -1
- package/dist/engine/nodeContext.d.ts +6 -2
- package/dist/engine/nodeContext.d.ts.map +1 -1
- package/dist/engine/nodeContext.js +42 -10
- package/dist/engine/nodeContext.js.map +1 -1
- package/dist/engine/nodeLock.d.ts +15 -0
- package/dist/engine/nodeLock.d.ts.map +1 -1
- package/dist/engine/nodeLock.js +24 -4
- package/dist/engine/nodeLock.js.map +1 -1
- package/dist/engine/placement.d.ts +27 -0
- package/dist/engine/placement.d.ts.map +1 -1
- package/dist/engine/placement.js +43 -0
- package/dist/engine/placement.js.map +1 -1
- package/dist/routes/agent.d.ts.map +1 -1
- package/dist/routes/agent.js +4 -20
- package/dist/routes/agent.js.map +1 -1
- package/dist/routes/deliveryRouting.d.ts.map +1 -1
- package/dist/routes/deliveryRouting.js +11 -51
- package/dist/routes/deliveryRouting.js.map +1 -1
- package/dist/routes/fanout.d.ts +9 -0
- package/dist/routes/fanout.d.ts.map +1 -1
- package/dist/routes/fanout.js +39 -88
- package/dist/routes/fanout.js.map +1 -1
- package/dist/routes/node.d.ts.map +1 -1
- package/dist/routes/node.js +30 -10
- package/dist/routes/node.js.map +1 -1
- package/package.json +3 -3
package/dist/engine/node.d.ts
CHANGED
|
@@ -35,6 +35,7 @@ declare function publicNode(row: NodeRow): {
|
|
|
35
35
|
name: string;
|
|
36
36
|
kind: string;
|
|
37
37
|
role: string;
|
|
38
|
+
machine_id: string | null;
|
|
38
39
|
delivery_adapter: string;
|
|
39
40
|
delivery: Record<string, unknown> | null;
|
|
40
41
|
capabilities: {
|
|
@@ -58,6 +59,7 @@ declare function publicNode(row: NodeRow): {
|
|
|
58
59
|
export declare function createNodeToken(db: Db, workspaceId: string, data: {
|
|
59
60
|
node_id?: string;
|
|
60
61
|
name: string;
|
|
62
|
+
machine_id?: string;
|
|
61
63
|
kind?: NodeKind;
|
|
62
64
|
role?: NodeRole;
|
|
63
65
|
delivery_adapter?: string;
|
|
@@ -72,6 +74,7 @@ export declare function createNodeToken(db: Db, workspaceId: string, data: {
|
|
|
72
74
|
name: string;
|
|
73
75
|
kind: string;
|
|
74
76
|
role: string;
|
|
77
|
+
machine_id: string | null;
|
|
75
78
|
delivery_adapter: string;
|
|
76
79
|
delivery: Record<string, unknown> | null;
|
|
77
80
|
capabilities: {
|
|
@@ -119,6 +122,7 @@ export declare function getNodeByTokenHash(db: Db, tokenHash: string): Promise<{
|
|
|
119
122
|
load: number;
|
|
120
123
|
loadReported: boolean;
|
|
121
124
|
lastHeartbeatAt: Date | null;
|
|
125
|
+
provenLiveAt: Date | null;
|
|
122
126
|
createdAt: Date;
|
|
123
127
|
}>;
|
|
124
128
|
export declare function getNodeByName(db: Db, workspaceId: string, name: string): Promise<{
|
|
@@ -148,6 +152,7 @@ export declare function getNodeByName(db: Db, workspaceId: string, name: string)
|
|
|
148
152
|
load: number;
|
|
149
153
|
loadReported: boolean;
|
|
150
154
|
lastHeartbeatAt: Date | null;
|
|
155
|
+
provenLiveAt: Date | null;
|
|
151
156
|
createdAt: Date;
|
|
152
157
|
}>;
|
|
153
158
|
export declare function getNodeById(db: Db, workspaceId: string, nodeId: string): Promise<{
|
|
@@ -177,15 +182,102 @@ export declare function getNodeById(db: Db, workspaceId: string, nodeId: string)
|
|
|
177
182
|
load: number;
|
|
178
183
|
loadReported: boolean;
|
|
179
184
|
lastHeartbeatAt: Date | null;
|
|
185
|
+
provenLiveAt: Date | null;
|
|
180
186
|
createdAt: Date;
|
|
181
187
|
}>;
|
|
188
|
+
/**
|
|
189
|
+
* The broker a machine already has on the roster and is no longer running.
|
|
190
|
+
*
|
|
191
|
+
* Scoped to `broker` deliberately: a broker is the node-of-many fleet host and
|
|
192
|
+
* a machine runs one, so machine_id identifies it. A `direct` node is a
|
|
193
|
+
* node-of-one delivery host and a single machine legitimately runs many of
|
|
194
|
+
* them, so machine_id is not a key there and must never collapse them.
|
|
195
|
+
*
|
|
196
|
+
* A LIVE incumbent is never returned. Adopting one would rename a running
|
|
197
|
+
* broker and rotate its token out from under it — the caller cannot be that
|
|
198
|
+
* host coming back, because that host has not left. Only a node that is gone
|
|
199
|
+
* can be the one re-enrolling. Skipping the live row lets the caller fall
|
|
200
|
+
* through to a new node instead, which is the right answer for the legitimate
|
|
201
|
+
* duplicate too: a VM cloned from a snapshot, or containers baked from one
|
|
202
|
+
* image, carry the same machine_id and run CONCURRENTLY, so they are genuinely
|
|
203
|
+
* separate nodes. Declining the match costs an extra roster row; rejecting the
|
|
204
|
+
* enrollment outright would break a whole fleet booted from one image.
|
|
205
|
+
*
|
|
206
|
+
* Ordered oldest-first so a roster that already holds several rows for one
|
|
207
|
+
* machine converges onto its earliest reusable row instead of picking
|
|
208
|
+
* arbitrarily.
|
|
209
|
+
*/
|
|
210
|
+
export declare function getBrokerNodeByMachineId(db: Db, workspaceId: string, machineId: string): Promise<{
|
|
211
|
+
id: string;
|
|
212
|
+
workspaceId: string;
|
|
213
|
+
name: string;
|
|
214
|
+
tokenHash: string;
|
|
215
|
+
machineId: string | null;
|
|
216
|
+
kind: string;
|
|
217
|
+
role: string;
|
|
218
|
+
deliveryAdapter: string;
|
|
219
|
+
deliveryConfig: Record<string, unknown> | null;
|
|
220
|
+
capabilities: {
|
|
221
|
+
name: string;
|
|
222
|
+
kind?: string | undefined;
|
|
223
|
+
global?: boolean | undefined;
|
|
224
|
+
queue?: boolean | undefined;
|
|
225
|
+
metadata?: Record<string, import("@relaycast/types").FleetWireJsonValue> | undefined;
|
|
226
|
+
}[];
|
|
227
|
+
maxAgents: number;
|
|
228
|
+
activeAgents: number;
|
|
229
|
+
reservedAgents: number;
|
|
230
|
+
tags: string[];
|
|
231
|
+
version: string;
|
|
232
|
+
status: string;
|
|
233
|
+
handlersLive: boolean;
|
|
234
|
+
load: number;
|
|
235
|
+
loadReported: boolean;
|
|
236
|
+
lastHeartbeatAt: Date | null;
|
|
237
|
+
provenLiveAt: Date | null;
|
|
238
|
+
createdAt: Date;
|
|
239
|
+
} | null>;
|
|
240
|
+
/**
|
|
241
|
+
* The role an enroll is asking for, derived from the request alone.
|
|
242
|
+
*
|
|
243
|
+
* Enrollment's role default depends on `kind` (`ws` is a broker; `http_push`
|
|
244
|
+
* and `poll` are direct) and on `max_agents`, and the machine lookup has to
|
|
245
|
+
* know it *before* it runs. Otherwise a machine that already has a broker has
|
|
246
|
+
* that broker rotated -- and its transport rewritten to http_push or poll --
|
|
247
|
+
* by a node enrolling alongside it without an explicit `role`. That failure
|
|
248
|
+
* would be silent and would move a live node's identity, which is worse than
|
|
249
|
+
* the roster growth this dedupe exists to stop.
|
|
250
|
+
*/
|
|
251
|
+
export declare function requestedNodeRole(data: {
|
|
252
|
+
kind?: string;
|
|
253
|
+
role?: NodeRole;
|
|
254
|
+
max_agents?: number;
|
|
255
|
+
}): NodeRole;
|
|
182
256
|
/**
|
|
183
257
|
* Resolve the node an enroll (POST /v1/nodes) targets: by node_id when
|
|
184
|
-
* supplied, by name
|
|
258
|
+
* supplied, then by name, then by machine_id.
|
|
259
|
+
*
|
|
260
|
+
* The machine_id step is what stops the roster refilling. A host that enrolls
|
|
261
|
+
* without a persisted node_id arrives under a fresh name on every boot, and
|
|
262
|
+
* each of those names used to mint a brand-new row that nothing ever reclaimed.
|
|
263
|
+
* Falling back to the machine's existing broker turns re-enrollment into a
|
|
264
|
+
* token rotation on the row that is already there.
|
|
265
|
+
*
|
|
266
|
+
* node_id and name still win, so a caller that pins either keeps the exact
|
|
267
|
+
* identity it asked for; passing node_id is the way to opt out of machine
|
|
268
|
+
* grouping and run two brokers on one host.
|
|
269
|
+
*
|
|
270
|
+
* The machine step only ever matches a node that is not live — see
|
|
271
|
+
* getBrokerNodeByMachineId — so a running broker is never adopted by another
|
|
272
|
+
* caller presenting its machine_id.
|
|
185
273
|
*/
|
|
186
274
|
export declare function resolveNodeForEnroll(db: Db, workspaceId: string, data: {
|
|
187
275
|
node_id?: string;
|
|
188
276
|
name: string;
|
|
277
|
+
machine_id?: string;
|
|
278
|
+
kind?: string;
|
|
279
|
+
role?: NodeRole;
|
|
280
|
+
max_agents?: number;
|
|
189
281
|
}): Promise<{
|
|
190
282
|
id: string;
|
|
191
283
|
workspaceId: string;
|
|
@@ -213,8 +305,9 @@ export declare function resolveNodeForEnroll(db: Db, workspaceId: string, data:
|
|
|
213
305
|
load: number;
|
|
214
306
|
loadReported: boolean;
|
|
215
307
|
lastHeartbeatAt: Date | null;
|
|
308
|
+
provenLiveAt: Date | null;
|
|
216
309
|
createdAt: Date;
|
|
217
|
-
}>;
|
|
310
|
+
} | null>;
|
|
218
311
|
export interface RegisterNodeResult {
|
|
219
312
|
node: ReturnType<typeof publicNode>;
|
|
220
313
|
acceptance: FleetCapabilityAcceptance[];
|
|
@@ -226,6 +319,7 @@ export declare function heartbeatNode(db: Db, workspaceId: string, nodeId: strin
|
|
|
226
319
|
name: string;
|
|
227
320
|
kind: string;
|
|
228
321
|
role: string;
|
|
322
|
+
machine_id: string | null;
|
|
229
323
|
delivery_adapter: string;
|
|
230
324
|
delivery: Record<string, unknown> | null;
|
|
231
325
|
capabilities: {
|
|
@@ -272,6 +366,7 @@ export declare function ensureDirectNodeForAgent(db: Db, workspaceId: string, ag
|
|
|
272
366
|
name: string;
|
|
273
367
|
kind: string;
|
|
274
368
|
role: string;
|
|
369
|
+
machine_id: string | null;
|
|
275
370
|
delivery_adapter: string;
|
|
276
371
|
delivery: Record<string, unknown> | null;
|
|
277
372
|
capabilities: {
|
|
@@ -381,6 +476,7 @@ export declare function listNodes(db: Db, workspaceId: string, filters?: {
|
|
|
381
476
|
name: string;
|
|
382
477
|
kind: string;
|
|
383
478
|
role: string;
|
|
479
|
+
machine_id: string | null;
|
|
384
480
|
delivery_adapter: string;
|
|
385
481
|
delivery: Record<string, unknown> | null;
|
|
386
482
|
capabilities: {
|
|
@@ -406,6 +502,7 @@ export declare function getPublicNode(db: Db, workspaceId: string, name: string)
|
|
|
406
502
|
name: string;
|
|
407
503
|
kind: string;
|
|
408
504
|
role: string;
|
|
505
|
+
machine_id: string | null;
|
|
409
506
|
delivery_adapter: string;
|
|
410
507
|
delivery: Record<string, unknown> | null;
|
|
411
508
|
capabilities: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"node.d.ts","sourceRoot":"","sources":["../../src/engine/node.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,wBAAwB,EACxB,yBAAyB,EAEzB,yBAAyB,EACzB,mBAAmB,EACnB,yBAAyB,EACzB,wBAAwB,EACxB,qBAAqB,EACrB,eAAe,EACf,sBAAsB,EACvB,MAAM,kBAAkB,CAAC;AAK1B,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAqB,MAAM,EAA8D,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAI/H,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAgC,KAAK,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAwBjG,OAAO,KAAK,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAG7F,KAAK,EAAE,GAAG,UAAU,CAAC,OAAO,KAAK,CAAC,CAAC;AACnC,KAAK,OAAO,GAAG,OAAO,KAAK,CAAC,YAAY,CAAC;AACzC,KAAK,QAAQ,GAAG,OAAO,MAAM,CAAC,YAAY,CAAC;AAC3C,KAAK,QAAQ,GAAG,IAAI,GAAG,WAAW,GAAG,MAAM,CAAC;AAC5C,KAAK,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEpC,oFAAoF;AACpF,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,MAAM,WAAW,4BAA4B;IAC3C,EAAE,EAAE,QAAQ,CAAC;IACb,QAAQ,EAAE,sBAAsB,CAAC;IACjC,cAAc,CAAC,EAAE,wBAAwB,CAAC;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;CACb;AAUD,KAAK,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAE/C,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE5D;AA+CD,iBAAS,UAAU,CAAC,GAAG,EAAE,OAAO
|
|
1
|
+
{"version":3,"file":"node.d.ts","sourceRoot":"","sources":["../../src/engine/node.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,wBAAwB,EACxB,yBAAyB,EAEzB,yBAAyB,EACzB,mBAAmB,EACnB,yBAAyB,EACzB,wBAAwB,EACxB,qBAAqB,EACrB,eAAe,EACf,sBAAsB,EACvB,MAAM,kBAAkB,CAAC;AAK1B,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAqB,MAAM,EAA8D,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAI/H,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAgC,KAAK,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAwBjG,OAAO,KAAK,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAG7F,KAAK,EAAE,GAAG,UAAU,CAAC,OAAO,KAAK,CAAC,CAAC;AACnC,KAAK,OAAO,GAAG,OAAO,KAAK,CAAC,YAAY,CAAC;AACzC,KAAK,QAAQ,GAAG,OAAO,MAAM,CAAC,YAAY,CAAC;AAC3C,KAAK,QAAQ,GAAG,IAAI,GAAG,WAAW,GAAG,MAAM,CAAC;AAC5C,KAAK,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEpC,oFAAoF;AACpF,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,MAAM,WAAW,4BAA4B;IAC3C,EAAE,EAAE,QAAQ,CAAC;IACb,QAAQ,EAAE,sBAAsB,CAAC;IACjC,cAAc,CAAC,EAAE,wBAAwB,CAAC;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;CACb;AAUD,KAAK,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAE/C,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE5D;AA+CD,iBAAS,UAAU,CAAC,GAAG,EAAE,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;EAsB/B;AAyDD,wBAAsB,eAAe,CACnC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,YAAY,CAAC,EAAE,cAAc,EAAE,CAAC;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;;;;;;;;;;;;;;;;;;;;;;;;;;GAqGF;AAED,wBAAsB,kBAAkB,CAAC,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAGjE;AAED,wBAAsB,aAAa,CAAC,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAO5E;AAED,wBAAsB,WAAW,CAAC,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAM5E;AASD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,wBAAwB,CAAC,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAqC5F;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,QAAQ,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,QAAQ,CAMzG;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,oBAAoB,CACxC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,QAAQ,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAWnH;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;IACpC,UAAU,EAAE,yBAAyB,EAAE,CAAC;IACxC,QAAQ,EAAE,qBAAqB,CAAC;CACjC;AAED,wBAAsB,YAAY,CAChC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,mBAAmB,EAAE,MAAM,EAC3B,OAAO,EAAE,wBAAwB,EACjC,QAAQ,EAAE,qBAAqB,GAC9B,OAAO,CAAC,kBAAkB,CAAC,CAwF7B;AAED,wBAAsB,aAAa,CACjC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;UA4FnC;AAED,wBAAsB,eAAe,CACnC,EAAE,EAAE,EAAE,EACN,QAAQ,EAAE,sBAAsB,EAChC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,wBAAwB,CAAC;IAAC,MAAM,CAAC,EAAE,iBAAiB,CAAA;CAAE,iBAiD1E;AAED;;;;GAIG;AACH,wBAAsB,wBAAwB,CAC5C,EAAE,EAAE,EAAE,EACN,QAAQ,EAAE,sBAAsB,EAChC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,uBAAuB,EAAE,OAAO,EAChC,IAAI,CAAC,EAAE,wBAAwB,iBAsBhC;AAED;;;;GAIG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,EAAE,EACN,QAAQ,EAAE,sBAAsB,EAChC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,IAAI,CAAC,EAAE,wBAAwB,iBAgDhC;AAED,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,EAAE,EACN,QAAQ,EAAE,sBAAsB,EAChC,IAAI,CAAC,EAAE,wBAAwB,GAC9B,OAAO,CAAC,MAAM,CAAC,CAmBjB;AAgHD,wBAAsB,wBAAwB,CAC5C,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,MAAM,GAAG,gBAAgB,CAAC,EACvD,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAO;;;;;;;;;;;;;;;;;;;;;;;;;UAG7E;AA+ID,wBAAsB,6BAA6B,CACjD,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,iBAahB;AAgCD,wBAAsB,eAAe,CACnC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE;IAAE,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO;;;;;;;;;;;;;GA0D9D;AAED,wBAAsB,mBAAmB,CAAC,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,oBA6BzG;AAED,wBAAsB,cAAc,CAAC,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;;;;;;;;;;;;;YA2BjF;AAED,wBAAsB,oBAAoB,CACxC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,yBAAyB,EAClC,OAAO,GAAE;IAAE,uBAAuB,CAAC,EAAE,OAAO,CAAA;CAAO,GAClD,OAAO,CAAC,sBAAsB,CAAC,CAgFjC;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CACvC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,wBAAwB,EACjC,OAAO,GAAE;IAAE,uBAAuB,CAAC,EAAE,OAAO,CAAA;CAAO,GAClD,OAAO,CAAC,sBAAsB,CAAC,CA8HjC;AAED,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,EAC7C,IAAI,CAAC,EAAE,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;UAmDhC;AAED,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,EAAE,EACN,QAAQ,EAAE,sBAAsB,EAChC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,eAAe,EAAE,mBAAmB,EAAE,EACtC,cAAc,CAAC,EAAE,wBAAwB;;;;;;;;;GAiN1C;AAED,wBAAsB,SAAS,CAC7B,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO;;;;;;;;;;;;;;;;;;;;;;;;;KAOrD;AAED,wBAAsB,aAAa,CAAC,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;UAG5E;AAwGD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,4BAA4B,iBAiWhF"}
|
package/dist/engine/node.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { and, eq, inArray, ne, or, sql } from 'drizzle-orm';
|
|
1
|
+
import { and, asc, eq, inArray, isNotNull, isNull, lt, lte, ne, or, sql } from 'drizzle-orm';
|
|
2
2
|
import { FLEET_DELIVERY_CURSOR_CAPABILITY, parseFleetBrokerToRelaycastMessage, } from '@relaycast/types';
|
|
3
3
|
import { actionInvocations, agents, agentNodeBindings, channelMembers, channels, nodeProviders, nodes } from '../db/schema.js';
|
|
4
4
|
import { randomHex, sha256Hex } from '../lib/crypto.js';
|
|
@@ -8,7 +8,7 @@ import { isProviderAgentDeliveryReady } from '../ports/realtime.js';
|
|
|
8
8
|
import { generateId } from './snowflake.js';
|
|
9
9
|
import { assertRegistrableAgentName } from './agent.js';
|
|
10
10
|
import { rotateAgentIdentity } from './agentIdentity.js';
|
|
11
|
-
import { isNodeLive, nodeHasCapacity, nodeHasCapability } from './placement.js';
|
|
11
|
+
import { isNodeLive, isReusableForMachineMatch, nodeHasCapacity, nodeHasCapability, NODE_LIVENESS_TTL_MS } from './placement.js';
|
|
12
12
|
import { DEFAULT_PROVIDER_NAME, capabilityKind, heartbeatProvider, isProviderLive, markProviderOffline, materializeProviderActions, recomputeNodeAggregate, removeProvider, upsertProvider, } from './nodeProvider.js';
|
|
13
13
|
import { serializeNodeOp } from './nodeLock.js';
|
|
14
14
|
import { completeNodeInvocation, dispatchCapacitySpawn, rescheduleInvocationsForLostNode, rescheduleNodeInvocation, } from './action.js';
|
|
@@ -65,6 +65,7 @@ function publicNode(row) {
|
|
|
65
65
|
name: row.name,
|
|
66
66
|
kind: row.kind,
|
|
67
67
|
role: row.role,
|
|
68
|
+
machine_id: row.machineId,
|
|
68
69
|
delivery_adapter: row.deliveryAdapter,
|
|
69
70
|
delivery: redactDeliveryConfig(row.deliveryConfig),
|
|
70
71
|
capabilities: row.capabilities,
|
|
@@ -191,6 +192,12 @@ export async function createNodeToken(db, workspaceId, data) {
|
|
|
191
192
|
deliveryAdapter,
|
|
192
193
|
deliveryConfig,
|
|
193
194
|
capabilities: normalizeCapabilities(data.capabilities ?? existing.capabilities),
|
|
195
|
+
machineId: data.machine_id ?? existing.machineId,
|
|
196
|
+
// The credential just changed, so whatever the previous holder proved
|
|
197
|
+
// no longer applies to this row. Without clearing it, a row reused once
|
|
198
|
+
// still looks proven and could be reused again immediately, revoking
|
|
199
|
+
// the token this call is handing out.
|
|
200
|
+
provenLiveAt: null,
|
|
194
201
|
maxAgents,
|
|
195
202
|
tags: data.tags ?? existing.tags,
|
|
196
203
|
version: data.version ?? existing.version,
|
|
@@ -215,6 +222,7 @@ export async function createNodeToken(db, workspaceId, data) {
|
|
|
215
222
|
deliveryAdapter,
|
|
216
223
|
deliveryConfig,
|
|
217
224
|
capabilities: normalizeCapabilities(data.capabilities ?? []),
|
|
225
|
+
machineId: data.machine_id ?? null,
|
|
218
226
|
maxAgents,
|
|
219
227
|
tags: data.tags ?? [],
|
|
220
228
|
version: data.version ?? 'unknown',
|
|
@@ -247,15 +255,108 @@ export async function getNodeById(db, workspaceId, nodeId) {
|
|
|
247
255
|
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
|
|
248
256
|
return node ?? null;
|
|
249
257
|
}
|
|
258
|
+
/**
|
|
259
|
+
* How many *reusable* rows sharing one machine_id the enroll lookup will
|
|
260
|
+
* consider. Live rows are excluded in SQL before this bound applies, so a
|
|
261
|
+
* machine with many live brokers cannot push its one offline row out of range.
|
|
262
|
+
*/
|
|
263
|
+
const MACHINE_MATCH_SCAN_LIMIT = 20;
|
|
264
|
+
/**
|
|
265
|
+
* The broker a machine already has on the roster and is no longer running.
|
|
266
|
+
*
|
|
267
|
+
* Scoped to `broker` deliberately: a broker is the node-of-many fleet host and
|
|
268
|
+
* a machine runs one, so machine_id identifies it. A `direct` node is a
|
|
269
|
+
* node-of-one delivery host and a single machine legitimately runs many of
|
|
270
|
+
* them, so machine_id is not a key there and must never collapse them.
|
|
271
|
+
*
|
|
272
|
+
* A LIVE incumbent is never returned. Adopting one would rename a running
|
|
273
|
+
* broker and rotate its token out from under it — the caller cannot be that
|
|
274
|
+
* host coming back, because that host has not left. Only a node that is gone
|
|
275
|
+
* can be the one re-enrolling. Skipping the live row lets the caller fall
|
|
276
|
+
* through to a new node instead, which is the right answer for the legitimate
|
|
277
|
+
* duplicate too: a VM cloned from a snapshot, or containers baked from one
|
|
278
|
+
* image, carry the same machine_id and run CONCURRENTLY, so they are genuinely
|
|
279
|
+
* separate nodes. Declining the match costs an extra roster row; rejecting the
|
|
280
|
+
* enrollment outright would break a whole fleet booted from one image.
|
|
281
|
+
*
|
|
282
|
+
* Ordered oldest-first so a roster that already holds several rows for one
|
|
283
|
+
* machine converges onto its earliest reusable row instead of picking
|
|
284
|
+
* arbitrarily.
|
|
285
|
+
*/
|
|
286
|
+
export async function getBrokerNodeByMachineId(db, workspaceId, machineId) {
|
|
287
|
+
// Exclude live rows in SQL so the bound below counts only reusable
|
|
288
|
+
// candidates. Bounding first and filtering after would let a machine with
|
|
289
|
+
// enough live brokers hide its one offline row past the limit, and enrollment
|
|
290
|
+
// would insert a new row instead of reusing it.
|
|
291
|
+
const now = new Date();
|
|
292
|
+
const liveCutoff = new Date(now.getTime() - NODE_LIVENESS_TTL_MS);
|
|
293
|
+
const candidates = await db
|
|
294
|
+
.select()
|
|
295
|
+
.from(nodes)
|
|
296
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.machineId, machineId), eq(nodes.role, 'broker'),
|
|
297
|
+
// Mirrors isReusableForMachineMatch in full, so the scan bound below
|
|
298
|
+
// counts only rows that are actually reusable. A stale proof alone is not
|
|
299
|
+
// enough: a broker that registered recently has a fresh lastHeartbeatAt
|
|
300
|
+
// and is therefore live, and admitting those here would let them crowd
|
|
301
|
+
// the window and hide the one reusable row behind them.
|
|
302
|
+
isNotNull(nodes.provenLiveAt), lt(nodes.provenLiveAt, liveCutoff),
|
|
303
|
+
// not live — mirrors isNodeLive
|
|
304
|
+
or(ne(nodes.status, 'online'), isNull(nodes.lastHeartbeatAt), lt(nodes.lastHeartbeatAt, liveCutoff)),
|
|
305
|
+
// no future-dated heartbeat — server clock rollback
|
|
306
|
+
or(isNull(nodes.lastHeartbeatAt), lte(nodes.lastHeartbeatAt, now))))
|
|
307
|
+
.orderBy(asc(nodes.createdAt), asc(nodes.id))
|
|
308
|
+
.limit(MACHINE_MATCH_SCAN_LIMIT);
|
|
309
|
+
// The SQL above is a pre-filter for `isReusableForMachineMatch`, which is the
|
|
310
|
+
// single definition of reusability and mirrors it exactly — including
|
|
311
|
+
// treating an `online` row with a future heartbeat as live. Re-checking here
|
|
312
|
+
// keeps that definition authoritative rather than trusting the query alone.
|
|
313
|
+
return candidates.find((node) => isReusableForMachineMatch(node)) ?? null;
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* The role an enroll is asking for, derived from the request alone.
|
|
317
|
+
*
|
|
318
|
+
* Enrollment's role default depends on `kind` (`ws` is a broker; `http_push`
|
|
319
|
+
* and `poll` are direct) and on `max_agents`, and the machine lookup has to
|
|
320
|
+
* know it *before* it runs. Otherwise a machine that already has a broker has
|
|
321
|
+
* that broker rotated -- and its transport rewritten to http_push or poll --
|
|
322
|
+
* by a node enrolling alongside it without an explicit `role`. That failure
|
|
323
|
+
* would be silent and would move a live node's identity, which is worse than
|
|
324
|
+
* the roster growth this dedupe exists to stop.
|
|
325
|
+
*/
|
|
326
|
+
export function requestedNodeRole(data) {
|
|
327
|
+
if (data.role)
|
|
328
|
+
return data.role;
|
|
329
|
+
return normalizeLegacyNodeShape(data.kind ?? 'ws', data.max_agents !== undefined && data.max_agents > 1 ? 'broker' : undefined).role;
|
|
330
|
+
}
|
|
250
331
|
/**
|
|
251
332
|
* Resolve the node an enroll (POST /v1/nodes) targets: by node_id when
|
|
252
|
-
* supplied, by name
|
|
333
|
+
* supplied, then by name, then by machine_id.
|
|
334
|
+
*
|
|
335
|
+
* The machine_id step is what stops the roster refilling. A host that enrolls
|
|
336
|
+
* without a persisted node_id arrives under a fresh name on every boot, and
|
|
337
|
+
* each of those names used to mint a brand-new row that nothing ever reclaimed.
|
|
338
|
+
* Falling back to the machine's existing broker turns re-enrollment into a
|
|
339
|
+
* token rotation on the row that is already there.
|
|
340
|
+
*
|
|
341
|
+
* node_id and name still win, so a caller that pins either keeps the exact
|
|
342
|
+
* identity it asked for; passing node_id is the way to opt out of machine
|
|
343
|
+
* grouping and run two brokers on one host.
|
|
344
|
+
*
|
|
345
|
+
* The machine step only ever matches a node that is not live — see
|
|
346
|
+
* getBrokerNodeByMachineId — so a running broker is never adopted by another
|
|
347
|
+
* caller presenting its machine_id.
|
|
253
348
|
*/
|
|
254
349
|
export async function resolveNodeForEnroll(db, workspaceId, data) {
|
|
255
350
|
if (data.node_id !== undefined) {
|
|
256
351
|
return getNodeById(db, workspaceId, data.node_id);
|
|
257
352
|
}
|
|
258
|
-
|
|
353
|
+
const byName = await getNodeByName(db, workspaceId, data.name);
|
|
354
|
+
if (byName)
|
|
355
|
+
return byName;
|
|
356
|
+
if (data.machine_id !== undefined && requestedNodeRole(data) === 'broker') {
|
|
357
|
+
return getBrokerNodeByMachineId(db, workspaceId, data.machine_id);
|
|
358
|
+
}
|
|
359
|
+
return null;
|
|
259
360
|
}
|
|
260
361
|
export async function registerNode(db, workspaceId, authenticatedNodeId, message, provider) {
|
|
261
362
|
if (message.node_id !== authenticatedNodeId) {
|
|
@@ -370,6 +471,9 @@ export async function heartbeatNode(db, workspaceId, nodeId, providerName, messa
|
|
|
370
471
|
activeAgents: 1,
|
|
371
472
|
handlersLive: false,
|
|
372
473
|
lastHeartbeatAt: new Date(),
|
|
474
|
+
// A heartbeat frame arrived: this is proof of life, unlike the
|
|
475
|
+
// registration and disconnect writes that also touch lastHeartbeatAt.
|
|
476
|
+
provenLiveAt: new Date(),
|
|
373
477
|
})
|
|
374
478
|
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)))
|
|
375
479
|
.returning();
|
|
@@ -410,6 +514,13 @@ export async function heartbeatNode(db, workspaceId, nodeId, providerName, messa
|
|
|
410
514
|
await tx.update(nodes).set({ name: message.name }).where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
|
|
411
515
|
}
|
|
412
516
|
await recomputeNodeAggregate(tx, workspaceId, nodeId, message.version !== undefined ? { version: message.version } : {});
|
|
517
|
+
// Stamped after the aggregate so it cannot be overwritten by it: the
|
|
518
|
+
// aggregate derives lastHeartbeatAt from provider rows, which registration
|
|
519
|
+
// also writes, whereas this column is only ever set by an arriving frame.
|
|
520
|
+
await tx
|
|
521
|
+
.update(nodes)
|
|
522
|
+
.set({ provenLiveAt: new Date() })
|
|
523
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
|
|
413
524
|
});
|
|
414
525
|
const [updated] = await db
|
|
415
526
|
.select()
|