@relaycast/engine 5.1.1 → 6.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/node/database.d.ts.map +1 -1
- package/dist/adapters/node/database.js +204 -27
- package/dist/adapters/node/database.js.map +1 -1
- package/dist/adapters/node/realtime.d.ts +29 -1
- package/dist/adapters/node/realtime.d.ts.map +1 -1
- package/dist/adapters/node/realtime.js +241 -50
- package/dist/adapters/node/realtime.js.map +1 -1
- package/dist/auth/tokenKind.d.ts +1 -1
- package/dist/auth/tokenKind.d.ts.map +1 -1
- package/dist/auth/tokenKind.js +4 -1
- package/dist/auth/tokenKind.js.map +1 -1
- package/dist/db/migrations/0028_node_providers.sql +55 -0
- package/dist/db/schema.d.ts +376 -0
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +59 -1
- package/dist/db/schema.js.map +1 -1
- package/dist/engine/action.d.ts +47 -1
- package/dist/engine/action.d.ts.map +1 -1
- package/dist/engine/action.js +342 -43
- package/dist/engine/action.js.map +1 -1
- package/dist/engine/delivery.d.ts.map +1 -1
- package/dist/engine/delivery.js +2 -1
- package/dist/engine/delivery.js.map +1 -1
- package/dist/engine/node.d.ts +71 -28
- package/dist/engine/node.d.ts.map +1 -1
- package/dist/engine/node.js +357 -189
- package/dist/engine/node.js.map +1 -1
- package/dist/engine/nodeContext.d.ts.map +1 -1
- package/dist/engine/nodeContext.js +17 -6
- package/dist/engine/nodeContext.js.map +1 -1
- package/dist/engine/nodeDeliver.d.ts.map +1 -1
- package/dist/engine/nodeDeliver.js +4 -1
- package/dist/engine/nodeDeliver.js.map +1 -1
- package/dist/engine/nodeLock.d.ts +2 -0
- package/dist/engine/nodeLock.d.ts.map +1 -0
- package/dist/engine/nodeLock.js +31 -0
- package/dist/engine/nodeLock.js.map +1 -0
- package/dist/engine/nodeProvider.d.ts +62 -0
- package/dist/engine/nodeProvider.d.ts.map +1 -0
- package/dist/engine/nodeProvider.js +256 -0
- package/dist/engine/nodeProvider.js.map +1 -0
- package/dist/engine/placement.d.ts +10 -0
- package/dist/engine/placement.d.ts.map +1 -1
- package/dist/engine/placement.js +10 -0
- package/dist/engine/placement.js.map +1 -1
- package/dist/engine/trigger.d.ts.map +1 -1
- package/dist/engine/trigger.js +17 -3
- package/dist/engine/trigger.js.map +1 -1
- package/dist/middleware/auth.d.ts +6 -0
- package/dist/middleware/auth.d.ts.map +1 -1
- package/dist/middleware/auth.js +6 -0
- package/dist/middleware/auth.js.map +1 -1
- package/dist/node-control.d.ts +1 -1
- package/dist/node-control.d.ts.map +1 -1
- package/dist/node-control.js +11 -1
- package/dist/node-control.js.map +1 -1
- package/dist/ports/auth.d.ts +1 -1
- package/dist/ports/auth.d.ts.map +1 -1
- package/dist/ports/realtime.d.ts +30 -2
- package/dist/ports/realtime.d.ts.map +1 -1
- package/dist/routes/deliveryRouting.d.ts.map +1 -1
- package/dist/routes/deliveryRouting.js +6 -1
- package/dist/routes/deliveryRouting.js.map +1 -1
- package/dist/routes/message.d.ts.map +1 -1
- package/dist/routes/message.js +44 -14
- package/dist/routes/message.js.map +1 -1
- package/dist/routes/node.d.ts.map +1 -1
- package/dist/routes/node.js +92 -3
- package/dist/routes/node.js.map +1 -1
- package/package.json +3 -3
package/dist/engine/node.js
CHANGED
|
@@ -1,14 +1,21 @@
|
|
|
1
1
|
import { and, eq, inArray, ne, or, sql } from 'drizzle-orm';
|
|
2
2
|
import { parseFleetBrokerToRelaycastMessage } from '@relaycast/types';
|
|
3
|
-
import { actionInvocations,
|
|
3
|
+
import { actionInvocations, agents, agentNodeBindings, channelMembers, channels, nodeProviders, nodes } from '../db/schema.js';
|
|
4
4
|
import { randomHex, sha256Hex } from '../lib/crypto.js';
|
|
5
5
|
import { codedError } from '../lib/httpError.js';
|
|
6
6
|
import { runAtomic } from '../ports/database.js';
|
|
7
7
|
import { generateId } from './snowflake.js';
|
|
8
8
|
import { isNodeLive, nodeHasCapability } from './placement.js';
|
|
9
|
-
import {
|
|
9
|
+
import { DEFAULT_PROVIDER_NAME, capabilityKind, heartbeatProvider, markProviderOffline, materializeProviderActions, recomputeNodeAggregate, removeProvider, upsertProvider, } from './nodeProvider.js';
|
|
10
|
+
import { serializeNodeOp } from './nodeLock.js';
|
|
11
|
+
import { completeNodeInvocation, dispatchCapacitySpawn, rescheduleInvocationsForLostNode, rescheduleNodeInvocation, } from './action.js';
|
|
10
12
|
import { emitInvocationCompletionEffects } from './invocationCompletion.js';
|
|
11
13
|
import { ackDeliveriesUpToSeq, deliverPendingToNode } from './delivery.js';
|
|
14
|
+
function resolveProviderIdentity(message, connectionId) {
|
|
15
|
+
if (message.provider)
|
|
16
|
+
return message.provider;
|
|
17
|
+
return { name: DEFAULT_PROVIDER_NAME, instance_id: connectionId ?? generateId() };
|
|
18
|
+
}
|
|
12
19
|
export function directNodeIdForAgent(agentId) {
|
|
13
20
|
return `node_direct_${agentId}`;
|
|
14
21
|
}
|
|
@@ -18,11 +25,6 @@ function directNodeNameForAgent(agentId) {
|
|
|
18
25
|
function isImplicitDirectLocation(agent) {
|
|
19
26
|
return agent.locationNodeId === directNodeIdForAgent(agent.id);
|
|
20
27
|
}
|
|
21
|
-
function capabilityName(capability) {
|
|
22
|
-
if (typeof capability === 'string')
|
|
23
|
-
return capability;
|
|
24
|
-
return capability?.name ?? null;
|
|
25
|
-
}
|
|
26
28
|
function normalizeCapabilities(capabilities) {
|
|
27
29
|
return capabilities.map((capability) => (typeof capability === 'string' ? { name: capability } : capability));
|
|
28
30
|
}
|
|
@@ -110,36 +112,6 @@ function normalizeLegacyAdapter(adapter) {
|
|
|
110
112
|
return 'ws.node.v1';
|
|
111
113
|
return adapter;
|
|
112
114
|
}
|
|
113
|
-
async function ensureCapabilityActions(db, workspaceId, nodeId, capabilities) {
|
|
114
|
-
for (const capability of capabilities) {
|
|
115
|
-
const name = capabilityName(capability);
|
|
116
|
-
if (!name || name.startsWith('spawn:'))
|
|
117
|
-
continue;
|
|
118
|
-
const [existing] = await db
|
|
119
|
-
.select()
|
|
120
|
-
.from(actions)
|
|
121
|
-
.where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, name)));
|
|
122
|
-
if (!existing) {
|
|
123
|
-
await db.insert(actions).values({
|
|
124
|
-
id: `act_${generateId()}`,
|
|
125
|
-
workspaceId,
|
|
126
|
-
name,
|
|
127
|
-
description: `Node handler ${name}`,
|
|
128
|
-
handlerAgentId: null,
|
|
129
|
-
handlerNodeId: nodeId,
|
|
130
|
-
inputSchema: {},
|
|
131
|
-
outputSchema: {},
|
|
132
|
-
availableTo: null,
|
|
133
|
-
});
|
|
134
|
-
}
|
|
135
|
-
else if (!existing.handlerAgentId && (!existing.handlerNodeId || existing.handlerNodeId === nodeId)) {
|
|
136
|
-
await db
|
|
137
|
-
.update(actions)
|
|
138
|
-
.set({ handlerNodeId: nodeId, isActive: true })
|
|
139
|
-
.where(eq(actions.id, existing.id));
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
115
|
export async function createNodeToken(db, workspaceId, data) {
|
|
144
116
|
const token = `nt_live_${randomHex(24)}`;
|
|
145
117
|
const tokenHash = await sha256Hex(token);
|
|
@@ -220,7 +192,7 @@ export async function getNodeByName(db, workspaceId, name) {
|
|
|
220
192
|
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.name, normalized)));
|
|
221
193
|
return node ?? null;
|
|
222
194
|
}
|
|
223
|
-
export async function registerNode(db, workspaceId, authenticatedNodeId, message) {
|
|
195
|
+
export async function registerNode(db, workspaceId, authenticatedNodeId, message, provider) {
|
|
224
196
|
if (message.node_id !== authenticatedNodeId) {
|
|
225
197
|
throw codedError('node_id does not match the authenticated node token', 'node_id_mismatch', 403);
|
|
226
198
|
}
|
|
@@ -239,41 +211,66 @@ export async function registerNode(db, workspaceId, authenticatedNodeId, message
|
|
|
239
211
|
if (existingByName && existingByName.id !== authenticatedNodeId) {
|
|
240
212
|
throw codedError(`Node name "${message.name}" is already enrolled`, 'node_name_conflict', 409);
|
|
241
213
|
}
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
if (role === 'broker') {
|
|
265
|
-
await ensureCapabilityActions(db, workspaceId, updated.id, capabilities);
|
|
214
|
+
// Direct nodes are the per-agent delivery hosts: they never carry providers.
|
|
215
|
+
if (existing.role === 'direct') {
|
|
216
|
+
const [updated] = await db
|
|
217
|
+
.update(nodes)
|
|
218
|
+
.set({
|
|
219
|
+
name: message.name,
|
|
220
|
+
capabilities: [],
|
|
221
|
+
kind: 'ws',
|
|
222
|
+
role: 'direct',
|
|
223
|
+
deliveryAdapter: 'ws.node.v1',
|
|
224
|
+
deliveryConfig: existing.deliveryConfig,
|
|
225
|
+
maxAgents: 1,
|
|
226
|
+
activeAgents: 1,
|
|
227
|
+
tags: existing.tags,
|
|
228
|
+
version: message.version,
|
|
229
|
+
status: 'online',
|
|
230
|
+
handlersLive: false,
|
|
231
|
+
lastHeartbeatAt: now,
|
|
232
|
+
})
|
|
233
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, authenticatedNodeId)))
|
|
234
|
+
.returning();
|
|
235
|
+
return { node: publicNode(updated), acceptance: [], provider };
|
|
266
236
|
}
|
|
267
|
-
|
|
237
|
+
const capabilities = normalizeCapabilities(message.capabilities);
|
|
238
|
+
await runAtomic(db, async (tx) => {
|
|
239
|
+
await upsertProvider(tx, workspaceId, authenticatedNodeId, {
|
|
240
|
+
name: provider.name,
|
|
241
|
+
instanceId: provider.instance_id,
|
|
242
|
+
capabilities,
|
|
243
|
+
maxAgents: message.max_agents,
|
|
244
|
+
version: message.version,
|
|
245
|
+
handlersLive: capabilities.length > 0,
|
|
246
|
+
});
|
|
247
|
+
await materializeProviderActions(tx, workspaceId, authenticatedNodeId, provider.name, capabilities);
|
|
248
|
+
await tx
|
|
249
|
+
.update(nodes)
|
|
250
|
+
.set({ name: message.name, kind: 'ws', role: 'broker', deliveryAdapter: 'ws.node.v1', deliveryConfig: null, tags: message.tags })
|
|
251
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, authenticatedNodeId)));
|
|
252
|
+
await recomputeNodeAggregate(tx, workspaceId, authenticatedNodeId, {
|
|
253
|
+
version: message.version,
|
|
254
|
+
machineId: message.machine_id ?? null,
|
|
255
|
+
});
|
|
256
|
+
});
|
|
257
|
+
const [updated] = await db
|
|
258
|
+
.select()
|
|
259
|
+
.from(nodes)
|
|
260
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, authenticatedNodeId)));
|
|
261
|
+
const acceptance = capabilities.map((cap) => ({
|
|
262
|
+
name: cap.name,
|
|
263
|
+
kind: capabilityKind(cap),
|
|
264
|
+
accepted: true,
|
|
265
|
+
}));
|
|
266
|
+
return { node: publicNode(updated), acceptance, provider };
|
|
268
267
|
}
|
|
269
|
-
export async function heartbeatNode(db, workspaceId, nodeId, message) {
|
|
270
|
-
// A heartbeat
|
|
271
|
-
//
|
|
272
|
-
// e.g. after an engine restart where the
|
|
273
|
-
//
|
|
274
|
-
//
|
|
275
|
-
// server-side here as the authoritative receipt time — the broker never sends
|
|
276
|
-
// it, and any client-supplied value would be ignored.
|
|
268
|
+
export async function heartbeatNode(db, workspaceId, nodeId, providerName, message) {
|
|
269
|
+
// A heartbeat is provider-scoped by connection. It may carry the provider's
|
|
270
|
+
// roster snapshot so the engine can refresh the descriptor between (or in the
|
|
271
|
+
// absence of a fresh) node.register — e.g. after an engine restart where the
|
|
272
|
+
// provider keeps heartbeating. lastHeartbeatAt is always stamped server-side
|
|
273
|
+
// as the authoritative receipt time; the client never sends it.
|
|
277
274
|
if (message.node_id !== undefined && message.node_id !== nodeId) {
|
|
278
275
|
throw codedError('node_id does not match the authenticated node token', 'node_id_mismatch', 403);
|
|
279
276
|
}
|
|
@@ -281,35 +278,66 @@ export async function heartbeatNode(db, workspaceId, nodeId, message) {
|
|
|
281
278
|
.select({ role: nodes.role })
|
|
282
279
|
.from(nodes)
|
|
283
280
|
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
if (
|
|
287
|
-
rosterUpdate
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
281
|
+
if (!nodeState)
|
|
282
|
+
return null;
|
|
283
|
+
if (nodeState.role === 'direct') {
|
|
284
|
+
const rosterUpdate = {};
|
|
285
|
+
if (message.name !== undefined)
|
|
286
|
+
rosterUpdate.name = message.name;
|
|
287
|
+
if (message.version !== undefined)
|
|
288
|
+
rosterUpdate.version = message.version;
|
|
289
|
+
const [updated] = await db
|
|
290
|
+
.update(nodes)
|
|
291
|
+
.set({
|
|
292
|
+
...rosterUpdate,
|
|
293
|
+
status: 'online',
|
|
294
|
+
load: message.load,
|
|
295
|
+
activeAgents: 1,
|
|
296
|
+
handlersLive: false,
|
|
297
|
+
lastHeartbeatAt: new Date(),
|
|
298
|
+
})
|
|
299
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)))
|
|
300
|
+
.returning();
|
|
301
|
+
return updated ? publicNode(updated) : null;
|
|
295
302
|
}
|
|
296
|
-
|
|
297
|
-
|
|
303
|
+
await runAtomic(db, async (tx) => {
|
|
304
|
+
if (message.capabilities !== undefined) {
|
|
305
|
+
const caps = normalizeCapabilities(message.capabilities);
|
|
306
|
+
const [existingProvider] = await tx
|
|
307
|
+
.select({ instanceId: nodeProviders.instanceId, maxAgents: nodeProviders.maxAgents, version: nodeProviders.version })
|
|
308
|
+
.from(nodeProviders)
|
|
309
|
+
.where(and(eq(nodeProviders.workspaceId, workspaceId), eq(nodeProviders.nodeId, nodeId), eq(nodeProviders.name, providerName)));
|
|
310
|
+
await upsertProvider(tx, workspaceId, nodeId, {
|
|
311
|
+
name: providerName,
|
|
312
|
+
instanceId: existingProvider?.instanceId ?? `${providerName}:heartbeat`,
|
|
313
|
+
capabilities: caps,
|
|
314
|
+
maxAgents: message.max_agents ?? existingProvider?.maxAgents ?? 0,
|
|
315
|
+
version: message.version ?? existingProvider?.version ?? 'unknown',
|
|
316
|
+
handlersLive: message.handlers_live,
|
|
317
|
+
});
|
|
318
|
+
await heartbeatProvider(tx, workspaceId, nodeId, providerName, {
|
|
319
|
+
load: message.load,
|
|
320
|
+
activeAgents: message.active_agents,
|
|
321
|
+
handlersLive: message.handlers_live,
|
|
322
|
+
});
|
|
323
|
+
await materializeProviderActions(tx, workspaceId, nodeId, providerName, caps);
|
|
324
|
+
}
|
|
325
|
+
else {
|
|
326
|
+
await heartbeatProvider(tx, workspaceId, nodeId, providerName, {
|
|
327
|
+
load: message.load,
|
|
328
|
+
activeAgents: message.active_agents,
|
|
329
|
+
handlersLive: message.handlers_live,
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
if (message.name !== undefined) {
|
|
333
|
+
await tx.update(nodes).set({ name: message.name }).where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
|
|
334
|
+
}
|
|
335
|
+
await recomputeNodeAggregate(tx, workspaceId, nodeId, message.version !== undefined ? { version: message.version } : {});
|
|
336
|
+
});
|
|
298
337
|
const [updated] = await db
|
|
299
|
-
.
|
|
300
|
-
.
|
|
301
|
-
|
|
302
|
-
status: 'online',
|
|
303
|
-
load: message.load,
|
|
304
|
-
activeAgents: role === 'direct' ? 1 : message.active_agents,
|
|
305
|
-
handlersLive: role !== 'direct' && message.handlers_live,
|
|
306
|
-
lastHeartbeatAt: new Date(),
|
|
307
|
-
})
|
|
308
|
-
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)))
|
|
309
|
-
.returning();
|
|
310
|
-
if (updated && updated.role !== 'direct' && message.capabilities !== undefined) {
|
|
311
|
-
await ensureCapabilityActions(db, workspaceId, updated.id, message.capabilities);
|
|
312
|
-
}
|
|
338
|
+
.select()
|
|
339
|
+
.from(nodes)
|
|
340
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
|
|
313
341
|
return updated ? publicNode(updated) : null;
|
|
314
342
|
}
|
|
315
343
|
export async function markNodeOffline(db, registry, workspaceId, nodeId) {
|
|
@@ -323,22 +351,89 @@ export async function markNodeOffline(db, registry, workspaceId, nodeId) {
|
|
|
323
351
|
lastHeartbeatAt: new Date(),
|
|
324
352
|
})
|
|
325
353
|
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
|
|
354
|
+
// Provider capability sets persist (offline nodes still show their manifest);
|
|
355
|
+
// only their liveness flips. Zero activeAgents too, so a later
|
|
356
|
+
// recomputeNodeAggregate never resurrects a dropped provider's agent count.
|
|
357
|
+
await db
|
|
358
|
+
.update(nodeProviders)
|
|
359
|
+
.set({ status: 'offline', handlersLive: false, load: 0, activeAgents: 0, lastHeartbeatAt: new Date() })
|
|
360
|
+
.where(and(eq(nodeProviders.workspaceId, workspaceId), eq(nodeProviders.nodeId, nodeId)));
|
|
326
361
|
await db
|
|
327
362
|
.update(agents)
|
|
328
363
|
.set({ status: 'offline', lastSeen: new Date() })
|
|
329
364
|
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.locationType, 'via_node'), eq(agents.locationNodeId, nodeId)));
|
|
330
365
|
await rescheduleInvocationsForLostNode(db, registry, workspaceId, nodeId);
|
|
331
366
|
}
|
|
332
|
-
|
|
333
|
-
|
|
367
|
+
/**
|
|
368
|
+
* A provider's control connection dropped. Its capability set persists (marked
|
|
369
|
+
* offline) so the node still shows the manifest; agents it hosted go offline.
|
|
370
|
+
* When it was the node's last connection the whole node goes offline.
|
|
371
|
+
*/
|
|
372
|
+
export async function handleProviderDisconnect(db, registry, workspaceId, nodeId, providerName, hasRemainingConnections) {
|
|
373
|
+
// Serialized with node-control operations so a teardown can't race a
|
|
374
|
+
// concurrent register's provider upsert / aggregate recompute (or, on
|
|
375
|
+
// better-sqlite3, deadlock its isolated transaction).
|
|
376
|
+
return serializeNodeOp(workspaceId, nodeId, async () => {
|
|
377
|
+
if (!hasRemainingConnections) {
|
|
378
|
+
await markNodeOffline(db, registry, workspaceId, nodeId);
|
|
379
|
+
return;
|
|
380
|
+
}
|
|
381
|
+
await markProviderOffline(db, workspaceId, nodeId, providerName);
|
|
382
|
+
await db
|
|
383
|
+
.update(agents)
|
|
384
|
+
.set({ status: 'offline', lastSeen: new Date() })
|
|
385
|
+
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.locationType, 'via_node'), eq(agents.locationNodeId, nodeId), eq(agents.providerName, providerName)));
|
|
386
|
+
await recomputeNodeAggregate(db, workspaceId, nodeId);
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
/**
|
|
390
|
+
* Provider-scoped deregister: removes the provider's attachment, persisted
|
|
391
|
+
* capability set, and materialized actions. When it was the node's last
|
|
392
|
+
* provider the node goes fully offline.
|
|
393
|
+
*/
|
|
394
|
+
export async function deregisterProvider(db, registry, workspaceId, nodeId, providerName) {
|
|
395
|
+
await removeProvider(db, workspaceId, nodeId, providerName);
|
|
396
|
+
registry.detachProvider(workspaceId, nodeId, providerName);
|
|
397
|
+
await db
|
|
398
|
+
.update(agents)
|
|
399
|
+
.set({ status: 'offline', lastSeen: new Date() })
|
|
400
|
+
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.locationType, 'via_node'), eq(agents.locationNodeId, nodeId), eq(agents.providerName, providerName)));
|
|
401
|
+
const [remaining] = await db
|
|
402
|
+
.select({ count: sql `count(*)` })
|
|
403
|
+
.from(nodeProviders)
|
|
404
|
+
.where(and(eq(nodeProviders.workspaceId, workspaceId), eq(nodeProviders.nodeId, nodeId)));
|
|
405
|
+
if (!remaining || remaining.count === 0) {
|
|
406
|
+
await markNodeOffline(db, registry, workspaceId, nodeId);
|
|
407
|
+
}
|
|
408
|
+
else {
|
|
409
|
+
// Other providers remain: recompute the node aggregate but don't reschedule
|
|
410
|
+
// node-wide — that would disturb the surviving providers' in-flight invokes.
|
|
411
|
+
// Work dispatched to the removed provider is caught by the dispatch-timeout
|
|
412
|
+
// sweep, matching the provider-disconnect path.
|
|
413
|
+
await recomputeNodeAggregate(db, workspaceId, nodeId);
|
|
414
|
+
}
|
|
334
415
|
}
|
|
335
416
|
export async function sweepOfflineNodes(db, registry) {
|
|
336
417
|
const rows = await db.select().from(nodes).where(eq(nodes.status, 'online'));
|
|
337
418
|
const stale = rows.filter((node) => !isNodeLive(node));
|
|
419
|
+
let swept = 0;
|
|
338
420
|
for (const node of stale) {
|
|
339
|
-
await
|
|
421
|
+
const offlined = await serializeNodeOp(node.workspaceId, node.id, async () => {
|
|
422
|
+
// Re-read under the lock: a heartbeat/register queued ahead of this sweep
|
|
423
|
+
// may have refreshed the node since the stale snapshot was taken.
|
|
424
|
+
const [current] = await db
|
|
425
|
+
.select()
|
|
426
|
+
.from(nodes)
|
|
427
|
+
.where(and(eq(nodes.workspaceId, node.workspaceId), eq(nodes.id, node.id)));
|
|
428
|
+
if (!current || current.status !== 'online' || isNodeLive(current))
|
|
429
|
+
return false;
|
|
430
|
+
await markNodeOffline(db, registry, node.workspaceId, node.id);
|
|
431
|
+
return true;
|
|
432
|
+
});
|
|
433
|
+
if (offlined)
|
|
434
|
+
swept++;
|
|
340
435
|
}
|
|
341
|
-
return
|
|
436
|
+
return swept;
|
|
342
437
|
}
|
|
343
438
|
async function autoJoinGeneral(db, workspaceId, agentId) {
|
|
344
439
|
const [generalChannel] = await db
|
|
@@ -684,7 +779,7 @@ export async function listNodeAgents(db, workspaceId, nodeName) {
|
|
|
684
779
|
.where(and(eq(agentNodeBindings.workspaceId, workspaceId), eq(agentNodeBindings.nodeId, node.id), eq(agentNodeBindings.status, 'active')));
|
|
685
780
|
return rows.map(serializeBinding);
|
|
686
781
|
}
|
|
687
|
-
export async function registerAgentViaNode(db, workspaceId, nodeId, message) {
|
|
782
|
+
export async function registerAgentViaNode(db, workspaceId, nodeId, providerName, message) {
|
|
688
783
|
return runAtomic(db, async (tx) => {
|
|
689
784
|
const [node] = await tx
|
|
690
785
|
.select()
|
|
@@ -714,6 +809,7 @@ export async function registerAgentViaNode(db, workspaceId, nodeId, message) {
|
|
|
714
809
|
metadata: { fleet: { node_id: nodeId, invocation_id: message.invocation_id ?? null, registered_at: now } },
|
|
715
810
|
locationType: 'via_node',
|
|
716
811
|
locationNodeId: nodeId,
|
|
812
|
+
providerName,
|
|
717
813
|
originNodeId: nodeId,
|
|
718
814
|
resumable: message.resumable ?? false,
|
|
719
815
|
sessionRef: message.session_ref ?? null,
|
|
@@ -727,6 +823,7 @@ export async function registerAgentViaNode(db, workspaceId, nodeId, message) {
|
|
|
727
823
|
metadata: sql `json_patch(COALESCE(${agents.metadata}, '{}'), ${fleetMetadata})`,
|
|
728
824
|
locationType: 'via_node',
|
|
729
825
|
locationNodeId: nodeId,
|
|
826
|
+
providerName,
|
|
730
827
|
originNodeId: sql `COALESCE(${agents.originNodeId}, ${nodeId})`,
|
|
731
828
|
resumable: message.resumable ?? false,
|
|
732
829
|
sessionRef: message.session_ref ?? null,
|
|
@@ -919,6 +1016,7 @@ export async function reconcileInventory(db, registry, workspaceId, nodeId, inve
|
|
|
919
1016
|
input: actionInvocations.input,
|
|
920
1017
|
status: actionInvocations.status,
|
|
921
1018
|
dispatchedNodeId: actionInvocations.dispatchedNodeId,
|
|
1019
|
+
spawnReservedAt: actionInvocations.spawnReservedAt,
|
|
922
1020
|
attemptedNodeIds: actionInvocations.attemptedNodeIds,
|
|
923
1021
|
dispatchAttempts: actionInvocations.dispatchAttempts,
|
|
924
1022
|
})
|
|
@@ -989,94 +1087,164 @@ export async function handleNodeControlMessage(args) {
|
|
|
989
1087
|
});
|
|
990
1088
|
return;
|
|
991
1089
|
}
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1090
|
+
// A provider binds its name to the connection at node.register; later frames
|
|
1091
|
+
// (heartbeat, deregister, agent.register, results) resolve it from the
|
|
1092
|
+
// connection. Absent a bound provider (or when driven without a registry
|
|
1093
|
+
// connection), fall back to the synthetic `default` provider.
|
|
1094
|
+
//
|
|
1095
|
+
// Resolve this at frame ARRIVAL, before entering the per-node queue: if the
|
|
1096
|
+
// socket closes while this frame is queued, `onNodeConnectionClose` clears the
|
|
1097
|
+
// connection's provider binding synchronously, and a lookup inside the callback
|
|
1098
|
+
// would then fall back to `default` and act on the wrong provider.
|
|
1099
|
+
const boundProviderName = args.connectionId
|
|
1100
|
+
? args.registry.providerNameForConnection?.(args.connectionId)
|
|
1101
|
+
: undefined;
|
|
1102
|
+
// The connection's registered identity is authoritative — a provider can't act
|
|
1103
|
+
// on another provider by putting a different name in the frame. The frame's
|
|
1104
|
+
// provider is only a hint for unbound/back-compat traffic.
|
|
1105
|
+
const frameProviderName = boundProviderName
|
|
1106
|
+
?? ('provider' in message && message.provider ? message.provider.name : undefined)
|
|
1107
|
+
?? DEFAULT_PROVIDER_NAME;
|
|
1108
|
+
// Serialize all control operations for a node so a concurrent register and
|
|
1109
|
+
// teardown (or two providers attaching at once) can't race the duplicate
|
|
1110
|
+
// check, the provider-row upsert, or the aggregate recompute.
|
|
1111
|
+
return serializeNodeOp(args.workspaceId, args.nodeId, async () => {
|
|
1112
|
+
try {
|
|
1113
|
+
switch (message.type) {
|
|
1114
|
+
case 'node.register': {
|
|
1115
|
+
const provider = resolveProviderIdentity(message, args.connectionId);
|
|
1116
|
+
if (args.connectionId && args.registry.providerAttachConflict) {
|
|
1117
|
+
const conflict = args.registry.providerAttachConflict(args.workspaceId, args.nodeId, provider.name, provider.instance_id, args.connectionId);
|
|
1118
|
+
if (conflict) {
|
|
1119
|
+
throw codedError(conflict.message, conflict.code, 409);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
const registered = await registerNode(args.db, args.workspaceId, args.nodeId, message, provider);
|
|
1123
|
+
if (args.connectionId && args.registry.attachProvider) {
|
|
1124
|
+
args.registry.attachProvider(args.workspaceId, args.nodeId, provider.name, provider.instance_id, args.connectionId);
|
|
1125
|
+
}
|
|
1126
|
+
sendControl(args.socket, {
|
|
1127
|
+
v: 1,
|
|
1128
|
+
id: requestId(message),
|
|
1129
|
+
type: 'reply',
|
|
1130
|
+
ok: true,
|
|
1131
|
+
data: {
|
|
1132
|
+
...registered.node,
|
|
1133
|
+
provider: registered.provider,
|
|
1134
|
+
accepted_capabilities: registered.acceptance,
|
|
1135
|
+
},
|
|
1136
|
+
});
|
|
1137
|
+
// Node is now marked online: flush any queued action.invoke frames so
|
|
1138
|
+
// spawns queued while it was offline can reserve capacity and dispatch.
|
|
1139
|
+
await args.registry.drainNode(args.workspaceId, args.nodeId);
|
|
1140
|
+
// The success reply was already sent above; keep the pending flush
|
|
1141
|
+
// best-effort so a delivery error cannot trigger a second error reply
|
|
1142
|
+
// for the same request id from the outer catch.
|
|
1143
|
+
await deliverPendingToNode(args.db, args.registry, args.workspaceId, args.nodeId).catch(() => { });
|
|
1144
|
+
return;
|
|
1145
|
+
}
|
|
1146
|
+
case 'node.heartbeat':
|
|
1147
|
+
await heartbeatNode(args.db, args.workspaceId, args.nodeId, frameProviderName, message);
|
|
1148
|
+
// Heartbeat refreshes online/capacity state; re-drain as a backstop in
|
|
1149
|
+
// case a queued spawn could not reserve capacity at register time.
|
|
1150
|
+
await args.registry.drainNode(args.workspaceId, args.nodeId);
|
|
1151
|
+
return;
|
|
1152
|
+
case 'node.deregister':
|
|
1153
|
+
await deregisterProvider(args.db, args.registry, args.workspaceId, args.nodeId, frameProviderName);
|
|
1154
|
+
return;
|
|
1155
|
+
case 'node.spawn': {
|
|
1156
|
+
// Handler-context `ctx.spawnAgent`: capacity-direct delegation to this
|
|
1157
|
+
// connection's own node capacity executor (the broker provider).
|
|
1158
|
+
// Bypasses action dispatch so a `spawn:<harness>` shadow handler cannot
|
|
1159
|
+
// re-enter itself. The target is always the connection's node — a node
|
|
1160
|
+
// credential cannot direct a spawn at another node.
|
|
1161
|
+
const result = await dispatchCapacitySpawn(args.db, args.workspaceId, {
|
|
1162
|
+
input: message.input,
|
|
1163
|
+
target_node_id: args.nodeId,
|
|
1164
|
+
}, { nodeConnections: args.registry });
|
|
1165
|
+
sendControl(args.socket, {
|
|
1166
|
+
v: 1,
|
|
1167
|
+
id: requestId(message),
|
|
1168
|
+
type: 'reply',
|
|
1169
|
+
ok: true,
|
|
1170
|
+
data: result,
|
|
1171
|
+
});
|
|
1172
|
+
return;
|
|
1173
|
+
}
|
|
1174
|
+
case 'agent.register': {
|
|
1175
|
+
const registered = await registerAgentViaNode(args.db, args.workspaceId, args.nodeId, frameProviderName, message);
|
|
1176
|
+
// The relay broker's node_control client awaits a `reply` frame keyed by
|
|
1177
|
+
// the request id (it matches `pending_agent_registrations` by `reply.id`
|
|
1178
|
+
// and parses `data` as {agent_id, token, name} with deny_unknown_fields).
|
|
1179
|
+
// A bare `agent.registered` frame leaves the token-authority handshake
|
|
1180
|
+
// hanging until the 30s timeout, so spawn never completes. Reply in the
|
|
1181
|
+
// shape the shipped broker consumes; the broker already holds the
|
|
1182
|
+
// invocation_id/session_ref it sent, so only the minted identity is echoed.
|
|
1183
|
+
sendControl(args.socket, {
|
|
1184
|
+
v: 1,
|
|
1185
|
+
id: requestId(message),
|
|
1186
|
+
type: 'reply',
|
|
1187
|
+
ok: true,
|
|
1188
|
+
data: { agent_id: registered.agent_id, token: registered.token, name: registered.name },
|
|
1189
|
+
});
|
|
1190
|
+
await deliverPendingToNode(args.db, args.registry, args.workspaceId, args.nodeId);
|
|
1191
|
+
return;
|
|
1192
|
+
}
|
|
1193
|
+
case 'agent.deregister':
|
|
1194
|
+
await deregisterAgentViaNode(args.db, args.workspaceId, args.nodeId, message);
|
|
1195
|
+
return;
|
|
1196
|
+
case 'inventory.sync': {
|
|
1197
|
+
const result = await reconcileInventory(args.db, args.registry, args.workspaceId, args.nodeId, message.agents, args.completionDeps);
|
|
1198
|
+
sendControl(args.socket, {
|
|
1199
|
+
v: 1,
|
|
1200
|
+
id: requestId(message),
|
|
1201
|
+
type: 'reply',
|
|
1202
|
+
ok: true,
|
|
1203
|
+
data: result,
|
|
1204
|
+
});
|
|
1205
|
+
await deliverPendingToNode(args.db, args.registry, args.workspaceId, args.nodeId);
|
|
1206
|
+
return;
|
|
1207
|
+
}
|
|
1208
|
+
case 'action.result': {
|
|
1209
|
+
const completed = await completeNodeInvocation(args.db, args.registry, args.workspaceId, args.nodeId, message.invocation_id, {
|
|
1210
|
+
...(Object.prototype.hasOwnProperty.call(message, 'output') ? { output: message.output } : {}),
|
|
1211
|
+
...(message.error ? { error: message.error } : {}),
|
|
1212
|
+
});
|
|
1213
|
+
if (completed && args.completionDeps) {
|
|
1214
|
+
await emitInvocationCompletionEffects(args.completionDeps, args.workspaceId, completed);
|
|
1215
|
+
}
|
|
1216
|
+
return;
|
|
1062
1217
|
}
|
|
1063
|
-
|
|
1218
|
+
case 'delivery.ack':
|
|
1219
|
+
await ackDeliveriesUpToSeq(args.db, args.workspaceId, args.nodeId, message.agent, message.up_to_seq);
|
|
1220
|
+
return;
|
|
1064
1221
|
}
|
|
1065
|
-
case 'delivery.ack':
|
|
1066
|
-
await ackDeliveriesUpToSeq(args.db, args.workspaceId, args.nodeId, message.agent, message.up_to_seq);
|
|
1067
|
-
return;
|
|
1068
1222
|
}
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1223
|
+
catch (err) {
|
|
1224
|
+
const error = err;
|
|
1225
|
+
const code = error.code ?? 'node_control_failed';
|
|
1226
|
+
// The rejection reaches the client only as the error frame below. Log it so
|
|
1227
|
+
// it is not invisible server-side: a rejected node.register/node.heartbeat
|
|
1228
|
+
// (e.g. node_name_conflict, a UNIQUE violation) otherwise leaves a node
|
|
1229
|
+
// running half-registered with no signal to operators. Warn, not error —
|
|
1230
|
+
// these are client/protocol rejections, not server faults.
|
|
1231
|
+
console.warn('[node.control] rejected message', {
|
|
1232
|
+
type: message.type,
|
|
1233
|
+
code,
|
|
1234
|
+
workspaceId: args.workspaceId,
|
|
1235
|
+
nodeId: args.nodeId,
|
|
1236
|
+
provider: frameProviderName,
|
|
1237
|
+
message: error.message,
|
|
1238
|
+
});
|
|
1239
|
+
sendControl(args.socket, {
|
|
1240
|
+
v: 1,
|
|
1241
|
+
id: requestId(message),
|
|
1242
|
+
type: 'error',
|
|
1243
|
+
ok: false,
|
|
1244
|
+
code,
|
|
1245
|
+
message: error.message,
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
});
|
|
1081
1249
|
}
|
|
1082
1250
|
//# sourceMappingURL=node.js.map
|