@relaycast/engine 8.11.2 → 8.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/node/realtime.d.ts +1 -0
- package/dist/adapters/node/realtime.d.ts.map +1 -1
- package/dist/adapters/node/realtime.js +16 -0
- package/dist/adapters/node/realtime.js.map +1 -1
- package/dist/engine/node.d.ts +46 -9
- package/dist/engine/node.d.ts.map +1 -1
- package/dist/engine/node.js +315 -69
- package/dist/engine/node.js.map +1 -1
- package/dist/engine/nodeContext.d.ts +11 -0
- package/dist/engine/nodeContext.d.ts.map +1 -1
- package/dist/engine/nodeContext.js +66 -12
- package/dist/engine/nodeContext.js.map +1 -1
- package/dist/ports/realtime.d.ts +12 -0
- package/dist/ports/realtime.d.ts.map +1 -1
- package/dist/ports/realtime.js.map +1 -1
- package/dist/routes/node.js +8 -2
- package/dist/routes/node.js.map +1 -1
- package/package.json +3 -3
package/dist/engine/node.js
CHANGED
|
@@ -6,8 +6,7 @@ import { actionInvocations, agents, agentNodeBindings, channelMembers, channels,
|
|
|
6
6
|
import { randomHex, sha256Hex } from '../lib/crypto.js';
|
|
7
7
|
import { codedError } from '../lib/httpError.js';
|
|
8
8
|
import { acceptTaskInvocation, completeTaskInvocation } from './taskInvocation.js';
|
|
9
|
-
import { runAtomic } from '../ports/database.js';
|
|
10
|
-
import { isProviderAgentDeliveryReady } from '../ports/realtime.js';
|
|
9
|
+
import { runAtomic, runAtomicWrites } from '../ports/database.js';
|
|
11
10
|
import { generateId } from './snowflake.js';
|
|
12
11
|
import { assertRegistrableAgentName } from './agent.js';
|
|
13
12
|
import { rotateAgentIdentity } from './agentIdentity.js';
|
|
@@ -31,6 +30,35 @@ function directNodeNameForAgent(agentId) {
|
|
|
31
30
|
function isImplicitDirectLocation(agent) {
|
|
32
31
|
return agent.locationNodeId === directNodeIdForAgent(agent.id);
|
|
33
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* An agent adopted onto a node keeps a provider identity that node can serve:
|
|
35
|
+
* deliveries are pushed to the `(node, provider)` socket named by
|
|
36
|
+
* `agents.provider_name`, so a provider the node does not serve is a silent
|
|
37
|
+
* dead end. Precedence, most specific first:
|
|
38
|
+
*
|
|
39
|
+
* 1. The node already serves the agent's provider — keep it. It routes today,
|
|
40
|
+
* and it is the closest thing to evidence of which provider owns the agent;
|
|
41
|
+
* rewriting it would hand the agent to a sibling provider on the same host.
|
|
42
|
+
* 2. The node serves exactly one provider — adopt it; there is no other answer.
|
|
43
|
+
* 3. The node serves the synthetic `default` — adopt it as the generic
|
|
44
|
+
* fallback, since the agent's own provider is not served here.
|
|
45
|
+
* 4. Otherwise (several named providers, none of them the agent's) nothing
|
|
46
|
+
* identifies the owner, so the existing provider stands rather than guessing.
|
|
47
|
+
*
|
|
48
|
+
* A node with no live providers at all falls through to (4): no socket exists
|
|
49
|
+
* to adopt, and rewriting to a name nobody serves would not help. Callers pass
|
|
50
|
+
* only live provider names — a persisted-but-offline row is a historical
|
|
51
|
+
* manifest entry, not a connection deliveries can reach.
|
|
52
|
+
*/
|
|
53
|
+
function adoptNodeProviderName(current, nodeProviderNames) {
|
|
54
|
+
if (nodeProviderNames.includes(current))
|
|
55
|
+
return current;
|
|
56
|
+
if (nodeProviderNames.length === 1)
|
|
57
|
+
return nodeProviderNames[0];
|
|
58
|
+
if (nodeProviderNames.includes(DEFAULT_PROVIDER_NAME))
|
|
59
|
+
return DEFAULT_PROVIDER_NAME;
|
|
60
|
+
return current;
|
|
61
|
+
}
|
|
34
62
|
function normalizeCapabilities(capabilities) {
|
|
35
63
|
return capabilities.map((capability) => (typeof capability === 'string' ? { name: capability } : capability));
|
|
36
64
|
}
|
|
@@ -95,6 +123,69 @@ function supportsProviderDeliveryReadiness(registry) {
|
|
|
95
123
|
&& typeof registry.markProviderAgentsDeliveryReady === 'function'
|
|
96
124
|
&& typeof registry.isProviderAgentDeliveryReady === 'function';
|
|
97
125
|
}
|
|
126
|
+
/**
|
|
127
|
+
* Whether a provider's current registration advertised the delivery-cursor
|
|
128
|
+
* capability. Registration persists the advertised set, so later frames on the
|
|
129
|
+
* same provider can recover the handshake `node.register` negotiated without
|
|
130
|
+
* the client re-asserting it.
|
|
131
|
+
*/
|
|
132
|
+
async function providerAdvertisesDeliveryCursor(db, workspaceId, nodeId, providerName) {
|
|
133
|
+
const [provider] = await db
|
|
134
|
+
.select({ capabilities: nodeProviders.capabilities })
|
|
135
|
+
.from(nodeProviders)
|
|
136
|
+
.where(and(eq(nodeProviders.workspaceId, workspaceId), eq(nodeProviders.nodeId, nodeId), eq(nodeProviders.name, providerName)));
|
|
137
|
+
return provider?.capabilities?.some((capability) => capability.name === FLEET_DELIVERY_CURSOR_CAPABILITY) ?? false;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Capabilities that are NEGOTIATED at `node.register` (answered in its
|
|
141
|
+
* acceptance list and wired into the connection's delivery mode) rather than
|
|
142
|
+
* merely advertised. A heartbeat refreshes the provider's roster of spawn /
|
|
143
|
+
* action capacity; it is not a renegotiation, so it may neither drop nor
|
|
144
|
+
* introduce one of these. Dropping would demote a live cursor-gated connection
|
|
145
|
+
* that was deliberately denied the register-time flush — stranding its backlog;
|
|
146
|
+
* introducing would promote an immediate connection and replay frames that
|
|
147
|
+
* flush already sent. The registered value stands until the next
|
|
148
|
+
* `node.register`.
|
|
149
|
+
*/
|
|
150
|
+
const NEGOTIATED_PROTOCOL_CAPABILITIES = [FLEET_DELIVERY_CURSOR_CAPABILITY];
|
|
151
|
+
function isNegotiatedProtocolCapability(capability) {
|
|
152
|
+
return NEGOTIATED_PROTOCOL_CAPABILITIES.includes(capability.name);
|
|
153
|
+
}
|
|
154
|
+
function withRegisteredProtocolCapabilities(heartbeatCapabilities, registeredCapabilities) {
|
|
155
|
+
return [
|
|
156
|
+
...heartbeatCapabilities.filter((capability) => !isNegotiatedProtocolCapability(capability)),
|
|
157
|
+
...registeredCapabilities.filter(isNegotiatedProtocolCapability),
|
|
158
|
+
];
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Whether the provider's LIVE connection negotiated cursor-gated delivery at
|
|
162
|
+
* `node.register` — i.e. whether it was denied the register-time flush and so
|
|
163
|
+
* depends on a later certification to replay.
|
|
164
|
+
*
|
|
165
|
+
* The registry owns that answer: `node.register` hands it the negotiated mode
|
|
166
|
+
* per connection, and a reconnect starts a fresh one. Ask it first. Only a
|
|
167
|
+
* registry that does not expose the mode (an out-of-process owner on an older
|
|
168
|
+
* contract) falls back to the persisted advertisement — which heartbeats keep
|
|
169
|
+
* off-limits (see `heartbeatNode`) precisely so the fallback answers for the
|
|
170
|
+
* registration rather than for the latest roster snapshot.
|
|
171
|
+
*
|
|
172
|
+
* When the method DOES exist, its `undefined` is an answer too: the connection
|
|
173
|
+
* asking is not the provider's current owner (a superseded connection whose
|
|
174
|
+
* frame queued behind the replacement's register). Reading the persisted
|
|
175
|
+
* advertisement then would classify a stale connection's sync by whichever
|
|
176
|
+
* registration owns the provider now — replaying certified deliveries to a
|
|
177
|
+
* connection that has not certified those sessions. A registry that can
|
|
178
|
+
* answer gets the last word; the fallback is only for one that cannot.
|
|
179
|
+
*/
|
|
180
|
+
async function connectionNegotiatedDeliveryCursor(db, registry, workspaceId, nodeId, providerName, connectionId) {
|
|
181
|
+
if (!supportsProviderDeliveryReadiness(registry))
|
|
182
|
+
return false;
|
|
183
|
+
if (registry.providerDeliveryReadinessMode) {
|
|
184
|
+
return registry.providerDeliveryReadinessMode(workspaceId, nodeId, providerName, connectionId)
|
|
185
|
+
=== 'agent_scoped';
|
|
186
|
+
}
|
|
187
|
+
return providerAdvertisesDeliveryCursor(db, workspaceId, nodeId, providerName);
|
|
188
|
+
}
|
|
98
189
|
function requestId(message) {
|
|
99
190
|
return message.id ?? generateId();
|
|
100
191
|
}
|
|
@@ -540,11 +631,16 @@ export async function heartbeatNode(db, workspaceId, nodeId, providerName, messa
|
|
|
540
631
|
}
|
|
541
632
|
await runAtomic(db, async (tx) => {
|
|
542
633
|
if (message.capabilities !== undefined) {
|
|
543
|
-
const caps = normalizeCapabilities(message.capabilities);
|
|
544
634
|
const [existingProvider] = await tx
|
|
545
|
-
.select({
|
|
635
|
+
.select({
|
|
636
|
+
instanceId: nodeProviders.instanceId,
|
|
637
|
+
maxAgents: nodeProviders.maxAgents,
|
|
638
|
+
version: nodeProviders.version,
|
|
639
|
+
capabilities: nodeProviders.capabilities,
|
|
640
|
+
})
|
|
546
641
|
.from(nodeProviders)
|
|
547
642
|
.where(and(eq(nodeProviders.workspaceId, workspaceId), eq(nodeProviders.nodeId, nodeId), eq(nodeProviders.name, providerName)));
|
|
643
|
+
const caps = withRegisteredProtocolCapabilities(normalizeCapabilities(message.capabilities), existingProvider?.capabilities ?? []);
|
|
548
644
|
await upsertProvider(tx, workspaceId, nodeId, {
|
|
549
645
|
name: providerName,
|
|
550
646
|
instanceId: existingProvider?.instanceId ?? `${providerName}:heartbeat`,
|
|
@@ -733,44 +829,73 @@ async function autoJoinGeneral(db, workspaceId, agentId) {
|
|
|
733
829
|
}).onConflictDoNothing();
|
|
734
830
|
}
|
|
735
831
|
}
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
832
|
+
/**
|
|
833
|
+
* Statements that bind `agent` to `nodeId` and move its routable location onto
|
|
834
|
+
* that node. Built, not executed: a caller that owns the whole move hands them
|
|
835
|
+
* to {@link runAtomicWrites} so the binding row, the location move and the
|
|
836
|
+
* retirement of the bindings left behind commit as one unit — on D1 too, where
|
|
837
|
+
* there is no interactive transaction to roll a half-applied move back.
|
|
838
|
+
*
|
|
839
|
+
* Statement order is the last-resort story for a caller that still runs them
|
|
840
|
+
* one at a time inside its own transaction: every prefix has to leave the agent
|
|
841
|
+
* routable, so the new binding is inserted first (inert on its own — delivery
|
|
842
|
+
* joins bindings only where `agents.location_node_id` matches), then the agent
|
|
843
|
+
* moves onto it, then the bindings it left behind are retired (inert once the
|
|
844
|
+
* agent has moved off them). Deactivating first, or moving the agent before its
|
|
845
|
+
* binding exists, would strand the agent with no active binding at its
|
|
846
|
+
* location. Callers that mutate `agents` themselves must write through
|
|
847
|
+
* `providerName`/`originNodeId` here rather than updating the row up front,
|
|
848
|
+
* for the same reason.
|
|
849
|
+
*/
|
|
850
|
+
function agentNodeBindingWrites(db, workspaceId, agent, nodeId, opts = {}) {
|
|
851
|
+
const now = new Date();
|
|
852
|
+
const writes = [
|
|
853
|
+
db
|
|
854
|
+
.insert(agentNodeBindings)
|
|
855
|
+
.values({
|
|
856
|
+
id: `anb_${generateId()}`,
|
|
857
|
+
workspaceId,
|
|
858
|
+
agentId: agent.id,
|
|
859
|
+
nodeId,
|
|
758
860
|
status: 'active',
|
|
759
861
|
sessionRef: opts.sessionRef ?? null,
|
|
760
862
|
priority: opts.priority ?? 0,
|
|
761
|
-
updatedAt:
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
863
|
+
updatedAt: now,
|
|
864
|
+
})
|
|
865
|
+
.onConflictDoUpdate({
|
|
866
|
+
target: [agentNodeBindings.agentId, agentNodeBindings.nodeId],
|
|
867
|
+
set: {
|
|
868
|
+
status: 'active',
|
|
869
|
+
sessionRef: opts.sessionRef ?? null,
|
|
870
|
+
priority: opts.priority ?? 0,
|
|
871
|
+
updatedAt: now,
|
|
872
|
+
},
|
|
873
|
+
}),
|
|
874
|
+
db
|
|
875
|
+
.update(agents)
|
|
876
|
+
.set({
|
|
877
|
+
locationType: 'via_node',
|
|
878
|
+
locationNodeId: nodeId,
|
|
879
|
+
sessionRef: opts.sessionRef ?? undefined,
|
|
880
|
+
providerName: opts.providerName ?? undefined,
|
|
881
|
+
originNodeId: opts.originNodeId ?? undefined,
|
|
882
|
+
status: 'active',
|
|
883
|
+
lastSeen: now,
|
|
884
|
+
})
|
|
885
|
+
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.id, agent.id))),
|
|
886
|
+
];
|
|
887
|
+
if (opts.deactivateExisting ?? true) {
|
|
888
|
+
writes.push(db
|
|
889
|
+
.update(agentNodeBindings)
|
|
890
|
+
.set({ status: 'inactive', updatedAt: now })
|
|
891
|
+
.where(and(eq(agentNodeBindings.workspaceId, workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.status, 'active'), ne(agentNodeBindings.nodeId, nodeId))));
|
|
892
|
+
}
|
|
893
|
+
return writes;
|
|
894
|
+
}
|
|
895
|
+
/** Run {@link agentNodeBindingWrites} one statement at a time. */
|
|
896
|
+
async function upsertAgentNodeBinding(db, workspaceId, agent, nodeId, opts = {}) {
|
|
897
|
+
for (const write of agentNodeBindingWrites(db, workspaceId, agent, nodeId, opts))
|
|
898
|
+
await write;
|
|
774
899
|
}
|
|
775
900
|
async function activeBindingNodeIdsForAgent(db, workspaceId, agentId) {
|
|
776
901
|
const rows = await db
|
|
@@ -834,16 +959,28 @@ async function rejectMigrationCanceledSpawnRegistration(db, workspaceId, registr
|
|
|
834
959
|
throw codedError(`Spawn invocation "${canceled.id}" was canceled during migration`, 'spawn_invocation_canceled', 409);
|
|
835
960
|
}
|
|
836
961
|
}
|
|
837
|
-
|
|
962
|
+
/**
|
|
963
|
+
* The statement (at most one) that gives back one agent slot on each of
|
|
964
|
+
* `nodeIds`. Built, not executed, so a caller can commit the decrement in the
|
|
965
|
+
* same atomic unit as the binding retirement that justifies it: a decrement
|
|
966
|
+
* that lands without the retirement lets the node overshoot `maxAgents`, and a
|
|
967
|
+
* retirement that lands without the decrement charges capacity to a binding
|
|
968
|
+
* nothing will ever release.
|
|
969
|
+
*/
|
|
970
|
+
function releaseNodeAgentSlotWrites(db, workspaceId, nodeIds) {
|
|
838
971
|
const uniqueNodeIds = [...new Set(nodeIds)];
|
|
839
972
|
if (uniqueNodeIds.length === 0)
|
|
840
|
-
return;
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
973
|
+
return [];
|
|
974
|
+
return [db
|
|
975
|
+
.update(nodes)
|
|
976
|
+
.set({
|
|
977
|
+
activeAgents: sql `CASE WHEN ${nodes.activeAgents} > 0 THEN ${nodes.activeAgents} - 1 ELSE 0 END`,
|
|
978
|
+
})
|
|
979
|
+
.where(and(eq(nodes.workspaceId, workspaceId), inArray(nodes.id, uniqueNodeIds)))];
|
|
980
|
+
}
|
|
981
|
+
async function releaseNodeAgentSlots(db, workspaceId, nodeIds) {
|
|
982
|
+
for (const write of releaseNodeAgentSlotWrites(db, workspaceId, nodeIds))
|
|
983
|
+
await write;
|
|
847
984
|
}
|
|
848
985
|
export async function ensureDirectNodeForAgent(db, workspaceId, agent, opts = {}) {
|
|
849
986
|
return runAtomic(db, (tx) => ensureDirectNodeForAgentInTx(tx, workspaceId, agent, opts));
|
|
@@ -1004,11 +1141,24 @@ function serializeBinding(row) {
|
|
|
1004
1141
|
status: row.status,
|
|
1005
1142
|
session_ref: row.sessionRef,
|
|
1006
1143
|
priority: row.priority,
|
|
1144
|
+
// Only the bind response selects this: it hands the caller the agent's
|
|
1145
|
+
// authoritative delivery cursor, the same contract `agent.register` and
|
|
1146
|
+
// `agent.recover` replies carry for cursor-aware providers.
|
|
1147
|
+
...(row.deliveryAckSeq !== undefined && row.deliveryAckSeq !== null
|
|
1148
|
+
? { delivery_ack_seq: row.deliveryAckSeq }
|
|
1149
|
+
: {}),
|
|
1007
1150
|
created_at: row.createdAt.toISOString(),
|
|
1008
1151
|
updated_at: row.updatedAt?.toISOString() ?? null,
|
|
1009
1152
|
};
|
|
1010
1153
|
}
|
|
1011
1154
|
export async function bindAgentToNode(db, workspaceId, nodeName, agentName, opts = {}) {
|
|
1155
|
+
// `runAtomic` below already owns a transaction on an adapter that has one, so
|
|
1156
|
+
// the move's statements must not re-enter `withTransaction` (the shared
|
|
1157
|
+
// connection serializes transactions and would deadlock on itself) — they run
|
|
1158
|
+
// in order inside it instead. Without one, the move has to be a D1 batch, and
|
|
1159
|
+
// a handle offering neither is refused before a single statement lands rather
|
|
1160
|
+
// than committing half a move it cannot undo.
|
|
1161
|
+
const hasInteractiveTransaction = typeof db.withTransaction === 'function';
|
|
1012
1162
|
return runAtomic(db, async (tx) => {
|
|
1013
1163
|
const node = await getNodeByName(tx, workspaceId, nodeName);
|
|
1014
1164
|
if (!node)
|
|
@@ -1019,6 +1169,43 @@ export async function bindAgentToNode(db, workspaceId, nodeName, agentName, opts
|
|
|
1019
1169
|
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.name, agentName)));
|
|
1020
1170
|
if (!agent)
|
|
1021
1171
|
throw codedError(`Agent "${agentName}" not found`, 'agent_not_found', 404);
|
|
1172
|
+
// A bind moves the agent's routable location onto this node. Delivery
|
|
1173
|
+
// routing joins bindings only where `agents.location_node_id` matches the
|
|
1174
|
+
// bound node, so a binding row alone leaves an HTTP-registered agent
|
|
1175
|
+
// stranded on its implicit `direct-*` pseudo-node and never woken — the
|
|
1176
|
+
// exact shape of the broker's create-only register falling back to this
|
|
1177
|
+
// endpoint. `unbindAgentFromNode` already assumes this move happened.
|
|
1178
|
+
if (agent.status === 'active'
|
|
1179
|
+
&& agent.locationNodeId
|
|
1180
|
+
&& agent.locationNodeId !== node.id
|
|
1181
|
+
&& !isImplicitDirectLocation(agent)) {
|
|
1182
|
+
const [locatedNode] = await tx
|
|
1183
|
+
.select()
|
|
1184
|
+
.from(nodes)
|
|
1185
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, agent.locationNodeId)));
|
|
1186
|
+
// A dead or missing prior location is not a conflict — it is exactly what
|
|
1187
|
+
// a bind is for. Only a live one still owns the identity.
|
|
1188
|
+
if (locatedNode && isNodeLive(locatedNode)) {
|
|
1189
|
+
throw codedError(`Agent "${agentName}" is already active on another live location`, 'agent_location_conflict', 409);
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
// Provider rows persist after a disconnect so the node keeps its capability
|
|
1193
|
+
// manifest; they are history, not sockets. Adopt only among providers that
|
|
1194
|
+
// can actually receive a push right now — keeping or choosing an offline
|
|
1195
|
+
// row while a live sibling serves the node would point the agent at a
|
|
1196
|
+
// provider connection that does not exist.
|
|
1197
|
+
const liveProviderNames = (await tx
|
|
1198
|
+
.select({
|
|
1199
|
+
name: nodeProviders.name,
|
|
1200
|
+
status: nodeProviders.status,
|
|
1201
|
+
handlersLive: nodeProviders.handlersLive,
|
|
1202
|
+
lastHeartbeatAt: nodeProviders.lastHeartbeatAt,
|
|
1203
|
+
})
|
|
1204
|
+
.from(nodeProviders)
|
|
1205
|
+
.where(and(eq(nodeProviders.workspaceId, workspaceId), eq(nodeProviders.nodeId, node.id))))
|
|
1206
|
+
.filter((provider) => isProviderLive(provider))
|
|
1207
|
+
.map((row) => row.name);
|
|
1208
|
+
const adoptedProviderName = adoptNodeProviderName(agent.providerName, liveProviderNames);
|
|
1022
1209
|
const activeNodeIds = await activeBindingNodeIdsForAgent(tx, workspaceId, agent.id);
|
|
1023
1210
|
const targetWasActive = activeNodeIds.includes(node.id);
|
|
1024
1211
|
let reservedTargetSlot = false;
|
|
@@ -1027,13 +1214,33 @@ export async function bindAgentToNode(db, workspaceId, nodeName, agentName, opts
|
|
|
1027
1214
|
await reserveNodeAgentSlot(tx, workspaceId, node);
|
|
1028
1215
|
reservedTargetSlot = true;
|
|
1029
1216
|
}
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1217
|
+
// The binding row, the location move, the provider adoption, the
|
|
1218
|
+
// retirement of the bindings left behind and the slots those bindings
|
|
1219
|
+
// held are one mutation. Committing any subset is a corruption that no
|
|
1220
|
+
// retry repairs: a binding without its reservation lets the node exceed
|
|
1221
|
+
// `maxAgents`, and a retired binding whose slot was never given back
|
|
1222
|
+
// charges the old node forever, because the retry that would fix either
|
|
1223
|
+
// reads the half-applied state as already done.
|
|
1224
|
+
const buildMoveWrites = (writeDb) => [
|
|
1225
|
+
...agentNodeBindingWrites(writeDb, workspaceId, agent, node.id, {
|
|
1226
|
+
sessionRef: opts.session_ref ?? null,
|
|
1227
|
+
priority: opts.priority ?? 0,
|
|
1228
|
+
providerName: adoptedProviderName,
|
|
1229
|
+
originNodeId: agent.originNodeId ?? node.id,
|
|
1230
|
+
}),
|
|
1231
|
+
...releaseNodeAgentSlotWrites(writeDb, workspaceId, activeNodeIds.filter((nodeId) => nodeId !== node.id)),
|
|
1232
|
+
];
|
|
1233
|
+
if (hasInteractiveTransaction) {
|
|
1234
|
+
for (const write of buildMoveWrites(tx))
|
|
1235
|
+
await write;
|
|
1236
|
+
}
|
|
1237
|
+
else {
|
|
1238
|
+
await runAtomicWrites(tx, buildMoveWrites, { requireAtomic: true });
|
|
1239
|
+
}
|
|
1035
1240
|
}
|
|
1036
1241
|
catch (err) {
|
|
1242
|
+
// Every statement of the move either landed or did not, so a throw means
|
|
1243
|
+
// none of it did and the slot reserved for it belongs to nobody.
|
|
1037
1244
|
if (reservedTargetSlot) {
|
|
1038
1245
|
await releaseNodeAgentSlots(tx, workspaceId, [node.id]);
|
|
1039
1246
|
}
|
|
@@ -1051,6 +1258,7 @@ export async function bindAgentToNode(db, workspaceId, nodeName, agentName, opts
|
|
|
1051
1258
|
status: agentNodeBindings.status,
|
|
1052
1259
|
sessionRef: agentNodeBindings.sessionRef,
|
|
1053
1260
|
priority: agentNodeBindings.priority,
|
|
1261
|
+
deliveryAckSeq: agents.deliveryAckSeq,
|
|
1054
1262
|
createdAt: agentNodeBindings.createdAt,
|
|
1055
1263
|
updatedAt: agentNodeBindings.updatedAt,
|
|
1056
1264
|
})
|
|
@@ -1058,7 +1266,25 @@ export async function bindAgentToNode(db, workspaceId, nodeName, agentName, opts
|
|
|
1058
1266
|
.innerJoin(agents, eq(agentNodeBindings.agentId, agents.id))
|
|
1059
1267
|
.innerJoin(nodes, eq(agentNodeBindings.nodeId, nodes.id))
|
|
1060
1268
|
.where(and(eq(agentNodeBindings.workspaceId, workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.nodeId, node.id)));
|
|
1061
|
-
return
|
|
1269
|
+
return {
|
|
1270
|
+
binding: serializeBinding(binding),
|
|
1271
|
+
move: { nodeId: node.id, agentId: agent.id, providerName: adoptedProviderName },
|
|
1272
|
+
};
|
|
1273
|
+
});
|
|
1274
|
+
}
|
|
1275
|
+
/**
|
|
1276
|
+
* The second half of a bind, run by the caller only after the cursor-bearing
|
|
1277
|
+
* response is on the wire — `agent.register`/`agent.recover` order their
|
|
1278
|
+
* cursor reply ahead of the ready mark and replay for the same reason.
|
|
1279
|
+
* Cursor-aware providers gate every delivery on a per-identity ready mark,
|
|
1280
|
+
* so a bound agent needs this transition or it points at a socket that will
|
|
1281
|
+
* never push to it; immediate-mode providers treat the mark as a no-op.
|
|
1282
|
+
*/
|
|
1283
|
+
export async function completeBoundAgentDelivery(db, registry, workspaceId, move) {
|
|
1284
|
+
registry.markProviderAgentsDeliveryReady?.(workspaceId, move.nodeId, move.providerName, undefined, [move.agentId]);
|
|
1285
|
+
await deliverPendingToNode(db, registry, workspaceId, move.nodeId, {
|
|
1286
|
+
providerName: move.providerName,
|
|
1287
|
+
agentIds: [move.agentId],
|
|
1062
1288
|
});
|
|
1063
1289
|
}
|
|
1064
1290
|
export async function unbindAgentFromNode(db, workspaceId, nodeName, agentName) {
|
|
@@ -1126,11 +1352,8 @@ export async function registerAgentViaNode(db, workspaceId, nodeId, providerName
|
|
|
1126
1352
|
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
|
|
1127
1353
|
if (!node)
|
|
1128
1354
|
throw codedError(`Node "${nodeId}" not found`, 'node_not_found', 404);
|
|
1129
|
-
const
|
|
1130
|
-
|
|
1131
|
-
.from(nodeProviders)
|
|
1132
|
-
.where(and(eq(nodeProviders.workspaceId, workspaceId), eq(nodeProviders.nodeId, nodeId), eq(nodeProviders.name, providerName)));
|
|
1133
|
-
const cursorHandshake = (options.deliveryCursorSupported ?? true) && (provider?.capabilities?.some((capability) => capability.name === FLEET_DELIVERY_CURSOR_CAPABILITY) ?? false);
|
|
1355
|
+
const cursorHandshake = (options.deliveryCursorSupported ?? true)
|
|
1356
|
+
&& await providerAdvertisesDeliveryCursor(tx, workspaceId, nodeId, providerName);
|
|
1134
1357
|
// Preserve the established identity conflict even when this node is full.
|
|
1135
1358
|
// A concurrent create after this read is still caught by the unique insert
|
|
1136
1359
|
// below, whose reserved slot is then compensated in the catch path.
|
|
@@ -1230,11 +1453,8 @@ export async function recoverAgentViaNode(db, workspaceId, nodeId, providerName,
|
|
|
1230
1453
|
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
|
|
1231
1454
|
if (!node)
|
|
1232
1455
|
throw codedError(`Node "${nodeId}" not found`, 'node_not_found', 404);
|
|
1233
|
-
const
|
|
1234
|
-
|
|
1235
|
-
.from(nodeProviders)
|
|
1236
|
-
.where(and(eq(nodeProviders.workspaceId, workspaceId), eq(nodeProviders.nodeId, nodeId), eq(nodeProviders.name, providerName)));
|
|
1237
|
-
const cursorHandshake = (options.deliveryCursorSupported ?? true) && (provider?.capabilities?.some((capability) => capability.name === FLEET_DELIVERY_CURSOR_CAPABILITY) ?? false);
|
|
1456
|
+
const cursorHandshake = (options.deliveryCursorSupported ?? true)
|
|
1457
|
+
&& await providerAdvertisesDeliveryCursor(tx, workspaceId, nodeId, providerName);
|
|
1238
1458
|
const [target] = await tx
|
|
1239
1459
|
.select()
|
|
1240
1460
|
.from(agents)
|
|
@@ -2009,14 +2229,40 @@ export async function handleNodeControlMessage(args) {
|
|
|
2009
2229
|
});
|
|
2010
2230
|
if (!replySent)
|
|
2011
2231
|
return;
|
|
2012
|
-
const newlyReadyAgentIds = result.reconciledAgentIds.filter((agentId) => (!isProviderAgentDeliveryReady(args.registry, args.workspaceId, args.nodeId, frameProviderName, agentId)));
|
|
2013
2232
|
args.registry.markProviderAgentsDeliveryReady?.(args.workspaceId, args.nodeId, frameProviderName, args.connectionId, result.reconciledAgentIds);
|
|
2014
2233
|
// Inventory represents sessions that survived a transport reconnect and
|
|
2015
|
-
// therefore retain their in-memory cursors.
|
|
2016
|
-
//
|
|
2017
|
-
//
|
|
2018
|
-
|
|
2019
|
-
|
|
2234
|
+
// therefore retain their in-memory cursors. Restarted sessions not in
|
|
2235
|
+
// the inventory stay excluded either way — they become ready
|
|
2236
|
+
// individually through `agent.register`.
|
|
2237
|
+
//
|
|
2238
|
+
// A cursor-negotiated connection receives NO replay at `node.register`
|
|
2239
|
+
// (each identity is seeded by its own cursor-bearing reply), so this
|
|
2240
|
+
// certification is the whole node's only reconnect-replay trigger and
|
|
2241
|
+
// must cover every certified identity. Scoping it to a readiness/routing
|
|
2242
|
+
// TRANSITION instead stranded a reconnecting node's entire outage
|
|
2243
|
+
// backlog whenever the socket owner already reported those identities as
|
|
2244
|
+
// delivery-ready — an out-of-process owner whose ready-set is keyed per
|
|
2245
|
+
// node+provider rather than per connection. The diff came back empty,
|
|
2246
|
+
// the drain ran with an empty scope, and rows queued during the outage
|
|
2247
|
+
// sat unsent until their mailbox TTL.
|
|
2248
|
+
//
|
|
2249
|
+
// A legacy immediate-delivery connection already had the whole node
|
|
2250
|
+
// flushed to it at `node.register` and gates nothing afterwards, so only
|
|
2251
|
+
// identities this sync newly routed here still need a push; re-sending
|
|
2252
|
+
// the rest would duplicate that flush.
|
|
2253
|
+
//
|
|
2254
|
+
// Either way replay stays bounded, ordered oldest-first, gated per
|
|
2255
|
+
// identity on delivery readiness inside the drain, and deduped by the
|
|
2256
|
+
// cumulative delivery cursor, so an identity that is already drained and
|
|
2257
|
+
// acked re-sends nothing.
|
|
2258
|
+
// The mode is the CONNECTION's, recovered from the registry that
|
|
2259
|
+
// `node.register` configured it on — not re-derived from roster state
|
|
2260
|
+
// that a heartbeat can rewrite between registration and certification.
|
|
2261
|
+
const cursorGated = await connectionNegotiatedDeliveryCursor(args.db, args.registry, args.workspaceId, args.nodeId, frameProviderName, args.connectionId);
|
|
2262
|
+
await deliverPendingToNode(args.db, args.registry, args.workspaceId, args.nodeId, {
|
|
2263
|
+
providerName: frameProviderName,
|
|
2264
|
+
agentIds: cursorGated ? result.reconciledAgentIds : result.newlyRoutedAgentIds,
|
|
2265
|
+
});
|
|
2020
2266
|
return;
|
|
2021
2267
|
}
|
|
2022
2268
|
case 'action.accept': {
|