@relaycast/engine 8.4.0 → 8.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/node/realtime.d.ts +2 -2
- package/dist/adapters/node/realtime.d.ts.map +1 -1
- package/dist/adapters/node/realtime.js +111 -28
- package/dist/adapters/node/realtime.js.map +1 -1
- package/dist/db/migrations/0046_action_invocation_origin.sql +51 -0
- package/dist/db/migrations/0047_action_invocation_provider_acceptance.sql +15 -0
- package/dist/db/schema.d.ts +37 -0
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +11 -0
- package/dist/db/schema.js.map +1 -1
- package/dist/engine/action.d.ts +24 -1
- package/dist/engine/action.d.ts.map +1 -1
- package/dist/engine/action.js +823 -111
- package/dist/engine/action.js.map +1 -1
- package/dist/engine/node.d.ts.map +1 -1
- package/dist/engine/node.js +127 -43
- package/dist/engine/node.js.map +1 -1
- package/dist/engine/nodeProvider.d.ts.map +1 -1
- package/dist/engine/nodeProvider.js +15 -8
- package/dist/engine/nodeProvider.js.map +1 -1
- package/dist/ports/realtime.d.ts +35 -4
- package/dist/ports/realtime.d.ts.map +1 -1
- package/dist/ports/realtime.js.map +1 -1
- package/dist/routes/agent.js +5 -3
- package/dist/routes/agent.js.map +1 -1
- package/package.json +3 -3
package/dist/engine/action.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { and, asc, eq, inArray, isNotNull, isNull, lte, or, sql } from 'drizzle-orm';
|
|
2
2
|
import { actions, actionInvocations, agents, agentNodeBindings, channelMembers, dmParticipants, nodes } from '../db/schema.js';
|
|
3
3
|
import { generateId } from './snowflake.js';
|
|
4
|
-
import { RELEASED_AGENT_STATUS, releasedAgentName } from './agent.js';
|
|
4
|
+
import { assertRegistrableAgentName, RELEASED_AGENT_STATUS, releasedAgentName } from './agent.js';
|
|
5
5
|
import { randomHex, sha256Hex } from '../lib/crypto.js';
|
|
6
6
|
import { codedError } from '../lib/httpError.js';
|
|
7
7
|
import { D1_SAFE_IN_QUERY_CHUNK_SIZE } from '../lib/queryChunks.js';
|
|
@@ -10,6 +10,7 @@ import { emitAgentExitedEffects, emitInvocationCompletionEffects, fleetInvocatio
|
|
|
10
10
|
import { runAtomic, runAtomicWrites } from '../ports/database.js';
|
|
11
11
|
import { claimSpawnNode, chooseNodeForAction, isNodeLive, releaseNodeCapacity, reserveNodeCapacity } from './placement.js';
|
|
12
12
|
import { DEFAULT_PROVIDER_NAME, capacityProviderName, getProvider, isProviderLive } from './nodeProvider.js';
|
|
13
|
+
import { AGENT_TOKEN_HASH_PATTERN } from '@relaycast/types';
|
|
13
14
|
const OPEN_INVOCATION_STATUSES = ['pending', 'dispatched', 'invoked'];
|
|
14
15
|
const REPLAYED_INVOCATION = Symbol('replayed-action-invocation');
|
|
15
16
|
export const ACTION_DISPATCH_TIMEOUT_MS = 30_000;
|
|
@@ -62,6 +63,42 @@ function isSpawnInvocation(actionName) {
|
|
|
62
63
|
function isReleaseInvocation(actionName) {
|
|
63
64
|
return actionName === 'release';
|
|
64
65
|
}
|
|
66
|
+
function isBuiltinReleaseInvocation(invocation) {
|
|
67
|
+
return invocation.invocationOrigin === 'builtin' && isReleaseInvocation(invocation.actionName);
|
|
68
|
+
}
|
|
69
|
+
function isDeletedRegisteredActionInvocation(invocation) {
|
|
70
|
+
// For a registered-action origin, actionId is cleared only by the FK when its
|
|
71
|
+
// exact action row is deleted. The immutable origin prevents that orphan
|
|
72
|
+
// from being mistaken for a built-in, and this predicate prevents it from
|
|
73
|
+
// being rebound by name during drain/retry/reconciliation.
|
|
74
|
+
return invocation.invocationOrigin === 'registered_action'
|
|
75
|
+
&& invocation.actionId === null
|
|
76
|
+
&& invocation.providerAcceptedAttempt !== invocation.dispatchAttempts;
|
|
77
|
+
}
|
|
78
|
+
function isAcceptedDeletedRegisteredActionInvocation(invocation) {
|
|
79
|
+
return invocation.invocationOrigin === 'registered_action'
|
|
80
|
+
&& invocation.actionId === null
|
|
81
|
+
&& invocation.providerAcceptedAttempt !== null
|
|
82
|
+
&& invocation.providerAcceptedAttempt === invocation.dispatchAttempts;
|
|
83
|
+
}
|
|
84
|
+
const RELEASE_GENERATION_CONFLICT_CODE = 'agent_release_generation_conflict';
|
|
85
|
+
function releaseExpectedTokenHash(input) {
|
|
86
|
+
const value = input.expected_token_hash;
|
|
87
|
+
if (value === undefined)
|
|
88
|
+
return null;
|
|
89
|
+
if (typeof value !== 'string' || !AGENT_TOKEN_HASH_PATTERN.test(value)) {
|
|
90
|
+
throw codedError('release action input.expected_token_hash must be a lowercase SHA-256 hash', 'invalid_release_request', 400);
|
|
91
|
+
}
|
|
92
|
+
return value;
|
|
93
|
+
}
|
|
94
|
+
function releaseGenerationStillCurrent(workspaceId, agentId, expectedTokenHash) {
|
|
95
|
+
return sql `EXISTS (
|
|
96
|
+
SELECT 1 FROM ${agents}
|
|
97
|
+
WHERE ${agents.workspaceId} = ${workspaceId}
|
|
98
|
+
AND ${agents.id} = ${agentId}
|
|
99
|
+
${expectedTokenHash ? sql `AND ${agents.tokenHash} = ${expectedTokenHash}` : sql ``}
|
|
100
|
+
)`;
|
|
101
|
+
}
|
|
65
102
|
function dispatchActionNameForInvocation(actionName, input) {
|
|
66
103
|
if (!isSpawnInvocation(actionName))
|
|
67
104
|
return actionName;
|
|
@@ -257,13 +294,16 @@ export async function registerAction(db, workspaceId, data, options = {}) {
|
|
|
257
294
|
: { target: [actions.workspaceId, actions.name], targetWhere: isNull(actions.handlerNodeId), set: refresh })
|
|
258
295
|
.returning();
|
|
259
296
|
// A refresh that moves the handler to a DIFFERENT agent strands
|
|
260
|
-
// invocations
|
|
261
|
-
//
|
|
262
|
-
//
|
|
263
|
-
//
|
|
264
|
-
// fail them; the callers are told after commit.
|
|
297
|
+
// invocations still waiting for provider acceptance. Once the previous
|
|
298
|
+
// handler accepted an exact attempt, detach that invocation from the
|
|
299
|
+
// mutable action row instead: its snapshotted route remains authorized
|
|
300
|
+
// to complete and timeout recovery cannot retarget it to the replacement.
|
|
265
301
|
let strandedRows = [];
|
|
266
302
|
if (previous && previous.handlerAgentId && previous.handlerAgentId !== handlerAgentId) {
|
|
303
|
+
await tx
|
|
304
|
+
.update(actionInvocations)
|
|
305
|
+
.set({ actionId: null })
|
|
306
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.actionId, previous.id), eq(actionInvocations.invocationOrigin, 'registered_action'), isNotNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} = ${actionInvocations.dispatchAttempts}`, inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
267
307
|
const strandedIds = await openInvocationIdsForActions(tx, workspaceId, [previous.id]);
|
|
268
308
|
strandedRows = await failOpenInvocationRows(tx, workspaceId, strandedIds, 'handler_unavailable');
|
|
269
309
|
}
|
|
@@ -322,7 +362,7 @@ export async function listActions(db, workspaceId, callerName) {
|
|
|
322
362
|
.map(publicAction);
|
|
323
363
|
}
|
|
324
364
|
export async function getAction(db, workspaceId, name, callerName) {
|
|
325
|
-
const
|
|
365
|
+
const rows = await db
|
|
326
366
|
.select({
|
|
327
367
|
id: actions.id,
|
|
328
368
|
name: actions.name,
|
|
@@ -330,6 +370,7 @@ export async function getAction(db, workspaceId, name, callerName) {
|
|
|
330
370
|
handlerAgentName: agents.name,
|
|
331
371
|
handlerNodeName: nodes.name,
|
|
332
372
|
handlerNodeId: actions.handlerNodeId,
|
|
373
|
+
isGlobal: actions.isGlobal,
|
|
333
374
|
inputSchema: actions.inputSchema,
|
|
334
375
|
outputSchema: actions.outputSchema,
|
|
335
376
|
availableTo: actions.availableTo,
|
|
@@ -340,6 +381,8 @@ export async function getAction(db, workspaceId, name, callerName) {
|
|
|
340
381
|
.leftJoin(agents, eq(actions.handlerAgentId, agents.id))
|
|
341
382
|
.leftJoin(nodes, eq(actions.handlerNodeId, nodes.id))
|
|
342
383
|
.where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, name)));
|
|
384
|
+
const row = rows.sort((a, b) => actionResolutionRank(a) - actionResolutionRank(b) ||
|
|
385
|
+
(a.id < b.id ? -1 : a.id > b.id ? 1 : 0))[0];
|
|
343
386
|
if (!row || !isActionVisibleToCaller(row.availableTo ?? null, callerName))
|
|
344
387
|
return null;
|
|
345
388
|
return publicAction(row);
|
|
@@ -378,6 +421,7 @@ async function createInvocation(db, workspaceId, action, data) {
|
|
|
378
421
|
workspaceId,
|
|
379
422
|
actionId: action?.id ?? null,
|
|
380
423
|
actionName: expectedActionName,
|
|
424
|
+
invocationOrigin: action ? 'registered_action' : 'builtin',
|
|
381
425
|
callerId: data.caller_id ?? null,
|
|
382
426
|
callerName: data.caller_name ?? null,
|
|
383
427
|
handlerAgentId: data.handler_agent_id ?? action?.handlerAgentId ?? null,
|
|
@@ -439,11 +483,29 @@ async function idempotentInvocationId(workspaceId, callerId, actionName, key) {
|
|
|
439
483
|
async function waitForInvocationReplayOutcome(db, workspaceId, invocation, options = {}) {
|
|
440
484
|
let current = invocation;
|
|
441
485
|
const deadline = Date.now() + 500;
|
|
442
|
-
while (
|
|
443
|
-
|
|
486
|
+
while (true) {
|
|
487
|
+
// A guarded release can lose its generation CAS after provider dispatch.
|
|
488
|
+
// Surface that durable terminal conflict before the generic dispatched-row
|
|
489
|
+
// replay shortcut, otherwise the same idempotency key changes 409 into 201.
|
|
490
|
+
if (isBuiltinReleaseInvocation(current)
|
|
491
|
+
&& current.status === 'failed'
|
|
492
|
+
&& current.error === RELEASE_GENERATION_CONFLICT_CODE) {
|
|
493
|
+
throw codedError('Action invocation failed because the release generation changed', RELEASE_GENERATION_CONFLICT_CODE, 409);
|
|
494
|
+
}
|
|
495
|
+
// Registered node actions only enter action_deleted before the provider
|
|
496
|
+
// accepts the frame. Surface that durable result before the route snapshot
|
|
497
|
+
// shortcut so the winning request and every replay return the same 503.
|
|
498
|
+
if (current.status === 'failed' && current.error === 'action_deleted') {
|
|
499
|
+
throw codedError('Action invocation failed before provider dispatch completed', 'action_deleted', 503);
|
|
500
|
+
}
|
|
501
|
+
if (current.dispatchedNodeId
|
|
502
|
+
|| (options.acceptDispatchStarted && current.dispatchAttempts > 0)) {
|
|
503
|
+
return current;
|
|
504
|
+
}
|
|
444
505
|
if (!OPEN_INVOCATION_STATUSES.some((status) => status === current.status)) {
|
|
445
506
|
if (current.status === 'failed') {
|
|
446
|
-
|
|
507
|
+
const errorCode = current.error ?? 'handler_unavailable';
|
|
508
|
+
throw codedError('Action invocation failed before provider dispatch completed', errorCode, isBuiltinReleaseInvocation(current) && errorCode === RELEASE_GENERATION_CONFLICT_CODE ? 409 : 503);
|
|
447
509
|
}
|
|
448
510
|
return current;
|
|
449
511
|
}
|
|
@@ -467,7 +529,6 @@ async function waitForInvocationReplayOutcome(db, workspaceId, invocation, optio
|
|
|
467
529
|
}
|
|
468
530
|
current = updated;
|
|
469
531
|
}
|
|
470
|
-
return current;
|
|
471
532
|
}
|
|
472
533
|
/**
|
|
473
534
|
* Resolve a losing durable claim from the immutable invocation snapshots.
|
|
@@ -479,7 +540,7 @@ async function waitForInvocationReplayOutcome(db, workspaceId, invocation, optio
|
|
|
479
540
|
*/
|
|
480
541
|
async function replayInvocationClaim(db, workspaceId, invocation) {
|
|
481
542
|
if (isSpawnInvocation(invocation.actionName)
|
|
482
|
-
||
|
|
543
|
+
|| isBuiltinReleaseInvocation(invocation)
|
|
483
544
|
|| (!!invocation.handlerNodeId && !invocation.handlerAgentId)) {
|
|
484
545
|
return waitForInvocationReplayOutcome(db, workspaceId, invocation);
|
|
485
546
|
}
|
|
@@ -631,7 +692,7 @@ async function openInvocationIdsForActions(db, workspaceId, actionIds) {
|
|
|
631
692
|
const rows = await db
|
|
632
693
|
.select({ id: actionInvocations.id })
|
|
633
694
|
.from(actionInvocations)
|
|
634
|
-
.where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.actionId, actionIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
695
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.actionId, actionIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`)));
|
|
635
696
|
return rows.map((row) => row.id);
|
|
636
697
|
}
|
|
637
698
|
/**
|
|
@@ -658,12 +719,22 @@ async function dispatchNodeProviderInvocation(args, options = {}) {
|
|
|
658
719
|
// The DB is the authoritative offline queue. Do not call sendToProvider
|
|
659
720
|
// here: a connected provider whose heartbeat says handlers_live=false still
|
|
660
721
|
// has a socket, and sendToProvider would deliver the frame immediately.
|
|
722
|
+
const expectedActionId = args.invocationOrigin === 'registered_action'
|
|
723
|
+
? (args.expectedActionId ?? args.actionId)
|
|
724
|
+
: null;
|
|
661
725
|
const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, {
|
|
662
726
|
providerName: args.providerName,
|
|
663
727
|
pending: true,
|
|
664
728
|
reservationHeld: args.reservationHeld,
|
|
665
|
-
|
|
729
|
+
expectedAction: expectedActionId
|
|
730
|
+
? { id: expectedActionId, name: args.action }
|
|
731
|
+
: undefined,
|
|
732
|
+
targetActionId: args.actionId ?? undefined,
|
|
666
733
|
});
|
|
734
|
+
if (!accepted && args.invocationOrigin === 'registered_action') {
|
|
735
|
+
const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
|
|
736
|
+
return { accepted: false, pending: false, settled };
|
|
737
|
+
}
|
|
667
738
|
return { accepted, pending: true };
|
|
668
739
|
}
|
|
669
740
|
return dispatchNodeInvocation({ ...args });
|
|
@@ -674,11 +745,21 @@ async function dispatchRelease(args) {
|
|
|
674
745
|
if (!name) {
|
|
675
746
|
throw codedError('release action input.name is required', 'invalid_release_request', 400);
|
|
676
747
|
}
|
|
748
|
+
const expectedTokenHash = releaseExpectedTokenHash(input);
|
|
677
749
|
const [agent] = await args.db
|
|
678
750
|
.select()
|
|
679
751
|
.from(agents)
|
|
680
|
-
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name)));
|
|
752
|
+
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name), ...(expectedTokenHash ? [eq(agents.tokenHash, expectedTokenHash)] : [])));
|
|
681
753
|
if (!agent) {
|
|
754
|
+
if (expectedTokenHash) {
|
|
755
|
+
const [sameName] = await args.db
|
|
756
|
+
.select({ id: agents.id })
|
|
757
|
+
.from(agents)
|
|
758
|
+
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name)));
|
|
759
|
+
if (sameName) {
|
|
760
|
+
throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
|
|
761
|
+
}
|
|
762
|
+
}
|
|
682
763
|
throw codedError(`Agent "${name}" not found`, 'agent_not_found', 404);
|
|
683
764
|
}
|
|
684
765
|
const { invocation, replayed } = await createInvocation(args.db, args.workspaceId, null, {
|
|
@@ -702,23 +783,87 @@ async function dispatchRelease(args) {
|
|
|
702
783
|
// The release time is preserved in `metadata.release.releasedAt`.
|
|
703
784
|
const releasedName = releasedAgentName(agent.name, agent.id);
|
|
704
785
|
const releasedTokenHash = await sha256Hex(`released:${agent.id}:${randomHex(16)}`);
|
|
705
|
-
const
|
|
786
|
+
const invocationCompleted = sql `EXISTS (
|
|
706
787
|
SELECT 1 FROM ${actionInvocations}
|
|
707
788
|
WHERE ${actionInvocations.workspaceId} = ${args.workspaceId}
|
|
708
789
|
AND ${actionInvocations.id} = ${invocation.id}
|
|
709
|
-
AND ${actionInvocations.status}
|
|
790
|
+
AND ${actionInvocations.status} = 'completed'
|
|
791
|
+
)`;
|
|
792
|
+
const generationStillCurrent = releaseGenerationStillCurrent(args.workspaceId, agent.id, expectedTokenHash);
|
|
793
|
+
const atomicExitNodeId = sql `COALESCE(
|
|
794
|
+
(
|
|
795
|
+
SELECT ${agentNodeBindings.nodeId}
|
|
796
|
+
FROM ${agentNodeBindings}
|
|
797
|
+
WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
|
|
798
|
+
AND ${agentNodeBindings.agentId} = ${agent.id}
|
|
799
|
+
AND ${agentNodeBindings.status} = 'active'
|
|
800
|
+
ORDER BY
|
|
801
|
+
CASE
|
|
802
|
+
WHEN ${agentNodeBindings.nodeId} = (
|
|
803
|
+
SELECT ${agents.locationNodeId}
|
|
804
|
+
FROM ${agents}
|
|
805
|
+
WHERE ${agents.workspaceId} = ${args.workspaceId}
|
|
806
|
+
AND ${agents.id} = ${agent.id}
|
|
807
|
+
) THEN 0
|
|
808
|
+
WHEN ${agentNodeBindings.nodeId} = ${`node_direct_${agent.id}`} THEN 1
|
|
809
|
+
ELSE 2
|
|
810
|
+
END,
|
|
811
|
+
${agentNodeBindings.priority} DESC,
|
|
812
|
+
${agentNodeBindings.nodeId}
|
|
813
|
+
LIMIT 1
|
|
814
|
+
),
|
|
815
|
+
(
|
|
816
|
+
SELECT ${agents.locationNodeId}
|
|
817
|
+
FROM ${agents}
|
|
818
|
+
WHERE ${agents.workspaceId} = ${args.workspaceId}
|
|
819
|
+
AND ${agents.id} = ${agent.id}
|
|
820
|
+
),
|
|
821
|
+
${exitNodeId}
|
|
710
822
|
)`;
|
|
711
823
|
const results = await runAtomicWrites(args.db, (writeDb) => {
|
|
712
824
|
const writes = [];
|
|
713
|
-
//
|
|
825
|
+
// If the optional generation lease no longer matches, settle the
|
|
826
|
+
// invocation as a durable conflict in the same atomic unit in which all
|
|
827
|
+
// lifecycle mutations are conditioned to no-op.
|
|
828
|
+
writes.push(writeDb
|
|
829
|
+
.update(actionInvocations)
|
|
830
|
+
.set({
|
|
831
|
+
status: 'failed',
|
|
832
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
833
|
+
completedAt,
|
|
834
|
+
})
|
|
835
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), expectedTokenHash ? sql `NOT (${generationStillCurrent})` : sql `0`))
|
|
836
|
+
.returning({ id: actionInvocations.id }));
|
|
837
|
+
// Win completion and snapshot the current binding before deactivating
|
|
838
|
+
// it. The returned node drives the response and agent.exited event, so a
|
|
839
|
+
// concurrent rebind cannot leave either pointing at the stale host read
|
|
840
|
+
// before this atomic unit began.
|
|
841
|
+
writes.push(writeDb
|
|
842
|
+
.update(actionInvocations)
|
|
843
|
+
.set({
|
|
844
|
+
status: 'completed',
|
|
845
|
+
handlerNodeId: atomicExitNodeId,
|
|
846
|
+
output: {
|
|
847
|
+
released: true,
|
|
848
|
+
// The roster row is retained as a tombstone so the agent's history
|
|
849
|
+
// keeps its author; the name is what the caller gets back.
|
|
850
|
+
deleted: false,
|
|
851
|
+
reaped_locally: true,
|
|
852
|
+
released_name: releasedName,
|
|
853
|
+
},
|
|
854
|
+
completedAt,
|
|
855
|
+
})
|
|
856
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
|
|
857
|
+
.returning({ id: actionInvocations.id, handlerNodeId: actionInvocations.handlerNodeId }));
|
|
858
|
+
// Resolve every active binding in this atomic unit instead of from a
|
|
714
859
|
// pre-transaction snapshot. A concurrent rebind therefore decrements
|
|
715
|
-
// the node that
|
|
860
|
+
// exactly the node or nodes that are deactivated below.
|
|
716
861
|
writes.push(writeDb
|
|
717
862
|
.update(nodes)
|
|
718
863
|
.set({
|
|
719
864
|
activeAgents: sql `CASE WHEN ${nodes.activeAgents} > 0 THEN ${nodes.activeAgents} - 1 ELSE 0 END`,
|
|
720
865
|
})
|
|
721
|
-
.where(and(eq(nodes.workspaceId, args.workspaceId),
|
|
866
|
+
.where(and(eq(nodes.workspaceId, args.workspaceId), invocationCompleted, generationStillCurrent, sql `EXISTS (
|
|
722
867
|
SELECT 1 FROM ${agentNodeBindings}
|
|
723
868
|
WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
|
|
724
869
|
AND ${agentNodeBindings.agentId} = ${agent.id}
|
|
@@ -728,38 +873,32 @@ async function dispatchRelease(args) {
|
|
|
728
873
|
writes.push(writeDb
|
|
729
874
|
.update(agentNodeBindings)
|
|
730
875
|
.set({ status: 'inactive', updatedAt: completedAt })
|
|
731
|
-
.where(and(eq(agentNodeBindings.workspaceId, args.workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.status, 'active'),
|
|
732
|
-
//
|
|
733
|
-
//
|
|
734
|
-
//
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
//
|
|
745
|
-
//
|
|
876
|
+
.where(and(eq(agentNodeBindings.workspaceId, args.workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.status, 'active'), invocationCompleted, generationStillCurrent)));
|
|
877
|
+
// `channel_members` and `dm_participants` cascade on DELETE; the
|
|
878
|
+
// tombstone's UPDATE does not fire that cascade, so drop the memberships
|
|
879
|
+
// in the SAME atomic unit or the released agent stays a delivery target.
|
|
880
|
+
writes.push(writeDb
|
|
881
|
+
.delete(channelMembers)
|
|
882
|
+
.where(and(eq(channelMembers.agentId, agent.id), invocationCompleted, generationStillCurrent)));
|
|
883
|
+
writes.push(writeDb
|
|
884
|
+
.delete(dmParticipants)
|
|
885
|
+
.where(and(eq(dmParticipants.agentId, agent.id), invocationCompleted, generationStillCurrent)));
|
|
886
|
+
writes.push(writeDb
|
|
887
|
+
.delete(nodes)
|
|
888
|
+
.where(and(eq(nodes.workspaceId, args.workspaceId), eq(nodes.id, `node_direct_${agent.id}`), invocationCompleted, generationStillCurrent)));
|
|
889
|
+
// Tombstone after the invocation has won its completion CAS. The whole
|
|
890
|
+
// batch/transaction rolls back together, while this ordering lets the
|
|
891
|
+
// success predicate inspect the still-current token hash before we rotate
|
|
892
|
+
// it to an unusable released credential.
|
|
746
893
|
writes.push(writeDb
|
|
747
894
|
.update(agents)
|
|
748
895
|
.set({
|
|
749
896
|
name: releasedName,
|
|
750
897
|
handle: `@${releasedName}`,
|
|
751
898
|
status: RELEASED_AGENT_STATUS,
|
|
752
|
-
// The row survives, so its credential must not. `token_hash` is
|
|
753
|
-
// NOT NULL UNIQUE and cannot be cleared, so rotate it to a value
|
|
754
|
-
// nobody holds; the released agent's old token stops authenticating.
|
|
755
899
|
tokenHash: releasedTokenHash,
|
|
756
|
-
// Clear the rotation grace slot too, otherwise any token issued by
|
|
757
|
-
// the last live rotation would keep authenticating for its grace
|
|
758
|
-
// window on an agent that is supposed to be gone. See 0035_agent_token_grace.
|
|
759
900
|
previousTokenHash: null,
|
|
760
901
|
previousTokenExpiresAt: null,
|
|
761
|
-
// Same `release` shape the dispatched path writes, so an audit does
|
|
762
|
-
// not have to know which path released the agent.
|
|
763
902
|
metadata: sql `json_patch(COALESCE(${agents.metadata}, '{}'), ${JSON.stringify({
|
|
764
903
|
release: {
|
|
765
904
|
reason: typeof input.reason === 'string' ? input.reason : null,
|
|
@@ -768,48 +907,22 @@ async function dispatchRelease(args) {
|
|
|
768
907
|
},
|
|
769
908
|
})})`,
|
|
770
909
|
})
|
|
771
|
-
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.id, agent.id),
|
|
772
|
-
// `channel_members` and `dm_participants` cascade on DELETE; the
|
|
773
|
-
// tombstone's UPDATE does not fire that cascade, so drop the memberships
|
|
774
|
-
// in the SAME atomic unit or the released agent stays a delivery target.
|
|
775
|
-
writes.push(writeDb
|
|
776
|
-
.delete(channelMembers)
|
|
777
|
-
.where(eq(channelMembers.agentId, agent.id)));
|
|
778
|
-
writes.push(writeDb
|
|
779
|
-
.delete(dmParticipants)
|
|
780
|
-
.where(eq(dmParticipants.agentId, agent.id)));
|
|
781
|
-
writes.push(writeDb
|
|
782
|
-
.delete(nodes)
|
|
783
|
-
.where(and(eq(nodes.workspaceId, args.workspaceId), eq(nodes.id, `node_direct_${agent.id}`), invocationIsOpen)));
|
|
784
|
-
writes.push(writeDb
|
|
785
|
-
.update(actionInvocations)
|
|
786
|
-
.set({
|
|
787
|
-
status: 'completed',
|
|
788
|
-
// This is part of the same atomic unit as completion so a lost 201
|
|
789
|
-
// replays the exact handler node returned below.
|
|
790
|
-
handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${exitNodeId})`,
|
|
791
|
-
output: {
|
|
792
|
-
released: true,
|
|
793
|
-
// The roster row is retained as a tombstone so the agent's history
|
|
794
|
-
// keeps its author; the name is what the caller gets back.
|
|
795
|
-
deleted: false,
|
|
796
|
-
reaped_locally: true,
|
|
797
|
-
released_name: releasedName,
|
|
798
|
-
},
|
|
799
|
-
completedAt,
|
|
800
|
-
})
|
|
801
|
-
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
|
|
802
|
-
.returning({ id: actionInvocations.id }));
|
|
910
|
+
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.id, agent.id), ...(expectedTokenHash ? [eq(agents.tokenHash, expectedTokenHash)] : []), invocationCompleted)));
|
|
803
911
|
return writes;
|
|
804
|
-
});
|
|
805
|
-
const
|
|
912
|
+
}, expectedTokenHash ? { requireAtomic: true } : undefined);
|
|
913
|
+
const generationConflict = results[0];
|
|
914
|
+
const completed = results[1];
|
|
915
|
+
const completedExitNodeId = completed[0]?.handlerNodeId ?? null;
|
|
916
|
+
if (expectedTokenHash && (generationConflict.length > 0 || completed.length === 0)) {
|
|
917
|
+
throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
|
|
918
|
+
}
|
|
806
919
|
// External completion effects belong after the durable atomic unit: an
|
|
807
920
|
// aborted local reap must never publish agent.exited.
|
|
808
|
-
if (completed.length > 0 && args.completionDeps &&
|
|
921
|
+
if (completed.length > 0 && args.completionDeps && completedExitNodeId) {
|
|
809
922
|
await emitAgentExitedEffects(args.completionDeps, args.workspaceId, {
|
|
810
923
|
agentId: agent.id,
|
|
811
924
|
agentName: agent.name,
|
|
812
|
-
nodeId:
|
|
925
|
+
nodeId: completedExitNodeId,
|
|
813
926
|
invocationId: fleetInvocationId(agent.metadata),
|
|
814
927
|
reason: 'released',
|
|
815
928
|
});
|
|
@@ -818,7 +931,7 @@ async function dispatchRelease(args) {
|
|
|
818
931
|
invocation_id: invocation.id,
|
|
819
932
|
action_name: 'release',
|
|
820
933
|
handler_agent_id: null,
|
|
821
|
-
handler_node_id:
|
|
934
|
+
handler_node_id: completedExitNodeId,
|
|
822
935
|
dispatched_node_id: null,
|
|
823
936
|
input,
|
|
824
937
|
status: 'completed',
|
|
@@ -826,10 +939,36 @@ async function dispatchRelease(args) {
|
|
|
826
939
|
};
|
|
827
940
|
};
|
|
828
941
|
const failClosed = async () => {
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
942
|
+
const completedAt = new Date();
|
|
943
|
+
if (expectedTokenHash) {
|
|
944
|
+
const generationStillCurrent = releaseGenerationStillCurrent(args.workspaceId, agent.id, expectedTokenHash);
|
|
945
|
+
const results = await runAtomicWrites(args.db, (writeDb) => [
|
|
946
|
+
writeDb
|
|
947
|
+
.update(actionInvocations)
|
|
948
|
+
.set({
|
|
949
|
+
status: 'failed',
|
|
950
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
951
|
+
completedAt,
|
|
952
|
+
})
|
|
953
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (${generationStillCurrent})`))
|
|
954
|
+
.returning({ id: actionInvocations.id }),
|
|
955
|
+
writeDb
|
|
956
|
+
.update(actionInvocations)
|
|
957
|
+
.set({ status: 'failed', error: 'agent_host_unavailable', completedAt })
|
|
958
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
|
|
959
|
+
.returning({ id: actionInvocations.id }),
|
|
960
|
+
], { requireAtomic: true });
|
|
961
|
+
const generationConflict = results[0];
|
|
962
|
+
if (generationConflict.length > 0) {
|
|
963
|
+
throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
else {
|
|
967
|
+
await args.db
|
|
968
|
+
.update(actionInvocations)
|
|
969
|
+
.set({ status: 'failed', error: 'agent_host_unavailable', completedAt })
|
|
970
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
971
|
+
}
|
|
833
972
|
throw codedError(`Agent "${name}" has no live host node; cannot dispatch release`, 'agent_host_unavailable', 503);
|
|
834
973
|
};
|
|
835
974
|
const registry = args.registry;
|
|
@@ -858,10 +997,35 @@ async function dispatchRelease(args) {
|
|
|
858
997
|
providerName: agent.providerName,
|
|
859
998
|
action: 'release',
|
|
860
999
|
input,
|
|
1000
|
+
actionId: null,
|
|
1001
|
+
invocationOrigin: 'builtin',
|
|
861
1002
|
});
|
|
1003
|
+
// Some socket owners durably record or complete an accepted frame before
|
|
1004
|
+
// this process resumes to stamp its own dispatch attempt. Return that
|
|
1005
|
+
// terminal winner instead of treating the failed post-send stamp as a
|
|
1006
|
+
// rejected send and entering local fallback.
|
|
1007
|
+
if (dispatched.settled) {
|
|
1008
|
+
if (dispatched.settled.status === 'failed') {
|
|
1009
|
+
const errorCode = dispatched.settled.error ?? 'handler_unavailable';
|
|
1010
|
+
throw codedError('Release invocation failed after provider dispatch', errorCode, errorCode === RELEASE_GENERATION_CONFLICT_CODE ? 409 : 503);
|
|
1011
|
+
}
|
|
1012
|
+
return invocationAck(dispatched.settled, { actionName: 'release' });
|
|
1013
|
+
}
|
|
862
1014
|
// The provider can disconnect between the liveness check and send. Complete
|
|
863
1015
|
// the DB lifecycle locally instead of creating an ownerless pending request.
|
|
864
1016
|
if (!dispatched.accepted) {
|
|
1017
|
+
if (expectedTokenHash) {
|
|
1018
|
+
const [settled] = await args.db
|
|
1019
|
+
.select({ error: actionInvocations.error })
|
|
1020
|
+
.from(actionInvocations)
|
|
1021
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id)));
|
|
1022
|
+
if (settled?.error === RELEASE_GENERATION_CONFLICT_CODE) {
|
|
1023
|
+
throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
|
|
1024
|
+
}
|
|
1025
|
+
if (settled?.error === 'node_dispatch_unavailable') {
|
|
1026
|
+
throw codedError('Node adapter does not support generation-authorized release dispatch', 'node_dispatch_unavailable', 503);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
865
1029
|
return input.delete_agent === true ? completeLocally() : failClosed();
|
|
866
1030
|
}
|
|
867
1031
|
return {
|
|
@@ -875,6 +1039,21 @@ async function dispatchRelease(args) {
|
|
|
875
1039
|
created_at: invocation.createdAt.toISOString(),
|
|
876
1040
|
};
|
|
877
1041
|
}
|
|
1042
|
+
/**
|
|
1043
|
+
* Agent-lifecycle release used by `POST /v1/agents/release`. Unlike the
|
|
1044
|
+
* generic action endpoint, this path must always enforce the built-in release
|
|
1045
|
+
* lifecycle and its optional generation guard; a user-defined `release`
|
|
1046
|
+
* action must not be able to shadow it.
|
|
1047
|
+
*/
|
|
1048
|
+
export async function dispatchAgentRelease(db, workspaceId, data, options = {}) {
|
|
1049
|
+
return dispatchRelease({
|
|
1050
|
+
db,
|
|
1051
|
+
registry: options.nodeConnections,
|
|
1052
|
+
completionDeps: options.completionDeps,
|
|
1053
|
+
workspaceId,
|
|
1054
|
+
data,
|
|
1055
|
+
});
|
|
1056
|
+
}
|
|
878
1057
|
function spawnResult(invocation, nodeId, dispatched) {
|
|
879
1058
|
return {
|
|
880
1059
|
invocation_id: invocation.id,
|
|
@@ -892,6 +1071,11 @@ async function dispatchSpawn(args) {
|
|
|
892
1071
|
throw codedError('Node dispatch is not available', 'node_dispatch_unavailable', 503);
|
|
893
1072
|
}
|
|
894
1073
|
const input = recordInput(args.data.input);
|
|
1074
|
+
const requestedName = typeof input.name === 'string' ? input.name : '';
|
|
1075
|
+
if (requestedName.trim().length === 0) {
|
|
1076
|
+
throw codedError('spawn action input.name is required', 'invalid_spawn_request', 400);
|
|
1077
|
+
}
|
|
1078
|
+
assertRegistrableAgentName(requestedName);
|
|
895
1079
|
const capability = dispatchActionNameForInvocation('spawn', input);
|
|
896
1080
|
const { invocation, replayed } = await createInvocation(args.db, args.workspaceId, null, {
|
|
897
1081
|
input: args.data.input,
|
|
@@ -937,20 +1121,40 @@ async function dispatchSpawn(args) {
|
|
|
937
1121
|
if (!args.bypassShadow) {
|
|
938
1122
|
const shadow = await fetchNodeAction(args.db, args.workspaceId, nodeId, capability);
|
|
939
1123
|
if (shadow && shadow.handlerProvider) {
|
|
940
|
-
|
|
941
|
-
await releaseNodeCapacity(args.db, args.workspaceId, nodeId);
|
|
942
|
-
const dispatched = await dispatchNodeProviderInvocation({
|
|
943
|
-
db: args.db,
|
|
944
|
-
registry: args.registry,
|
|
1124
|
+
const bound = await bindInvocationToRegisteredNodeAction(args.db, {
|
|
945
1125
|
workspaceId: args.workspaceId,
|
|
946
1126
|
invocationId: invocation.id,
|
|
947
1127
|
nodeId,
|
|
948
1128
|
providerName: shadow.handlerProvider,
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
queue: shadow.queue,
|
|
1129
|
+
actionId: shadow.id,
|
|
1130
|
+
actionName: capability,
|
|
952
1131
|
});
|
|
953
|
-
|
|
1132
|
+
if (bound) {
|
|
1133
|
+
if (!placement.queued)
|
|
1134
|
+
await releaseNodeCapacity(args.db, args.workspaceId, nodeId);
|
|
1135
|
+
const dispatched = await dispatchNodeProviderInvocation({
|
|
1136
|
+
db: args.db,
|
|
1137
|
+
registry: args.registry,
|
|
1138
|
+
workspaceId: args.workspaceId,
|
|
1139
|
+
invocationId: invocation.id,
|
|
1140
|
+
nodeId,
|
|
1141
|
+
providerName: shadow.handlerProvider,
|
|
1142
|
+
action: capability,
|
|
1143
|
+
input,
|
|
1144
|
+
queue: shadow.queue,
|
|
1145
|
+
actionId: shadow.id,
|
|
1146
|
+
expectedActionId: shadow.id,
|
|
1147
|
+
invocationOrigin: 'registered_action',
|
|
1148
|
+
});
|
|
1149
|
+
if (dispatched.settled) {
|
|
1150
|
+
const settled = await waitForInvocationReplayOutcome(args.db, args.workspaceId, dispatched.settled);
|
|
1151
|
+
return invocationAck(settled, { actionName: 'spawn', handlerNodeId: nodeId });
|
|
1152
|
+
}
|
|
1153
|
+
return spawnResult(invocation, nodeId, dispatched);
|
|
1154
|
+
}
|
|
1155
|
+
// The shadow disappeared before its exact identity could be claimed.
|
|
1156
|
+
// Keep the native reservation and continue through native capacity; a
|
|
1157
|
+
// stale shadow snapshot must never receive an unbound invocation.
|
|
954
1158
|
}
|
|
955
1159
|
}
|
|
956
1160
|
const capProvider = (await capacityProviderName(args.db, args.workspaceId, nodeId, capability)) ?? DEFAULT_PROVIDER_NAME;
|
|
@@ -963,6 +1167,8 @@ async function dispatchSpawn(args) {
|
|
|
963
1167
|
providerName: capProvider,
|
|
964
1168
|
action: capability,
|
|
965
1169
|
input,
|
|
1170
|
+
actionId: null,
|
|
1171
|
+
invocationOrigin: 'builtin',
|
|
966
1172
|
pending: placement.queued,
|
|
967
1173
|
reservationHeld: !placement.queued,
|
|
968
1174
|
});
|
|
@@ -1031,8 +1237,14 @@ export async function invokeNodeAction(db, workspaceId, nodeId, actionName, data
|
|
|
1031
1237
|
action: action.name,
|
|
1032
1238
|
input: recordInput(invocation.input),
|
|
1033
1239
|
queue: action.queue,
|
|
1240
|
+
actionId: action.id,
|
|
1241
|
+
invocationOrigin: 'registered_action',
|
|
1034
1242
|
reservationHeld: !!reservedNode,
|
|
1035
1243
|
});
|
|
1244
|
+
if (dispatched.settled) {
|
|
1245
|
+
const settled = await waitForInvocationReplayOutcome(db, workspaceId, dispatched.settled);
|
|
1246
|
+
return invocationAck(settled, { actionName, handlerNodeId: nodeId });
|
|
1247
|
+
}
|
|
1036
1248
|
return {
|
|
1037
1249
|
invocation_id: invocation.id,
|
|
1038
1250
|
action_name: actionName,
|
|
@@ -1127,8 +1339,14 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
|
|
|
1127
1339
|
action: action.name,
|
|
1128
1340
|
input: recordInput(invocation.input),
|
|
1129
1341
|
queue: action.queue || providerName === DEFAULT_PROVIDER_NAME,
|
|
1342
|
+
actionId: action.id,
|
|
1343
|
+
invocationOrigin: 'registered_action',
|
|
1130
1344
|
reservationHeld: !!reservedNode,
|
|
1131
1345
|
});
|
|
1346
|
+
if (dispatched.settled) {
|
|
1347
|
+
const settled = await waitForInvocationReplayOutcome(db, workspaceId, dispatched.settled);
|
|
1348
|
+
return invocationAck(settled, { actionName, handlerNodeId: action.handlerNodeId });
|
|
1349
|
+
}
|
|
1132
1350
|
return {
|
|
1133
1351
|
invocation_id: invocation.id,
|
|
1134
1352
|
action_name: actionName,
|
|
@@ -1211,6 +1429,7 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
|
|
|
1211
1429
|
input: recordInput(invocation.input),
|
|
1212
1430
|
agent: { id: handlerAgent.id, name: handlerAgent.name },
|
|
1213
1431
|
actionId: action.id,
|
|
1432
|
+
invocationOrigin: 'registered_action',
|
|
1214
1433
|
});
|
|
1215
1434
|
if (dispatched.sent && !dispatched.accepted) {
|
|
1216
1435
|
// The provider owns the frame once the send returns true. A fast completion
|
|
@@ -1295,8 +1514,196 @@ function publicInvocation(row) {
|
|
|
1295
1514
|
completed_at: row.completedAt?.toISOString() ?? null,
|
|
1296
1515
|
};
|
|
1297
1516
|
}
|
|
1517
|
+
async function completeGuardedReleaseNodeInvocation(db, workspaceId, nodeId, providerName, invocation, data, deps) {
|
|
1518
|
+
const input = recordInput(invocation.input);
|
|
1519
|
+
const name = typeof input.name === 'string' ? input.name : null;
|
|
1520
|
+
const persistedTokenHash = input.expected_token_hash;
|
|
1521
|
+
const expectedTokenHash = typeof persistedTokenHash === 'string'
|
|
1522
|
+
&& AGENT_TOKEN_HASH_PATTERN.test(persistedTokenHash)
|
|
1523
|
+
? persistedTokenHash
|
|
1524
|
+
: null;
|
|
1525
|
+
if (!name || !expectedTokenHash) {
|
|
1526
|
+
const [failed] = await db
|
|
1527
|
+
.update(actionInvocations)
|
|
1528
|
+
.set({
|
|
1529
|
+
status: 'failed',
|
|
1530
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
1531
|
+
completedAt: new Date(),
|
|
1532
|
+
spawnReservedAt: null,
|
|
1533
|
+
})
|
|
1534
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
|
|
1535
|
+
.returning();
|
|
1536
|
+
return failed ? publicInvocation(failed) : null;
|
|
1537
|
+
}
|
|
1538
|
+
const [agent] = await db
|
|
1539
|
+
.select()
|
|
1540
|
+
.from(agents)
|
|
1541
|
+
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.name, name)));
|
|
1542
|
+
if (!agent) {
|
|
1543
|
+
const [failed] = await db
|
|
1544
|
+
.update(actionInvocations)
|
|
1545
|
+
.set({
|
|
1546
|
+
status: 'failed',
|
|
1547
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
1548
|
+
completedAt: new Date(),
|
|
1549
|
+
spawnReservedAt: null,
|
|
1550
|
+
})
|
|
1551
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
|
|
1552
|
+
.returning();
|
|
1553
|
+
return failed ? publicInvocation(failed) : null;
|
|
1554
|
+
}
|
|
1555
|
+
const completedAt = new Date();
|
|
1556
|
+
const generationStillCurrent = releaseGenerationStillCurrent(workspaceId, agent.id, expectedTokenHash);
|
|
1557
|
+
const activeBindingStillCurrent = sql `EXISTS (
|
|
1558
|
+
SELECT 1 FROM ${agentNodeBindings}
|
|
1559
|
+
WHERE ${agentNodeBindings.workspaceId} = ${workspaceId}
|
|
1560
|
+
AND ${agentNodeBindings.agentId} = ${agent.id}
|
|
1561
|
+
AND ${agentNodeBindings.nodeId} = ${nodeId}
|
|
1562
|
+
AND ${agentNodeBindings.status} = 'active'
|
|
1563
|
+
)`;
|
|
1564
|
+
const invocationIsOpen = sql `EXISTS (
|
|
1565
|
+
SELECT 1 FROM ${actionInvocations}
|
|
1566
|
+
WHERE ${actionInvocations.workspaceId} = ${workspaceId}
|
|
1567
|
+
AND ${actionInvocations.id} = ${invocation.id}
|
|
1568
|
+
AND ${actionInvocations.dispatchedNodeId} = ${nodeId}
|
|
1569
|
+
AND ${actionInvocations.dispatchedProvider} = ${providerName}
|
|
1570
|
+
AND ${actionInvocations.status} IN ('pending', 'dispatched', 'invoked')
|
|
1571
|
+
)`;
|
|
1572
|
+
const invocationCompleted = sql `EXISTS (
|
|
1573
|
+
SELECT 1 FROM ${actionInvocations}
|
|
1574
|
+
WHERE ${actionInvocations.workspaceId} = ${workspaceId}
|
|
1575
|
+
AND ${actionInvocations.id} = ${invocation.id}
|
|
1576
|
+
AND ${actionInvocations.dispatchedNodeId} = ${nodeId}
|
|
1577
|
+
AND ${actionInvocations.dispatchedProvider} = ${providerName}
|
|
1578
|
+
AND ${actionInvocations.status} = 'completed'
|
|
1579
|
+
)`;
|
|
1580
|
+
const releaseCanApply = sql `(${generationStillCurrent}) AND (${activeBindingStillCurrent})`;
|
|
1581
|
+
const releasedName = releasedAgentName(agent.name, agent.id);
|
|
1582
|
+
const releasedTokenHash = await sha256Hex(`released:${agent.id}:${randomHex(16)}`);
|
|
1583
|
+
let successIndex = -1;
|
|
1584
|
+
const results = await runAtomicWrites(db, (writeDb) => {
|
|
1585
|
+
const writes = [];
|
|
1586
|
+
// The conflict settlement comes first. In a sequential SQLite batch it
|
|
1587
|
+
// closes the invocation before every mutation below if either the token
|
|
1588
|
+
// generation or node binding changed; on transactional adapters the same
|
|
1589
|
+
// statement ordering and rollback guarantee apply.
|
|
1590
|
+
writes.push(writeDb
|
|
1591
|
+
.update(actionInvocations)
|
|
1592
|
+
.set({
|
|
1593
|
+
status: 'failed',
|
|
1594
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
1595
|
+
completedAt,
|
|
1596
|
+
spawnReservedAt: null,
|
|
1597
|
+
})
|
|
1598
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (${releaseCanApply})`))
|
|
1599
|
+
.returning());
|
|
1600
|
+
writes.push(writeDb
|
|
1601
|
+
.update(nodes)
|
|
1602
|
+
.set({
|
|
1603
|
+
activeAgents: sql `CASE WHEN ${nodes.activeAgents} > 0 THEN ${nodes.activeAgents} - 1 ELSE 0 END`,
|
|
1604
|
+
})
|
|
1605
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId), invocationIsOpen, generationStillCurrent, activeBindingStillCurrent)));
|
|
1606
|
+
writes.push(writeDb
|
|
1607
|
+
.update(agentNodeBindings)
|
|
1608
|
+
.set({ status: 'inactive', updatedAt: completedAt })
|
|
1609
|
+
.where(and(eq(agentNodeBindings.workspaceId, workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.nodeId, nodeId), eq(agentNodeBindings.status, 'active'), invocationIsOpen, generationStillCurrent)));
|
|
1610
|
+
if (input.delete_agent === true) {
|
|
1611
|
+
writes.push(writeDb
|
|
1612
|
+
.delete(channelMembers)
|
|
1613
|
+
.where(and(eq(channelMembers.agentId, agent.id), invocationIsOpen, generationStillCurrent)));
|
|
1614
|
+
writes.push(writeDb
|
|
1615
|
+
.delete(dmParticipants)
|
|
1616
|
+
.where(and(eq(dmParticipants.agentId, agent.id), invocationIsOpen, generationStillCurrent)));
|
|
1617
|
+
}
|
|
1618
|
+
successIndex = writes.length;
|
|
1619
|
+
writes.push(writeDb
|
|
1620
|
+
.update(actionInvocations)
|
|
1621
|
+
.set({
|
|
1622
|
+
output: data.output ?? null,
|
|
1623
|
+
error: null,
|
|
1624
|
+
status: 'completed',
|
|
1625
|
+
completedAt,
|
|
1626
|
+
spawnReservedAt: null,
|
|
1627
|
+
})
|
|
1628
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
|
|
1629
|
+
.returning());
|
|
1630
|
+
const currentAgent = and(eq(agents.workspaceId, workspaceId), eq(agents.id, agent.id), eq(agents.name, name), eq(agents.tokenHash, expectedTokenHash), invocationCompleted);
|
|
1631
|
+
if (input.delete_agent === true) {
|
|
1632
|
+
writes.push(writeDb
|
|
1633
|
+
.update(agents)
|
|
1634
|
+
.set({
|
|
1635
|
+
name: releasedName,
|
|
1636
|
+
handle: `@${releasedName}`,
|
|
1637
|
+
status: RELEASED_AGENT_STATUS,
|
|
1638
|
+
tokenHash: releasedTokenHash,
|
|
1639
|
+
previousTokenHash: null,
|
|
1640
|
+
previousTokenExpiresAt: null,
|
|
1641
|
+
locationType: 'self_connected',
|
|
1642
|
+
locationNodeId: null,
|
|
1643
|
+
lastSeen: completedAt,
|
|
1644
|
+
metadata: sql `json_patch(COALESCE(${agents.metadata}, '{}'), ${JSON.stringify({
|
|
1645
|
+
release: {
|
|
1646
|
+
reason: typeof input.reason === 'string' ? input.reason : null,
|
|
1647
|
+
released_at: completedAt.toISOString(),
|
|
1648
|
+
previous_name: agent.name,
|
|
1649
|
+
},
|
|
1650
|
+
})})`,
|
|
1651
|
+
})
|
|
1652
|
+
.where(currentAgent));
|
|
1653
|
+
}
|
|
1654
|
+
else {
|
|
1655
|
+
const existingMetadata = agent.metadata ?? {};
|
|
1656
|
+
const { spawn: _spawn, cli: _cli, ...restMetadata } = existingMetadata;
|
|
1657
|
+
writes.push(writeDb
|
|
1658
|
+
.update(agents)
|
|
1659
|
+
.set({
|
|
1660
|
+
status: 'offline',
|
|
1661
|
+
locationType: 'self_connected',
|
|
1662
|
+
locationNodeId: null,
|
|
1663
|
+
lastSeen: completedAt,
|
|
1664
|
+
metadata: {
|
|
1665
|
+
...restMetadata,
|
|
1666
|
+
release: {
|
|
1667
|
+
reason: typeof input.reason === 'string' ? input.reason : null,
|
|
1668
|
+
released_at: completedAt.toISOString(),
|
|
1669
|
+
},
|
|
1670
|
+
},
|
|
1671
|
+
})
|
|
1672
|
+
.where(currentAgent));
|
|
1673
|
+
}
|
|
1674
|
+
if (input.delete_agent === true) {
|
|
1675
|
+
// Deleting the implicit node clears action_invocations.dispatched_node_id
|
|
1676
|
+
// via its FK. Do this only after the owned completion and agent CAS have
|
|
1677
|
+
// both succeeded, or their node predicates could no longer match.
|
|
1678
|
+
writes.push(writeDb
|
|
1679
|
+
.delete(nodes)
|
|
1680
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, `node_direct_${agent.id}`), sql `EXISTS (
|
|
1681
|
+
SELECT 1 FROM ${actionInvocations}
|
|
1682
|
+
WHERE ${actionInvocations.workspaceId} = ${workspaceId}
|
|
1683
|
+
AND ${actionInvocations.id} = ${invocation.id}
|
|
1684
|
+
AND ${actionInvocations.status} = 'completed'
|
|
1685
|
+
)`)));
|
|
1686
|
+
}
|
|
1687
|
+
return writes;
|
|
1688
|
+
}, { requireAtomic: true });
|
|
1689
|
+
const completed = results[successIndex];
|
|
1690
|
+
const generationConflict = results[0];
|
|
1691
|
+
const settled = completed[0] ?? generationConflict[0];
|
|
1692
|
+
if (!settled)
|
|
1693
|
+
return null;
|
|
1694
|
+
if (completed[0] && deps) {
|
|
1695
|
+
await emitAgentExitedEffects(deps, workspaceId, {
|
|
1696
|
+
agentId: agent.id,
|
|
1697
|
+
agentName: agent.name,
|
|
1698
|
+
nodeId,
|
|
1699
|
+
invocationId: fleetInvocationId(agent.metadata),
|
|
1700
|
+
reason: 'released',
|
|
1701
|
+
});
|
|
1702
|
+
}
|
|
1703
|
+
return publicInvocation(settled);
|
|
1704
|
+
}
|
|
1298
1705
|
async function applyReleaseCompletionEffect(db, workspaceId, nodeId, invocation, data, deps, options = {}) {
|
|
1299
|
-
if (!
|
|
1706
|
+
if (!isBuiltinReleaseInvocation(invocation) || data.error)
|
|
1300
1707
|
return false;
|
|
1301
1708
|
const input = recordInput(invocation.input);
|
|
1302
1709
|
const name = typeof input.name === 'string' ? input.name : null;
|
|
@@ -1424,13 +1831,137 @@ async function dispatchNodeAttempt(db, workspaceId, invocationId, nodeId, opts)
|
|
|
1424
1831
|
dispatchedNodeId: nodeId,
|
|
1425
1832
|
dispatchedProvider: opts.providerName,
|
|
1426
1833
|
spawnReservedAt: opts.reservationHeld ? new Date() : null,
|
|
1427
|
-
...(opts.
|
|
1834
|
+
...(opts.targetActionId ? { actionId: opts.targetActionId } : {}),
|
|
1428
1835
|
...attemptFields,
|
|
1429
1836
|
})
|
|
1430
|
-
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)
|
|
1837
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedAction ? [
|
|
1838
|
+
eq(actionInvocations.invocationOrigin, 'registered_action'),
|
|
1839
|
+
eq(actionInvocations.actionId, opts.expectedAction.id),
|
|
1840
|
+
sql `EXISTS (
|
|
1841
|
+
SELECT 1 FROM ${actions}
|
|
1842
|
+
WHERE ${actions.workspaceId} = ${workspaceId}
|
|
1843
|
+
AND ${actions.id} = ${opts.expectedAction.id}
|
|
1844
|
+
)`,
|
|
1845
|
+
...(opts.targetActionId || opts.expectedAction.handlerAgentId ? [
|
|
1846
|
+
registeredActionDispatchPredicate({
|
|
1847
|
+
workspaceId,
|
|
1848
|
+
nodeId,
|
|
1849
|
+
providerName: opts.providerName,
|
|
1850
|
+
actionId: opts.targetActionId ?? opts.expectedAction.id,
|
|
1851
|
+
actionName: opts.expectedAction.name,
|
|
1852
|
+
handlerAgentId: opts.expectedAction.handlerAgentId,
|
|
1853
|
+
}),
|
|
1854
|
+
] : []),
|
|
1855
|
+
] : [])))
|
|
1431
1856
|
.returning();
|
|
1432
1857
|
return !!updated;
|
|
1433
1858
|
}
|
|
1859
|
+
function registeredActionDispatchPredicate(args) {
|
|
1860
|
+
const handlerPredicate = args.handlerAgentId
|
|
1861
|
+
? sql `AND ${actions.handlerAgentId} = ${args.handlerAgentId} AND ${actions.handlerNodeId} IS NULL`
|
|
1862
|
+
: sql `AND ${actions.handlerNodeId} = ${args.nodeId}
|
|
1863
|
+
AND COALESCE(${actions.handlerProvider}, ${DEFAULT_PROVIDER_NAME}) = ${args.providerName}`;
|
|
1864
|
+
return sql `EXISTS (
|
|
1865
|
+
SELECT 1 FROM ${actions}
|
|
1866
|
+
WHERE ${actions.workspaceId} = ${args.workspaceId}
|
|
1867
|
+
AND ${actions.id} = ${args.actionId}
|
|
1868
|
+
AND ${actions.name} = ${args.actionName}
|
|
1869
|
+
AND ${actions.isActive} = 1
|
|
1870
|
+
${handlerPredicate}
|
|
1871
|
+
)`;
|
|
1872
|
+
}
|
|
1873
|
+
async function claimRegisteredActionHandoff(db, args) {
|
|
1874
|
+
const claimedActionId = args.targetActionId ?? args.expectedActionId;
|
|
1875
|
+
const stateFields = args.pending
|
|
1876
|
+
? { status: 'pending', dispatchedAt: null, retryAfterAt: args.retryAfterAt ?? null }
|
|
1877
|
+
: dispatchedStateFields({ retryAfterAt: args.retryAfterAt });
|
|
1878
|
+
const attemptFields = args.skipIncrementAttempts
|
|
1879
|
+
? {}
|
|
1880
|
+
: {
|
|
1881
|
+
attemptedNodeIds: sql `json_insert(COALESCE(${actionInvocations.attemptedNodeIds}, '[]'), '$[#]', ${args.nodeId})`,
|
|
1882
|
+
dispatchAttempts: sql `COALESCE(${actionInvocations.dispatchAttempts}, 0) + 1`,
|
|
1883
|
+
};
|
|
1884
|
+
const [claimed] = await db
|
|
1885
|
+
.update(actionInvocations)
|
|
1886
|
+
// This CAS is the durable dispatch linearization point. Claim the exact
|
|
1887
|
+
// action generation, route, and attempt before awaiting the provider. A
|
|
1888
|
+
// retry may hand off between same-name registrations only while it owns the
|
|
1889
|
+
// source generation; a later rescheduler observes the new route instead of
|
|
1890
|
+
// sending the same attempt again.
|
|
1891
|
+
.set({
|
|
1892
|
+
...stateFields,
|
|
1893
|
+
handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${args.nodeId})`,
|
|
1894
|
+
dispatchedNodeId: args.nodeId,
|
|
1895
|
+
dispatchedProvider: args.providerName,
|
|
1896
|
+
spawnReservedAt: args.reservationHeld ? new Date() : null,
|
|
1897
|
+
actionId: claimedActionId,
|
|
1898
|
+
...attemptFields,
|
|
1899
|
+
})
|
|
1900
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.expectedActionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (
|
|
1901
|
+
${actionInvocations.actionId} = ${claimedActionId}
|
|
1902
|
+
AND COALESCE(${actionInvocations.dispatchedNodeId}, '') = ${args.nodeId}
|
|
1903
|
+
AND COALESCE(${actionInvocations.dispatchedProvider}, ${DEFAULT_PROVIDER_NAME}) = ${args.providerName}
|
|
1904
|
+
AND ${actionInvocations.status} IN ('dispatched', 'invoked')
|
|
1905
|
+
)`, sql `EXISTS (
|
|
1906
|
+
SELECT 1 FROM ${actions}
|
|
1907
|
+
WHERE ${actions.workspaceId} = ${args.workspaceId}
|
|
1908
|
+
AND ${actions.id} = ${args.expectedActionId}
|
|
1909
|
+
)`, ...(args.targetActionId ? [registeredActionDispatchPredicate({
|
|
1910
|
+
workspaceId: args.workspaceId,
|
|
1911
|
+
nodeId: args.nodeId,
|
|
1912
|
+
providerName: args.providerName,
|
|
1913
|
+
actionId: args.targetActionId,
|
|
1914
|
+
actionName: args.actionName,
|
|
1915
|
+
})] : [])))
|
|
1916
|
+
.returning({
|
|
1917
|
+
actionId: actionInvocations.actionId,
|
|
1918
|
+
actionName: actionInvocations.actionName,
|
|
1919
|
+
dispatchAttempts: actionInvocations.dispatchAttempts,
|
|
1920
|
+
});
|
|
1921
|
+
return claimed?.actionId ? {
|
|
1922
|
+
actionId: claimed.actionId,
|
|
1923
|
+
actionName: claimed.actionName,
|
|
1924
|
+
dispatchAttempts: claimed.dispatchAttempts,
|
|
1925
|
+
} : null;
|
|
1926
|
+
}
|
|
1927
|
+
async function bindInvocationToRegisteredNodeAction(db, args) {
|
|
1928
|
+
const [bound] = await db
|
|
1929
|
+
.update(actionInvocations)
|
|
1930
|
+
.set({ actionId: args.actionId, invocationOrigin: 'registered_action' })
|
|
1931
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'builtin'), isNull(actionInvocations.actionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), eq(actionInvocations.dispatchAttempts, 0), registeredActionDispatchPredicate(args)))
|
|
1932
|
+
.returning({ id: actionInvocations.id });
|
|
1933
|
+
return !!bound;
|
|
1934
|
+
}
|
|
1935
|
+
async function settleDeletedRegisteredAction(db, workspaceId, invocationId) {
|
|
1936
|
+
// A failed handoff can also mean another dispatcher already advanced the
|
|
1937
|
+
// same invocation to a different live action. Fail only the FK-null state
|
|
1938
|
+
// that proves pruning won; never clobber that concurrent dispatch winner.
|
|
1939
|
+
const [failed] = await db
|
|
1940
|
+
.update(actionInvocations)
|
|
1941
|
+
.set({
|
|
1942
|
+
status: 'failed',
|
|
1943
|
+
error: 'action_deleted',
|
|
1944
|
+
completedAt: new Date(),
|
|
1945
|
+
spawnReservedAt: null,
|
|
1946
|
+
})
|
|
1947
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), isNull(actionInvocations.actionId), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
|
|
1948
|
+
.returning();
|
|
1949
|
+
if (failed)
|
|
1950
|
+
return failed;
|
|
1951
|
+
const [settled] = await db
|
|
1952
|
+
.select()
|
|
1953
|
+
.from(actionInvocations)
|
|
1954
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId)));
|
|
1955
|
+
return settled;
|
|
1956
|
+
}
|
|
1957
|
+
async function settleRegisteredActionSendFailure(db, args) {
|
|
1958
|
+
const [failed] = await db
|
|
1959
|
+
.update(actionInvocations)
|
|
1960
|
+
.set({ status: 'failed', error: 'node_dispatch_unavailable', completedAt: new Date() })
|
|
1961
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.actionId), eq(actionInvocations.dispatchedNodeId, args.nodeId), eq(actionInvocations.dispatchedProvider, args.providerName), eq(actionInvocations.dispatchAttempts, args.dispatchAttempts), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
|
|
1962
|
+
.returning();
|
|
1963
|
+
return failed ?? settleDeletedRegisteredAction(db, args.workspaceId, args.invocationId);
|
|
1964
|
+
}
|
|
1434
1965
|
async function snapshotInvocationHandlerNode(db, workspaceId, invocationId, nodeId) {
|
|
1435
1966
|
const [updated] = await db
|
|
1436
1967
|
.update(actionInvocations)
|
|
@@ -1460,6 +1991,44 @@ export async function markDrainedInvocationDispatched(db, workspaceId, invocatio
|
|
|
1460
1991
|
return !!updated;
|
|
1461
1992
|
}
|
|
1462
1993
|
async function dispatchNodeInvocation(args) {
|
|
1994
|
+
const guardedReleaseHash = args.invocationOrigin === 'builtin' && isReleaseInvocation(args.action)
|
|
1995
|
+
? releaseExpectedTokenHash(args.input)
|
|
1996
|
+
: null;
|
|
1997
|
+
if (guardedReleaseHash) {
|
|
1998
|
+
const name = typeof args.input.name === 'string' ? args.input.name : '';
|
|
1999
|
+
const [current] = await args.db
|
|
2000
|
+
.select({ id: agents.id })
|
|
2001
|
+
.from(agents)
|
|
2002
|
+
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name), eq(agents.tokenHash, guardedReleaseHash), sql `EXISTS (
|
|
2003
|
+
SELECT 1 FROM ${agentNodeBindings}
|
|
2004
|
+
WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
|
|
2005
|
+
AND ${agentNodeBindings.agentId} = ${agents.id}
|
|
2006
|
+
AND ${agentNodeBindings.nodeId} = ${args.nodeId}
|
|
2007
|
+
AND ${agentNodeBindings.status} = 'active'
|
|
2008
|
+
)`));
|
|
2009
|
+
if (!current) {
|
|
2010
|
+
await args.db
|
|
2011
|
+
.update(actionInvocations)
|
|
2012
|
+
.set({
|
|
2013
|
+
status: 'failed',
|
|
2014
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
2015
|
+
completedAt: new Date(),
|
|
2016
|
+
})
|
|
2017
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
2018
|
+
return { accepted: false, pending: false, sent: false };
|
|
2019
|
+
}
|
|
2020
|
+
if (!args.registry.sendAuthorizedActionToProvider) {
|
|
2021
|
+
await args.db
|
|
2022
|
+
.update(actionInvocations)
|
|
2023
|
+
.set({
|
|
2024
|
+
status: 'failed',
|
|
2025
|
+
error: 'node_dispatch_unavailable',
|
|
2026
|
+
completedAt: new Date(),
|
|
2027
|
+
})
|
|
2028
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
2029
|
+
return { accepted: false, pending: false, sent: false };
|
|
2030
|
+
}
|
|
2031
|
+
}
|
|
1463
2032
|
const frame = {
|
|
1464
2033
|
v: 1,
|
|
1465
2034
|
type: 'action.invoke',
|
|
@@ -1477,6 +2046,32 @@ async function dispatchNodeInvocation(args) {
|
|
|
1477
2046
|
if (!snapshotted)
|
|
1478
2047
|
return { accepted: false, pending: false, sent: false };
|
|
1479
2048
|
const connectedBefore = args.registry.isProviderConnected(args.workspaceId, args.nodeId, args.providerName);
|
|
2049
|
+
const pending = !!args.pending || !connectedBefore;
|
|
2050
|
+
const expectedActionId = args.invocationOrigin === 'registered_action'
|
|
2051
|
+
? (args.expectedActionId ?? args.actionId)
|
|
2052
|
+
: null;
|
|
2053
|
+
let registeredNodeClaim = null;
|
|
2054
|
+
if (args.invocationOrigin === 'registered_action' && !args.agent) {
|
|
2055
|
+
registeredNodeClaim = expectedActionId
|
|
2056
|
+
? await claimRegisteredActionHandoff(args.db, {
|
|
2057
|
+
workspaceId: args.workspaceId,
|
|
2058
|
+
invocationId: args.invocationId,
|
|
2059
|
+
nodeId: args.nodeId,
|
|
2060
|
+
providerName: args.providerName,
|
|
2061
|
+
expectedActionId,
|
|
2062
|
+
targetActionId: args.actionId ?? undefined,
|
|
2063
|
+
actionName: args.action,
|
|
2064
|
+
pending,
|
|
2065
|
+
retryAfterAt: args.retryAfterAt,
|
|
2066
|
+
reservationHeld: args.reservationHeld,
|
|
2067
|
+
skipIncrementAttempts: args.skipIncrementAttempts,
|
|
2068
|
+
})
|
|
2069
|
+
: null;
|
|
2070
|
+
if (!registeredNodeClaim) {
|
|
2071
|
+
const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
|
|
2072
|
+
return { accepted: false, pending: false, sent: false, settled };
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
1480
2075
|
const sent = args.agent && args.actionId
|
|
1481
2076
|
? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
|
|
1482
2077
|
kind: 'agent-action-v1',
|
|
@@ -1485,18 +2080,88 @@ async function dispatchNodeInvocation(args) {
|
|
|
1485
2080
|
handlerAgentId: args.agent.id,
|
|
1486
2081
|
recordAttempt: !args.skipIncrementAttempts,
|
|
1487
2082
|
}) ?? false)
|
|
1488
|
-
:
|
|
1489
|
-
|
|
2083
|
+
: registeredNodeClaim
|
|
2084
|
+
? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
|
|
2085
|
+
kind: 'registered-node-action-v2',
|
|
2086
|
+
invocationId: args.invocationId,
|
|
2087
|
+
actionId: registeredNodeClaim.actionId,
|
|
2088
|
+
dispatchAttempt: registeredNodeClaim.dispatchAttempts,
|
|
2089
|
+
invocationActionName: registeredNodeClaim.actionName,
|
|
2090
|
+
actionName: args.action,
|
|
2091
|
+
}) ?? false)
|
|
2092
|
+
: guardedReleaseHash
|
|
2093
|
+
? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
|
|
2094
|
+
kind: 'release-generation-v1',
|
|
2095
|
+
invocationId: args.invocationId,
|
|
2096
|
+
agentName: typeof args.input.name === 'string' ? args.input.name : '',
|
|
2097
|
+
expectedTokenHash: guardedReleaseHash,
|
|
2098
|
+
}) ?? false)
|
|
2099
|
+
: await args.registry.sendToProvider(args.workspaceId, args.nodeId, args.providerName, frame);
|
|
2100
|
+
if (!sent && guardedReleaseHash) {
|
|
2101
|
+
// An adapter compiled against the older owner-authorization contract may
|
|
2102
|
+
// expose the method but reject the new proof kind. Settle any still-open
|
|
2103
|
+
// row before the caller considers local fallback, so a guarded delete can
|
|
2104
|
+
// never degrade to an unguarded lifecycle mutation.
|
|
2105
|
+
await args.db
|
|
2106
|
+
.update(actionInvocations)
|
|
2107
|
+
.set({
|
|
2108
|
+
status: 'failed',
|
|
2109
|
+
error: 'node_dispatch_unavailable',
|
|
2110
|
+
completedAt: new Date(),
|
|
2111
|
+
})
|
|
2112
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
2113
|
+
}
|
|
2114
|
+
if (!sent) {
|
|
2115
|
+
if (registeredNodeClaim) {
|
|
2116
|
+
const settled = await settleRegisteredActionSendFailure(args.db, {
|
|
2117
|
+
workspaceId: args.workspaceId,
|
|
2118
|
+
invocationId: args.invocationId,
|
|
2119
|
+
nodeId: args.nodeId,
|
|
2120
|
+
providerName: args.providerName,
|
|
2121
|
+
actionId: registeredNodeClaim.actionId,
|
|
2122
|
+
dispatchAttempts: registeredNodeClaim.dispatchAttempts,
|
|
2123
|
+
});
|
|
2124
|
+
return { accepted: false, pending: false, sent: false, settled };
|
|
2125
|
+
}
|
|
1490
2126
|
return { accepted: false, pending: false, sent: false };
|
|
1491
|
-
|
|
2127
|
+
}
|
|
2128
|
+
// Registered node-action state was committed by the pre-send handoff CAS.
|
|
2129
|
+
// Pruning after provider acceptance may clear the action FK, but it cannot
|
|
2130
|
+
// revoke work the provider already received or prevent route-owned completion.
|
|
2131
|
+
if (registeredNodeClaim) {
|
|
2132
|
+
return { accepted: true, pending, sent: true };
|
|
2133
|
+
}
|
|
2134
|
+
const finalExpectedActionId = expectedActionId;
|
|
1492
2135
|
const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, {
|
|
1493
2136
|
pending,
|
|
1494
2137
|
providerName: args.providerName,
|
|
1495
2138
|
retryAfterAt: args.retryAfterAt,
|
|
1496
2139
|
reservationHeld: args.reservationHeld,
|
|
1497
2140
|
skipIncrementAttempts: args.skipIncrementAttempts || !!args.agent,
|
|
1498
|
-
|
|
2141
|
+
expectedAction: finalExpectedActionId
|
|
2142
|
+
? {
|
|
2143
|
+
id: finalExpectedActionId,
|
|
2144
|
+
name: args.action,
|
|
2145
|
+
...(args.agent ? { handlerAgentId: args.agent.id } : {}),
|
|
2146
|
+
}
|
|
2147
|
+
: undefined,
|
|
2148
|
+
targetActionId: args.agent || args.actionId
|
|
2149
|
+
? finalExpectedActionId ?? undefined
|
|
2150
|
+
: undefined,
|
|
1499
2151
|
});
|
|
2152
|
+
if (!accepted && args.invocationOrigin === 'registered_action') {
|
|
2153
|
+
const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
|
|
2154
|
+
return { accepted: false, pending: false, sent: true, settled };
|
|
2155
|
+
}
|
|
2156
|
+
if (!accepted && guardedReleaseHash) {
|
|
2157
|
+
const [settled] = await args.db
|
|
2158
|
+
.select()
|
|
2159
|
+
.from(actionInvocations)
|
|
2160
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId)));
|
|
2161
|
+
if (settled && !OPEN_INVOCATION_STATUSES.some((status) => status === settled.status)) {
|
|
2162
|
+
return { accepted: false, pending: false, sent: true, settled };
|
|
2163
|
+
}
|
|
2164
|
+
}
|
|
1500
2165
|
return { accepted, pending, sent: true };
|
|
1501
2166
|
}
|
|
1502
2167
|
function attemptedNodeSet(invocation) {
|
|
@@ -1551,12 +2216,16 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
|
|
|
1551
2216
|
const rows = await db
|
|
1552
2217
|
.select({
|
|
1553
2218
|
id: actionInvocations.id,
|
|
1554
|
-
actionId:
|
|
2219
|
+
actionId: actionInvocations.actionId,
|
|
2220
|
+
invocationOrigin: actionInvocations.invocationOrigin,
|
|
1555
2221
|
workspaceId: actionInvocations.workspaceId,
|
|
1556
2222
|
actionName: actionInvocations.actionName,
|
|
1557
2223
|
input: actionInvocations.input,
|
|
2224
|
+
providerAcceptedAttempt: actionInvocations.providerAcceptedAttempt,
|
|
2225
|
+
dispatchAttempts: actionInvocations.dispatchAttempts,
|
|
1558
2226
|
spawnReservedAt: actionInvocations.spawnReservedAt,
|
|
1559
2227
|
dispatchedNodeId: actionInvocations.dispatchedNodeId,
|
|
2228
|
+
dispatchedProvider: actionInvocations.dispatchedProvider,
|
|
1560
2229
|
handlerNodeId: actions.handlerNodeId,
|
|
1561
2230
|
handlerProvider: actions.handlerProvider,
|
|
1562
2231
|
handlerAgentId: agents.id,
|
|
@@ -1574,6 +2243,10 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
|
|
|
1574
2243
|
.orderBy(asc(actionInvocations.createdAt));
|
|
1575
2244
|
let drained = 0;
|
|
1576
2245
|
for (const row of rows) {
|
|
2246
|
+
if (isDeletedRegisteredActionInvocation(row)) {
|
|
2247
|
+
await failOpenInvocationRows(db, workspaceId, [row.id], 'action_deleted');
|
|
2248
|
+
continue;
|
|
2249
|
+
}
|
|
1577
2250
|
const targetAgent = row.handlerAgentId
|
|
1578
2251
|
&& row.handlerAgentName
|
|
1579
2252
|
&& row.handlerAgentLocationType === 'via_node'
|
|
@@ -1596,10 +2269,12 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
|
|
|
1596
2269
|
? row.handlerAgentProvider ?? DEFAULT_PROVIDER_NAME
|
|
1597
2270
|
: row.handlerNodeId
|
|
1598
2271
|
? row.handlerProvider ?? DEFAULT_PROVIDER_NAME
|
|
1599
|
-
:
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
2272
|
+
: isBuiltinReleaseInvocation(row)
|
|
2273
|
+
? row.dispatchedProvider ?? DEFAULT_PROVIDER_NAME
|
|
2274
|
+
: shadowProvider
|
|
2275
|
+
?? (isSpawnInvocation(row.actionName)
|
|
2276
|
+
? (await capacityProviderName(db, workspaceId, nodeId, dispatchActionNameForInvocation(row.actionName, input))) ?? DEFAULT_PROVIDER_NAME
|
|
2277
|
+
: DEFAULT_PROVIDER_NAME);
|
|
1603
2278
|
const nativeSpawn = isSpawnInvocation(row.actionName) && !shadowProvider;
|
|
1604
2279
|
// Skip draining to a provider that is disconnected or whose heartbeat says
|
|
1605
2280
|
// handlers are not live. A missing provider row is a legacy single-socket
|
|
@@ -1641,7 +2316,9 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
|
|
|
1641
2316
|
action: dispatchActionNameForInvocation(row.actionName, input),
|
|
1642
2317
|
input,
|
|
1643
2318
|
agent: targetAgent ? { id: targetAgent.id, name: targetAgent.name } : null,
|
|
1644
|
-
actionId:
|
|
2319
|
+
actionId: row.actionId,
|
|
2320
|
+
expectedActionId: row.actionId,
|
|
2321
|
+
invocationOrigin: row.invocationOrigin,
|
|
1645
2322
|
retryAfterAt: new Date(Date.now() + ACTION_DISPATCH_TIMEOUT_MS),
|
|
1646
2323
|
reservationHeld,
|
|
1647
2324
|
skipIncrementAttempts: row.dispatchedNodeId === nodeId,
|
|
@@ -1663,6 +2340,17 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
|
|
|
1663
2340
|
return drained;
|
|
1664
2341
|
}
|
|
1665
2342
|
export async function rescheduleNodeInvocation(db, registry, invocation, opts = {}) {
|
|
2343
|
+
// Once the exact provider accepted this generation, pruning only clears its
|
|
2344
|
+
// materialized action FK. It must not make timeout, disconnect, or inventory
|
|
2345
|
+
// recovery move the invocation to a same-name replacement and invalidate the
|
|
2346
|
+
// original provider's result.
|
|
2347
|
+
if (isAcceptedDeletedRegisteredActionInvocation(invocation)) {
|
|
2348
|
+
return false;
|
|
2349
|
+
}
|
|
2350
|
+
if (isDeletedRegisteredActionInvocation(invocation)) {
|
|
2351
|
+
await failOpenInvocationRows(db, invocation.workspaceId, [invocation.id], 'action_deleted');
|
|
2352
|
+
return false;
|
|
2353
|
+
}
|
|
1666
2354
|
// Only release capacity the invocation actually holds. A shadowed spawn
|
|
1667
2355
|
// dropped its native reservation at dispatch (spawnReservedAt is null), so it
|
|
1668
2356
|
// must not release capacity that now belongs to another spawn.
|
|
@@ -1697,6 +2385,8 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
|
|
|
1697
2385
|
input: recordInput(invocation.input),
|
|
1698
2386
|
agent: { id: targetAgent.agentId, name: targetAgent.agentName },
|
|
1699
2387
|
actionId: targetAgent.actionId,
|
|
2388
|
+
expectedActionId: invocation.actionId,
|
|
2389
|
+
invocationOrigin: invocation.invocationOrigin,
|
|
1700
2390
|
retryAfterAt: opts.retryAfterAt ?? null,
|
|
1701
2391
|
});
|
|
1702
2392
|
return dispatched.accepted;
|
|
@@ -1704,7 +2394,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
|
|
|
1704
2394
|
// Release invocations carry no actionId, so targetAgentForInvocation() returns
|
|
1705
2395
|
// null. They must never fall through to generic node placement, which could
|
|
1706
2396
|
// route a release to an unrelated node that doesn't own the agent.
|
|
1707
|
-
if (
|
|
2397
|
+
if (isBuiltinReleaseInvocation(invocation)) {
|
|
1708
2398
|
return false;
|
|
1709
2399
|
}
|
|
1710
2400
|
const attempted = attemptedNodeSet(invocation);
|
|
@@ -1741,6 +2431,8 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
|
|
|
1741
2431
|
providerName: target.handlerProvider,
|
|
1742
2432
|
action: actionToSend,
|
|
1743
2433
|
actionId: target.id,
|
|
2434
|
+
expectedActionId: invocation.actionId,
|
|
2435
|
+
invocationOrigin: invocation.invocationOrigin,
|
|
1744
2436
|
input,
|
|
1745
2437
|
queue: target.queue,
|
|
1746
2438
|
reservationHeld: false,
|
|
@@ -1758,14 +2450,23 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
|
|
|
1758
2450
|
? (await capacityProviderName(db, invocation.workspaceId, placement.node.id, actionToSend)) ?? DEFAULT_PROVIDER_NAME
|
|
1759
2451
|
: DEFAULT_PROVIDER_NAME;
|
|
1760
2452
|
if (placement.queued) {
|
|
1761
|
-
|
|
2453
|
+
const expectedActionId = invocation.invocationOrigin === 'registered_action'
|
|
2454
|
+
? invocation.actionId
|
|
2455
|
+
: null;
|
|
2456
|
+
const accepted = await dispatchNodeAttempt(db, invocation.workspaceId, invocation.id, placement.node.id, {
|
|
1762
2457
|
providerName,
|
|
1763
2458
|
pending: true,
|
|
1764
2459
|
retryAfterAt: opts.retryAfterAt ?? null,
|
|
1765
2460
|
reservationHeld: false,
|
|
1766
|
-
|
|
2461
|
+
expectedAction: expectedActionId
|
|
2462
|
+
? { id: expectedActionId, name: actionToSend }
|
|
2463
|
+
: undefined,
|
|
2464
|
+
targetActionId: target?.id,
|
|
1767
2465
|
});
|
|
1768
|
-
|
|
2466
|
+
if (!accepted && invocation.invocationOrigin === 'registered_action') {
|
|
2467
|
+
await settleDeletedRegisteredAction(db, invocation.workspaceId, invocation.id);
|
|
2468
|
+
}
|
|
2469
|
+
return accepted;
|
|
1769
2470
|
}
|
|
1770
2471
|
const dispatched = await dispatchNodeInvocation({
|
|
1771
2472
|
db,
|
|
@@ -1775,7 +2476,9 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
|
|
|
1775
2476
|
nodeId: placement.node.id,
|
|
1776
2477
|
providerName,
|
|
1777
2478
|
action: actionToSend,
|
|
1778
|
-
actionId: target?.id,
|
|
2479
|
+
actionId: target?.id ?? null,
|
|
2480
|
+
expectedActionId: invocation.actionId,
|
|
2481
|
+
invocationOrigin: invocation.invocationOrigin,
|
|
1779
2482
|
input,
|
|
1780
2483
|
reservationHeld,
|
|
1781
2484
|
});
|
|
@@ -1879,6 +2582,8 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
|
|
|
1879
2582
|
.select({
|
|
1880
2583
|
id: actionInvocations.id,
|
|
1881
2584
|
workspaceId: actionInvocations.workspaceId,
|
|
2585
|
+
actionId: actionInvocations.actionId,
|
|
2586
|
+
invocationOrigin: actionInvocations.invocationOrigin,
|
|
1882
2587
|
actionName: actionInvocations.actionName,
|
|
1883
2588
|
callerId: actionInvocations.callerId,
|
|
1884
2589
|
input: actionInvocations.input,
|
|
@@ -1888,6 +2593,7 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
|
|
|
1888
2593
|
durationMs: actionInvocations.durationMs,
|
|
1889
2594
|
dispatchedNodeId: actionInvocations.dispatchedNodeId,
|
|
1890
2595
|
dispatchedAt: actionInvocations.dispatchedAt,
|
|
2596
|
+
providerAcceptedAttempt: actionInvocations.providerAcceptedAttempt,
|
|
1891
2597
|
spawnReservedAt: actionInvocations.spawnReservedAt,
|
|
1892
2598
|
attemptedNodeIds: actionInvocations.attemptedNodeIds,
|
|
1893
2599
|
dispatchAttempts: actionInvocations.dispatchAttempts,
|
|
@@ -1918,6 +2624,12 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
|
|
|
1918
2624
|
// Fall through and fail the invocation if no eligible node exists.
|
|
1919
2625
|
}
|
|
1920
2626
|
}
|
|
2627
|
+
const existingInput = recordInput(existing.input);
|
|
2628
|
+
if (!data.error
|
|
2629
|
+
&& isBuiltinReleaseInvocation(existing)
|
|
2630
|
+
&& existingInput.expected_token_hash !== undefined) {
|
|
2631
|
+
return completeGuardedReleaseNodeInvocation(db, workspaceId, nodeId, providerName, existing, data, deps);
|
|
2632
|
+
}
|
|
1921
2633
|
const [updated] = await db
|
|
1922
2634
|
.update(actionInvocations)
|
|
1923
2635
|
.set({
|