@relaycast/engine 8.4.0 → 8.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  import { and, asc, eq, inArray, isNotNull, isNull, lte, or, sql } from 'drizzle-orm';
2
2
  import { actions, actionInvocations, agents, agentNodeBindings, channelMembers, dmParticipants, nodes } from '../db/schema.js';
3
3
  import { generateId } from './snowflake.js';
4
- import { RELEASED_AGENT_STATUS, releasedAgentName } from './agent.js';
4
+ import { assertRegistrableAgentName, RELEASED_AGENT_STATUS, releasedAgentName } from './agent.js';
5
5
  import { randomHex, sha256Hex } from '../lib/crypto.js';
6
6
  import { codedError } from '../lib/httpError.js';
7
7
  import { D1_SAFE_IN_QUERY_CHUNK_SIZE } from '../lib/queryChunks.js';
@@ -10,6 +10,7 @@ import { emitAgentExitedEffects, emitInvocationCompletionEffects, fleetInvocatio
10
10
  import { runAtomic, runAtomicWrites } from '../ports/database.js';
11
11
  import { claimSpawnNode, chooseNodeForAction, isNodeLive, releaseNodeCapacity, reserveNodeCapacity } from './placement.js';
12
12
  import { DEFAULT_PROVIDER_NAME, capacityProviderName, getProvider, isProviderLive } from './nodeProvider.js';
13
+ import { AGENT_TOKEN_HASH_PATTERN } from '@relaycast/types';
13
14
  const OPEN_INVOCATION_STATUSES = ['pending', 'dispatched', 'invoked'];
14
15
  const REPLAYED_INVOCATION = Symbol('replayed-action-invocation');
15
16
  export const ACTION_DISPATCH_TIMEOUT_MS = 30_000;
@@ -62,6 +63,42 @@ function isSpawnInvocation(actionName) {
62
63
  function isReleaseInvocation(actionName) {
63
64
  return actionName === 'release';
64
65
  }
66
+ function isBuiltinReleaseInvocation(invocation) {
67
+ return invocation.invocationOrigin === 'builtin' && isReleaseInvocation(invocation.actionName);
68
+ }
69
+ function isDeletedRegisteredActionInvocation(invocation) {
70
+ // For a registered-action origin, actionId is cleared only by the FK when its
71
+ // exact action row is deleted. The immutable origin prevents that orphan
72
+ // from being mistaken for a built-in, and this predicate prevents it from
73
+ // being rebound by name during drain/retry/reconciliation.
74
+ return invocation.invocationOrigin === 'registered_action'
75
+ && invocation.actionId === null
76
+ && invocation.providerAcceptedAttempt !== invocation.dispatchAttempts;
77
+ }
78
+ function isAcceptedDeletedRegisteredActionInvocation(invocation) {
79
+ return invocation.invocationOrigin === 'registered_action'
80
+ && invocation.actionId === null
81
+ && invocation.providerAcceptedAttempt !== null
82
+ && invocation.providerAcceptedAttempt === invocation.dispatchAttempts;
83
+ }
84
+ const RELEASE_GENERATION_CONFLICT_CODE = 'agent_release_generation_conflict';
85
+ function releaseExpectedTokenHash(input) {
86
+ const value = input.expected_token_hash;
87
+ if (value === undefined)
88
+ return null;
89
+ if (typeof value !== 'string' || !AGENT_TOKEN_HASH_PATTERN.test(value)) {
90
+ throw codedError('release action input.expected_token_hash must be a lowercase SHA-256 hash', 'invalid_release_request', 400);
91
+ }
92
+ return value;
93
+ }
94
+ function releaseGenerationStillCurrent(workspaceId, agentId, expectedTokenHash) {
95
+ return sql `EXISTS (
96
+ SELECT 1 FROM ${agents}
97
+ WHERE ${agents.workspaceId} = ${workspaceId}
98
+ AND ${agents.id} = ${agentId}
99
+ ${expectedTokenHash ? sql `AND ${agents.tokenHash} = ${expectedTokenHash}` : sql ``}
100
+ )`;
101
+ }
65
102
  function dispatchActionNameForInvocation(actionName, input) {
66
103
  if (!isSpawnInvocation(actionName))
67
104
  return actionName;
@@ -257,13 +294,16 @@ export async function registerAction(db, workspaceId, data, options = {}) {
257
294
  : { target: [actions.workspaceId, actions.name], targetWhere: isNull(actions.handlerNodeId), set: refresh })
258
295
  .returning();
259
296
  // A refresh that moves the handler to a DIFFERENT agent strands
260
- // invocations already in flight toward the previous handler: completion
261
- // authorization follows actions.handlerAgentId (the old handler's
262
- // completion would 403) and the timeout sweep would redeliver them to the
263
- // new handler, which never received the original dispatch. Terminally
264
- // fail them; the callers are told after commit.
297
+ // invocations still waiting for provider acceptance. Once the previous
298
+ // handler accepted an exact attempt, detach that invocation from the
299
+ // mutable action row instead: its snapshotted route remains authorized
300
+ // to complete and timeout recovery cannot retarget it to the replacement.
265
301
  let strandedRows = [];
266
302
  if (previous && previous.handlerAgentId && previous.handlerAgentId !== handlerAgentId) {
303
+ await tx
304
+ .update(actionInvocations)
305
+ .set({ actionId: null })
306
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.actionId, previous.id), eq(actionInvocations.invocationOrigin, 'registered_action'), isNotNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} = ${actionInvocations.dispatchAttempts}`, inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
267
307
  const strandedIds = await openInvocationIdsForActions(tx, workspaceId, [previous.id]);
268
308
  strandedRows = await failOpenInvocationRows(tx, workspaceId, strandedIds, 'handler_unavailable');
269
309
  }
@@ -322,7 +362,7 @@ export async function listActions(db, workspaceId, callerName) {
322
362
  .map(publicAction);
323
363
  }
324
364
  export async function getAction(db, workspaceId, name, callerName) {
325
- const [row] = await db
365
+ const rows = await db
326
366
  .select({
327
367
  id: actions.id,
328
368
  name: actions.name,
@@ -330,6 +370,7 @@ export async function getAction(db, workspaceId, name, callerName) {
330
370
  handlerAgentName: agents.name,
331
371
  handlerNodeName: nodes.name,
332
372
  handlerNodeId: actions.handlerNodeId,
373
+ isGlobal: actions.isGlobal,
333
374
  inputSchema: actions.inputSchema,
334
375
  outputSchema: actions.outputSchema,
335
376
  availableTo: actions.availableTo,
@@ -340,6 +381,8 @@ export async function getAction(db, workspaceId, name, callerName) {
340
381
  .leftJoin(agents, eq(actions.handlerAgentId, agents.id))
341
382
  .leftJoin(nodes, eq(actions.handlerNodeId, nodes.id))
342
383
  .where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, name)));
384
+ const row = rows.sort((a, b) => actionResolutionRank(a) - actionResolutionRank(b) ||
385
+ (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))[0];
343
386
  if (!row || !isActionVisibleToCaller(row.availableTo ?? null, callerName))
344
387
  return null;
345
388
  return publicAction(row);
@@ -378,6 +421,7 @@ async function createInvocation(db, workspaceId, action, data) {
378
421
  workspaceId,
379
422
  actionId: action?.id ?? null,
380
423
  actionName: expectedActionName,
424
+ invocationOrigin: action ? 'registered_action' : 'builtin',
381
425
  callerId: data.caller_id ?? null,
382
426
  callerName: data.caller_name ?? null,
383
427
  handlerAgentId: data.handler_agent_id ?? action?.handlerAgentId ?? null,
@@ -439,11 +483,29 @@ async function idempotentInvocationId(workspaceId, callerId, actionName, key) {
439
483
  async function waitForInvocationReplayOutcome(db, workspaceId, invocation, options = {}) {
440
484
  let current = invocation;
441
485
  const deadline = Date.now() + 500;
442
- while (!current.dispatchedNodeId
443
- && !(options.acceptDispatchStarted && current.dispatchAttempts > 0)) {
486
+ while (true) {
487
+ // A guarded release can lose its generation CAS after provider dispatch.
488
+ // Surface that durable terminal conflict before the generic dispatched-row
489
+ // replay shortcut, otherwise the same idempotency key changes 409 into 201.
490
+ if (isBuiltinReleaseInvocation(current)
491
+ && current.status === 'failed'
492
+ && current.error === RELEASE_GENERATION_CONFLICT_CODE) {
493
+ throw codedError('Action invocation failed because the release generation changed', RELEASE_GENERATION_CONFLICT_CODE, 409);
494
+ }
495
+ // Registered node actions only enter action_deleted before the provider
496
+ // accepts the frame. Surface that durable result before the route snapshot
497
+ // shortcut so the winning request and every replay return the same 503.
498
+ if (current.status === 'failed' && current.error === 'action_deleted') {
499
+ throw codedError('Action invocation failed before provider dispatch completed', 'action_deleted', 503);
500
+ }
501
+ if (current.dispatchedNodeId
502
+ || (options.acceptDispatchStarted && current.dispatchAttempts > 0)) {
503
+ return current;
504
+ }
444
505
  if (!OPEN_INVOCATION_STATUSES.some((status) => status === current.status)) {
445
506
  if (current.status === 'failed') {
446
- throw codedError('Action invocation failed before provider dispatch completed', current.error ?? 'handler_unavailable', 503);
507
+ const errorCode = current.error ?? 'handler_unavailable';
508
+ throw codedError('Action invocation failed before provider dispatch completed', errorCode, isBuiltinReleaseInvocation(current) && errorCode === RELEASE_GENERATION_CONFLICT_CODE ? 409 : 503);
447
509
  }
448
510
  return current;
449
511
  }
@@ -467,7 +529,6 @@ async function waitForInvocationReplayOutcome(db, workspaceId, invocation, optio
467
529
  }
468
530
  current = updated;
469
531
  }
470
- return current;
471
532
  }
472
533
  /**
473
534
  * Resolve a losing durable claim from the immutable invocation snapshots.
@@ -479,7 +540,7 @@ async function waitForInvocationReplayOutcome(db, workspaceId, invocation, optio
479
540
  */
480
541
  async function replayInvocationClaim(db, workspaceId, invocation) {
481
542
  if (isSpawnInvocation(invocation.actionName)
482
- || isReleaseInvocation(invocation.actionName)
543
+ || isBuiltinReleaseInvocation(invocation)
483
544
  || (!!invocation.handlerNodeId && !invocation.handlerAgentId)) {
484
545
  return waitForInvocationReplayOutcome(db, workspaceId, invocation);
485
546
  }
@@ -631,7 +692,7 @@ async function openInvocationIdsForActions(db, workspaceId, actionIds) {
631
692
  const rows = await db
632
693
  .select({ id: actionInvocations.id })
633
694
  .from(actionInvocations)
634
- .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.actionId, actionIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
695
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.actionId, actionIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`)));
635
696
  return rows.map((row) => row.id);
636
697
  }
637
698
  /**
@@ -658,12 +719,22 @@ async function dispatchNodeProviderInvocation(args, options = {}) {
658
719
  // The DB is the authoritative offline queue. Do not call sendToProvider
659
720
  // here: a connected provider whose heartbeat says handlers_live=false still
660
721
  // has a socket, and sendToProvider would deliver the frame immediately.
722
+ const expectedActionId = args.invocationOrigin === 'registered_action'
723
+ ? (args.expectedActionId ?? args.actionId)
724
+ : null;
661
725
  const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, {
662
726
  providerName: args.providerName,
663
727
  pending: true,
664
728
  reservationHeld: args.reservationHeld,
665
- actionId: args.actionId,
729
+ expectedAction: expectedActionId
730
+ ? { id: expectedActionId, name: args.action }
731
+ : undefined,
732
+ targetActionId: args.actionId ?? undefined,
666
733
  });
734
+ if (!accepted && args.invocationOrigin === 'registered_action') {
735
+ const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
736
+ return { accepted: false, pending: false, settled };
737
+ }
667
738
  return { accepted, pending: true };
668
739
  }
669
740
  return dispatchNodeInvocation({ ...args });
@@ -674,11 +745,21 @@ async function dispatchRelease(args) {
674
745
  if (!name) {
675
746
  throw codedError('release action input.name is required', 'invalid_release_request', 400);
676
747
  }
748
+ const expectedTokenHash = releaseExpectedTokenHash(input);
677
749
  const [agent] = await args.db
678
750
  .select()
679
751
  .from(agents)
680
- .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name)));
752
+ .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name), ...(expectedTokenHash ? [eq(agents.tokenHash, expectedTokenHash)] : [])));
681
753
  if (!agent) {
754
+ if (expectedTokenHash) {
755
+ const [sameName] = await args.db
756
+ .select({ id: agents.id })
757
+ .from(agents)
758
+ .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name)));
759
+ if (sameName) {
760
+ throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
761
+ }
762
+ }
682
763
  throw codedError(`Agent "${name}" not found`, 'agent_not_found', 404);
683
764
  }
684
765
  const { invocation, replayed } = await createInvocation(args.db, args.workspaceId, null, {
@@ -702,23 +783,87 @@ async function dispatchRelease(args) {
702
783
  // The release time is preserved in `metadata.release.releasedAt`.
703
784
  const releasedName = releasedAgentName(agent.name, agent.id);
704
785
  const releasedTokenHash = await sha256Hex(`released:${agent.id}:${randomHex(16)}`);
705
- const invocationIsOpen = sql `EXISTS (
786
+ const invocationCompleted = sql `EXISTS (
706
787
  SELECT 1 FROM ${actionInvocations}
707
788
  WHERE ${actionInvocations.workspaceId} = ${args.workspaceId}
708
789
  AND ${actionInvocations.id} = ${invocation.id}
709
- AND ${actionInvocations.status} IN ('pending', 'dispatched', 'invoked')
790
+ AND ${actionInvocations.status} = 'completed'
791
+ )`;
792
+ const generationStillCurrent = releaseGenerationStillCurrent(args.workspaceId, agent.id, expectedTokenHash);
793
+ const atomicExitNodeId = sql `COALESCE(
794
+ (
795
+ SELECT ${agentNodeBindings.nodeId}
796
+ FROM ${agentNodeBindings}
797
+ WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
798
+ AND ${agentNodeBindings.agentId} = ${agent.id}
799
+ AND ${agentNodeBindings.status} = 'active'
800
+ ORDER BY
801
+ CASE
802
+ WHEN ${agentNodeBindings.nodeId} = (
803
+ SELECT ${agents.locationNodeId}
804
+ FROM ${agents}
805
+ WHERE ${agents.workspaceId} = ${args.workspaceId}
806
+ AND ${agents.id} = ${agent.id}
807
+ ) THEN 0
808
+ WHEN ${agentNodeBindings.nodeId} = ${`node_direct_${agent.id}`} THEN 1
809
+ ELSE 2
810
+ END,
811
+ ${agentNodeBindings.priority} DESC,
812
+ ${agentNodeBindings.nodeId}
813
+ LIMIT 1
814
+ ),
815
+ (
816
+ SELECT ${agents.locationNodeId}
817
+ FROM ${agents}
818
+ WHERE ${agents.workspaceId} = ${args.workspaceId}
819
+ AND ${agents.id} = ${agent.id}
820
+ ),
821
+ ${exitNodeId}
710
822
  )`;
711
823
  const results = await runAtomicWrites(args.db, (writeDb) => {
712
824
  const writes = [];
713
- // Resolve the active binding in this atomic unit instead of from a
825
+ // If the optional generation lease no longer matches, settle the
826
+ // invocation as a durable conflict in the same atomic unit in which all
827
+ // lifecycle mutations are conditioned to no-op.
828
+ writes.push(writeDb
829
+ .update(actionInvocations)
830
+ .set({
831
+ status: 'failed',
832
+ error: RELEASE_GENERATION_CONFLICT_CODE,
833
+ completedAt,
834
+ })
835
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), expectedTokenHash ? sql `NOT (${generationStillCurrent})` : sql `0`))
836
+ .returning({ id: actionInvocations.id }));
837
+ // Win completion and snapshot the current binding before deactivating
838
+ // it. The returned node drives the response and agent.exited event, so a
839
+ // concurrent rebind cannot leave either pointing at the stale host read
840
+ // before this atomic unit began.
841
+ writes.push(writeDb
842
+ .update(actionInvocations)
843
+ .set({
844
+ status: 'completed',
845
+ handlerNodeId: atomicExitNodeId,
846
+ output: {
847
+ released: true,
848
+ // The roster row is retained as a tombstone so the agent's history
849
+ // keeps its author; the name is what the caller gets back.
850
+ deleted: false,
851
+ reaped_locally: true,
852
+ released_name: releasedName,
853
+ },
854
+ completedAt,
855
+ })
856
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
857
+ .returning({ id: actionInvocations.id, handlerNodeId: actionInvocations.handlerNodeId }));
858
+ // Resolve every active binding in this atomic unit instead of from a
714
859
  // pre-transaction snapshot. A concurrent rebind therefore decrements
715
- // the node that is actually deactivated below.
860
+ // exactly the node or nodes that are deactivated below.
716
861
  writes.push(writeDb
717
862
  .update(nodes)
718
863
  .set({
719
864
  activeAgents: sql `CASE WHEN ${nodes.activeAgents} > 0 THEN ${nodes.activeAgents} - 1 ELSE 0 END`,
720
865
  })
721
- .where(and(eq(nodes.workspaceId, args.workspaceId), invocationIsOpen, sql `EXISTS (
866
+ .where(and(eq(nodes.workspaceId, args.workspaceId), invocationCompleted, generationStillCurrent, sql `EXISTS (
722
867
  SELECT 1 FROM ${agentNodeBindings}
723
868
  WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
724
869
  AND ${agentNodeBindings.agentId} = ${agent.id}
@@ -728,38 +873,32 @@ async function dispatchRelease(args) {
728
873
  writes.push(writeDb
729
874
  .update(agentNodeBindings)
730
875
  .set({ status: 'inactive', updatedAt: completedAt })
731
- .where(and(eq(agentNodeBindings.workspaceId, args.workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.status, 'active'), invocationIsOpen)));
732
- // This helper is only used for delete_agent releases. Non-delete
733
- // releases fail closed when no live host can receive the invocation.
734
- //
735
- // Tombstone-rename rather than DELETE (relaycast#309). Four FKs reference
736
- // `agents.id` without `onDelete` — channels.created_by (schema.ts:455),
737
- // messages.agent_id (:503), files.uploaded_by (:666),
738
- // webhooks.created_by (:759) — so a bare DELETE is refused for any agent
739
- // that has ever spoken, and inside this atomic unit that refusal aborts
740
- // the binding update and the invocation completion along with it. Cascade
741
- // is not an option either: it would destroy the agent's message history,
742
- // and `messages.agent_id` is NOT NULL so `set null` cannot apply.
743
- //
744
- // Renaming frees the unique `(workspace_id, name)` immediately while
745
- // every FK target stays valid and every message keeps its sender.
876
+ .where(and(eq(agentNodeBindings.workspaceId, args.workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.status, 'active'), invocationCompleted, generationStillCurrent)));
877
+ // `channel_members` and `dm_participants` cascade on DELETE; the
878
+ // tombstone's UPDATE does not fire that cascade, so drop the memberships
879
+ // in the SAME atomic unit or the released agent stays a delivery target.
880
+ writes.push(writeDb
881
+ .delete(channelMembers)
882
+ .where(and(eq(channelMembers.agentId, agent.id), invocationCompleted, generationStillCurrent)));
883
+ writes.push(writeDb
884
+ .delete(dmParticipants)
885
+ .where(and(eq(dmParticipants.agentId, agent.id), invocationCompleted, generationStillCurrent)));
886
+ writes.push(writeDb
887
+ .delete(nodes)
888
+ .where(and(eq(nodes.workspaceId, args.workspaceId), eq(nodes.id, `node_direct_${agent.id}`), invocationCompleted, generationStillCurrent)));
889
+ // Tombstone after the invocation has won its completion CAS. The whole
890
+ // batch/transaction rolls back together, while this ordering lets the
891
+ // success predicate inspect the still-current token hash before we rotate
892
+ // it to an unusable released credential.
746
893
  writes.push(writeDb
747
894
  .update(agents)
748
895
  .set({
749
896
  name: releasedName,
750
897
  handle: `@${releasedName}`,
751
898
  status: RELEASED_AGENT_STATUS,
752
- // The row survives, so its credential must not. `token_hash` is
753
- // NOT NULL UNIQUE and cannot be cleared, so rotate it to a value
754
- // nobody holds; the released agent's old token stops authenticating.
755
899
  tokenHash: releasedTokenHash,
756
- // Clear the rotation grace slot too, otherwise any token issued by
757
- // the last live rotation would keep authenticating for its grace
758
- // window on an agent that is supposed to be gone. See 0035_agent_token_grace.
759
900
  previousTokenHash: null,
760
901
  previousTokenExpiresAt: null,
761
- // Same `release` shape the dispatched path writes, so an audit does
762
- // not have to know which path released the agent.
763
902
  metadata: sql `json_patch(COALESCE(${agents.metadata}, '{}'), ${JSON.stringify({
764
903
  release: {
765
904
  reason: typeof input.reason === 'string' ? input.reason : null,
@@ -768,48 +907,22 @@ async function dispatchRelease(args) {
768
907
  },
769
908
  })})`,
770
909
  })
771
- .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.id, agent.id), invocationIsOpen)));
772
- // `channel_members` and `dm_participants` cascade on DELETE; the
773
- // tombstone's UPDATE does not fire that cascade, so drop the memberships
774
- // in the SAME atomic unit or the released agent stays a delivery target.
775
- writes.push(writeDb
776
- .delete(channelMembers)
777
- .where(eq(channelMembers.agentId, agent.id)));
778
- writes.push(writeDb
779
- .delete(dmParticipants)
780
- .where(eq(dmParticipants.agentId, agent.id)));
781
- writes.push(writeDb
782
- .delete(nodes)
783
- .where(and(eq(nodes.workspaceId, args.workspaceId), eq(nodes.id, `node_direct_${agent.id}`), invocationIsOpen)));
784
- writes.push(writeDb
785
- .update(actionInvocations)
786
- .set({
787
- status: 'completed',
788
- // This is part of the same atomic unit as completion so a lost 201
789
- // replays the exact handler node returned below.
790
- handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${exitNodeId})`,
791
- output: {
792
- released: true,
793
- // The roster row is retained as a tombstone so the agent's history
794
- // keeps its author; the name is what the caller gets back.
795
- deleted: false,
796
- reaped_locally: true,
797
- released_name: releasedName,
798
- },
799
- completedAt,
800
- })
801
- .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
802
- .returning({ id: actionInvocations.id }));
910
+ .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.id, agent.id), ...(expectedTokenHash ? [eq(agents.tokenHash, expectedTokenHash)] : []), invocationCompleted)));
803
911
  return writes;
804
- });
805
- const completed = results.at(-1);
912
+ }, expectedTokenHash ? { requireAtomic: true } : undefined);
913
+ const generationConflict = results[0];
914
+ const completed = results[1];
915
+ const completedExitNodeId = completed[0]?.handlerNodeId ?? null;
916
+ if (expectedTokenHash && (generationConflict.length > 0 || completed.length === 0)) {
917
+ throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
918
+ }
806
919
  // External completion effects belong after the durable atomic unit: an
807
920
  // aborted local reap must never publish agent.exited.
808
- if (completed.length > 0 && args.completionDeps && exitNodeId) {
921
+ if (completed.length > 0 && args.completionDeps && completedExitNodeId) {
809
922
  await emitAgentExitedEffects(args.completionDeps, args.workspaceId, {
810
923
  agentId: agent.id,
811
924
  agentName: agent.name,
812
- nodeId: exitNodeId,
925
+ nodeId: completedExitNodeId,
813
926
  invocationId: fleetInvocationId(agent.metadata),
814
927
  reason: 'released',
815
928
  });
@@ -818,7 +931,7 @@ async function dispatchRelease(args) {
818
931
  invocation_id: invocation.id,
819
932
  action_name: 'release',
820
933
  handler_agent_id: null,
821
- handler_node_id: exitNodeId,
934
+ handler_node_id: completedExitNodeId,
822
935
  dispatched_node_id: null,
823
936
  input,
824
937
  status: 'completed',
@@ -826,10 +939,36 @@ async function dispatchRelease(args) {
826
939
  };
827
940
  };
828
941
  const failClosed = async () => {
829
- await args.db
830
- .update(actionInvocations)
831
- .set({ status: 'failed', error: 'agent_host_unavailable', completedAt: new Date() })
832
- .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
942
+ const completedAt = new Date();
943
+ if (expectedTokenHash) {
944
+ const generationStillCurrent = releaseGenerationStillCurrent(args.workspaceId, agent.id, expectedTokenHash);
945
+ const results = await runAtomicWrites(args.db, (writeDb) => [
946
+ writeDb
947
+ .update(actionInvocations)
948
+ .set({
949
+ status: 'failed',
950
+ error: RELEASE_GENERATION_CONFLICT_CODE,
951
+ completedAt,
952
+ })
953
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (${generationStillCurrent})`))
954
+ .returning({ id: actionInvocations.id }),
955
+ writeDb
956
+ .update(actionInvocations)
957
+ .set({ status: 'failed', error: 'agent_host_unavailable', completedAt })
958
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
959
+ .returning({ id: actionInvocations.id }),
960
+ ], { requireAtomic: true });
961
+ const generationConflict = results[0];
962
+ if (generationConflict.length > 0) {
963
+ throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
964
+ }
965
+ }
966
+ else {
967
+ await args.db
968
+ .update(actionInvocations)
969
+ .set({ status: 'failed', error: 'agent_host_unavailable', completedAt })
970
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
971
+ }
833
972
  throw codedError(`Agent "${name}" has no live host node; cannot dispatch release`, 'agent_host_unavailable', 503);
834
973
  };
835
974
  const registry = args.registry;
@@ -858,10 +997,35 @@ async function dispatchRelease(args) {
858
997
  providerName: agent.providerName,
859
998
  action: 'release',
860
999
  input,
1000
+ actionId: null,
1001
+ invocationOrigin: 'builtin',
861
1002
  });
1003
+ // Some socket owners durably record or complete an accepted frame before
1004
+ // this process resumes to stamp its own dispatch attempt. Return that
1005
+ // terminal winner instead of treating the failed post-send stamp as a
1006
+ // rejected send and entering local fallback.
1007
+ if (dispatched.settled) {
1008
+ if (dispatched.settled.status === 'failed') {
1009
+ const errorCode = dispatched.settled.error ?? 'handler_unavailable';
1010
+ throw codedError('Release invocation failed after provider dispatch', errorCode, errorCode === RELEASE_GENERATION_CONFLICT_CODE ? 409 : 503);
1011
+ }
1012
+ return invocationAck(dispatched.settled, { actionName: 'release' });
1013
+ }
862
1014
  // The provider can disconnect between the liveness check and send. Complete
863
1015
  // the DB lifecycle locally instead of creating an ownerless pending request.
864
1016
  if (!dispatched.accepted) {
1017
+ if (expectedTokenHash) {
1018
+ const [settled] = await args.db
1019
+ .select({ error: actionInvocations.error })
1020
+ .from(actionInvocations)
1021
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id)));
1022
+ if (settled?.error === RELEASE_GENERATION_CONFLICT_CODE) {
1023
+ throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
1024
+ }
1025
+ if (settled?.error === 'node_dispatch_unavailable') {
1026
+ throw codedError('Node adapter does not support generation-authorized release dispatch', 'node_dispatch_unavailable', 503);
1027
+ }
1028
+ }
865
1029
  return input.delete_agent === true ? completeLocally() : failClosed();
866
1030
  }
867
1031
  return {
@@ -875,6 +1039,21 @@ async function dispatchRelease(args) {
875
1039
  created_at: invocation.createdAt.toISOString(),
876
1040
  };
877
1041
  }
1042
+ /**
1043
+ * Agent-lifecycle release used by `POST /v1/agents/release`. Unlike the
1044
+ * generic action endpoint, this path must always enforce the built-in release
1045
+ * lifecycle and its optional generation guard; a user-defined `release`
1046
+ * action must not be able to shadow it.
1047
+ */
1048
+ export async function dispatchAgentRelease(db, workspaceId, data, options = {}) {
1049
+ return dispatchRelease({
1050
+ db,
1051
+ registry: options.nodeConnections,
1052
+ completionDeps: options.completionDeps,
1053
+ workspaceId,
1054
+ data,
1055
+ });
1056
+ }
878
1057
  function spawnResult(invocation, nodeId, dispatched) {
879
1058
  return {
880
1059
  invocation_id: invocation.id,
@@ -892,6 +1071,11 @@ async function dispatchSpawn(args) {
892
1071
  throw codedError('Node dispatch is not available', 'node_dispatch_unavailable', 503);
893
1072
  }
894
1073
  const input = recordInput(args.data.input);
1074
+ const requestedName = typeof input.name === 'string' ? input.name : '';
1075
+ if (requestedName.trim().length === 0) {
1076
+ throw codedError('spawn action input.name is required', 'invalid_spawn_request', 400);
1077
+ }
1078
+ assertRegistrableAgentName(requestedName);
895
1079
  const capability = dispatchActionNameForInvocation('spawn', input);
896
1080
  const { invocation, replayed } = await createInvocation(args.db, args.workspaceId, null, {
897
1081
  input: args.data.input,
@@ -937,20 +1121,40 @@ async function dispatchSpawn(args) {
937
1121
  if (!args.bypassShadow) {
938
1122
  const shadow = await fetchNodeAction(args.db, args.workspaceId, nodeId, capability);
939
1123
  if (shadow && shadow.handlerProvider) {
940
- if (!placement.queued)
941
- await releaseNodeCapacity(args.db, args.workspaceId, nodeId);
942
- const dispatched = await dispatchNodeProviderInvocation({
943
- db: args.db,
944
- registry: args.registry,
1124
+ const bound = await bindInvocationToRegisteredNodeAction(args.db, {
945
1125
  workspaceId: args.workspaceId,
946
1126
  invocationId: invocation.id,
947
1127
  nodeId,
948
1128
  providerName: shadow.handlerProvider,
949
- action: capability,
950
- input,
951
- queue: shadow.queue,
1129
+ actionId: shadow.id,
1130
+ actionName: capability,
952
1131
  });
953
- return spawnResult(invocation, nodeId, dispatched);
1132
+ if (bound) {
1133
+ if (!placement.queued)
1134
+ await releaseNodeCapacity(args.db, args.workspaceId, nodeId);
1135
+ const dispatched = await dispatchNodeProviderInvocation({
1136
+ db: args.db,
1137
+ registry: args.registry,
1138
+ workspaceId: args.workspaceId,
1139
+ invocationId: invocation.id,
1140
+ nodeId,
1141
+ providerName: shadow.handlerProvider,
1142
+ action: capability,
1143
+ input,
1144
+ queue: shadow.queue,
1145
+ actionId: shadow.id,
1146
+ expectedActionId: shadow.id,
1147
+ invocationOrigin: 'registered_action',
1148
+ });
1149
+ if (dispatched.settled) {
1150
+ const settled = await waitForInvocationReplayOutcome(args.db, args.workspaceId, dispatched.settled);
1151
+ return invocationAck(settled, { actionName: 'spawn', handlerNodeId: nodeId });
1152
+ }
1153
+ return spawnResult(invocation, nodeId, dispatched);
1154
+ }
1155
+ // The shadow disappeared before its exact identity could be claimed.
1156
+ // Keep the native reservation and continue through native capacity; a
1157
+ // stale shadow snapshot must never receive an unbound invocation.
954
1158
  }
955
1159
  }
956
1160
  const capProvider = (await capacityProviderName(args.db, args.workspaceId, nodeId, capability)) ?? DEFAULT_PROVIDER_NAME;
@@ -963,6 +1167,8 @@ async function dispatchSpawn(args) {
963
1167
  providerName: capProvider,
964
1168
  action: capability,
965
1169
  input,
1170
+ actionId: null,
1171
+ invocationOrigin: 'builtin',
966
1172
  pending: placement.queued,
967
1173
  reservationHeld: !placement.queued,
968
1174
  });
@@ -1031,8 +1237,14 @@ export async function invokeNodeAction(db, workspaceId, nodeId, actionName, data
1031
1237
  action: action.name,
1032
1238
  input: recordInput(invocation.input),
1033
1239
  queue: action.queue,
1240
+ actionId: action.id,
1241
+ invocationOrigin: 'registered_action',
1034
1242
  reservationHeld: !!reservedNode,
1035
1243
  });
1244
+ if (dispatched.settled) {
1245
+ const settled = await waitForInvocationReplayOutcome(db, workspaceId, dispatched.settled);
1246
+ return invocationAck(settled, { actionName, handlerNodeId: nodeId });
1247
+ }
1036
1248
  return {
1037
1249
  invocation_id: invocation.id,
1038
1250
  action_name: actionName,
@@ -1127,8 +1339,14 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
1127
1339
  action: action.name,
1128
1340
  input: recordInput(invocation.input),
1129
1341
  queue: action.queue || providerName === DEFAULT_PROVIDER_NAME,
1342
+ actionId: action.id,
1343
+ invocationOrigin: 'registered_action',
1130
1344
  reservationHeld: !!reservedNode,
1131
1345
  });
1346
+ if (dispatched.settled) {
1347
+ const settled = await waitForInvocationReplayOutcome(db, workspaceId, dispatched.settled);
1348
+ return invocationAck(settled, { actionName, handlerNodeId: action.handlerNodeId });
1349
+ }
1132
1350
  return {
1133
1351
  invocation_id: invocation.id,
1134
1352
  action_name: actionName,
@@ -1211,6 +1429,7 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
1211
1429
  input: recordInput(invocation.input),
1212
1430
  agent: { id: handlerAgent.id, name: handlerAgent.name },
1213
1431
  actionId: action.id,
1432
+ invocationOrigin: 'registered_action',
1214
1433
  });
1215
1434
  if (dispatched.sent && !dispatched.accepted) {
1216
1435
  // The provider owns the frame once the send returns true. A fast completion
@@ -1295,8 +1514,196 @@ function publicInvocation(row) {
1295
1514
  completed_at: row.completedAt?.toISOString() ?? null,
1296
1515
  };
1297
1516
  }
1517
+ async function completeGuardedReleaseNodeInvocation(db, workspaceId, nodeId, providerName, invocation, data, deps) {
1518
+ const input = recordInput(invocation.input);
1519
+ const name = typeof input.name === 'string' ? input.name : null;
1520
+ const persistedTokenHash = input.expected_token_hash;
1521
+ const expectedTokenHash = typeof persistedTokenHash === 'string'
1522
+ && AGENT_TOKEN_HASH_PATTERN.test(persistedTokenHash)
1523
+ ? persistedTokenHash
1524
+ : null;
1525
+ if (!name || !expectedTokenHash) {
1526
+ const [failed] = await db
1527
+ .update(actionInvocations)
1528
+ .set({
1529
+ status: 'failed',
1530
+ error: RELEASE_GENERATION_CONFLICT_CODE,
1531
+ completedAt: new Date(),
1532
+ spawnReservedAt: null,
1533
+ })
1534
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
1535
+ .returning();
1536
+ return failed ? publicInvocation(failed) : null;
1537
+ }
1538
+ const [agent] = await db
1539
+ .select()
1540
+ .from(agents)
1541
+ .where(and(eq(agents.workspaceId, workspaceId), eq(agents.name, name)));
1542
+ if (!agent) {
1543
+ const [failed] = await db
1544
+ .update(actionInvocations)
1545
+ .set({
1546
+ status: 'failed',
1547
+ error: RELEASE_GENERATION_CONFLICT_CODE,
1548
+ completedAt: new Date(),
1549
+ spawnReservedAt: null,
1550
+ })
1551
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
1552
+ .returning();
1553
+ return failed ? publicInvocation(failed) : null;
1554
+ }
1555
+ const completedAt = new Date();
1556
+ const generationStillCurrent = releaseGenerationStillCurrent(workspaceId, agent.id, expectedTokenHash);
1557
+ const activeBindingStillCurrent = sql `EXISTS (
1558
+ SELECT 1 FROM ${agentNodeBindings}
1559
+ WHERE ${agentNodeBindings.workspaceId} = ${workspaceId}
1560
+ AND ${agentNodeBindings.agentId} = ${agent.id}
1561
+ AND ${agentNodeBindings.nodeId} = ${nodeId}
1562
+ AND ${agentNodeBindings.status} = 'active'
1563
+ )`;
1564
+ const invocationIsOpen = sql `EXISTS (
1565
+ SELECT 1 FROM ${actionInvocations}
1566
+ WHERE ${actionInvocations.workspaceId} = ${workspaceId}
1567
+ AND ${actionInvocations.id} = ${invocation.id}
1568
+ AND ${actionInvocations.dispatchedNodeId} = ${nodeId}
1569
+ AND ${actionInvocations.dispatchedProvider} = ${providerName}
1570
+ AND ${actionInvocations.status} IN ('pending', 'dispatched', 'invoked')
1571
+ )`;
1572
+ const invocationCompleted = sql `EXISTS (
1573
+ SELECT 1 FROM ${actionInvocations}
1574
+ WHERE ${actionInvocations.workspaceId} = ${workspaceId}
1575
+ AND ${actionInvocations.id} = ${invocation.id}
1576
+ AND ${actionInvocations.dispatchedNodeId} = ${nodeId}
1577
+ AND ${actionInvocations.dispatchedProvider} = ${providerName}
1578
+ AND ${actionInvocations.status} = 'completed'
1579
+ )`;
1580
+ const releaseCanApply = sql `(${generationStillCurrent}) AND (${activeBindingStillCurrent})`;
1581
+ const releasedName = releasedAgentName(agent.name, agent.id);
1582
+ const releasedTokenHash = await sha256Hex(`released:${agent.id}:${randomHex(16)}`);
1583
+ let successIndex = -1;
1584
+ const results = await runAtomicWrites(db, (writeDb) => {
1585
+ const writes = [];
1586
+ // The conflict settlement comes first. In a sequential SQLite batch it
1587
+ // closes the invocation before every mutation below if either the token
1588
+ // generation or node binding changed; on transactional adapters the same
1589
+ // statement ordering and rollback guarantee apply.
1590
+ writes.push(writeDb
1591
+ .update(actionInvocations)
1592
+ .set({
1593
+ status: 'failed',
1594
+ error: RELEASE_GENERATION_CONFLICT_CODE,
1595
+ completedAt,
1596
+ spawnReservedAt: null,
1597
+ })
1598
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (${releaseCanApply})`))
1599
+ .returning());
1600
+ writes.push(writeDb
1601
+ .update(nodes)
1602
+ .set({
1603
+ activeAgents: sql `CASE WHEN ${nodes.activeAgents} > 0 THEN ${nodes.activeAgents} - 1 ELSE 0 END`,
1604
+ })
1605
+ .where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId), invocationIsOpen, generationStillCurrent, activeBindingStillCurrent)));
1606
+ writes.push(writeDb
1607
+ .update(agentNodeBindings)
1608
+ .set({ status: 'inactive', updatedAt: completedAt })
1609
+ .where(and(eq(agentNodeBindings.workspaceId, workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.nodeId, nodeId), eq(agentNodeBindings.status, 'active'), invocationIsOpen, generationStillCurrent)));
1610
+ if (input.delete_agent === true) {
1611
+ writes.push(writeDb
1612
+ .delete(channelMembers)
1613
+ .where(and(eq(channelMembers.agentId, agent.id), invocationIsOpen, generationStillCurrent)));
1614
+ writes.push(writeDb
1615
+ .delete(dmParticipants)
1616
+ .where(and(eq(dmParticipants.agentId, agent.id), invocationIsOpen, generationStillCurrent)));
1617
+ }
1618
+ successIndex = writes.length;
1619
+ writes.push(writeDb
1620
+ .update(actionInvocations)
1621
+ .set({
1622
+ output: data.output ?? null,
1623
+ error: null,
1624
+ status: 'completed',
1625
+ completedAt,
1626
+ spawnReservedAt: null,
1627
+ })
1628
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
1629
+ .returning());
1630
+ const currentAgent = and(eq(agents.workspaceId, workspaceId), eq(agents.id, agent.id), eq(agents.name, name), eq(agents.tokenHash, expectedTokenHash), invocationCompleted);
1631
+ if (input.delete_agent === true) {
1632
+ writes.push(writeDb
1633
+ .update(agents)
1634
+ .set({
1635
+ name: releasedName,
1636
+ handle: `@${releasedName}`,
1637
+ status: RELEASED_AGENT_STATUS,
1638
+ tokenHash: releasedTokenHash,
1639
+ previousTokenHash: null,
1640
+ previousTokenExpiresAt: null,
1641
+ locationType: 'self_connected',
1642
+ locationNodeId: null,
1643
+ lastSeen: completedAt,
1644
+ metadata: sql `json_patch(COALESCE(${agents.metadata}, '{}'), ${JSON.stringify({
1645
+ release: {
1646
+ reason: typeof input.reason === 'string' ? input.reason : null,
1647
+ released_at: completedAt.toISOString(),
1648
+ previous_name: agent.name,
1649
+ },
1650
+ })})`,
1651
+ })
1652
+ .where(currentAgent));
1653
+ }
1654
+ else {
1655
+ const existingMetadata = agent.metadata ?? {};
1656
+ const { spawn: _spawn, cli: _cli, ...restMetadata } = existingMetadata;
1657
+ writes.push(writeDb
1658
+ .update(agents)
1659
+ .set({
1660
+ status: 'offline',
1661
+ locationType: 'self_connected',
1662
+ locationNodeId: null,
1663
+ lastSeen: completedAt,
1664
+ metadata: {
1665
+ ...restMetadata,
1666
+ release: {
1667
+ reason: typeof input.reason === 'string' ? input.reason : null,
1668
+ released_at: completedAt.toISOString(),
1669
+ },
1670
+ },
1671
+ })
1672
+ .where(currentAgent));
1673
+ }
1674
+ if (input.delete_agent === true) {
1675
+ // Deleting the implicit node clears action_invocations.dispatched_node_id
1676
+ // via its FK. Do this only after the owned completion and agent CAS have
1677
+ // both succeeded, or their node predicates could no longer match.
1678
+ writes.push(writeDb
1679
+ .delete(nodes)
1680
+ .where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, `node_direct_${agent.id}`), sql `EXISTS (
1681
+ SELECT 1 FROM ${actionInvocations}
1682
+ WHERE ${actionInvocations.workspaceId} = ${workspaceId}
1683
+ AND ${actionInvocations.id} = ${invocation.id}
1684
+ AND ${actionInvocations.status} = 'completed'
1685
+ )`)));
1686
+ }
1687
+ return writes;
1688
+ }, { requireAtomic: true });
1689
+ const completed = results[successIndex];
1690
+ const generationConflict = results[0];
1691
+ const settled = completed[0] ?? generationConflict[0];
1692
+ if (!settled)
1693
+ return null;
1694
+ if (completed[0] && deps) {
1695
+ await emitAgentExitedEffects(deps, workspaceId, {
1696
+ agentId: agent.id,
1697
+ agentName: agent.name,
1698
+ nodeId,
1699
+ invocationId: fleetInvocationId(agent.metadata),
1700
+ reason: 'released',
1701
+ });
1702
+ }
1703
+ return publicInvocation(settled);
1704
+ }
1298
1705
  async function applyReleaseCompletionEffect(db, workspaceId, nodeId, invocation, data, deps, options = {}) {
1299
- if (!isReleaseInvocation(invocation.actionName) || data.error)
1706
+ if (!isBuiltinReleaseInvocation(invocation) || data.error)
1300
1707
  return false;
1301
1708
  const input = recordInput(invocation.input);
1302
1709
  const name = typeof input.name === 'string' ? input.name : null;
@@ -1424,13 +1831,137 @@ async function dispatchNodeAttempt(db, workspaceId, invocationId, nodeId, opts)
1424
1831
  dispatchedNodeId: nodeId,
1425
1832
  dispatchedProvider: opts.providerName,
1426
1833
  spawnReservedAt: opts.reservationHeld ? new Date() : null,
1427
- ...(opts.actionId ? { actionId: opts.actionId } : {}),
1834
+ ...(opts.targetActionId ? { actionId: opts.targetActionId } : {}),
1428
1835
  ...attemptFields,
1429
1836
  })
1430
- .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
1837
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedAction ? [
1838
+ eq(actionInvocations.invocationOrigin, 'registered_action'),
1839
+ eq(actionInvocations.actionId, opts.expectedAction.id),
1840
+ sql `EXISTS (
1841
+ SELECT 1 FROM ${actions}
1842
+ WHERE ${actions.workspaceId} = ${workspaceId}
1843
+ AND ${actions.id} = ${opts.expectedAction.id}
1844
+ )`,
1845
+ ...(opts.targetActionId || opts.expectedAction.handlerAgentId ? [
1846
+ registeredActionDispatchPredicate({
1847
+ workspaceId,
1848
+ nodeId,
1849
+ providerName: opts.providerName,
1850
+ actionId: opts.targetActionId ?? opts.expectedAction.id,
1851
+ actionName: opts.expectedAction.name,
1852
+ handlerAgentId: opts.expectedAction.handlerAgentId,
1853
+ }),
1854
+ ] : []),
1855
+ ] : [])))
1431
1856
  .returning();
1432
1857
  return !!updated;
1433
1858
  }
1859
+ function registeredActionDispatchPredicate(args) {
1860
+ const handlerPredicate = args.handlerAgentId
1861
+ ? sql `AND ${actions.handlerAgentId} = ${args.handlerAgentId} AND ${actions.handlerNodeId} IS NULL`
1862
+ : sql `AND ${actions.handlerNodeId} = ${args.nodeId}
1863
+ AND COALESCE(${actions.handlerProvider}, ${DEFAULT_PROVIDER_NAME}) = ${args.providerName}`;
1864
+ return sql `EXISTS (
1865
+ SELECT 1 FROM ${actions}
1866
+ WHERE ${actions.workspaceId} = ${args.workspaceId}
1867
+ AND ${actions.id} = ${args.actionId}
1868
+ AND ${actions.name} = ${args.actionName}
1869
+ AND ${actions.isActive} = 1
1870
+ ${handlerPredicate}
1871
+ )`;
1872
+ }
1873
+ async function claimRegisteredActionHandoff(db, args) {
1874
+ const claimedActionId = args.targetActionId ?? args.expectedActionId;
1875
+ const stateFields = args.pending
1876
+ ? { status: 'pending', dispatchedAt: null, retryAfterAt: args.retryAfterAt ?? null }
1877
+ : dispatchedStateFields({ retryAfterAt: args.retryAfterAt });
1878
+ const attemptFields = args.skipIncrementAttempts
1879
+ ? {}
1880
+ : {
1881
+ attemptedNodeIds: sql `json_insert(COALESCE(${actionInvocations.attemptedNodeIds}, '[]'), '$[#]', ${args.nodeId})`,
1882
+ dispatchAttempts: sql `COALESCE(${actionInvocations.dispatchAttempts}, 0) + 1`,
1883
+ };
1884
+ const [claimed] = await db
1885
+ .update(actionInvocations)
1886
+ // This CAS is the durable dispatch linearization point. Claim the exact
1887
+ // action generation, route, and attempt before awaiting the provider. A
1888
+ // retry may hand off between same-name registrations only while it owns the
1889
+ // source generation; a later rescheduler observes the new route instead of
1890
+ // sending the same attempt again.
1891
+ .set({
1892
+ ...stateFields,
1893
+ handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${args.nodeId})`,
1894
+ dispatchedNodeId: args.nodeId,
1895
+ dispatchedProvider: args.providerName,
1896
+ spawnReservedAt: args.reservationHeld ? new Date() : null,
1897
+ actionId: claimedActionId,
1898
+ ...attemptFields,
1899
+ })
1900
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.expectedActionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (
1901
+ ${actionInvocations.actionId} = ${claimedActionId}
1902
+ AND COALESCE(${actionInvocations.dispatchedNodeId}, '') = ${args.nodeId}
1903
+ AND COALESCE(${actionInvocations.dispatchedProvider}, ${DEFAULT_PROVIDER_NAME}) = ${args.providerName}
1904
+ AND ${actionInvocations.status} IN ('dispatched', 'invoked')
1905
+ )`, sql `EXISTS (
1906
+ SELECT 1 FROM ${actions}
1907
+ WHERE ${actions.workspaceId} = ${args.workspaceId}
1908
+ AND ${actions.id} = ${args.expectedActionId}
1909
+ )`, ...(args.targetActionId ? [registeredActionDispatchPredicate({
1910
+ workspaceId: args.workspaceId,
1911
+ nodeId: args.nodeId,
1912
+ providerName: args.providerName,
1913
+ actionId: args.targetActionId,
1914
+ actionName: args.actionName,
1915
+ })] : [])))
1916
+ .returning({
1917
+ actionId: actionInvocations.actionId,
1918
+ actionName: actionInvocations.actionName,
1919
+ dispatchAttempts: actionInvocations.dispatchAttempts,
1920
+ });
1921
+ return claimed?.actionId ? {
1922
+ actionId: claimed.actionId,
1923
+ actionName: claimed.actionName,
1924
+ dispatchAttempts: claimed.dispatchAttempts,
1925
+ } : null;
1926
+ }
1927
+ async function bindInvocationToRegisteredNodeAction(db, args) {
1928
+ const [bound] = await db
1929
+ .update(actionInvocations)
1930
+ .set({ actionId: args.actionId, invocationOrigin: 'registered_action' })
1931
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'builtin'), isNull(actionInvocations.actionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), eq(actionInvocations.dispatchAttempts, 0), registeredActionDispatchPredicate(args)))
1932
+ .returning({ id: actionInvocations.id });
1933
+ return !!bound;
1934
+ }
1935
+ async function settleDeletedRegisteredAction(db, workspaceId, invocationId) {
1936
+ // A failed handoff can also mean another dispatcher already advanced the
1937
+ // same invocation to a different live action. Fail only the FK-null state
1938
+ // that proves pruning won; never clobber that concurrent dispatch winner.
1939
+ const [failed] = await db
1940
+ .update(actionInvocations)
1941
+ .set({
1942
+ status: 'failed',
1943
+ error: 'action_deleted',
1944
+ completedAt: new Date(),
1945
+ spawnReservedAt: null,
1946
+ })
1947
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), isNull(actionInvocations.actionId), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
1948
+ .returning();
1949
+ if (failed)
1950
+ return failed;
1951
+ const [settled] = await db
1952
+ .select()
1953
+ .from(actionInvocations)
1954
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId)));
1955
+ return settled;
1956
+ }
1957
+ async function settleRegisteredActionSendFailure(db, args) {
1958
+ const [failed] = await db
1959
+ .update(actionInvocations)
1960
+ .set({ status: 'failed', error: 'node_dispatch_unavailable', completedAt: new Date() })
1961
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.actionId), eq(actionInvocations.dispatchedNodeId, args.nodeId), eq(actionInvocations.dispatchedProvider, args.providerName), eq(actionInvocations.dispatchAttempts, args.dispatchAttempts), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
1962
+ .returning();
1963
+ return failed ?? settleDeletedRegisteredAction(db, args.workspaceId, args.invocationId);
1964
+ }
1434
1965
  async function snapshotInvocationHandlerNode(db, workspaceId, invocationId, nodeId) {
1435
1966
  const [updated] = await db
1436
1967
  .update(actionInvocations)
@@ -1460,6 +1991,44 @@ export async function markDrainedInvocationDispatched(db, workspaceId, invocatio
1460
1991
  return !!updated;
1461
1992
  }
1462
1993
  async function dispatchNodeInvocation(args) {
1994
+ const guardedReleaseHash = args.invocationOrigin === 'builtin' && isReleaseInvocation(args.action)
1995
+ ? releaseExpectedTokenHash(args.input)
1996
+ : null;
1997
+ if (guardedReleaseHash) {
1998
+ const name = typeof args.input.name === 'string' ? args.input.name : '';
1999
+ const [current] = await args.db
2000
+ .select({ id: agents.id })
2001
+ .from(agents)
2002
+ .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name), eq(agents.tokenHash, guardedReleaseHash), sql `EXISTS (
2003
+ SELECT 1 FROM ${agentNodeBindings}
2004
+ WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
2005
+ AND ${agentNodeBindings.agentId} = ${agents.id}
2006
+ AND ${agentNodeBindings.nodeId} = ${args.nodeId}
2007
+ AND ${agentNodeBindings.status} = 'active'
2008
+ )`));
2009
+ if (!current) {
2010
+ await args.db
2011
+ .update(actionInvocations)
2012
+ .set({
2013
+ status: 'failed',
2014
+ error: RELEASE_GENERATION_CONFLICT_CODE,
2015
+ completedAt: new Date(),
2016
+ })
2017
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
2018
+ return { accepted: false, pending: false, sent: false };
2019
+ }
2020
+ if (!args.registry.sendAuthorizedActionToProvider) {
2021
+ await args.db
2022
+ .update(actionInvocations)
2023
+ .set({
2024
+ status: 'failed',
2025
+ error: 'node_dispatch_unavailable',
2026
+ completedAt: new Date(),
2027
+ })
2028
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
2029
+ return { accepted: false, pending: false, sent: false };
2030
+ }
2031
+ }
1463
2032
  const frame = {
1464
2033
  v: 1,
1465
2034
  type: 'action.invoke',
@@ -1477,6 +2046,32 @@ async function dispatchNodeInvocation(args) {
1477
2046
  if (!snapshotted)
1478
2047
  return { accepted: false, pending: false, sent: false };
1479
2048
  const connectedBefore = args.registry.isProviderConnected(args.workspaceId, args.nodeId, args.providerName);
2049
+ const pending = !!args.pending || !connectedBefore;
2050
+ const expectedActionId = args.invocationOrigin === 'registered_action'
2051
+ ? (args.expectedActionId ?? args.actionId)
2052
+ : null;
2053
+ let registeredNodeClaim = null;
2054
+ if (args.invocationOrigin === 'registered_action' && !args.agent) {
2055
+ registeredNodeClaim = expectedActionId
2056
+ ? await claimRegisteredActionHandoff(args.db, {
2057
+ workspaceId: args.workspaceId,
2058
+ invocationId: args.invocationId,
2059
+ nodeId: args.nodeId,
2060
+ providerName: args.providerName,
2061
+ expectedActionId,
2062
+ targetActionId: args.actionId ?? undefined,
2063
+ actionName: args.action,
2064
+ pending,
2065
+ retryAfterAt: args.retryAfterAt,
2066
+ reservationHeld: args.reservationHeld,
2067
+ skipIncrementAttempts: args.skipIncrementAttempts,
2068
+ })
2069
+ : null;
2070
+ if (!registeredNodeClaim) {
2071
+ const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
2072
+ return { accepted: false, pending: false, sent: false, settled };
2073
+ }
2074
+ }
1480
2075
  const sent = args.agent && args.actionId
1481
2076
  ? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
1482
2077
  kind: 'agent-action-v1',
@@ -1485,18 +2080,88 @@ async function dispatchNodeInvocation(args) {
1485
2080
  handlerAgentId: args.agent.id,
1486
2081
  recordAttempt: !args.skipIncrementAttempts,
1487
2082
  }) ?? false)
1488
- : await args.registry.sendToProvider(args.workspaceId, args.nodeId, args.providerName, frame);
1489
- if (!sent)
2083
+ : registeredNodeClaim
2084
+ ? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
2085
+ kind: 'registered-node-action-v2',
2086
+ invocationId: args.invocationId,
2087
+ actionId: registeredNodeClaim.actionId,
2088
+ dispatchAttempt: registeredNodeClaim.dispatchAttempts,
2089
+ invocationActionName: registeredNodeClaim.actionName,
2090
+ actionName: args.action,
2091
+ }) ?? false)
2092
+ : guardedReleaseHash
2093
+ ? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
2094
+ kind: 'release-generation-v1',
2095
+ invocationId: args.invocationId,
2096
+ agentName: typeof args.input.name === 'string' ? args.input.name : '',
2097
+ expectedTokenHash: guardedReleaseHash,
2098
+ }) ?? false)
2099
+ : await args.registry.sendToProvider(args.workspaceId, args.nodeId, args.providerName, frame);
2100
+ if (!sent && guardedReleaseHash) {
2101
+ // An adapter compiled against the older owner-authorization contract may
2102
+ // expose the method but reject the new proof kind. Settle any still-open
2103
+ // row before the caller considers local fallback, so a guarded delete can
2104
+ // never degrade to an unguarded lifecycle mutation.
2105
+ await args.db
2106
+ .update(actionInvocations)
2107
+ .set({
2108
+ status: 'failed',
2109
+ error: 'node_dispatch_unavailable',
2110
+ completedAt: new Date(),
2111
+ })
2112
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
2113
+ }
2114
+ if (!sent) {
2115
+ if (registeredNodeClaim) {
2116
+ const settled = await settleRegisteredActionSendFailure(args.db, {
2117
+ workspaceId: args.workspaceId,
2118
+ invocationId: args.invocationId,
2119
+ nodeId: args.nodeId,
2120
+ providerName: args.providerName,
2121
+ actionId: registeredNodeClaim.actionId,
2122
+ dispatchAttempts: registeredNodeClaim.dispatchAttempts,
2123
+ });
2124
+ return { accepted: false, pending: false, sent: false, settled };
2125
+ }
1490
2126
  return { accepted: false, pending: false, sent: false };
1491
- const pending = !!args.pending || !connectedBefore;
2127
+ }
2128
+ // Registered node-action state was committed by the pre-send handoff CAS.
2129
+ // Pruning after provider acceptance may clear the action FK, but it cannot
2130
+ // revoke work the provider already received or prevent route-owned completion.
2131
+ if (registeredNodeClaim) {
2132
+ return { accepted: true, pending, sent: true };
2133
+ }
2134
+ const finalExpectedActionId = expectedActionId;
1492
2135
  const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, {
1493
2136
  pending,
1494
2137
  providerName: args.providerName,
1495
2138
  retryAfterAt: args.retryAfterAt,
1496
2139
  reservationHeld: args.reservationHeld,
1497
2140
  skipIncrementAttempts: args.skipIncrementAttempts || !!args.agent,
1498
- actionId: args.actionId,
2141
+ expectedAction: finalExpectedActionId
2142
+ ? {
2143
+ id: finalExpectedActionId,
2144
+ name: args.action,
2145
+ ...(args.agent ? { handlerAgentId: args.agent.id } : {}),
2146
+ }
2147
+ : undefined,
2148
+ targetActionId: args.agent || args.actionId
2149
+ ? finalExpectedActionId ?? undefined
2150
+ : undefined,
1499
2151
  });
2152
+ if (!accepted && args.invocationOrigin === 'registered_action') {
2153
+ const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
2154
+ return { accepted: false, pending: false, sent: true, settled };
2155
+ }
2156
+ if (!accepted && guardedReleaseHash) {
2157
+ const [settled] = await args.db
2158
+ .select()
2159
+ .from(actionInvocations)
2160
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId)));
2161
+ if (settled && !OPEN_INVOCATION_STATUSES.some((status) => status === settled.status)) {
2162
+ return { accepted: false, pending: false, sent: true, settled };
2163
+ }
2164
+ }
1500
2165
  return { accepted, pending, sent: true };
1501
2166
  }
1502
2167
  function attemptedNodeSet(invocation) {
@@ -1551,12 +2216,16 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
1551
2216
  const rows = await db
1552
2217
  .select({
1553
2218
  id: actionInvocations.id,
1554
- actionId: actions.id,
2219
+ actionId: actionInvocations.actionId,
2220
+ invocationOrigin: actionInvocations.invocationOrigin,
1555
2221
  workspaceId: actionInvocations.workspaceId,
1556
2222
  actionName: actionInvocations.actionName,
1557
2223
  input: actionInvocations.input,
2224
+ providerAcceptedAttempt: actionInvocations.providerAcceptedAttempt,
2225
+ dispatchAttempts: actionInvocations.dispatchAttempts,
1558
2226
  spawnReservedAt: actionInvocations.spawnReservedAt,
1559
2227
  dispatchedNodeId: actionInvocations.dispatchedNodeId,
2228
+ dispatchedProvider: actionInvocations.dispatchedProvider,
1560
2229
  handlerNodeId: actions.handlerNodeId,
1561
2230
  handlerProvider: actions.handlerProvider,
1562
2231
  handlerAgentId: agents.id,
@@ -1574,6 +2243,10 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
1574
2243
  .orderBy(asc(actionInvocations.createdAt));
1575
2244
  let drained = 0;
1576
2245
  for (const row of rows) {
2246
+ if (isDeletedRegisteredActionInvocation(row)) {
2247
+ await failOpenInvocationRows(db, workspaceId, [row.id], 'action_deleted');
2248
+ continue;
2249
+ }
1577
2250
  const targetAgent = row.handlerAgentId
1578
2251
  && row.handlerAgentName
1579
2252
  && row.handlerAgentLocationType === 'via_node'
@@ -1596,10 +2269,12 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
1596
2269
  ? row.handlerAgentProvider ?? DEFAULT_PROVIDER_NAME
1597
2270
  : row.handlerNodeId
1598
2271
  ? row.handlerProvider ?? DEFAULT_PROVIDER_NAME
1599
- : shadowProvider
1600
- ?? (isSpawnInvocation(row.actionName)
1601
- ? (await capacityProviderName(db, workspaceId, nodeId, dispatchActionNameForInvocation(row.actionName, input))) ?? DEFAULT_PROVIDER_NAME
1602
- : DEFAULT_PROVIDER_NAME);
2272
+ : isBuiltinReleaseInvocation(row)
2273
+ ? row.dispatchedProvider ?? DEFAULT_PROVIDER_NAME
2274
+ : shadowProvider
2275
+ ?? (isSpawnInvocation(row.actionName)
2276
+ ? (await capacityProviderName(db, workspaceId, nodeId, dispatchActionNameForInvocation(row.actionName, input))) ?? DEFAULT_PROVIDER_NAME
2277
+ : DEFAULT_PROVIDER_NAME);
1603
2278
  const nativeSpawn = isSpawnInvocation(row.actionName) && !shadowProvider;
1604
2279
  // Skip draining to a provider that is disconnected or whose heartbeat says
1605
2280
  // handlers are not live. A missing provider row is a legacy single-socket
@@ -1641,7 +2316,9 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
1641
2316
  action: dispatchActionNameForInvocation(row.actionName, input),
1642
2317
  input,
1643
2318
  agent: targetAgent ? { id: targetAgent.id, name: targetAgent.name } : null,
1644
- actionId: targetAgent ? row.actionId ?? undefined : undefined,
2319
+ actionId: row.actionId,
2320
+ expectedActionId: row.actionId,
2321
+ invocationOrigin: row.invocationOrigin,
1645
2322
  retryAfterAt: new Date(Date.now() + ACTION_DISPATCH_TIMEOUT_MS),
1646
2323
  reservationHeld,
1647
2324
  skipIncrementAttempts: row.dispatchedNodeId === nodeId,
@@ -1663,6 +2340,17 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
1663
2340
  return drained;
1664
2341
  }
1665
2342
  export async function rescheduleNodeInvocation(db, registry, invocation, opts = {}) {
2343
+ // Once the exact provider accepted this generation, pruning only clears its
2344
+ // materialized action FK. It must not make timeout, disconnect, or inventory
2345
+ // recovery move the invocation to a same-name replacement and invalidate the
2346
+ // original provider's result.
2347
+ if (isAcceptedDeletedRegisteredActionInvocation(invocation)) {
2348
+ return false;
2349
+ }
2350
+ if (isDeletedRegisteredActionInvocation(invocation)) {
2351
+ await failOpenInvocationRows(db, invocation.workspaceId, [invocation.id], 'action_deleted');
2352
+ return false;
2353
+ }
1666
2354
  // Only release capacity the invocation actually holds. A shadowed spawn
1667
2355
  // dropped its native reservation at dispatch (spawnReservedAt is null), so it
1668
2356
  // must not release capacity that now belongs to another spawn.
@@ -1697,6 +2385,8 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
1697
2385
  input: recordInput(invocation.input),
1698
2386
  agent: { id: targetAgent.agentId, name: targetAgent.agentName },
1699
2387
  actionId: targetAgent.actionId,
2388
+ expectedActionId: invocation.actionId,
2389
+ invocationOrigin: invocation.invocationOrigin,
1700
2390
  retryAfterAt: opts.retryAfterAt ?? null,
1701
2391
  });
1702
2392
  return dispatched.accepted;
@@ -1704,7 +2394,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
1704
2394
  // Release invocations carry no actionId, so targetAgentForInvocation() returns
1705
2395
  // null. They must never fall through to generic node placement, which could
1706
2396
  // route a release to an unrelated node that doesn't own the agent.
1707
- if (isReleaseInvocation(invocation.actionName)) {
2397
+ if (isBuiltinReleaseInvocation(invocation)) {
1708
2398
  return false;
1709
2399
  }
1710
2400
  const attempted = attemptedNodeSet(invocation);
@@ -1741,6 +2431,8 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
1741
2431
  providerName: target.handlerProvider,
1742
2432
  action: actionToSend,
1743
2433
  actionId: target.id,
2434
+ expectedActionId: invocation.actionId,
2435
+ invocationOrigin: invocation.invocationOrigin,
1744
2436
  input,
1745
2437
  queue: target.queue,
1746
2438
  reservationHeld: false,
@@ -1758,14 +2450,23 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
1758
2450
  ? (await capacityProviderName(db, invocation.workspaceId, placement.node.id, actionToSend)) ?? DEFAULT_PROVIDER_NAME
1759
2451
  : DEFAULT_PROVIDER_NAME;
1760
2452
  if (placement.queued) {
1761
- await dispatchNodeAttempt(db, invocation.workspaceId, invocation.id, placement.node.id, {
2453
+ const expectedActionId = invocation.invocationOrigin === 'registered_action'
2454
+ ? invocation.actionId
2455
+ : null;
2456
+ const accepted = await dispatchNodeAttempt(db, invocation.workspaceId, invocation.id, placement.node.id, {
1762
2457
  providerName,
1763
2458
  pending: true,
1764
2459
  retryAfterAt: opts.retryAfterAt ?? null,
1765
2460
  reservationHeld: false,
1766
- actionId: target?.id,
2461
+ expectedAction: expectedActionId
2462
+ ? { id: expectedActionId, name: actionToSend }
2463
+ : undefined,
2464
+ targetActionId: target?.id,
1767
2465
  });
1768
- return true;
2466
+ if (!accepted && invocation.invocationOrigin === 'registered_action') {
2467
+ await settleDeletedRegisteredAction(db, invocation.workspaceId, invocation.id);
2468
+ }
2469
+ return accepted;
1769
2470
  }
1770
2471
  const dispatched = await dispatchNodeInvocation({
1771
2472
  db,
@@ -1775,7 +2476,9 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
1775
2476
  nodeId: placement.node.id,
1776
2477
  providerName,
1777
2478
  action: actionToSend,
1778
- actionId: target?.id,
2479
+ actionId: target?.id ?? null,
2480
+ expectedActionId: invocation.actionId,
2481
+ invocationOrigin: invocation.invocationOrigin,
1779
2482
  input,
1780
2483
  reservationHeld,
1781
2484
  });
@@ -1879,6 +2582,8 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
1879
2582
  .select({
1880
2583
  id: actionInvocations.id,
1881
2584
  workspaceId: actionInvocations.workspaceId,
2585
+ actionId: actionInvocations.actionId,
2586
+ invocationOrigin: actionInvocations.invocationOrigin,
1882
2587
  actionName: actionInvocations.actionName,
1883
2588
  callerId: actionInvocations.callerId,
1884
2589
  input: actionInvocations.input,
@@ -1888,6 +2593,7 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
1888
2593
  durationMs: actionInvocations.durationMs,
1889
2594
  dispatchedNodeId: actionInvocations.dispatchedNodeId,
1890
2595
  dispatchedAt: actionInvocations.dispatchedAt,
2596
+ providerAcceptedAttempt: actionInvocations.providerAcceptedAttempt,
1891
2597
  spawnReservedAt: actionInvocations.spawnReservedAt,
1892
2598
  attemptedNodeIds: actionInvocations.attemptedNodeIds,
1893
2599
  dispatchAttempts: actionInvocations.dispatchAttempts,
@@ -1918,6 +2624,12 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
1918
2624
  // Fall through and fail the invocation if no eligible node exists.
1919
2625
  }
1920
2626
  }
2627
+ const existingInput = recordInput(existing.input);
2628
+ if (!data.error
2629
+ && isBuiltinReleaseInvocation(existing)
2630
+ && existingInput.expected_token_hash !== undefined) {
2631
+ return completeGuardedReleaseNodeInvocation(db, workspaceId, nodeId, providerName, existing, data, deps);
2632
+ }
1921
2633
  const [updated] = await db
1922
2634
  .update(actionInvocations)
1923
2635
  .set({