@relaycast/engine 8.3.1 → 8.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/adapters/node/realtime.d.ts +2 -2
  2. package/dist/adapters/node/realtime.d.ts.map +1 -1
  3. package/dist/adapters/node/realtime.js +118 -30
  4. package/dist/adapters/node/realtime.js.map +1 -1
  5. package/dist/db/migrations/0045_workspace_create_idempotency.sql +16 -0
  6. package/dist/db/migrations/0046_action_invocation_origin.sql +51 -0
  7. package/dist/db/migrations/0047_action_invocation_provider_acceptance.sql +15 -0
  8. package/dist/db/schema.d.ts +178 -0
  9. package/dist/db/schema.d.ts.map +1 -1
  10. package/dist/db/schema.js +28 -0
  11. package/dist/db/schema.js.map +1 -1
  12. package/dist/engine/action.d.ts +24 -1
  13. package/dist/engine/action.d.ts.map +1 -1
  14. package/dist/engine/action.js +898 -122
  15. package/dist/engine/action.js.map +1 -1
  16. package/dist/engine/node.d.ts +1 -1
  17. package/dist/engine/node.d.ts.map +1 -1
  18. package/dist/engine/node.js +127 -43
  19. package/dist/engine/node.js.map +1 -1
  20. package/dist/engine/nodeProvider.d.ts.map +1 -1
  21. package/dist/engine/nodeProvider.js +15 -8
  22. package/dist/engine/nodeProvider.js.map +1 -1
  23. package/dist/engine/workspace.d.ts +9 -0
  24. package/dist/engine/workspace.d.ts.map +1 -1
  25. package/dist/engine/workspace.js +110 -4
  26. package/dist/engine/workspace.js.map +1 -1
  27. package/dist/ports/realtime.d.ts +35 -4
  28. package/dist/ports/realtime.d.ts.map +1 -1
  29. package/dist/ports/realtime.js.map +1 -1
  30. package/dist/routes/agent.js +5 -3
  31. package/dist/routes/agent.js.map +1 -1
  32. package/dist/routes/workspace.d.ts.map +1 -1
  33. package/dist/routes/workspace.js +23 -0
  34. package/dist/routes/workspace.js.map +1 -1
  35. package/package.json +3 -3
@@ -1,7 +1,7 @@
1
1
  import { and, asc, eq, inArray, isNotNull, isNull, lte, or, sql } from 'drizzle-orm';
2
2
  import { actions, actionInvocations, agents, agentNodeBindings, channelMembers, dmParticipants, nodes } from '../db/schema.js';
3
3
  import { generateId } from './snowflake.js';
4
- import { RELEASED_AGENT_STATUS, releasedAgentName } from './agent.js';
4
+ import { assertRegistrableAgentName, RELEASED_AGENT_STATUS, releasedAgentName } from './agent.js';
5
5
  import { randomHex, sha256Hex } from '../lib/crypto.js';
6
6
  import { codedError } from '../lib/httpError.js';
7
7
  import { D1_SAFE_IN_QUERY_CHUNK_SIZE } from '../lib/queryChunks.js';
@@ -10,6 +10,7 @@ import { emitAgentExitedEffects, emitInvocationCompletionEffects, fleetInvocatio
10
10
  import { runAtomic, runAtomicWrites } from '../ports/database.js';
11
11
  import { claimSpawnNode, chooseNodeForAction, isNodeLive, releaseNodeCapacity, reserveNodeCapacity } from './placement.js';
12
12
  import { DEFAULT_PROVIDER_NAME, capacityProviderName, getProvider, isProviderLive } from './nodeProvider.js';
13
+ import { AGENT_TOKEN_HASH_PATTERN } from '@relaycast/types';
13
14
  const OPEN_INVOCATION_STATUSES = ['pending', 'dispatched', 'invoked'];
14
15
  const REPLAYED_INVOCATION = Symbol('replayed-action-invocation');
15
16
  export const ACTION_DISPATCH_TIMEOUT_MS = 30_000;
@@ -62,6 +63,42 @@ function isSpawnInvocation(actionName) {
62
63
  function isReleaseInvocation(actionName) {
63
64
  return actionName === 'release';
64
65
  }
66
+ function isBuiltinReleaseInvocation(invocation) {
67
+ return invocation.invocationOrigin === 'builtin' && isReleaseInvocation(invocation.actionName);
68
+ }
69
+ function isDeletedRegisteredActionInvocation(invocation) {
70
+ // For a registered-action origin, actionId is cleared only by the FK when its
71
+ // exact action row is deleted. The immutable origin prevents that orphan
72
+ // from being mistaken for a built-in, and this predicate prevents it from
73
+ // being rebound by name during drain/retry/reconciliation.
74
+ return invocation.invocationOrigin === 'registered_action'
75
+ && invocation.actionId === null
76
+ && invocation.providerAcceptedAttempt !== invocation.dispatchAttempts;
77
+ }
78
+ function isAcceptedDeletedRegisteredActionInvocation(invocation) {
79
+ return invocation.invocationOrigin === 'registered_action'
80
+ && invocation.actionId === null
81
+ && invocation.providerAcceptedAttempt !== null
82
+ && invocation.providerAcceptedAttempt === invocation.dispatchAttempts;
83
+ }
84
+ const RELEASE_GENERATION_CONFLICT_CODE = 'agent_release_generation_conflict';
85
+ function releaseExpectedTokenHash(input) {
86
+ const value = input.expected_token_hash;
87
+ if (value === undefined)
88
+ return null;
89
+ if (typeof value !== 'string' || !AGENT_TOKEN_HASH_PATTERN.test(value)) {
90
+ throw codedError('release action input.expected_token_hash must be a lowercase SHA-256 hash', 'invalid_release_request', 400);
91
+ }
92
+ return value;
93
+ }
94
+ function releaseGenerationStillCurrent(workspaceId, agentId, expectedTokenHash) {
95
+ return sql `EXISTS (
96
+ SELECT 1 FROM ${agents}
97
+ WHERE ${agents.workspaceId} = ${workspaceId}
98
+ AND ${agents.id} = ${agentId}
99
+ ${expectedTokenHash ? sql `AND ${agents.tokenHash} = ${expectedTokenHash}` : sql ``}
100
+ )`;
101
+ }
65
102
  function dispatchActionNameForInvocation(actionName, input) {
66
103
  if (!isSpawnInvocation(actionName))
67
104
  return actionName;
@@ -257,13 +294,16 @@ export async function registerAction(db, workspaceId, data, options = {}) {
257
294
  : { target: [actions.workspaceId, actions.name], targetWhere: isNull(actions.handlerNodeId), set: refresh })
258
295
  .returning();
259
296
  // A refresh that moves the handler to a DIFFERENT agent strands
260
- // invocations already in flight toward the previous handler: completion
261
- // authorization follows actions.handlerAgentId (the old handler's
262
- // completion would 403) and the timeout sweep would redeliver them to the
263
- // new handler, which never received the original dispatch. Terminally
264
- // fail them; the callers are told after commit.
297
+ // invocations still waiting for provider acceptance. Once the previous
298
+ // handler accepted an exact attempt, detach that invocation from the
299
+ // mutable action row instead: its snapshotted route remains authorized
300
+ // to complete and timeout recovery cannot retarget it to the replacement.
265
301
  let strandedRows = [];
266
302
  if (previous && previous.handlerAgentId && previous.handlerAgentId !== handlerAgentId) {
303
+ await tx
304
+ .update(actionInvocations)
305
+ .set({ actionId: null })
306
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.actionId, previous.id), eq(actionInvocations.invocationOrigin, 'registered_action'), isNotNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} = ${actionInvocations.dispatchAttempts}`, inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
267
307
  const strandedIds = await openInvocationIdsForActions(tx, workspaceId, [previous.id]);
268
308
  strandedRows = await failOpenInvocationRows(tx, workspaceId, strandedIds, 'handler_unavailable');
269
309
  }
@@ -322,7 +362,7 @@ export async function listActions(db, workspaceId, callerName) {
322
362
  .map(publicAction);
323
363
  }
324
364
  export async function getAction(db, workspaceId, name, callerName) {
325
- const [row] = await db
365
+ const rows = await db
326
366
  .select({
327
367
  id: actions.id,
328
368
  name: actions.name,
@@ -330,6 +370,7 @@ export async function getAction(db, workspaceId, name, callerName) {
330
370
  handlerAgentName: agents.name,
331
371
  handlerNodeName: nodes.name,
332
372
  handlerNodeId: actions.handlerNodeId,
373
+ isGlobal: actions.isGlobal,
333
374
  inputSchema: actions.inputSchema,
334
375
  outputSchema: actions.outputSchema,
335
376
  availableTo: actions.availableTo,
@@ -340,6 +381,8 @@ export async function getAction(db, workspaceId, name, callerName) {
340
381
  .leftJoin(agents, eq(actions.handlerAgentId, agents.id))
341
382
  .leftJoin(nodes, eq(actions.handlerNodeId, nodes.id))
342
383
  .where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, name)));
384
+ const row = rows.sort((a, b) => actionResolutionRank(a) - actionResolutionRank(b) ||
385
+ (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))[0];
343
386
  if (!row || !isActionVisibleToCaller(row.availableTo ?? null, callerName))
344
387
  return null;
345
388
  return publicAction(row);
@@ -378,6 +421,7 @@ async function createInvocation(db, workspaceId, action, data) {
378
421
  workspaceId,
379
422
  actionId: action?.id ?? null,
380
423
  actionName: expectedActionName,
424
+ invocationOrigin: action ? 'registered_action' : 'builtin',
381
425
  callerId: data.caller_id ?? null,
382
426
  callerName: data.caller_name ?? null,
383
427
  handlerAgentId: data.handler_agent_id ?? action?.handlerAgentId ?? null,
@@ -417,21 +461,62 @@ async function idempotentInvocationId(workspaceId, callerId, actionName, key) {
417
461
  const digest = await sha256Hex(['action-invoke-v1', workspaceId, callerId, actionName, key].join('\0'));
418
462
  return `inv_idem_${digest}`;
419
463
  }
464
+ /**
465
+ * Poll a durable idempotency claim until the winning request's dispatch is
466
+ * visible, for up to 500ms.
467
+ *
468
+ * Resolves as soon as the row leaves the pre-dispatch state: it returns the row
469
+ * once `dispatchedNodeId` is set (or, with `acceptDispatchStarted`, once an
470
+ * attempt has been recorded), returns a settled non-open row, and throws the
471
+ * row's own failure for one that failed before dispatch.
472
+ *
473
+ * `onDeadline` picks what a still-open row does when the 500ms deadline passes:
474
+ * `'throw'` (the default) raises a retryable 503 `idempotency_unavailable`,
475
+ * which is what a replay observing someone else's claim wants; `'return'` hands
476
+ * the still-open row back so a caller that owns the claim itself — the
477
+ * post-send takeover re-read below — can classify it.
478
+ *
479
+ * @param options.acceptDispatchStarted - treat a recorded attempt as dispatch.
480
+ * @param options.onDeadline - deadline behavior for a still-open row.
481
+ * @returns the invocation row as last read.
482
+ */
420
483
  async function waitForInvocationReplayOutcome(db, workspaceId, invocation, options = {}) {
421
484
  let current = invocation;
422
485
  const deadline = Date.now() + 500;
423
- while (!current.dispatchedNodeId
424
- && !(options.acceptDispatchStarted && current.dispatchAttempts > 0)) {
486
+ while (true) {
487
+ // A guarded release can lose its generation CAS after provider dispatch.
488
+ // Surface that durable terminal conflict before the generic dispatched-row
489
+ // replay shortcut, otherwise the same idempotency key changes 409 into 201.
490
+ if (isBuiltinReleaseInvocation(current)
491
+ && current.status === 'failed'
492
+ && current.error === RELEASE_GENERATION_CONFLICT_CODE) {
493
+ throw codedError('Action invocation failed because the release generation changed', RELEASE_GENERATION_CONFLICT_CODE, 409);
494
+ }
495
+ // Registered node actions only enter action_deleted before the provider
496
+ // accepts the frame. Surface that durable result before the route snapshot
497
+ // shortcut so the winning request and every replay return the same 503.
498
+ if (current.status === 'failed' && current.error === 'action_deleted') {
499
+ throw codedError('Action invocation failed before provider dispatch completed', 'action_deleted', 503);
500
+ }
501
+ if (current.dispatchedNodeId
502
+ || (options.acceptDispatchStarted && current.dispatchAttempts > 0)) {
503
+ return current;
504
+ }
425
505
  if (!OPEN_INVOCATION_STATUSES.some((status) => status === current.status)) {
426
506
  if (current.status === 'failed') {
427
- throw codedError('Action invocation failed before provider dispatch completed', current.error ?? 'handler_unavailable', 503);
507
+ const errorCode = current.error ?? 'handler_unavailable';
508
+ throw codedError('Action invocation failed before provider dispatch completed', errorCode, isBuiltinReleaseInvocation(current) && errorCode === RELEASE_GENERATION_CONFLICT_CODE ? 409 : 503);
428
509
  }
429
510
  return current;
430
511
  }
431
512
  if (Date.now() >= deadline) {
432
513
  // The winning request may have died after its durable claim but before
433
514
  // provider dispatch started or completed. Fail retryably without
434
- // releasing the key for another execution.
515
+ // releasing the key for another execution. Callers that own the claim
516
+ // themselves (the post-send takeover re-read) opt out and classify the
517
+ // still-open row instead.
518
+ if (options.onDeadline === 'return')
519
+ return current;
435
520
  throw codedError('Idempotent action dispatch is still pending', 'idempotency_unavailable', 503);
436
521
  }
437
522
  await new Promise((resolve) => setTimeout(resolve, 10));
@@ -444,7 +529,6 @@ async function waitForInvocationReplayOutcome(db, workspaceId, invocation, optio
444
529
  }
445
530
  current = updated;
446
531
  }
447
- return current;
448
532
  }
449
533
  /**
450
534
  * Resolve a losing durable claim from the immutable invocation snapshots.
@@ -456,7 +540,7 @@ async function waitForInvocationReplayOutcome(db, workspaceId, invocation, optio
456
540
  */
457
541
  async function replayInvocationClaim(db, workspaceId, invocation) {
458
542
  if (isSpawnInvocation(invocation.actionName)
459
- || isReleaseInvocation(invocation.actionName)
543
+ || isBuiltinReleaseInvocation(invocation)
460
544
  || (!!invocation.handlerNodeId && !invocation.handlerAgentId)) {
461
545
  return waitForInvocationReplayOutcome(db, workspaceId, invocation);
462
546
  }
@@ -520,6 +604,25 @@ async function failInvocationForUnavailableProvider(db, workspaceId, invocationI
520
604
  .set({ status: 'failed', error: 'handler_unavailable', completedAt: new Date(), spawnReservedAt: null })
521
605
  .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
522
606
  }
607
+ /**
608
+ * Terminally fail one invocation ONLY while it is still open AND was never
609
+ * dispatched — no recorded attempt and no dispatched node. The predicates make
610
+ * the failure atomic against a concurrent dispatcher (`drainNodeInvocations`
611
+ * on a handler reconnect): once that dispatcher's attempt lands, this update
612
+ * matches nothing and the live dispatch is left alone. Sets the same columns as
613
+ * {@link failInvocationForUnavailableProvider}; no spawn reservation can be
614
+ * held by a row with no dispatched node, so there is no capacity to release.
615
+ *
616
+ * @returns true when this call is the one that failed the row.
617
+ */
618
+ async function failNeverDispatchedInvocation(db, workspaceId, invocationId, error) {
619
+ const failed = await db
620
+ .update(actionInvocations)
621
+ .set({ status: 'failed', error, completedAt: new Date(), spawnReservedAt: null })
622
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), eq(actionInvocations.dispatchAttempts, 0), isNull(actionInvocations.dispatchedNodeId)))
623
+ .returning({ id: actionInvocations.id });
624
+ return failed.length > 0;
625
+ }
523
626
  /**
524
627
  * Terminally fail the given open invocations (no completion emission — call
525
628
  * {@link emitFailedInvocationEffects} with the returned rows after the
@@ -589,7 +692,7 @@ async function openInvocationIdsForActions(db, workspaceId, actionIds) {
589
692
  const rows = await db
590
693
  .select({ id: actionInvocations.id })
591
694
  .from(actionInvocations)
592
- .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.actionId, actionIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
695
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.actionId, actionIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`)));
593
696
  return rows.map((row) => row.id);
594
697
  }
595
698
  /**
@@ -616,12 +719,22 @@ async function dispatchNodeProviderInvocation(args, options = {}) {
616
719
  // The DB is the authoritative offline queue. Do not call sendToProvider
617
720
  // here: a connected provider whose heartbeat says handlers_live=false still
618
721
  // has a socket, and sendToProvider would deliver the frame immediately.
722
+ const expectedActionId = args.invocationOrigin === 'registered_action'
723
+ ? (args.expectedActionId ?? args.actionId)
724
+ : null;
619
725
  const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, {
620
726
  providerName: args.providerName,
621
727
  pending: true,
622
728
  reservationHeld: args.reservationHeld,
623
- actionId: args.actionId,
729
+ expectedAction: expectedActionId
730
+ ? { id: expectedActionId, name: args.action }
731
+ : undefined,
732
+ targetActionId: args.actionId ?? undefined,
624
733
  });
734
+ if (!accepted && args.invocationOrigin === 'registered_action') {
735
+ const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
736
+ return { accepted: false, pending: false, settled };
737
+ }
625
738
  return { accepted, pending: true };
626
739
  }
627
740
  return dispatchNodeInvocation({ ...args });
@@ -632,11 +745,21 @@ async function dispatchRelease(args) {
632
745
  if (!name) {
633
746
  throw codedError('release action input.name is required', 'invalid_release_request', 400);
634
747
  }
748
+ const expectedTokenHash = releaseExpectedTokenHash(input);
635
749
  const [agent] = await args.db
636
750
  .select()
637
751
  .from(agents)
638
- .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name)));
752
+ .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name), ...(expectedTokenHash ? [eq(agents.tokenHash, expectedTokenHash)] : [])));
639
753
  if (!agent) {
754
+ if (expectedTokenHash) {
755
+ const [sameName] = await args.db
756
+ .select({ id: agents.id })
757
+ .from(agents)
758
+ .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name)));
759
+ if (sameName) {
760
+ throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
761
+ }
762
+ }
640
763
  throw codedError(`Agent "${name}" not found`, 'agent_not_found', 404);
641
764
  }
642
765
  const { invocation, replayed } = await createInvocation(args.db, args.workspaceId, null, {
@@ -660,23 +783,87 @@ async function dispatchRelease(args) {
660
783
  // The release time is preserved in `metadata.release.releasedAt`.
661
784
  const releasedName = releasedAgentName(agent.name, agent.id);
662
785
  const releasedTokenHash = await sha256Hex(`released:${agent.id}:${randomHex(16)}`);
663
- const invocationIsOpen = sql `EXISTS (
786
+ const invocationCompleted = sql `EXISTS (
664
787
  SELECT 1 FROM ${actionInvocations}
665
788
  WHERE ${actionInvocations.workspaceId} = ${args.workspaceId}
666
789
  AND ${actionInvocations.id} = ${invocation.id}
667
- AND ${actionInvocations.status} IN ('pending', 'dispatched', 'invoked')
790
+ AND ${actionInvocations.status} = 'completed'
791
+ )`;
792
+ const generationStillCurrent = releaseGenerationStillCurrent(args.workspaceId, agent.id, expectedTokenHash);
793
+ const atomicExitNodeId = sql `COALESCE(
794
+ (
795
+ SELECT ${agentNodeBindings.nodeId}
796
+ FROM ${agentNodeBindings}
797
+ WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
798
+ AND ${agentNodeBindings.agentId} = ${agent.id}
799
+ AND ${agentNodeBindings.status} = 'active'
800
+ ORDER BY
801
+ CASE
802
+ WHEN ${agentNodeBindings.nodeId} = (
803
+ SELECT ${agents.locationNodeId}
804
+ FROM ${agents}
805
+ WHERE ${agents.workspaceId} = ${args.workspaceId}
806
+ AND ${agents.id} = ${agent.id}
807
+ ) THEN 0
808
+ WHEN ${agentNodeBindings.nodeId} = ${`node_direct_${agent.id}`} THEN 1
809
+ ELSE 2
810
+ END,
811
+ ${agentNodeBindings.priority} DESC,
812
+ ${agentNodeBindings.nodeId}
813
+ LIMIT 1
814
+ ),
815
+ (
816
+ SELECT ${agents.locationNodeId}
817
+ FROM ${agents}
818
+ WHERE ${agents.workspaceId} = ${args.workspaceId}
819
+ AND ${agents.id} = ${agent.id}
820
+ ),
821
+ ${exitNodeId}
668
822
  )`;
669
823
  const results = await runAtomicWrites(args.db, (writeDb) => {
670
824
  const writes = [];
671
- // Resolve the active binding in this atomic unit instead of from a
825
+ // If the optional generation lease no longer matches, settle the
826
+ // invocation as a durable conflict in the same atomic unit in which all
827
+ // lifecycle mutations are conditioned to no-op.
828
+ writes.push(writeDb
829
+ .update(actionInvocations)
830
+ .set({
831
+ status: 'failed',
832
+ error: RELEASE_GENERATION_CONFLICT_CODE,
833
+ completedAt,
834
+ })
835
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), expectedTokenHash ? sql `NOT (${generationStillCurrent})` : sql `0`))
836
+ .returning({ id: actionInvocations.id }));
837
+ // Win completion and snapshot the current binding before deactivating
838
+ // it. The returned node drives the response and agent.exited event, so a
839
+ // concurrent rebind cannot leave either pointing at the stale host read
840
+ // before this atomic unit began.
841
+ writes.push(writeDb
842
+ .update(actionInvocations)
843
+ .set({
844
+ status: 'completed',
845
+ handlerNodeId: atomicExitNodeId,
846
+ output: {
847
+ released: true,
848
+ // The roster row is retained as a tombstone so the agent's history
849
+ // keeps its author; the name is what the caller gets back.
850
+ deleted: false,
851
+ reaped_locally: true,
852
+ released_name: releasedName,
853
+ },
854
+ completedAt,
855
+ })
856
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
857
+ .returning({ id: actionInvocations.id, handlerNodeId: actionInvocations.handlerNodeId }));
858
+ // Resolve every active binding in this atomic unit instead of from a
672
859
  // pre-transaction snapshot. A concurrent rebind therefore decrements
673
- // the node that is actually deactivated below.
860
+ // exactly the node or nodes that are deactivated below.
674
861
  writes.push(writeDb
675
862
  .update(nodes)
676
863
  .set({
677
864
  activeAgents: sql `CASE WHEN ${nodes.activeAgents} > 0 THEN ${nodes.activeAgents} - 1 ELSE 0 END`,
678
865
  })
679
- .where(and(eq(nodes.workspaceId, args.workspaceId), invocationIsOpen, sql `EXISTS (
866
+ .where(and(eq(nodes.workspaceId, args.workspaceId), invocationCompleted, generationStillCurrent, sql `EXISTS (
680
867
  SELECT 1 FROM ${agentNodeBindings}
681
868
  WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
682
869
  AND ${agentNodeBindings.agentId} = ${agent.id}
@@ -686,38 +873,32 @@ async function dispatchRelease(args) {
686
873
  writes.push(writeDb
687
874
  .update(agentNodeBindings)
688
875
  .set({ status: 'inactive', updatedAt: completedAt })
689
- .where(and(eq(agentNodeBindings.workspaceId, args.workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.status, 'active'), invocationIsOpen)));
690
- // This helper is only used for delete_agent releases. Non-delete
691
- // releases fail closed when no live host can receive the invocation.
692
- //
693
- // Tombstone-rename rather than DELETE (relaycast#309). Four FKs reference
694
- // `agents.id` without `onDelete` — channels.created_by (schema.ts:455),
695
- // messages.agent_id (:503), files.uploaded_by (:666),
696
- // webhooks.created_by (:759) — so a bare DELETE is refused for any agent
697
- // that has ever spoken, and inside this atomic unit that refusal aborts
698
- // the binding update and the invocation completion along with it. Cascade
699
- // is not an option either: it would destroy the agent's message history,
700
- // and `messages.agent_id` is NOT NULL so `set null` cannot apply.
701
- //
702
- // Renaming frees the unique `(workspace_id, name)` immediately while
703
- // every FK target stays valid and every message keeps its sender.
876
+ .where(and(eq(agentNodeBindings.workspaceId, args.workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.status, 'active'), invocationCompleted, generationStillCurrent)));
877
+ // `channel_members` and `dm_participants` cascade on DELETE; the
878
+ // tombstone's UPDATE does not fire that cascade, so drop the memberships
879
+ // in the SAME atomic unit or the released agent stays a delivery target.
880
+ writes.push(writeDb
881
+ .delete(channelMembers)
882
+ .where(and(eq(channelMembers.agentId, agent.id), invocationCompleted, generationStillCurrent)));
883
+ writes.push(writeDb
884
+ .delete(dmParticipants)
885
+ .where(and(eq(dmParticipants.agentId, agent.id), invocationCompleted, generationStillCurrent)));
886
+ writes.push(writeDb
887
+ .delete(nodes)
888
+ .where(and(eq(nodes.workspaceId, args.workspaceId), eq(nodes.id, `node_direct_${agent.id}`), invocationCompleted, generationStillCurrent)));
889
+ // Tombstone after the invocation has won its completion CAS. The whole
890
+ // batch/transaction rolls back together, while this ordering lets the
891
+ // success predicate inspect the still-current token hash before we rotate
892
+ // it to an unusable released credential.
704
893
  writes.push(writeDb
705
894
  .update(agents)
706
895
  .set({
707
896
  name: releasedName,
708
897
  handle: `@${releasedName}`,
709
898
  status: RELEASED_AGENT_STATUS,
710
- // The row survives, so its credential must not. `token_hash` is
711
- // NOT NULL UNIQUE and cannot be cleared, so rotate it to a value
712
- // nobody holds; the released agent's old token stops authenticating.
713
899
  tokenHash: releasedTokenHash,
714
- // Clear the rotation grace slot too, otherwise any token issued by
715
- // the last live rotation would keep authenticating for its grace
716
- // window on an agent that is supposed to be gone. See 0035_agent_token_grace.
717
900
  previousTokenHash: null,
718
901
  previousTokenExpiresAt: null,
719
- // Same `release` shape the dispatched path writes, so an audit does
720
- // not have to know which path released the agent.
721
902
  metadata: sql `json_patch(COALESCE(${agents.metadata}, '{}'), ${JSON.stringify({
722
903
  release: {
723
904
  reason: typeof input.reason === 'string' ? input.reason : null,
@@ -726,48 +907,22 @@ async function dispatchRelease(args) {
726
907
  },
727
908
  })})`,
728
909
  })
729
- .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.id, agent.id), invocationIsOpen)));
730
- // `channel_members` and `dm_participants` cascade on DELETE; the
731
- // tombstone's UPDATE does not fire that cascade, so drop the memberships
732
- // in the SAME atomic unit or the released agent stays a delivery target.
733
- writes.push(writeDb
734
- .delete(channelMembers)
735
- .where(eq(channelMembers.agentId, agent.id)));
736
- writes.push(writeDb
737
- .delete(dmParticipants)
738
- .where(eq(dmParticipants.agentId, agent.id)));
739
- writes.push(writeDb
740
- .delete(nodes)
741
- .where(and(eq(nodes.workspaceId, args.workspaceId), eq(nodes.id, `node_direct_${agent.id}`), invocationIsOpen)));
742
- writes.push(writeDb
743
- .update(actionInvocations)
744
- .set({
745
- status: 'completed',
746
- // This is part of the same atomic unit as completion so a lost 201
747
- // replays the exact handler node returned below.
748
- handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${exitNodeId})`,
749
- output: {
750
- released: true,
751
- // The roster row is retained as a tombstone so the agent's history
752
- // keeps its author; the name is what the caller gets back.
753
- deleted: false,
754
- reaped_locally: true,
755
- released_name: releasedName,
756
- },
757
- completedAt,
758
- })
759
- .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
760
- .returning({ id: actionInvocations.id }));
910
+ .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.id, agent.id), ...(expectedTokenHash ? [eq(agents.tokenHash, expectedTokenHash)] : []), invocationCompleted)));
761
911
  return writes;
762
- });
763
- const completed = results.at(-1);
912
+ }, expectedTokenHash ? { requireAtomic: true } : undefined);
913
+ const generationConflict = results[0];
914
+ const completed = results[1];
915
+ const completedExitNodeId = completed[0]?.handlerNodeId ?? null;
916
+ if (expectedTokenHash && (generationConflict.length > 0 || completed.length === 0)) {
917
+ throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
918
+ }
764
919
  // External completion effects belong after the durable atomic unit: an
765
920
  // aborted local reap must never publish agent.exited.
766
- if (completed.length > 0 && args.completionDeps && exitNodeId) {
921
+ if (completed.length > 0 && args.completionDeps && completedExitNodeId) {
767
922
  await emitAgentExitedEffects(args.completionDeps, args.workspaceId, {
768
923
  agentId: agent.id,
769
924
  agentName: agent.name,
770
- nodeId: exitNodeId,
925
+ nodeId: completedExitNodeId,
771
926
  invocationId: fleetInvocationId(agent.metadata),
772
927
  reason: 'released',
773
928
  });
@@ -776,7 +931,7 @@ async function dispatchRelease(args) {
776
931
  invocation_id: invocation.id,
777
932
  action_name: 'release',
778
933
  handler_agent_id: null,
779
- handler_node_id: exitNodeId,
934
+ handler_node_id: completedExitNodeId,
780
935
  dispatched_node_id: null,
781
936
  input,
782
937
  status: 'completed',
@@ -784,10 +939,36 @@ async function dispatchRelease(args) {
784
939
  };
785
940
  };
786
941
  const failClosed = async () => {
787
- await args.db
788
- .update(actionInvocations)
789
- .set({ status: 'failed', error: 'agent_host_unavailable', completedAt: new Date() })
790
- .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
942
+ const completedAt = new Date();
943
+ if (expectedTokenHash) {
944
+ const generationStillCurrent = releaseGenerationStillCurrent(args.workspaceId, agent.id, expectedTokenHash);
945
+ const results = await runAtomicWrites(args.db, (writeDb) => [
946
+ writeDb
947
+ .update(actionInvocations)
948
+ .set({
949
+ status: 'failed',
950
+ error: RELEASE_GENERATION_CONFLICT_CODE,
951
+ completedAt,
952
+ })
953
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (${generationStillCurrent})`))
954
+ .returning({ id: actionInvocations.id }),
955
+ writeDb
956
+ .update(actionInvocations)
957
+ .set({ status: 'failed', error: 'agent_host_unavailable', completedAt })
958
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
959
+ .returning({ id: actionInvocations.id }),
960
+ ], { requireAtomic: true });
961
+ const generationConflict = results[0];
962
+ if (generationConflict.length > 0) {
963
+ throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
964
+ }
965
+ }
966
+ else {
967
+ await args.db
968
+ .update(actionInvocations)
969
+ .set({ status: 'failed', error: 'agent_host_unavailable', completedAt })
970
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
971
+ }
791
972
  throw codedError(`Agent "${name}" has no live host node; cannot dispatch release`, 'agent_host_unavailable', 503);
792
973
  };
793
974
  const registry = args.registry;
@@ -816,10 +997,35 @@ async function dispatchRelease(args) {
816
997
  providerName: agent.providerName,
817
998
  action: 'release',
818
999
  input,
1000
+ actionId: null,
1001
+ invocationOrigin: 'builtin',
819
1002
  });
1003
+ // Some socket owners durably record or complete an accepted frame before
1004
+ // this process resumes to stamp its own dispatch attempt. Return that
1005
+ // terminal winner instead of treating the failed post-send stamp as a
1006
+ // rejected send and entering local fallback.
1007
+ if (dispatched.settled) {
1008
+ if (dispatched.settled.status === 'failed') {
1009
+ const errorCode = dispatched.settled.error ?? 'handler_unavailable';
1010
+ throw codedError('Release invocation failed after provider dispatch', errorCode, errorCode === RELEASE_GENERATION_CONFLICT_CODE ? 409 : 503);
1011
+ }
1012
+ return invocationAck(dispatched.settled, { actionName: 'release' });
1013
+ }
820
1014
  // The provider can disconnect between the liveness check and send. Complete
821
1015
  // the DB lifecycle locally instead of creating an ownerless pending request.
822
1016
  if (!dispatched.accepted) {
1017
+ if (expectedTokenHash) {
1018
+ const [settled] = await args.db
1019
+ .select({ error: actionInvocations.error })
1020
+ .from(actionInvocations)
1021
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id)));
1022
+ if (settled?.error === RELEASE_GENERATION_CONFLICT_CODE) {
1023
+ throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
1024
+ }
1025
+ if (settled?.error === 'node_dispatch_unavailable') {
1026
+ throw codedError('Node adapter does not support generation-authorized release dispatch', 'node_dispatch_unavailable', 503);
1027
+ }
1028
+ }
823
1029
  return input.delete_agent === true ? completeLocally() : failClosed();
824
1030
  }
825
1031
  return {
@@ -833,6 +1039,21 @@ async function dispatchRelease(args) {
833
1039
  created_at: invocation.createdAt.toISOString(),
834
1040
  };
835
1041
  }
1042
+ /**
1043
+ * Agent-lifecycle release used by `POST /v1/agents/release`. Unlike the
1044
+ * generic action endpoint, this path must always enforce the built-in release
1045
+ * lifecycle and its optional generation guard; a user-defined `release`
1046
+ * action must not be able to shadow it.
1047
+ */
1048
+ export async function dispatchAgentRelease(db, workspaceId, data, options = {}) {
1049
+ return dispatchRelease({
1050
+ db,
1051
+ registry: options.nodeConnections,
1052
+ completionDeps: options.completionDeps,
1053
+ workspaceId,
1054
+ data,
1055
+ });
1056
+ }
836
1057
  function spawnResult(invocation, nodeId, dispatched) {
837
1058
  return {
838
1059
  invocation_id: invocation.id,
@@ -850,6 +1071,11 @@ async function dispatchSpawn(args) {
850
1071
  throw codedError('Node dispatch is not available', 'node_dispatch_unavailable', 503);
851
1072
  }
852
1073
  const input = recordInput(args.data.input);
1074
+ const requestedName = typeof input.name === 'string' ? input.name : '';
1075
+ if (requestedName.trim().length === 0) {
1076
+ throw codedError('spawn action input.name is required', 'invalid_spawn_request', 400);
1077
+ }
1078
+ assertRegistrableAgentName(requestedName);
853
1079
  const capability = dispatchActionNameForInvocation('spawn', input);
854
1080
  const { invocation, replayed } = await createInvocation(args.db, args.workspaceId, null, {
855
1081
  input: args.data.input,
@@ -895,20 +1121,40 @@ async function dispatchSpawn(args) {
895
1121
  if (!args.bypassShadow) {
896
1122
  const shadow = await fetchNodeAction(args.db, args.workspaceId, nodeId, capability);
897
1123
  if (shadow && shadow.handlerProvider) {
898
- if (!placement.queued)
899
- await releaseNodeCapacity(args.db, args.workspaceId, nodeId);
900
- const dispatched = await dispatchNodeProviderInvocation({
901
- db: args.db,
902
- registry: args.registry,
1124
+ const bound = await bindInvocationToRegisteredNodeAction(args.db, {
903
1125
  workspaceId: args.workspaceId,
904
1126
  invocationId: invocation.id,
905
1127
  nodeId,
906
1128
  providerName: shadow.handlerProvider,
907
- action: capability,
908
- input,
909
- queue: shadow.queue,
1129
+ actionId: shadow.id,
1130
+ actionName: capability,
910
1131
  });
911
- return spawnResult(invocation, nodeId, dispatched);
1132
+ if (bound) {
1133
+ if (!placement.queued)
1134
+ await releaseNodeCapacity(args.db, args.workspaceId, nodeId);
1135
+ const dispatched = await dispatchNodeProviderInvocation({
1136
+ db: args.db,
1137
+ registry: args.registry,
1138
+ workspaceId: args.workspaceId,
1139
+ invocationId: invocation.id,
1140
+ nodeId,
1141
+ providerName: shadow.handlerProvider,
1142
+ action: capability,
1143
+ input,
1144
+ queue: shadow.queue,
1145
+ actionId: shadow.id,
1146
+ expectedActionId: shadow.id,
1147
+ invocationOrigin: 'registered_action',
1148
+ });
1149
+ if (dispatched.settled) {
1150
+ const settled = await waitForInvocationReplayOutcome(args.db, args.workspaceId, dispatched.settled);
1151
+ return invocationAck(settled, { actionName: 'spawn', handlerNodeId: nodeId });
1152
+ }
1153
+ return spawnResult(invocation, nodeId, dispatched);
1154
+ }
1155
+ // The shadow disappeared before its exact identity could be claimed.
1156
+ // Keep the native reservation and continue through native capacity; a
1157
+ // stale shadow snapshot must never receive an unbound invocation.
912
1158
  }
913
1159
  }
914
1160
  const capProvider = (await capacityProviderName(args.db, args.workspaceId, nodeId, capability)) ?? DEFAULT_PROVIDER_NAME;
@@ -921,6 +1167,8 @@ async function dispatchSpawn(args) {
921
1167
  providerName: capProvider,
922
1168
  action: capability,
923
1169
  input,
1170
+ actionId: null,
1171
+ invocationOrigin: 'builtin',
924
1172
  pending: placement.queued,
925
1173
  reservationHeld: !placement.queued,
926
1174
  });
@@ -989,8 +1237,14 @@ export async function invokeNodeAction(db, workspaceId, nodeId, actionName, data
989
1237
  action: action.name,
990
1238
  input: recordInput(invocation.input),
991
1239
  queue: action.queue,
1240
+ actionId: action.id,
1241
+ invocationOrigin: 'registered_action',
992
1242
  reservationHeld: !!reservedNode,
993
1243
  });
1244
+ if (dispatched.settled) {
1245
+ const settled = await waitForInvocationReplayOutcome(db, workspaceId, dispatched.settled);
1246
+ return invocationAck(settled, { actionName, handlerNodeId: nodeId });
1247
+ }
994
1248
  return {
995
1249
  invocation_id: invocation.id,
996
1250
  action_name: actionName,
@@ -1085,8 +1339,14 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
1085
1339
  action: action.name,
1086
1340
  input: recordInput(invocation.input),
1087
1341
  queue: action.queue || providerName === DEFAULT_PROVIDER_NAME,
1342
+ actionId: action.id,
1343
+ invocationOrigin: 'registered_action',
1088
1344
  reservationHeld: !!reservedNode,
1089
1345
  });
1346
+ if (dispatched.settled) {
1347
+ const settled = await waitForInvocationReplayOutcome(db, workspaceId, dispatched.settled);
1348
+ return invocationAck(settled, { actionName, handlerNodeId: action.handlerNodeId });
1349
+ }
1090
1350
  return {
1091
1351
  invocation_id: invocation.id,
1092
1352
  action_name: actionName,
@@ -1169,6 +1429,7 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
1169
1429
  input: recordInput(invocation.input),
1170
1430
  agent: { id: handlerAgent.id, name: handlerAgent.name },
1171
1431
  actionId: action.id,
1432
+ invocationOrigin: 'registered_action',
1172
1433
  });
1173
1434
  if (dispatched.sent && !dispatched.accepted) {
1174
1435
  // The provider owns the frame once the send returns true. A fast completion
@@ -1189,19 +1450,41 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
1189
1450
  }
1190
1451
  if (!dispatched.sent) {
1191
1452
  if (invocationId !== undefined) {
1192
- // A keyed claim must remain durable across a transient disconnect. Re-read
1193
- // it so the original request and its replay agree on the same pre-send
1194
- // outcome without releasing the key for a second execution.
1195
- await waitForInvocationReplayOutcome(db, workspaceId, invocation, { acceptDispatchStarted: true });
1453
+ // A keyed claim must stay consistent with its replays. Re-read it so a
1454
+ // takeover that raced the last-moment adapter gate is observed here (a
1455
+ // row it already failed throws its own error) instead of this request
1456
+ // and its replay disagreeing on the pre-send outcome. A row still open
1457
+ // at the deadline is handed back for classification below rather than
1458
+ // reported as a retryable idempotency stall.
1459
+ await waitForInvocationReplayOutcome(db, workspaceId, invocation, { acceptDispatchStarted: true, onDeadline: 'return' });
1196
1460
  }
1197
- else {
1198
- // An unkeyed invocation has no replay contract to preserve. Leaving its
1199
- // pre-send row pending reports an idempotency failure to a caller that did
1200
- // not request idempotency and strands work no retry can address. Fail the
1201
- // row and classify the actual condition instead.
1202
- await failOpenInvocationRows(db, workspaceId, [invocation.id], 'handler_unavailable');
1461
+ // The host adapter could not deliver to the handler. A hosted adapter can
1462
+ // only learn that from the send itself, so the pre-dispatch liveness gate
1463
+ // above could not catch it. Resolve it the same way that gate would. The
1464
+ // failure is conditional on the row still being open AND never dispatched:
1465
+ // a handler that reconnected in the meantime may have had this very row
1466
+ // dispatched by `drainNodeInvocations`, and overwriting a live dispatch
1467
+ // with `handler_unavailable` would fail the caller while the handler runs
1468
+ // the action.
1469
+ const failed = await failNeverDispatchedInvocation(db, workspaceId, invocation.id, 'handler_unavailable');
1470
+ if (failed) {
1203
1471
  throw codedError(`Action "${actionName}" handler "${handlerAgent.name}" has no live connection`, 'handler_unavailable', 503);
1204
1472
  }
1473
+ // Someone else owns the row now: a dispatcher on handler reconnect, or a
1474
+ // takeover that already failed it. Report that outcome rather than this
1475
+ // request's failed send.
1476
+ const [current] = await db
1477
+ .select()
1478
+ .from(actionInvocations)
1479
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id)));
1480
+ if (!current || current.status === 'failed') {
1481
+ throw codedError('Action invocation failed before provider dispatch completed', current?.error ?? 'handler_unavailable', 503);
1482
+ }
1483
+ return invocationAck(current, {
1484
+ actionName,
1485
+ handlerAgentId: action.handlerAgentId,
1486
+ handlerNodeId: handlerAgent.locationNodeId,
1487
+ });
1205
1488
  }
1206
1489
  return {
1207
1490
  invocation_id: invocation.id,
@@ -1231,8 +1514,196 @@ function publicInvocation(row) {
1231
1514
  completed_at: row.completedAt?.toISOString() ?? null,
1232
1515
  };
1233
1516
  }
1517
+ async function completeGuardedReleaseNodeInvocation(db, workspaceId, nodeId, providerName, invocation, data, deps) {
1518
+ const input = recordInput(invocation.input);
1519
+ const name = typeof input.name === 'string' ? input.name : null;
1520
+ const persistedTokenHash = input.expected_token_hash;
1521
+ const expectedTokenHash = typeof persistedTokenHash === 'string'
1522
+ && AGENT_TOKEN_HASH_PATTERN.test(persistedTokenHash)
1523
+ ? persistedTokenHash
1524
+ : null;
1525
+ if (!name || !expectedTokenHash) {
1526
+ const [failed] = await db
1527
+ .update(actionInvocations)
1528
+ .set({
1529
+ status: 'failed',
1530
+ error: RELEASE_GENERATION_CONFLICT_CODE,
1531
+ completedAt: new Date(),
1532
+ spawnReservedAt: null,
1533
+ })
1534
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
1535
+ .returning();
1536
+ return failed ? publicInvocation(failed) : null;
1537
+ }
1538
+ const [agent] = await db
1539
+ .select()
1540
+ .from(agents)
1541
+ .where(and(eq(agents.workspaceId, workspaceId), eq(agents.name, name)));
1542
+ if (!agent) {
1543
+ const [failed] = await db
1544
+ .update(actionInvocations)
1545
+ .set({
1546
+ status: 'failed',
1547
+ error: RELEASE_GENERATION_CONFLICT_CODE,
1548
+ completedAt: new Date(),
1549
+ spawnReservedAt: null,
1550
+ })
1551
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
1552
+ .returning();
1553
+ return failed ? publicInvocation(failed) : null;
1554
+ }
1555
+ const completedAt = new Date();
1556
+ const generationStillCurrent = releaseGenerationStillCurrent(workspaceId, agent.id, expectedTokenHash);
1557
+ const activeBindingStillCurrent = sql `EXISTS (
1558
+ SELECT 1 FROM ${agentNodeBindings}
1559
+ WHERE ${agentNodeBindings.workspaceId} = ${workspaceId}
1560
+ AND ${agentNodeBindings.agentId} = ${agent.id}
1561
+ AND ${agentNodeBindings.nodeId} = ${nodeId}
1562
+ AND ${agentNodeBindings.status} = 'active'
1563
+ )`;
1564
+ const invocationIsOpen = sql `EXISTS (
1565
+ SELECT 1 FROM ${actionInvocations}
1566
+ WHERE ${actionInvocations.workspaceId} = ${workspaceId}
1567
+ AND ${actionInvocations.id} = ${invocation.id}
1568
+ AND ${actionInvocations.dispatchedNodeId} = ${nodeId}
1569
+ AND ${actionInvocations.dispatchedProvider} = ${providerName}
1570
+ AND ${actionInvocations.status} IN ('pending', 'dispatched', 'invoked')
1571
+ )`;
1572
+ const invocationCompleted = sql `EXISTS (
1573
+ SELECT 1 FROM ${actionInvocations}
1574
+ WHERE ${actionInvocations.workspaceId} = ${workspaceId}
1575
+ AND ${actionInvocations.id} = ${invocation.id}
1576
+ AND ${actionInvocations.dispatchedNodeId} = ${nodeId}
1577
+ AND ${actionInvocations.dispatchedProvider} = ${providerName}
1578
+ AND ${actionInvocations.status} = 'completed'
1579
+ )`;
1580
+ const releaseCanApply = sql `(${generationStillCurrent}) AND (${activeBindingStillCurrent})`;
1581
+ const releasedName = releasedAgentName(agent.name, agent.id);
1582
+ const releasedTokenHash = await sha256Hex(`released:${agent.id}:${randomHex(16)}`);
1583
+ let successIndex = -1;
1584
+ const results = await runAtomicWrites(db, (writeDb) => {
1585
+ const writes = [];
1586
+ // The conflict settlement comes first. In a sequential SQLite batch it
1587
+ // closes the invocation before every mutation below if either the token
1588
+ // generation or node binding changed; on transactional adapters the same
1589
+ // statement ordering and rollback guarantee apply.
1590
+ writes.push(writeDb
1591
+ .update(actionInvocations)
1592
+ .set({
1593
+ status: 'failed',
1594
+ error: RELEASE_GENERATION_CONFLICT_CODE,
1595
+ completedAt,
1596
+ spawnReservedAt: null,
1597
+ })
1598
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (${releaseCanApply})`))
1599
+ .returning());
1600
+ writes.push(writeDb
1601
+ .update(nodes)
1602
+ .set({
1603
+ activeAgents: sql `CASE WHEN ${nodes.activeAgents} > 0 THEN ${nodes.activeAgents} - 1 ELSE 0 END`,
1604
+ })
1605
+ .where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId), invocationIsOpen, generationStillCurrent, activeBindingStillCurrent)));
1606
+ writes.push(writeDb
1607
+ .update(agentNodeBindings)
1608
+ .set({ status: 'inactive', updatedAt: completedAt })
1609
+ .where(and(eq(agentNodeBindings.workspaceId, workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.nodeId, nodeId), eq(agentNodeBindings.status, 'active'), invocationIsOpen, generationStillCurrent)));
1610
+ if (input.delete_agent === true) {
1611
+ writes.push(writeDb
1612
+ .delete(channelMembers)
1613
+ .where(and(eq(channelMembers.agentId, agent.id), invocationIsOpen, generationStillCurrent)));
1614
+ writes.push(writeDb
1615
+ .delete(dmParticipants)
1616
+ .where(and(eq(dmParticipants.agentId, agent.id), invocationIsOpen, generationStillCurrent)));
1617
+ }
1618
+ successIndex = writes.length;
1619
+ writes.push(writeDb
1620
+ .update(actionInvocations)
1621
+ .set({
1622
+ output: data.output ?? null,
1623
+ error: null,
1624
+ status: 'completed',
1625
+ completedAt,
1626
+ spawnReservedAt: null,
1627
+ })
1628
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
1629
+ .returning());
1630
+ const currentAgent = and(eq(agents.workspaceId, workspaceId), eq(agents.id, agent.id), eq(agents.name, name), eq(agents.tokenHash, expectedTokenHash), invocationCompleted);
1631
+ if (input.delete_agent === true) {
1632
+ writes.push(writeDb
1633
+ .update(agents)
1634
+ .set({
1635
+ name: releasedName,
1636
+ handle: `@${releasedName}`,
1637
+ status: RELEASED_AGENT_STATUS,
1638
+ tokenHash: releasedTokenHash,
1639
+ previousTokenHash: null,
1640
+ previousTokenExpiresAt: null,
1641
+ locationType: 'self_connected',
1642
+ locationNodeId: null,
1643
+ lastSeen: completedAt,
1644
+ metadata: sql `json_patch(COALESCE(${agents.metadata}, '{}'), ${JSON.stringify({
1645
+ release: {
1646
+ reason: typeof input.reason === 'string' ? input.reason : null,
1647
+ released_at: completedAt.toISOString(),
1648
+ previous_name: agent.name,
1649
+ },
1650
+ })})`,
1651
+ })
1652
+ .where(currentAgent));
1653
+ }
1654
+ else {
1655
+ const existingMetadata = agent.metadata ?? {};
1656
+ const { spawn: _spawn, cli: _cli, ...restMetadata } = existingMetadata;
1657
+ writes.push(writeDb
1658
+ .update(agents)
1659
+ .set({
1660
+ status: 'offline',
1661
+ locationType: 'self_connected',
1662
+ locationNodeId: null,
1663
+ lastSeen: completedAt,
1664
+ metadata: {
1665
+ ...restMetadata,
1666
+ release: {
1667
+ reason: typeof input.reason === 'string' ? input.reason : null,
1668
+ released_at: completedAt.toISOString(),
1669
+ },
1670
+ },
1671
+ })
1672
+ .where(currentAgent));
1673
+ }
1674
+ if (input.delete_agent === true) {
1675
+ // Deleting the implicit node clears action_invocations.dispatched_node_id
1676
+ // via its FK. Do this only after the owned completion and agent CAS have
1677
+ // both succeeded, or their node predicates could no longer match.
1678
+ writes.push(writeDb
1679
+ .delete(nodes)
1680
+ .where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, `node_direct_${agent.id}`), sql `EXISTS (
1681
+ SELECT 1 FROM ${actionInvocations}
1682
+ WHERE ${actionInvocations.workspaceId} = ${workspaceId}
1683
+ AND ${actionInvocations.id} = ${invocation.id}
1684
+ AND ${actionInvocations.status} = 'completed'
1685
+ )`)));
1686
+ }
1687
+ return writes;
1688
+ }, { requireAtomic: true });
1689
+ const completed = results[successIndex];
1690
+ const generationConflict = results[0];
1691
+ const settled = completed[0] ?? generationConflict[0];
1692
+ if (!settled)
1693
+ return null;
1694
+ if (completed[0] && deps) {
1695
+ await emitAgentExitedEffects(deps, workspaceId, {
1696
+ agentId: agent.id,
1697
+ agentName: agent.name,
1698
+ nodeId,
1699
+ invocationId: fleetInvocationId(agent.metadata),
1700
+ reason: 'released',
1701
+ });
1702
+ }
1703
+ return publicInvocation(settled);
1704
+ }
1234
1705
  async function applyReleaseCompletionEffect(db, workspaceId, nodeId, invocation, data, deps, options = {}) {
1235
- if (!isReleaseInvocation(invocation.actionName) || data.error)
1706
+ if (!isBuiltinReleaseInvocation(invocation) || data.error)
1236
1707
  return false;
1237
1708
  const input = recordInput(invocation.input);
1238
1709
  const name = typeof input.name === 'string' ? input.name : null;
@@ -1360,13 +1831,137 @@ async function dispatchNodeAttempt(db, workspaceId, invocationId, nodeId, opts)
1360
1831
  dispatchedNodeId: nodeId,
1361
1832
  dispatchedProvider: opts.providerName,
1362
1833
  spawnReservedAt: opts.reservationHeld ? new Date() : null,
1363
- ...(opts.actionId ? { actionId: opts.actionId } : {}),
1834
+ ...(opts.targetActionId ? { actionId: opts.targetActionId } : {}),
1364
1835
  ...attemptFields,
1365
1836
  })
1366
- .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
1837
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedAction ? [
1838
+ eq(actionInvocations.invocationOrigin, 'registered_action'),
1839
+ eq(actionInvocations.actionId, opts.expectedAction.id),
1840
+ sql `EXISTS (
1841
+ SELECT 1 FROM ${actions}
1842
+ WHERE ${actions.workspaceId} = ${workspaceId}
1843
+ AND ${actions.id} = ${opts.expectedAction.id}
1844
+ )`,
1845
+ ...(opts.targetActionId || opts.expectedAction.handlerAgentId ? [
1846
+ registeredActionDispatchPredicate({
1847
+ workspaceId,
1848
+ nodeId,
1849
+ providerName: opts.providerName,
1850
+ actionId: opts.targetActionId ?? opts.expectedAction.id,
1851
+ actionName: opts.expectedAction.name,
1852
+ handlerAgentId: opts.expectedAction.handlerAgentId,
1853
+ }),
1854
+ ] : []),
1855
+ ] : [])))
1367
1856
  .returning();
1368
1857
  return !!updated;
1369
1858
  }
1859
+ function registeredActionDispatchPredicate(args) {
1860
+ const handlerPredicate = args.handlerAgentId
1861
+ ? sql `AND ${actions.handlerAgentId} = ${args.handlerAgentId} AND ${actions.handlerNodeId} IS NULL`
1862
+ : sql `AND ${actions.handlerNodeId} = ${args.nodeId}
1863
+ AND COALESCE(${actions.handlerProvider}, ${DEFAULT_PROVIDER_NAME}) = ${args.providerName}`;
1864
+ return sql `EXISTS (
1865
+ SELECT 1 FROM ${actions}
1866
+ WHERE ${actions.workspaceId} = ${args.workspaceId}
1867
+ AND ${actions.id} = ${args.actionId}
1868
+ AND ${actions.name} = ${args.actionName}
1869
+ AND ${actions.isActive} = 1
1870
+ ${handlerPredicate}
1871
+ )`;
1872
+ }
1873
+ async function claimRegisteredActionHandoff(db, args) {
1874
+ const claimedActionId = args.targetActionId ?? args.expectedActionId;
1875
+ const stateFields = args.pending
1876
+ ? { status: 'pending', dispatchedAt: null, retryAfterAt: args.retryAfterAt ?? null }
1877
+ : dispatchedStateFields({ retryAfterAt: args.retryAfterAt });
1878
+ const attemptFields = args.skipIncrementAttempts
1879
+ ? {}
1880
+ : {
1881
+ attemptedNodeIds: sql `json_insert(COALESCE(${actionInvocations.attemptedNodeIds}, '[]'), '$[#]', ${args.nodeId})`,
1882
+ dispatchAttempts: sql `COALESCE(${actionInvocations.dispatchAttempts}, 0) + 1`,
1883
+ };
1884
+ const [claimed] = await db
1885
+ .update(actionInvocations)
1886
+ // This CAS is the durable dispatch linearization point. Claim the exact
1887
+ // action generation, route, and attempt before awaiting the provider. A
1888
+ // retry may hand off between same-name registrations only while it owns the
1889
+ // source generation; a later rescheduler observes the new route instead of
1890
+ // sending the same attempt again.
1891
+ .set({
1892
+ ...stateFields,
1893
+ handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${args.nodeId})`,
1894
+ dispatchedNodeId: args.nodeId,
1895
+ dispatchedProvider: args.providerName,
1896
+ spawnReservedAt: args.reservationHeld ? new Date() : null,
1897
+ actionId: claimedActionId,
1898
+ ...attemptFields,
1899
+ })
1900
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.expectedActionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (
1901
+ ${actionInvocations.actionId} = ${claimedActionId}
1902
+ AND COALESCE(${actionInvocations.dispatchedNodeId}, '') = ${args.nodeId}
1903
+ AND COALESCE(${actionInvocations.dispatchedProvider}, ${DEFAULT_PROVIDER_NAME}) = ${args.providerName}
1904
+ AND ${actionInvocations.status} IN ('dispatched', 'invoked')
1905
+ )`, sql `EXISTS (
1906
+ SELECT 1 FROM ${actions}
1907
+ WHERE ${actions.workspaceId} = ${args.workspaceId}
1908
+ AND ${actions.id} = ${args.expectedActionId}
1909
+ )`, ...(args.targetActionId ? [registeredActionDispatchPredicate({
1910
+ workspaceId: args.workspaceId,
1911
+ nodeId: args.nodeId,
1912
+ providerName: args.providerName,
1913
+ actionId: args.targetActionId,
1914
+ actionName: args.actionName,
1915
+ })] : [])))
1916
+ .returning({
1917
+ actionId: actionInvocations.actionId,
1918
+ actionName: actionInvocations.actionName,
1919
+ dispatchAttempts: actionInvocations.dispatchAttempts,
1920
+ });
1921
+ return claimed?.actionId ? {
1922
+ actionId: claimed.actionId,
1923
+ actionName: claimed.actionName,
1924
+ dispatchAttempts: claimed.dispatchAttempts,
1925
+ } : null;
1926
+ }
1927
+ async function bindInvocationToRegisteredNodeAction(db, args) {
1928
+ const [bound] = await db
1929
+ .update(actionInvocations)
1930
+ .set({ actionId: args.actionId, invocationOrigin: 'registered_action' })
1931
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'builtin'), isNull(actionInvocations.actionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), eq(actionInvocations.dispatchAttempts, 0), registeredActionDispatchPredicate(args)))
1932
+ .returning({ id: actionInvocations.id });
1933
+ return !!bound;
1934
+ }
1935
+ async function settleDeletedRegisteredAction(db, workspaceId, invocationId) {
1936
+ // A failed handoff can also mean another dispatcher already advanced the
1937
+ // same invocation to a different live action. Fail only the FK-null state
1938
+ // that proves pruning won; never clobber that concurrent dispatch winner.
1939
+ const [failed] = await db
1940
+ .update(actionInvocations)
1941
+ .set({
1942
+ status: 'failed',
1943
+ error: 'action_deleted',
1944
+ completedAt: new Date(),
1945
+ spawnReservedAt: null,
1946
+ })
1947
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), isNull(actionInvocations.actionId), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
1948
+ .returning();
1949
+ if (failed)
1950
+ return failed;
1951
+ const [settled] = await db
1952
+ .select()
1953
+ .from(actionInvocations)
1954
+ .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId)));
1955
+ return settled;
1956
+ }
1957
+ async function settleRegisteredActionSendFailure(db, args) {
1958
+ const [failed] = await db
1959
+ .update(actionInvocations)
1960
+ .set({ status: 'failed', error: 'node_dispatch_unavailable', completedAt: new Date() })
1961
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.actionId), eq(actionInvocations.dispatchedNodeId, args.nodeId), eq(actionInvocations.dispatchedProvider, args.providerName), eq(actionInvocations.dispatchAttempts, args.dispatchAttempts), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
1962
+ .returning();
1963
+ return failed ?? settleDeletedRegisteredAction(db, args.workspaceId, args.invocationId);
1964
+ }
1370
1965
  async function snapshotInvocationHandlerNode(db, workspaceId, invocationId, nodeId) {
1371
1966
  const [updated] = await db
1372
1967
  .update(actionInvocations)
@@ -1396,6 +1991,44 @@ export async function markDrainedInvocationDispatched(db, workspaceId, invocatio
1396
1991
  return !!updated;
1397
1992
  }
1398
1993
  async function dispatchNodeInvocation(args) {
1994
+ const guardedReleaseHash = args.invocationOrigin === 'builtin' && isReleaseInvocation(args.action)
1995
+ ? releaseExpectedTokenHash(args.input)
1996
+ : null;
1997
+ if (guardedReleaseHash) {
1998
+ const name = typeof args.input.name === 'string' ? args.input.name : '';
1999
+ const [current] = await args.db
2000
+ .select({ id: agents.id })
2001
+ .from(agents)
2002
+ .where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name), eq(agents.tokenHash, guardedReleaseHash), sql `EXISTS (
2003
+ SELECT 1 FROM ${agentNodeBindings}
2004
+ WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
2005
+ AND ${agentNodeBindings.agentId} = ${agents.id}
2006
+ AND ${agentNodeBindings.nodeId} = ${args.nodeId}
2007
+ AND ${agentNodeBindings.status} = 'active'
2008
+ )`));
2009
+ if (!current) {
2010
+ await args.db
2011
+ .update(actionInvocations)
2012
+ .set({
2013
+ status: 'failed',
2014
+ error: RELEASE_GENERATION_CONFLICT_CODE,
2015
+ completedAt: new Date(),
2016
+ })
2017
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
2018
+ return { accepted: false, pending: false, sent: false };
2019
+ }
2020
+ if (!args.registry.sendAuthorizedActionToProvider) {
2021
+ await args.db
2022
+ .update(actionInvocations)
2023
+ .set({
2024
+ status: 'failed',
2025
+ error: 'node_dispatch_unavailable',
2026
+ completedAt: new Date(),
2027
+ })
2028
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
2029
+ return { accepted: false, pending: false, sent: false };
2030
+ }
2031
+ }
1399
2032
  const frame = {
1400
2033
  v: 1,
1401
2034
  type: 'action.invoke',
@@ -1413,6 +2046,32 @@ async function dispatchNodeInvocation(args) {
1413
2046
  if (!snapshotted)
1414
2047
  return { accepted: false, pending: false, sent: false };
1415
2048
  const connectedBefore = args.registry.isProviderConnected(args.workspaceId, args.nodeId, args.providerName);
2049
+ const pending = !!args.pending || !connectedBefore;
2050
+ const expectedActionId = args.invocationOrigin === 'registered_action'
2051
+ ? (args.expectedActionId ?? args.actionId)
2052
+ : null;
2053
+ let registeredNodeClaim = null;
2054
+ if (args.invocationOrigin === 'registered_action' && !args.agent) {
2055
+ registeredNodeClaim = expectedActionId
2056
+ ? await claimRegisteredActionHandoff(args.db, {
2057
+ workspaceId: args.workspaceId,
2058
+ invocationId: args.invocationId,
2059
+ nodeId: args.nodeId,
2060
+ providerName: args.providerName,
2061
+ expectedActionId,
2062
+ targetActionId: args.actionId ?? undefined,
2063
+ actionName: args.action,
2064
+ pending,
2065
+ retryAfterAt: args.retryAfterAt,
2066
+ reservationHeld: args.reservationHeld,
2067
+ skipIncrementAttempts: args.skipIncrementAttempts,
2068
+ })
2069
+ : null;
2070
+ if (!registeredNodeClaim) {
2071
+ const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
2072
+ return { accepted: false, pending: false, sent: false, settled };
2073
+ }
2074
+ }
1416
2075
  const sent = args.agent && args.actionId
1417
2076
  ? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
1418
2077
  kind: 'agent-action-v1',
@@ -1421,18 +2080,88 @@ async function dispatchNodeInvocation(args) {
1421
2080
  handlerAgentId: args.agent.id,
1422
2081
  recordAttempt: !args.skipIncrementAttempts,
1423
2082
  }) ?? false)
1424
- : await args.registry.sendToProvider(args.workspaceId, args.nodeId, args.providerName, frame);
1425
- if (!sent)
2083
+ : registeredNodeClaim
2084
+ ? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
2085
+ kind: 'registered-node-action-v2',
2086
+ invocationId: args.invocationId,
2087
+ actionId: registeredNodeClaim.actionId,
2088
+ dispatchAttempt: registeredNodeClaim.dispatchAttempts,
2089
+ invocationActionName: registeredNodeClaim.actionName,
2090
+ actionName: args.action,
2091
+ }) ?? false)
2092
+ : guardedReleaseHash
2093
+ ? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
2094
+ kind: 'release-generation-v1',
2095
+ invocationId: args.invocationId,
2096
+ agentName: typeof args.input.name === 'string' ? args.input.name : '',
2097
+ expectedTokenHash: guardedReleaseHash,
2098
+ }) ?? false)
2099
+ : await args.registry.sendToProvider(args.workspaceId, args.nodeId, args.providerName, frame);
2100
+ if (!sent && guardedReleaseHash) {
2101
+ // An adapter compiled against the older owner-authorization contract may
2102
+ // expose the method but reject the new proof kind. Settle any still-open
2103
+ // row before the caller considers local fallback, so a guarded delete can
2104
+ // never degrade to an unguarded lifecycle mutation.
2105
+ await args.db
2106
+ .update(actionInvocations)
2107
+ .set({
2108
+ status: 'failed',
2109
+ error: 'node_dispatch_unavailable',
2110
+ completedAt: new Date(),
2111
+ })
2112
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
2113
+ }
2114
+ if (!sent) {
2115
+ if (registeredNodeClaim) {
2116
+ const settled = await settleRegisteredActionSendFailure(args.db, {
2117
+ workspaceId: args.workspaceId,
2118
+ invocationId: args.invocationId,
2119
+ nodeId: args.nodeId,
2120
+ providerName: args.providerName,
2121
+ actionId: registeredNodeClaim.actionId,
2122
+ dispatchAttempts: registeredNodeClaim.dispatchAttempts,
2123
+ });
2124
+ return { accepted: false, pending: false, sent: false, settled };
2125
+ }
1426
2126
  return { accepted: false, pending: false, sent: false };
1427
- const pending = !!args.pending || !connectedBefore;
2127
+ }
2128
+ // Registered node-action state was committed by the pre-send handoff CAS.
2129
+ // Pruning after provider acceptance may clear the action FK, but it cannot
2130
+ // revoke work the provider already received or prevent route-owned completion.
2131
+ if (registeredNodeClaim) {
2132
+ return { accepted: true, pending, sent: true };
2133
+ }
2134
+ const finalExpectedActionId = expectedActionId;
1428
2135
  const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, {
1429
2136
  pending,
1430
2137
  providerName: args.providerName,
1431
2138
  retryAfterAt: args.retryAfterAt,
1432
2139
  reservationHeld: args.reservationHeld,
1433
2140
  skipIncrementAttempts: args.skipIncrementAttempts || !!args.agent,
1434
- actionId: args.actionId,
2141
+ expectedAction: finalExpectedActionId
2142
+ ? {
2143
+ id: finalExpectedActionId,
2144
+ name: args.action,
2145
+ ...(args.agent ? { handlerAgentId: args.agent.id } : {}),
2146
+ }
2147
+ : undefined,
2148
+ targetActionId: args.agent || args.actionId
2149
+ ? finalExpectedActionId ?? undefined
2150
+ : undefined,
1435
2151
  });
2152
+ if (!accepted && args.invocationOrigin === 'registered_action') {
2153
+ const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
2154
+ return { accepted: false, pending: false, sent: true, settled };
2155
+ }
2156
+ if (!accepted && guardedReleaseHash) {
2157
+ const [settled] = await args.db
2158
+ .select()
2159
+ .from(actionInvocations)
2160
+ .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId)));
2161
+ if (settled && !OPEN_INVOCATION_STATUSES.some((status) => status === settled.status)) {
2162
+ return { accepted: false, pending: false, sent: true, settled };
2163
+ }
2164
+ }
1436
2165
  return { accepted, pending, sent: true };
1437
2166
  }
1438
2167
  function attemptedNodeSet(invocation) {
@@ -1487,12 +2216,16 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
1487
2216
  const rows = await db
1488
2217
  .select({
1489
2218
  id: actionInvocations.id,
1490
- actionId: actions.id,
2219
+ actionId: actionInvocations.actionId,
2220
+ invocationOrigin: actionInvocations.invocationOrigin,
1491
2221
  workspaceId: actionInvocations.workspaceId,
1492
2222
  actionName: actionInvocations.actionName,
1493
2223
  input: actionInvocations.input,
2224
+ providerAcceptedAttempt: actionInvocations.providerAcceptedAttempt,
2225
+ dispatchAttempts: actionInvocations.dispatchAttempts,
1494
2226
  spawnReservedAt: actionInvocations.spawnReservedAt,
1495
2227
  dispatchedNodeId: actionInvocations.dispatchedNodeId,
2228
+ dispatchedProvider: actionInvocations.dispatchedProvider,
1496
2229
  handlerNodeId: actions.handlerNodeId,
1497
2230
  handlerProvider: actions.handlerProvider,
1498
2231
  handlerAgentId: agents.id,
@@ -1510,6 +2243,10 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
1510
2243
  .orderBy(asc(actionInvocations.createdAt));
1511
2244
  let drained = 0;
1512
2245
  for (const row of rows) {
2246
+ if (isDeletedRegisteredActionInvocation(row)) {
2247
+ await failOpenInvocationRows(db, workspaceId, [row.id], 'action_deleted');
2248
+ continue;
2249
+ }
1513
2250
  const targetAgent = row.handlerAgentId
1514
2251
  && row.handlerAgentName
1515
2252
  && row.handlerAgentLocationType === 'via_node'
@@ -1532,10 +2269,12 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
1532
2269
  ? row.handlerAgentProvider ?? DEFAULT_PROVIDER_NAME
1533
2270
  : row.handlerNodeId
1534
2271
  ? row.handlerProvider ?? DEFAULT_PROVIDER_NAME
1535
- : shadowProvider
1536
- ?? (isSpawnInvocation(row.actionName)
1537
- ? (await capacityProviderName(db, workspaceId, nodeId, dispatchActionNameForInvocation(row.actionName, input))) ?? DEFAULT_PROVIDER_NAME
1538
- : DEFAULT_PROVIDER_NAME);
2272
+ : isBuiltinReleaseInvocation(row)
2273
+ ? row.dispatchedProvider ?? DEFAULT_PROVIDER_NAME
2274
+ : shadowProvider
2275
+ ?? (isSpawnInvocation(row.actionName)
2276
+ ? (await capacityProviderName(db, workspaceId, nodeId, dispatchActionNameForInvocation(row.actionName, input))) ?? DEFAULT_PROVIDER_NAME
2277
+ : DEFAULT_PROVIDER_NAME);
1539
2278
  const nativeSpawn = isSpawnInvocation(row.actionName) && !shadowProvider;
1540
2279
  // Skip draining to a provider that is disconnected or whose heartbeat says
1541
2280
  // handlers are not live. A missing provider row is a legacy single-socket
@@ -1577,7 +2316,9 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
1577
2316
  action: dispatchActionNameForInvocation(row.actionName, input),
1578
2317
  input,
1579
2318
  agent: targetAgent ? { id: targetAgent.id, name: targetAgent.name } : null,
1580
- actionId: targetAgent ? row.actionId ?? undefined : undefined,
2319
+ actionId: row.actionId,
2320
+ expectedActionId: row.actionId,
2321
+ invocationOrigin: row.invocationOrigin,
1581
2322
  retryAfterAt: new Date(Date.now() + ACTION_DISPATCH_TIMEOUT_MS),
1582
2323
  reservationHeld,
1583
2324
  skipIncrementAttempts: row.dispatchedNodeId === nodeId,
@@ -1599,6 +2340,17 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
1599
2340
  return drained;
1600
2341
  }
1601
2342
  export async function rescheduleNodeInvocation(db, registry, invocation, opts = {}) {
2343
+ // Once the exact provider accepted this generation, pruning only clears its
2344
+ // materialized action FK. It must not make timeout, disconnect, or inventory
2345
+ // recovery move the invocation to a same-name replacement and invalidate the
2346
+ // original provider's result.
2347
+ if (isAcceptedDeletedRegisteredActionInvocation(invocation)) {
2348
+ return false;
2349
+ }
2350
+ if (isDeletedRegisteredActionInvocation(invocation)) {
2351
+ await failOpenInvocationRows(db, invocation.workspaceId, [invocation.id], 'action_deleted');
2352
+ return false;
2353
+ }
1602
2354
  // Only release capacity the invocation actually holds. A shadowed spawn
1603
2355
  // dropped its native reservation at dispatch (spawnReservedAt is null), so it
1604
2356
  // must not release capacity that now belongs to another spawn.
@@ -1633,6 +2385,8 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
1633
2385
  input: recordInput(invocation.input),
1634
2386
  agent: { id: targetAgent.agentId, name: targetAgent.agentName },
1635
2387
  actionId: targetAgent.actionId,
2388
+ expectedActionId: invocation.actionId,
2389
+ invocationOrigin: invocation.invocationOrigin,
1636
2390
  retryAfterAt: opts.retryAfterAt ?? null,
1637
2391
  });
1638
2392
  return dispatched.accepted;
@@ -1640,7 +2394,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
1640
2394
  // Release invocations carry no actionId, so targetAgentForInvocation() returns
1641
2395
  // null. They must never fall through to generic node placement, which could
1642
2396
  // route a release to an unrelated node that doesn't own the agent.
1643
- if (isReleaseInvocation(invocation.actionName)) {
2397
+ if (isBuiltinReleaseInvocation(invocation)) {
1644
2398
  return false;
1645
2399
  }
1646
2400
  const attempted = attemptedNodeSet(invocation);
@@ -1677,6 +2431,8 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
1677
2431
  providerName: target.handlerProvider,
1678
2432
  action: actionToSend,
1679
2433
  actionId: target.id,
2434
+ expectedActionId: invocation.actionId,
2435
+ invocationOrigin: invocation.invocationOrigin,
1680
2436
  input,
1681
2437
  queue: target.queue,
1682
2438
  reservationHeld: false,
@@ -1694,14 +2450,23 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
1694
2450
  ? (await capacityProviderName(db, invocation.workspaceId, placement.node.id, actionToSend)) ?? DEFAULT_PROVIDER_NAME
1695
2451
  : DEFAULT_PROVIDER_NAME;
1696
2452
  if (placement.queued) {
1697
- await dispatchNodeAttempt(db, invocation.workspaceId, invocation.id, placement.node.id, {
2453
+ const expectedActionId = invocation.invocationOrigin === 'registered_action'
2454
+ ? invocation.actionId
2455
+ : null;
2456
+ const accepted = await dispatchNodeAttempt(db, invocation.workspaceId, invocation.id, placement.node.id, {
1698
2457
  providerName,
1699
2458
  pending: true,
1700
2459
  retryAfterAt: opts.retryAfterAt ?? null,
1701
2460
  reservationHeld: false,
1702
- actionId: target?.id,
2461
+ expectedAction: expectedActionId
2462
+ ? { id: expectedActionId, name: actionToSend }
2463
+ : undefined,
2464
+ targetActionId: target?.id,
1703
2465
  });
1704
- return true;
2466
+ if (!accepted && invocation.invocationOrigin === 'registered_action') {
2467
+ await settleDeletedRegisteredAction(db, invocation.workspaceId, invocation.id);
2468
+ }
2469
+ return accepted;
1705
2470
  }
1706
2471
  const dispatched = await dispatchNodeInvocation({
1707
2472
  db,
@@ -1711,7 +2476,9 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
1711
2476
  nodeId: placement.node.id,
1712
2477
  providerName,
1713
2478
  action: actionToSend,
1714
- actionId: target?.id,
2479
+ actionId: target?.id ?? null,
2480
+ expectedActionId: invocation.actionId,
2481
+ invocationOrigin: invocation.invocationOrigin,
1715
2482
  input,
1716
2483
  reservationHeld,
1717
2484
  });
@@ -1815,6 +2582,8 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
1815
2582
  .select({
1816
2583
  id: actionInvocations.id,
1817
2584
  workspaceId: actionInvocations.workspaceId,
2585
+ actionId: actionInvocations.actionId,
2586
+ invocationOrigin: actionInvocations.invocationOrigin,
1818
2587
  actionName: actionInvocations.actionName,
1819
2588
  callerId: actionInvocations.callerId,
1820
2589
  input: actionInvocations.input,
@@ -1824,6 +2593,7 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
1824
2593
  durationMs: actionInvocations.durationMs,
1825
2594
  dispatchedNodeId: actionInvocations.dispatchedNodeId,
1826
2595
  dispatchedAt: actionInvocations.dispatchedAt,
2596
+ providerAcceptedAttempt: actionInvocations.providerAcceptedAttempt,
1827
2597
  spawnReservedAt: actionInvocations.spawnReservedAt,
1828
2598
  attemptedNodeIds: actionInvocations.attemptedNodeIds,
1829
2599
  dispatchAttempts: actionInvocations.dispatchAttempts,
@@ -1854,6 +2624,12 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
1854
2624
  // Fall through and fail the invocation if no eligible node exists.
1855
2625
  }
1856
2626
  }
2627
+ const existingInput = recordInput(existing.input);
2628
+ if (!data.error
2629
+ && isBuiltinReleaseInvocation(existing)
2630
+ && existingInput.expected_token_hash !== undefined) {
2631
+ return completeGuardedReleaseNodeInvocation(db, workspaceId, nodeId, providerName, existing, data, deps);
2632
+ }
1857
2633
  const [updated] = await db
1858
2634
  .update(actionInvocations)
1859
2635
  .set({