@relaycast/engine 8.3.1 → 8.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/node/realtime.d.ts +2 -2
- package/dist/adapters/node/realtime.d.ts.map +1 -1
- package/dist/adapters/node/realtime.js +118 -30
- package/dist/adapters/node/realtime.js.map +1 -1
- package/dist/db/migrations/0045_workspace_create_idempotency.sql +16 -0
- package/dist/db/migrations/0046_action_invocation_origin.sql +51 -0
- package/dist/db/migrations/0047_action_invocation_provider_acceptance.sql +15 -0
- package/dist/db/schema.d.ts +178 -0
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +28 -0
- package/dist/db/schema.js.map +1 -1
- package/dist/engine/action.d.ts +24 -1
- package/dist/engine/action.d.ts.map +1 -1
- package/dist/engine/action.js +898 -122
- package/dist/engine/action.js.map +1 -1
- package/dist/engine/node.d.ts +1 -1
- package/dist/engine/node.d.ts.map +1 -1
- package/dist/engine/node.js +127 -43
- package/dist/engine/node.js.map +1 -1
- package/dist/engine/nodeProvider.d.ts.map +1 -1
- package/dist/engine/nodeProvider.js +15 -8
- package/dist/engine/nodeProvider.js.map +1 -1
- package/dist/engine/workspace.d.ts +9 -0
- package/dist/engine/workspace.d.ts.map +1 -1
- package/dist/engine/workspace.js +110 -4
- package/dist/engine/workspace.js.map +1 -1
- package/dist/ports/realtime.d.ts +35 -4
- package/dist/ports/realtime.d.ts.map +1 -1
- package/dist/ports/realtime.js.map +1 -1
- package/dist/routes/agent.js +5 -3
- package/dist/routes/agent.js.map +1 -1
- package/dist/routes/workspace.d.ts.map +1 -1
- package/dist/routes/workspace.js +23 -0
- package/dist/routes/workspace.js.map +1 -1
- package/package.json +3 -3
package/dist/engine/action.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { and, asc, eq, inArray, isNotNull, isNull, lte, or, sql } from 'drizzle-orm';
|
|
2
2
|
import { actions, actionInvocations, agents, agentNodeBindings, channelMembers, dmParticipants, nodes } from '../db/schema.js';
|
|
3
3
|
import { generateId } from './snowflake.js';
|
|
4
|
-
import { RELEASED_AGENT_STATUS, releasedAgentName } from './agent.js';
|
|
4
|
+
import { assertRegistrableAgentName, RELEASED_AGENT_STATUS, releasedAgentName } from './agent.js';
|
|
5
5
|
import { randomHex, sha256Hex } from '../lib/crypto.js';
|
|
6
6
|
import { codedError } from '../lib/httpError.js';
|
|
7
7
|
import { D1_SAFE_IN_QUERY_CHUNK_SIZE } from '../lib/queryChunks.js';
|
|
@@ -10,6 +10,7 @@ import { emitAgentExitedEffects, emitInvocationCompletionEffects, fleetInvocatio
|
|
|
10
10
|
import { runAtomic, runAtomicWrites } from '../ports/database.js';
|
|
11
11
|
import { claimSpawnNode, chooseNodeForAction, isNodeLive, releaseNodeCapacity, reserveNodeCapacity } from './placement.js';
|
|
12
12
|
import { DEFAULT_PROVIDER_NAME, capacityProviderName, getProvider, isProviderLive } from './nodeProvider.js';
|
|
13
|
+
import { AGENT_TOKEN_HASH_PATTERN } from '@relaycast/types';
|
|
13
14
|
const OPEN_INVOCATION_STATUSES = ['pending', 'dispatched', 'invoked'];
|
|
14
15
|
const REPLAYED_INVOCATION = Symbol('replayed-action-invocation');
|
|
15
16
|
export const ACTION_DISPATCH_TIMEOUT_MS = 30_000;
|
|
@@ -62,6 +63,42 @@ function isSpawnInvocation(actionName) {
|
|
|
62
63
|
function isReleaseInvocation(actionName) {
|
|
63
64
|
return actionName === 'release';
|
|
64
65
|
}
|
|
66
|
+
function isBuiltinReleaseInvocation(invocation) {
|
|
67
|
+
return invocation.invocationOrigin === 'builtin' && isReleaseInvocation(invocation.actionName);
|
|
68
|
+
}
|
|
69
|
+
function isDeletedRegisteredActionInvocation(invocation) {
|
|
70
|
+
// For a registered-action origin, actionId is cleared only by the FK when its
|
|
71
|
+
// exact action row is deleted. The immutable origin prevents that orphan
|
|
72
|
+
// from being mistaken for a built-in, and this predicate prevents it from
|
|
73
|
+
// being rebound by name during drain/retry/reconciliation.
|
|
74
|
+
return invocation.invocationOrigin === 'registered_action'
|
|
75
|
+
&& invocation.actionId === null
|
|
76
|
+
&& invocation.providerAcceptedAttempt !== invocation.dispatchAttempts;
|
|
77
|
+
}
|
|
78
|
+
function isAcceptedDeletedRegisteredActionInvocation(invocation) {
|
|
79
|
+
return invocation.invocationOrigin === 'registered_action'
|
|
80
|
+
&& invocation.actionId === null
|
|
81
|
+
&& invocation.providerAcceptedAttempt !== null
|
|
82
|
+
&& invocation.providerAcceptedAttempt === invocation.dispatchAttempts;
|
|
83
|
+
}
|
|
84
|
+
const RELEASE_GENERATION_CONFLICT_CODE = 'agent_release_generation_conflict';
|
|
85
|
+
function releaseExpectedTokenHash(input) {
|
|
86
|
+
const value = input.expected_token_hash;
|
|
87
|
+
if (value === undefined)
|
|
88
|
+
return null;
|
|
89
|
+
if (typeof value !== 'string' || !AGENT_TOKEN_HASH_PATTERN.test(value)) {
|
|
90
|
+
throw codedError('release action input.expected_token_hash must be a lowercase SHA-256 hash', 'invalid_release_request', 400);
|
|
91
|
+
}
|
|
92
|
+
return value;
|
|
93
|
+
}
|
|
94
|
+
function releaseGenerationStillCurrent(workspaceId, agentId, expectedTokenHash) {
|
|
95
|
+
return sql `EXISTS (
|
|
96
|
+
SELECT 1 FROM ${agents}
|
|
97
|
+
WHERE ${agents.workspaceId} = ${workspaceId}
|
|
98
|
+
AND ${agents.id} = ${agentId}
|
|
99
|
+
${expectedTokenHash ? sql `AND ${agents.tokenHash} = ${expectedTokenHash}` : sql ``}
|
|
100
|
+
)`;
|
|
101
|
+
}
|
|
65
102
|
function dispatchActionNameForInvocation(actionName, input) {
|
|
66
103
|
if (!isSpawnInvocation(actionName))
|
|
67
104
|
return actionName;
|
|
@@ -257,13 +294,16 @@ export async function registerAction(db, workspaceId, data, options = {}) {
|
|
|
257
294
|
: { target: [actions.workspaceId, actions.name], targetWhere: isNull(actions.handlerNodeId), set: refresh })
|
|
258
295
|
.returning();
|
|
259
296
|
// A refresh that moves the handler to a DIFFERENT agent strands
|
|
260
|
-
// invocations
|
|
261
|
-
//
|
|
262
|
-
//
|
|
263
|
-
//
|
|
264
|
-
// fail them; the callers are told after commit.
|
|
297
|
+
// invocations still waiting for provider acceptance. Once the previous
|
|
298
|
+
// handler accepted an exact attempt, detach that invocation from the
|
|
299
|
+
// mutable action row instead: its snapshotted route remains authorized
|
|
300
|
+
// to complete and timeout recovery cannot retarget it to the replacement.
|
|
265
301
|
let strandedRows = [];
|
|
266
302
|
if (previous && previous.handlerAgentId && previous.handlerAgentId !== handlerAgentId) {
|
|
303
|
+
await tx
|
|
304
|
+
.update(actionInvocations)
|
|
305
|
+
.set({ actionId: null })
|
|
306
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.actionId, previous.id), eq(actionInvocations.invocationOrigin, 'registered_action'), isNotNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} = ${actionInvocations.dispatchAttempts}`, inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
267
307
|
const strandedIds = await openInvocationIdsForActions(tx, workspaceId, [previous.id]);
|
|
268
308
|
strandedRows = await failOpenInvocationRows(tx, workspaceId, strandedIds, 'handler_unavailable');
|
|
269
309
|
}
|
|
@@ -322,7 +362,7 @@ export async function listActions(db, workspaceId, callerName) {
|
|
|
322
362
|
.map(publicAction);
|
|
323
363
|
}
|
|
324
364
|
export async function getAction(db, workspaceId, name, callerName) {
|
|
325
|
-
const
|
|
365
|
+
const rows = await db
|
|
326
366
|
.select({
|
|
327
367
|
id: actions.id,
|
|
328
368
|
name: actions.name,
|
|
@@ -330,6 +370,7 @@ export async function getAction(db, workspaceId, name, callerName) {
|
|
|
330
370
|
handlerAgentName: agents.name,
|
|
331
371
|
handlerNodeName: nodes.name,
|
|
332
372
|
handlerNodeId: actions.handlerNodeId,
|
|
373
|
+
isGlobal: actions.isGlobal,
|
|
333
374
|
inputSchema: actions.inputSchema,
|
|
334
375
|
outputSchema: actions.outputSchema,
|
|
335
376
|
availableTo: actions.availableTo,
|
|
@@ -340,6 +381,8 @@ export async function getAction(db, workspaceId, name, callerName) {
|
|
|
340
381
|
.leftJoin(agents, eq(actions.handlerAgentId, agents.id))
|
|
341
382
|
.leftJoin(nodes, eq(actions.handlerNodeId, nodes.id))
|
|
342
383
|
.where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, name)));
|
|
384
|
+
const row = rows.sort((a, b) => actionResolutionRank(a) - actionResolutionRank(b) ||
|
|
385
|
+
(a.id < b.id ? -1 : a.id > b.id ? 1 : 0))[0];
|
|
343
386
|
if (!row || !isActionVisibleToCaller(row.availableTo ?? null, callerName))
|
|
344
387
|
return null;
|
|
345
388
|
return publicAction(row);
|
|
@@ -378,6 +421,7 @@ async function createInvocation(db, workspaceId, action, data) {
|
|
|
378
421
|
workspaceId,
|
|
379
422
|
actionId: action?.id ?? null,
|
|
380
423
|
actionName: expectedActionName,
|
|
424
|
+
invocationOrigin: action ? 'registered_action' : 'builtin',
|
|
381
425
|
callerId: data.caller_id ?? null,
|
|
382
426
|
callerName: data.caller_name ?? null,
|
|
383
427
|
handlerAgentId: data.handler_agent_id ?? action?.handlerAgentId ?? null,
|
|
@@ -417,21 +461,62 @@ async function idempotentInvocationId(workspaceId, callerId, actionName, key) {
|
|
|
417
461
|
const digest = await sha256Hex(['action-invoke-v1', workspaceId, callerId, actionName, key].join('\0'));
|
|
418
462
|
return `inv_idem_${digest}`;
|
|
419
463
|
}
|
|
464
|
+
/**
|
|
465
|
+
* Poll a durable idempotency claim until the winning request's dispatch is
|
|
466
|
+
* visible, for up to 500ms.
|
|
467
|
+
*
|
|
468
|
+
* Resolves as soon as the row leaves the pre-dispatch state: it returns the row
|
|
469
|
+
* once `dispatchedNodeId` is set (or, with `acceptDispatchStarted`, once an
|
|
470
|
+
* attempt has been recorded), returns a settled non-open row, and throws the
|
|
471
|
+
* row's own failure for one that failed before dispatch.
|
|
472
|
+
*
|
|
473
|
+
* `onDeadline` picks what a still-open row does when the 500ms deadline passes:
|
|
474
|
+
* `'throw'` (the default) raises a retryable 503 `idempotency_unavailable`,
|
|
475
|
+
* which is what a replay observing someone else's claim wants; `'return'` hands
|
|
476
|
+
* the still-open row back so a caller that owns the claim itself — the
|
|
477
|
+
* post-send takeover re-read below — can classify it.
|
|
478
|
+
*
|
|
479
|
+
* @param options.acceptDispatchStarted - treat a recorded attempt as dispatch.
|
|
480
|
+
* @param options.onDeadline - deadline behavior for a still-open row.
|
|
481
|
+
* @returns the invocation row as last read.
|
|
482
|
+
*/
|
|
420
483
|
async function waitForInvocationReplayOutcome(db, workspaceId, invocation, options = {}) {
|
|
421
484
|
let current = invocation;
|
|
422
485
|
const deadline = Date.now() + 500;
|
|
423
|
-
while (
|
|
424
|
-
|
|
486
|
+
while (true) {
|
|
487
|
+
// A guarded release can lose its generation CAS after provider dispatch.
|
|
488
|
+
// Surface that durable terminal conflict before the generic dispatched-row
|
|
489
|
+
// replay shortcut, otherwise the same idempotency key changes 409 into 201.
|
|
490
|
+
if (isBuiltinReleaseInvocation(current)
|
|
491
|
+
&& current.status === 'failed'
|
|
492
|
+
&& current.error === RELEASE_GENERATION_CONFLICT_CODE) {
|
|
493
|
+
throw codedError('Action invocation failed because the release generation changed', RELEASE_GENERATION_CONFLICT_CODE, 409);
|
|
494
|
+
}
|
|
495
|
+
// Registered node actions only enter action_deleted before the provider
|
|
496
|
+
// accepts the frame. Surface that durable result before the route snapshot
|
|
497
|
+
// shortcut so the winning request and every replay return the same 503.
|
|
498
|
+
if (current.status === 'failed' && current.error === 'action_deleted') {
|
|
499
|
+
throw codedError('Action invocation failed before provider dispatch completed', 'action_deleted', 503);
|
|
500
|
+
}
|
|
501
|
+
if (current.dispatchedNodeId
|
|
502
|
+
|| (options.acceptDispatchStarted && current.dispatchAttempts > 0)) {
|
|
503
|
+
return current;
|
|
504
|
+
}
|
|
425
505
|
if (!OPEN_INVOCATION_STATUSES.some((status) => status === current.status)) {
|
|
426
506
|
if (current.status === 'failed') {
|
|
427
|
-
|
|
507
|
+
const errorCode = current.error ?? 'handler_unavailable';
|
|
508
|
+
throw codedError('Action invocation failed before provider dispatch completed', errorCode, isBuiltinReleaseInvocation(current) && errorCode === RELEASE_GENERATION_CONFLICT_CODE ? 409 : 503);
|
|
428
509
|
}
|
|
429
510
|
return current;
|
|
430
511
|
}
|
|
431
512
|
if (Date.now() >= deadline) {
|
|
432
513
|
// The winning request may have died after its durable claim but before
|
|
433
514
|
// provider dispatch started or completed. Fail retryably without
|
|
434
|
-
// releasing the key for another execution.
|
|
515
|
+
// releasing the key for another execution. Callers that own the claim
|
|
516
|
+
// themselves (the post-send takeover re-read) opt out and classify the
|
|
517
|
+
// still-open row instead.
|
|
518
|
+
if (options.onDeadline === 'return')
|
|
519
|
+
return current;
|
|
435
520
|
throw codedError('Idempotent action dispatch is still pending', 'idempotency_unavailable', 503);
|
|
436
521
|
}
|
|
437
522
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
@@ -444,7 +529,6 @@ async function waitForInvocationReplayOutcome(db, workspaceId, invocation, optio
|
|
|
444
529
|
}
|
|
445
530
|
current = updated;
|
|
446
531
|
}
|
|
447
|
-
return current;
|
|
448
532
|
}
|
|
449
533
|
/**
|
|
450
534
|
* Resolve a losing durable claim from the immutable invocation snapshots.
|
|
@@ -456,7 +540,7 @@ async function waitForInvocationReplayOutcome(db, workspaceId, invocation, optio
|
|
|
456
540
|
*/
|
|
457
541
|
async function replayInvocationClaim(db, workspaceId, invocation) {
|
|
458
542
|
if (isSpawnInvocation(invocation.actionName)
|
|
459
|
-
||
|
|
543
|
+
|| isBuiltinReleaseInvocation(invocation)
|
|
460
544
|
|| (!!invocation.handlerNodeId && !invocation.handlerAgentId)) {
|
|
461
545
|
return waitForInvocationReplayOutcome(db, workspaceId, invocation);
|
|
462
546
|
}
|
|
@@ -520,6 +604,25 @@ async function failInvocationForUnavailableProvider(db, workspaceId, invocationI
|
|
|
520
604
|
.set({ status: 'failed', error: 'handler_unavailable', completedAt: new Date(), spawnReservedAt: null })
|
|
521
605
|
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
522
606
|
}
|
|
607
|
+
/**
|
|
608
|
+
* Terminally fail one invocation ONLY while it is still open AND was never
|
|
609
|
+
* dispatched — no recorded attempt and no dispatched node. The predicates make
|
|
610
|
+
* the failure atomic against a concurrent dispatcher (`drainNodeInvocations`
|
|
611
|
+
* on a handler reconnect): once that dispatcher's attempt lands, this update
|
|
612
|
+
* matches nothing and the live dispatch is left alone. Sets the same columns as
|
|
613
|
+
* {@link failInvocationForUnavailableProvider}; no spawn reservation can be
|
|
614
|
+
* held by a row with no dispatched node, so there is no capacity to release.
|
|
615
|
+
*
|
|
616
|
+
* @returns true when this call is the one that failed the row.
|
|
617
|
+
*/
|
|
618
|
+
async function failNeverDispatchedInvocation(db, workspaceId, invocationId, error) {
|
|
619
|
+
const failed = await db
|
|
620
|
+
.update(actionInvocations)
|
|
621
|
+
.set({ status: 'failed', error, completedAt: new Date(), spawnReservedAt: null })
|
|
622
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), eq(actionInvocations.dispatchAttempts, 0), isNull(actionInvocations.dispatchedNodeId)))
|
|
623
|
+
.returning({ id: actionInvocations.id });
|
|
624
|
+
return failed.length > 0;
|
|
625
|
+
}
|
|
523
626
|
/**
|
|
524
627
|
* Terminally fail the given open invocations (no completion emission — call
|
|
525
628
|
* {@link emitFailedInvocationEffects} with the returned rows after the
|
|
@@ -589,7 +692,7 @@ async function openInvocationIdsForActions(db, workspaceId, actionIds) {
|
|
|
589
692
|
const rows = await db
|
|
590
693
|
.select({ id: actionInvocations.id })
|
|
591
694
|
.from(actionInvocations)
|
|
592
|
-
.where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.actionId, actionIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
695
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.actionId, actionIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`)));
|
|
593
696
|
return rows.map((row) => row.id);
|
|
594
697
|
}
|
|
595
698
|
/**
|
|
@@ -616,12 +719,22 @@ async function dispatchNodeProviderInvocation(args, options = {}) {
|
|
|
616
719
|
// The DB is the authoritative offline queue. Do not call sendToProvider
|
|
617
720
|
// here: a connected provider whose heartbeat says handlers_live=false still
|
|
618
721
|
// has a socket, and sendToProvider would deliver the frame immediately.
|
|
722
|
+
const expectedActionId = args.invocationOrigin === 'registered_action'
|
|
723
|
+
? (args.expectedActionId ?? args.actionId)
|
|
724
|
+
: null;
|
|
619
725
|
const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, {
|
|
620
726
|
providerName: args.providerName,
|
|
621
727
|
pending: true,
|
|
622
728
|
reservationHeld: args.reservationHeld,
|
|
623
|
-
|
|
729
|
+
expectedAction: expectedActionId
|
|
730
|
+
? { id: expectedActionId, name: args.action }
|
|
731
|
+
: undefined,
|
|
732
|
+
targetActionId: args.actionId ?? undefined,
|
|
624
733
|
});
|
|
734
|
+
if (!accepted && args.invocationOrigin === 'registered_action') {
|
|
735
|
+
const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
|
|
736
|
+
return { accepted: false, pending: false, settled };
|
|
737
|
+
}
|
|
625
738
|
return { accepted, pending: true };
|
|
626
739
|
}
|
|
627
740
|
return dispatchNodeInvocation({ ...args });
|
|
@@ -632,11 +745,21 @@ async function dispatchRelease(args) {
|
|
|
632
745
|
if (!name) {
|
|
633
746
|
throw codedError('release action input.name is required', 'invalid_release_request', 400);
|
|
634
747
|
}
|
|
748
|
+
const expectedTokenHash = releaseExpectedTokenHash(input);
|
|
635
749
|
const [agent] = await args.db
|
|
636
750
|
.select()
|
|
637
751
|
.from(agents)
|
|
638
|
-
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name)));
|
|
752
|
+
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name), ...(expectedTokenHash ? [eq(agents.tokenHash, expectedTokenHash)] : [])));
|
|
639
753
|
if (!agent) {
|
|
754
|
+
if (expectedTokenHash) {
|
|
755
|
+
const [sameName] = await args.db
|
|
756
|
+
.select({ id: agents.id })
|
|
757
|
+
.from(agents)
|
|
758
|
+
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name)));
|
|
759
|
+
if (sameName) {
|
|
760
|
+
throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
|
|
761
|
+
}
|
|
762
|
+
}
|
|
640
763
|
throw codedError(`Agent "${name}" not found`, 'agent_not_found', 404);
|
|
641
764
|
}
|
|
642
765
|
const { invocation, replayed } = await createInvocation(args.db, args.workspaceId, null, {
|
|
@@ -660,23 +783,87 @@ async function dispatchRelease(args) {
|
|
|
660
783
|
// The release time is preserved in `metadata.release.releasedAt`.
|
|
661
784
|
const releasedName = releasedAgentName(agent.name, agent.id);
|
|
662
785
|
const releasedTokenHash = await sha256Hex(`released:${agent.id}:${randomHex(16)}`);
|
|
663
|
-
const
|
|
786
|
+
const invocationCompleted = sql `EXISTS (
|
|
664
787
|
SELECT 1 FROM ${actionInvocations}
|
|
665
788
|
WHERE ${actionInvocations.workspaceId} = ${args.workspaceId}
|
|
666
789
|
AND ${actionInvocations.id} = ${invocation.id}
|
|
667
|
-
AND ${actionInvocations.status}
|
|
790
|
+
AND ${actionInvocations.status} = 'completed'
|
|
791
|
+
)`;
|
|
792
|
+
const generationStillCurrent = releaseGenerationStillCurrent(args.workspaceId, agent.id, expectedTokenHash);
|
|
793
|
+
const atomicExitNodeId = sql `COALESCE(
|
|
794
|
+
(
|
|
795
|
+
SELECT ${agentNodeBindings.nodeId}
|
|
796
|
+
FROM ${agentNodeBindings}
|
|
797
|
+
WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
|
|
798
|
+
AND ${agentNodeBindings.agentId} = ${agent.id}
|
|
799
|
+
AND ${agentNodeBindings.status} = 'active'
|
|
800
|
+
ORDER BY
|
|
801
|
+
CASE
|
|
802
|
+
WHEN ${agentNodeBindings.nodeId} = (
|
|
803
|
+
SELECT ${agents.locationNodeId}
|
|
804
|
+
FROM ${agents}
|
|
805
|
+
WHERE ${agents.workspaceId} = ${args.workspaceId}
|
|
806
|
+
AND ${agents.id} = ${agent.id}
|
|
807
|
+
) THEN 0
|
|
808
|
+
WHEN ${agentNodeBindings.nodeId} = ${`node_direct_${agent.id}`} THEN 1
|
|
809
|
+
ELSE 2
|
|
810
|
+
END,
|
|
811
|
+
${agentNodeBindings.priority} DESC,
|
|
812
|
+
${agentNodeBindings.nodeId}
|
|
813
|
+
LIMIT 1
|
|
814
|
+
),
|
|
815
|
+
(
|
|
816
|
+
SELECT ${agents.locationNodeId}
|
|
817
|
+
FROM ${agents}
|
|
818
|
+
WHERE ${agents.workspaceId} = ${args.workspaceId}
|
|
819
|
+
AND ${agents.id} = ${agent.id}
|
|
820
|
+
),
|
|
821
|
+
${exitNodeId}
|
|
668
822
|
)`;
|
|
669
823
|
const results = await runAtomicWrites(args.db, (writeDb) => {
|
|
670
824
|
const writes = [];
|
|
671
|
-
//
|
|
825
|
+
// If the optional generation lease no longer matches, settle the
|
|
826
|
+
// invocation as a durable conflict in the same atomic unit in which all
|
|
827
|
+
// lifecycle mutations are conditioned to no-op.
|
|
828
|
+
writes.push(writeDb
|
|
829
|
+
.update(actionInvocations)
|
|
830
|
+
.set({
|
|
831
|
+
status: 'failed',
|
|
832
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
833
|
+
completedAt,
|
|
834
|
+
})
|
|
835
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), expectedTokenHash ? sql `NOT (${generationStillCurrent})` : sql `0`))
|
|
836
|
+
.returning({ id: actionInvocations.id }));
|
|
837
|
+
// Win completion and snapshot the current binding before deactivating
|
|
838
|
+
// it. The returned node drives the response and agent.exited event, so a
|
|
839
|
+
// concurrent rebind cannot leave either pointing at the stale host read
|
|
840
|
+
// before this atomic unit began.
|
|
841
|
+
writes.push(writeDb
|
|
842
|
+
.update(actionInvocations)
|
|
843
|
+
.set({
|
|
844
|
+
status: 'completed',
|
|
845
|
+
handlerNodeId: atomicExitNodeId,
|
|
846
|
+
output: {
|
|
847
|
+
released: true,
|
|
848
|
+
// The roster row is retained as a tombstone so the agent's history
|
|
849
|
+
// keeps its author; the name is what the caller gets back.
|
|
850
|
+
deleted: false,
|
|
851
|
+
reaped_locally: true,
|
|
852
|
+
released_name: releasedName,
|
|
853
|
+
},
|
|
854
|
+
completedAt,
|
|
855
|
+
})
|
|
856
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
|
|
857
|
+
.returning({ id: actionInvocations.id, handlerNodeId: actionInvocations.handlerNodeId }));
|
|
858
|
+
// Resolve every active binding in this atomic unit instead of from a
|
|
672
859
|
// pre-transaction snapshot. A concurrent rebind therefore decrements
|
|
673
|
-
// the node that
|
|
860
|
+
// exactly the node or nodes that are deactivated below.
|
|
674
861
|
writes.push(writeDb
|
|
675
862
|
.update(nodes)
|
|
676
863
|
.set({
|
|
677
864
|
activeAgents: sql `CASE WHEN ${nodes.activeAgents} > 0 THEN ${nodes.activeAgents} - 1 ELSE 0 END`,
|
|
678
865
|
})
|
|
679
|
-
.where(and(eq(nodes.workspaceId, args.workspaceId),
|
|
866
|
+
.where(and(eq(nodes.workspaceId, args.workspaceId), invocationCompleted, generationStillCurrent, sql `EXISTS (
|
|
680
867
|
SELECT 1 FROM ${agentNodeBindings}
|
|
681
868
|
WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
|
|
682
869
|
AND ${agentNodeBindings.agentId} = ${agent.id}
|
|
@@ -686,38 +873,32 @@ async function dispatchRelease(args) {
|
|
|
686
873
|
writes.push(writeDb
|
|
687
874
|
.update(agentNodeBindings)
|
|
688
875
|
.set({ status: 'inactive', updatedAt: completedAt })
|
|
689
|
-
.where(and(eq(agentNodeBindings.workspaceId, args.workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.status, 'active'),
|
|
690
|
-
//
|
|
691
|
-
//
|
|
692
|
-
//
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
//
|
|
703
|
-
//
|
|
876
|
+
.where(and(eq(agentNodeBindings.workspaceId, args.workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.status, 'active'), invocationCompleted, generationStillCurrent)));
|
|
877
|
+
// `channel_members` and `dm_participants` cascade on DELETE; the
|
|
878
|
+
// tombstone's UPDATE does not fire that cascade, so drop the memberships
|
|
879
|
+
// in the SAME atomic unit or the released agent stays a delivery target.
|
|
880
|
+
writes.push(writeDb
|
|
881
|
+
.delete(channelMembers)
|
|
882
|
+
.where(and(eq(channelMembers.agentId, agent.id), invocationCompleted, generationStillCurrent)));
|
|
883
|
+
writes.push(writeDb
|
|
884
|
+
.delete(dmParticipants)
|
|
885
|
+
.where(and(eq(dmParticipants.agentId, agent.id), invocationCompleted, generationStillCurrent)));
|
|
886
|
+
writes.push(writeDb
|
|
887
|
+
.delete(nodes)
|
|
888
|
+
.where(and(eq(nodes.workspaceId, args.workspaceId), eq(nodes.id, `node_direct_${agent.id}`), invocationCompleted, generationStillCurrent)));
|
|
889
|
+
// Tombstone after the invocation has won its completion CAS. The whole
|
|
890
|
+
// batch/transaction rolls back together, while this ordering lets the
|
|
891
|
+
// success predicate inspect the still-current token hash before we rotate
|
|
892
|
+
// it to an unusable released credential.
|
|
704
893
|
writes.push(writeDb
|
|
705
894
|
.update(agents)
|
|
706
895
|
.set({
|
|
707
896
|
name: releasedName,
|
|
708
897
|
handle: `@${releasedName}`,
|
|
709
898
|
status: RELEASED_AGENT_STATUS,
|
|
710
|
-
// The row survives, so its credential must not. `token_hash` is
|
|
711
|
-
// NOT NULL UNIQUE and cannot be cleared, so rotate it to a value
|
|
712
|
-
// nobody holds; the released agent's old token stops authenticating.
|
|
713
899
|
tokenHash: releasedTokenHash,
|
|
714
|
-
// Clear the rotation grace slot too, otherwise any token issued by
|
|
715
|
-
// the last live rotation would keep authenticating for its grace
|
|
716
|
-
// window on an agent that is supposed to be gone. See 0035_agent_token_grace.
|
|
717
900
|
previousTokenHash: null,
|
|
718
901
|
previousTokenExpiresAt: null,
|
|
719
|
-
// Same `release` shape the dispatched path writes, so an audit does
|
|
720
|
-
// not have to know which path released the agent.
|
|
721
902
|
metadata: sql `json_patch(COALESCE(${agents.metadata}, '{}'), ${JSON.stringify({
|
|
722
903
|
release: {
|
|
723
904
|
reason: typeof input.reason === 'string' ? input.reason : null,
|
|
@@ -726,48 +907,22 @@ async function dispatchRelease(args) {
|
|
|
726
907
|
},
|
|
727
908
|
})})`,
|
|
728
909
|
})
|
|
729
|
-
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.id, agent.id),
|
|
730
|
-
// `channel_members` and `dm_participants` cascade on DELETE; the
|
|
731
|
-
// tombstone's UPDATE does not fire that cascade, so drop the memberships
|
|
732
|
-
// in the SAME atomic unit or the released agent stays a delivery target.
|
|
733
|
-
writes.push(writeDb
|
|
734
|
-
.delete(channelMembers)
|
|
735
|
-
.where(eq(channelMembers.agentId, agent.id)));
|
|
736
|
-
writes.push(writeDb
|
|
737
|
-
.delete(dmParticipants)
|
|
738
|
-
.where(eq(dmParticipants.agentId, agent.id)));
|
|
739
|
-
writes.push(writeDb
|
|
740
|
-
.delete(nodes)
|
|
741
|
-
.where(and(eq(nodes.workspaceId, args.workspaceId), eq(nodes.id, `node_direct_${agent.id}`), invocationIsOpen)));
|
|
742
|
-
writes.push(writeDb
|
|
743
|
-
.update(actionInvocations)
|
|
744
|
-
.set({
|
|
745
|
-
status: 'completed',
|
|
746
|
-
// This is part of the same atomic unit as completion so a lost 201
|
|
747
|
-
// replays the exact handler node returned below.
|
|
748
|
-
handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${exitNodeId})`,
|
|
749
|
-
output: {
|
|
750
|
-
released: true,
|
|
751
|
-
// The roster row is retained as a tombstone so the agent's history
|
|
752
|
-
// keeps its author; the name is what the caller gets back.
|
|
753
|
-
deleted: false,
|
|
754
|
-
reaped_locally: true,
|
|
755
|
-
released_name: releasedName,
|
|
756
|
-
},
|
|
757
|
-
completedAt,
|
|
758
|
-
})
|
|
759
|
-
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
|
|
760
|
-
.returning({ id: actionInvocations.id }));
|
|
910
|
+
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.id, agent.id), ...(expectedTokenHash ? [eq(agents.tokenHash, expectedTokenHash)] : []), invocationCompleted)));
|
|
761
911
|
return writes;
|
|
762
|
-
});
|
|
763
|
-
const
|
|
912
|
+
}, expectedTokenHash ? { requireAtomic: true } : undefined);
|
|
913
|
+
const generationConflict = results[0];
|
|
914
|
+
const completed = results[1];
|
|
915
|
+
const completedExitNodeId = completed[0]?.handlerNodeId ?? null;
|
|
916
|
+
if (expectedTokenHash && (generationConflict.length > 0 || completed.length === 0)) {
|
|
917
|
+
throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
|
|
918
|
+
}
|
|
764
919
|
// External completion effects belong after the durable atomic unit: an
|
|
765
920
|
// aborted local reap must never publish agent.exited.
|
|
766
|
-
if (completed.length > 0 && args.completionDeps &&
|
|
921
|
+
if (completed.length > 0 && args.completionDeps && completedExitNodeId) {
|
|
767
922
|
await emitAgentExitedEffects(args.completionDeps, args.workspaceId, {
|
|
768
923
|
agentId: agent.id,
|
|
769
924
|
agentName: agent.name,
|
|
770
|
-
nodeId:
|
|
925
|
+
nodeId: completedExitNodeId,
|
|
771
926
|
invocationId: fleetInvocationId(agent.metadata),
|
|
772
927
|
reason: 'released',
|
|
773
928
|
});
|
|
@@ -776,7 +931,7 @@ async function dispatchRelease(args) {
|
|
|
776
931
|
invocation_id: invocation.id,
|
|
777
932
|
action_name: 'release',
|
|
778
933
|
handler_agent_id: null,
|
|
779
|
-
handler_node_id:
|
|
934
|
+
handler_node_id: completedExitNodeId,
|
|
780
935
|
dispatched_node_id: null,
|
|
781
936
|
input,
|
|
782
937
|
status: 'completed',
|
|
@@ -784,10 +939,36 @@ async function dispatchRelease(args) {
|
|
|
784
939
|
};
|
|
785
940
|
};
|
|
786
941
|
const failClosed = async () => {
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
942
|
+
const completedAt = new Date();
|
|
943
|
+
if (expectedTokenHash) {
|
|
944
|
+
const generationStillCurrent = releaseGenerationStillCurrent(args.workspaceId, agent.id, expectedTokenHash);
|
|
945
|
+
const results = await runAtomicWrites(args.db, (writeDb) => [
|
|
946
|
+
writeDb
|
|
947
|
+
.update(actionInvocations)
|
|
948
|
+
.set({
|
|
949
|
+
status: 'failed',
|
|
950
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
951
|
+
completedAt,
|
|
952
|
+
})
|
|
953
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (${generationStillCurrent})`))
|
|
954
|
+
.returning({ id: actionInvocations.id }),
|
|
955
|
+
writeDb
|
|
956
|
+
.update(actionInvocations)
|
|
957
|
+
.set({ status: 'failed', error: 'agent_host_unavailable', completedAt })
|
|
958
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
|
|
959
|
+
.returning({ id: actionInvocations.id }),
|
|
960
|
+
], { requireAtomic: true });
|
|
961
|
+
const generationConflict = results[0];
|
|
962
|
+
if (generationConflict.length > 0) {
|
|
963
|
+
throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
else {
|
|
967
|
+
await args.db
|
|
968
|
+
.update(actionInvocations)
|
|
969
|
+
.set({ status: 'failed', error: 'agent_host_unavailable', completedAt })
|
|
970
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
971
|
+
}
|
|
791
972
|
throw codedError(`Agent "${name}" has no live host node; cannot dispatch release`, 'agent_host_unavailable', 503);
|
|
792
973
|
};
|
|
793
974
|
const registry = args.registry;
|
|
@@ -816,10 +997,35 @@ async function dispatchRelease(args) {
|
|
|
816
997
|
providerName: agent.providerName,
|
|
817
998
|
action: 'release',
|
|
818
999
|
input,
|
|
1000
|
+
actionId: null,
|
|
1001
|
+
invocationOrigin: 'builtin',
|
|
819
1002
|
});
|
|
1003
|
+
// Some socket owners durably record or complete an accepted frame before
|
|
1004
|
+
// this process resumes to stamp its own dispatch attempt. Return that
|
|
1005
|
+
// terminal winner instead of treating the failed post-send stamp as a
|
|
1006
|
+
// rejected send and entering local fallback.
|
|
1007
|
+
if (dispatched.settled) {
|
|
1008
|
+
if (dispatched.settled.status === 'failed') {
|
|
1009
|
+
const errorCode = dispatched.settled.error ?? 'handler_unavailable';
|
|
1010
|
+
throw codedError('Release invocation failed after provider dispatch', errorCode, errorCode === RELEASE_GENERATION_CONFLICT_CODE ? 409 : 503);
|
|
1011
|
+
}
|
|
1012
|
+
return invocationAck(dispatched.settled, { actionName: 'release' });
|
|
1013
|
+
}
|
|
820
1014
|
// The provider can disconnect between the liveness check and send. Complete
|
|
821
1015
|
// the DB lifecycle locally instead of creating an ownerless pending request.
|
|
822
1016
|
if (!dispatched.accepted) {
|
|
1017
|
+
if (expectedTokenHash) {
|
|
1018
|
+
const [settled] = await args.db
|
|
1019
|
+
.select({ error: actionInvocations.error })
|
|
1020
|
+
.from(actionInvocations)
|
|
1021
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id)));
|
|
1022
|
+
if (settled?.error === RELEASE_GENERATION_CONFLICT_CODE) {
|
|
1023
|
+
throw codedError(`Agent "${name}" no longer matches the expected token generation`, RELEASE_GENERATION_CONFLICT_CODE, 409);
|
|
1024
|
+
}
|
|
1025
|
+
if (settled?.error === 'node_dispatch_unavailable') {
|
|
1026
|
+
throw codedError('Node adapter does not support generation-authorized release dispatch', 'node_dispatch_unavailable', 503);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
823
1029
|
return input.delete_agent === true ? completeLocally() : failClosed();
|
|
824
1030
|
}
|
|
825
1031
|
return {
|
|
@@ -833,6 +1039,21 @@ async function dispatchRelease(args) {
|
|
|
833
1039
|
created_at: invocation.createdAt.toISOString(),
|
|
834
1040
|
};
|
|
835
1041
|
}
|
|
1042
|
+
/**
|
|
1043
|
+
* Agent-lifecycle release used by `POST /v1/agents/release`. Unlike the
|
|
1044
|
+
* generic action endpoint, this path must always enforce the built-in release
|
|
1045
|
+
* lifecycle and its optional generation guard; a user-defined `release`
|
|
1046
|
+
* action must not be able to shadow it.
|
|
1047
|
+
*/
|
|
1048
|
+
export async function dispatchAgentRelease(db, workspaceId, data, options = {}) {
|
|
1049
|
+
return dispatchRelease({
|
|
1050
|
+
db,
|
|
1051
|
+
registry: options.nodeConnections,
|
|
1052
|
+
completionDeps: options.completionDeps,
|
|
1053
|
+
workspaceId,
|
|
1054
|
+
data,
|
|
1055
|
+
});
|
|
1056
|
+
}
|
|
836
1057
|
function spawnResult(invocation, nodeId, dispatched) {
|
|
837
1058
|
return {
|
|
838
1059
|
invocation_id: invocation.id,
|
|
@@ -850,6 +1071,11 @@ async function dispatchSpawn(args) {
|
|
|
850
1071
|
throw codedError('Node dispatch is not available', 'node_dispatch_unavailable', 503);
|
|
851
1072
|
}
|
|
852
1073
|
const input = recordInput(args.data.input);
|
|
1074
|
+
const requestedName = typeof input.name === 'string' ? input.name : '';
|
|
1075
|
+
if (requestedName.trim().length === 0) {
|
|
1076
|
+
throw codedError('spawn action input.name is required', 'invalid_spawn_request', 400);
|
|
1077
|
+
}
|
|
1078
|
+
assertRegistrableAgentName(requestedName);
|
|
853
1079
|
const capability = dispatchActionNameForInvocation('spawn', input);
|
|
854
1080
|
const { invocation, replayed } = await createInvocation(args.db, args.workspaceId, null, {
|
|
855
1081
|
input: args.data.input,
|
|
@@ -895,20 +1121,40 @@ async function dispatchSpawn(args) {
|
|
|
895
1121
|
if (!args.bypassShadow) {
|
|
896
1122
|
const shadow = await fetchNodeAction(args.db, args.workspaceId, nodeId, capability);
|
|
897
1123
|
if (shadow && shadow.handlerProvider) {
|
|
898
|
-
|
|
899
|
-
await releaseNodeCapacity(args.db, args.workspaceId, nodeId);
|
|
900
|
-
const dispatched = await dispatchNodeProviderInvocation({
|
|
901
|
-
db: args.db,
|
|
902
|
-
registry: args.registry,
|
|
1124
|
+
const bound = await bindInvocationToRegisteredNodeAction(args.db, {
|
|
903
1125
|
workspaceId: args.workspaceId,
|
|
904
1126
|
invocationId: invocation.id,
|
|
905
1127
|
nodeId,
|
|
906
1128
|
providerName: shadow.handlerProvider,
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
queue: shadow.queue,
|
|
1129
|
+
actionId: shadow.id,
|
|
1130
|
+
actionName: capability,
|
|
910
1131
|
});
|
|
911
|
-
|
|
1132
|
+
if (bound) {
|
|
1133
|
+
if (!placement.queued)
|
|
1134
|
+
await releaseNodeCapacity(args.db, args.workspaceId, nodeId);
|
|
1135
|
+
const dispatched = await dispatchNodeProviderInvocation({
|
|
1136
|
+
db: args.db,
|
|
1137
|
+
registry: args.registry,
|
|
1138
|
+
workspaceId: args.workspaceId,
|
|
1139
|
+
invocationId: invocation.id,
|
|
1140
|
+
nodeId,
|
|
1141
|
+
providerName: shadow.handlerProvider,
|
|
1142
|
+
action: capability,
|
|
1143
|
+
input,
|
|
1144
|
+
queue: shadow.queue,
|
|
1145
|
+
actionId: shadow.id,
|
|
1146
|
+
expectedActionId: shadow.id,
|
|
1147
|
+
invocationOrigin: 'registered_action',
|
|
1148
|
+
});
|
|
1149
|
+
if (dispatched.settled) {
|
|
1150
|
+
const settled = await waitForInvocationReplayOutcome(args.db, args.workspaceId, dispatched.settled);
|
|
1151
|
+
return invocationAck(settled, { actionName: 'spawn', handlerNodeId: nodeId });
|
|
1152
|
+
}
|
|
1153
|
+
return spawnResult(invocation, nodeId, dispatched);
|
|
1154
|
+
}
|
|
1155
|
+
// The shadow disappeared before its exact identity could be claimed.
|
|
1156
|
+
// Keep the native reservation and continue through native capacity; a
|
|
1157
|
+
// stale shadow snapshot must never receive an unbound invocation.
|
|
912
1158
|
}
|
|
913
1159
|
}
|
|
914
1160
|
const capProvider = (await capacityProviderName(args.db, args.workspaceId, nodeId, capability)) ?? DEFAULT_PROVIDER_NAME;
|
|
@@ -921,6 +1167,8 @@ async function dispatchSpawn(args) {
|
|
|
921
1167
|
providerName: capProvider,
|
|
922
1168
|
action: capability,
|
|
923
1169
|
input,
|
|
1170
|
+
actionId: null,
|
|
1171
|
+
invocationOrigin: 'builtin',
|
|
924
1172
|
pending: placement.queued,
|
|
925
1173
|
reservationHeld: !placement.queued,
|
|
926
1174
|
});
|
|
@@ -989,8 +1237,14 @@ export async function invokeNodeAction(db, workspaceId, nodeId, actionName, data
|
|
|
989
1237
|
action: action.name,
|
|
990
1238
|
input: recordInput(invocation.input),
|
|
991
1239
|
queue: action.queue,
|
|
1240
|
+
actionId: action.id,
|
|
1241
|
+
invocationOrigin: 'registered_action',
|
|
992
1242
|
reservationHeld: !!reservedNode,
|
|
993
1243
|
});
|
|
1244
|
+
if (dispatched.settled) {
|
|
1245
|
+
const settled = await waitForInvocationReplayOutcome(db, workspaceId, dispatched.settled);
|
|
1246
|
+
return invocationAck(settled, { actionName, handlerNodeId: nodeId });
|
|
1247
|
+
}
|
|
994
1248
|
return {
|
|
995
1249
|
invocation_id: invocation.id,
|
|
996
1250
|
action_name: actionName,
|
|
@@ -1085,8 +1339,14 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
|
|
|
1085
1339
|
action: action.name,
|
|
1086
1340
|
input: recordInput(invocation.input),
|
|
1087
1341
|
queue: action.queue || providerName === DEFAULT_PROVIDER_NAME,
|
|
1342
|
+
actionId: action.id,
|
|
1343
|
+
invocationOrigin: 'registered_action',
|
|
1088
1344
|
reservationHeld: !!reservedNode,
|
|
1089
1345
|
});
|
|
1346
|
+
if (dispatched.settled) {
|
|
1347
|
+
const settled = await waitForInvocationReplayOutcome(db, workspaceId, dispatched.settled);
|
|
1348
|
+
return invocationAck(settled, { actionName, handlerNodeId: action.handlerNodeId });
|
|
1349
|
+
}
|
|
1090
1350
|
return {
|
|
1091
1351
|
invocation_id: invocation.id,
|
|
1092
1352
|
action_name: actionName,
|
|
@@ -1169,6 +1429,7 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
|
|
|
1169
1429
|
input: recordInput(invocation.input),
|
|
1170
1430
|
agent: { id: handlerAgent.id, name: handlerAgent.name },
|
|
1171
1431
|
actionId: action.id,
|
|
1432
|
+
invocationOrigin: 'registered_action',
|
|
1172
1433
|
});
|
|
1173
1434
|
if (dispatched.sent && !dispatched.accepted) {
|
|
1174
1435
|
// The provider owns the frame once the send returns true. A fast completion
|
|
@@ -1189,19 +1450,41 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
|
|
|
1189
1450
|
}
|
|
1190
1451
|
if (!dispatched.sent) {
|
|
1191
1452
|
if (invocationId !== undefined) {
|
|
1192
|
-
// A keyed claim must
|
|
1193
|
-
//
|
|
1194
|
-
//
|
|
1195
|
-
|
|
1453
|
+
// A keyed claim must stay consistent with its replays. Re-read it so a
|
|
1454
|
+
// takeover that raced the last-moment adapter gate is observed here (a
|
|
1455
|
+
// row it already failed throws its own error) instead of this request
|
|
1456
|
+
// and its replay disagreeing on the pre-send outcome. A row still open
|
|
1457
|
+
// at the deadline is handed back for classification below rather than
|
|
1458
|
+
// reported as a retryable idempotency stall.
|
|
1459
|
+
await waitForInvocationReplayOutcome(db, workspaceId, invocation, { acceptDispatchStarted: true, onDeadline: 'return' });
|
|
1196
1460
|
}
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1461
|
+
// The host adapter could not deliver to the handler. A hosted adapter can
|
|
1462
|
+
// only learn that from the send itself, so the pre-dispatch liveness gate
|
|
1463
|
+
// above could not catch it. Resolve it the same way that gate would. The
|
|
1464
|
+
// failure is conditional on the row still being open AND never dispatched:
|
|
1465
|
+
// a handler that reconnected in the meantime may have had this very row
|
|
1466
|
+
// dispatched by `drainNodeInvocations`, and overwriting a live dispatch
|
|
1467
|
+
// with `handler_unavailable` would fail the caller while the handler runs
|
|
1468
|
+
// the action.
|
|
1469
|
+
const failed = await failNeverDispatchedInvocation(db, workspaceId, invocation.id, 'handler_unavailable');
|
|
1470
|
+
if (failed) {
|
|
1203
1471
|
throw codedError(`Action "${actionName}" handler "${handlerAgent.name}" has no live connection`, 'handler_unavailable', 503);
|
|
1204
1472
|
}
|
|
1473
|
+
// Someone else owns the row now: a dispatcher on handler reconnect, or a
|
|
1474
|
+
// takeover that already failed it. Report that outcome rather than this
|
|
1475
|
+
// request's failed send.
|
|
1476
|
+
const [current] = await db
|
|
1477
|
+
.select()
|
|
1478
|
+
.from(actionInvocations)
|
|
1479
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id)));
|
|
1480
|
+
if (!current || current.status === 'failed') {
|
|
1481
|
+
throw codedError('Action invocation failed before provider dispatch completed', current?.error ?? 'handler_unavailable', 503);
|
|
1482
|
+
}
|
|
1483
|
+
return invocationAck(current, {
|
|
1484
|
+
actionName,
|
|
1485
|
+
handlerAgentId: action.handlerAgentId,
|
|
1486
|
+
handlerNodeId: handlerAgent.locationNodeId,
|
|
1487
|
+
});
|
|
1205
1488
|
}
|
|
1206
1489
|
return {
|
|
1207
1490
|
invocation_id: invocation.id,
|
|
@@ -1231,8 +1514,196 @@ function publicInvocation(row) {
|
|
|
1231
1514
|
completed_at: row.completedAt?.toISOString() ?? null,
|
|
1232
1515
|
};
|
|
1233
1516
|
}
|
|
1517
|
+
async function completeGuardedReleaseNodeInvocation(db, workspaceId, nodeId, providerName, invocation, data, deps) {
|
|
1518
|
+
const input = recordInput(invocation.input);
|
|
1519
|
+
const name = typeof input.name === 'string' ? input.name : null;
|
|
1520
|
+
const persistedTokenHash = input.expected_token_hash;
|
|
1521
|
+
const expectedTokenHash = typeof persistedTokenHash === 'string'
|
|
1522
|
+
&& AGENT_TOKEN_HASH_PATTERN.test(persistedTokenHash)
|
|
1523
|
+
? persistedTokenHash
|
|
1524
|
+
: null;
|
|
1525
|
+
if (!name || !expectedTokenHash) {
|
|
1526
|
+
const [failed] = await db
|
|
1527
|
+
.update(actionInvocations)
|
|
1528
|
+
.set({
|
|
1529
|
+
status: 'failed',
|
|
1530
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
1531
|
+
completedAt: new Date(),
|
|
1532
|
+
spawnReservedAt: null,
|
|
1533
|
+
})
|
|
1534
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
|
|
1535
|
+
.returning();
|
|
1536
|
+
return failed ? publicInvocation(failed) : null;
|
|
1537
|
+
}
|
|
1538
|
+
const [agent] = await db
|
|
1539
|
+
.select()
|
|
1540
|
+
.from(agents)
|
|
1541
|
+
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.name, name)));
|
|
1542
|
+
if (!agent) {
|
|
1543
|
+
const [failed] = await db
|
|
1544
|
+
.update(actionInvocations)
|
|
1545
|
+
.set({
|
|
1546
|
+
status: 'failed',
|
|
1547
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
1548
|
+
completedAt: new Date(),
|
|
1549
|
+
spawnReservedAt: null,
|
|
1550
|
+
})
|
|
1551
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
|
|
1552
|
+
.returning();
|
|
1553
|
+
return failed ? publicInvocation(failed) : null;
|
|
1554
|
+
}
|
|
1555
|
+
const completedAt = new Date();
|
|
1556
|
+
const generationStillCurrent = releaseGenerationStillCurrent(workspaceId, agent.id, expectedTokenHash);
|
|
1557
|
+
const activeBindingStillCurrent = sql `EXISTS (
|
|
1558
|
+
SELECT 1 FROM ${agentNodeBindings}
|
|
1559
|
+
WHERE ${agentNodeBindings.workspaceId} = ${workspaceId}
|
|
1560
|
+
AND ${agentNodeBindings.agentId} = ${agent.id}
|
|
1561
|
+
AND ${agentNodeBindings.nodeId} = ${nodeId}
|
|
1562
|
+
AND ${agentNodeBindings.status} = 'active'
|
|
1563
|
+
)`;
|
|
1564
|
+
const invocationIsOpen = sql `EXISTS (
|
|
1565
|
+
SELECT 1 FROM ${actionInvocations}
|
|
1566
|
+
WHERE ${actionInvocations.workspaceId} = ${workspaceId}
|
|
1567
|
+
AND ${actionInvocations.id} = ${invocation.id}
|
|
1568
|
+
AND ${actionInvocations.dispatchedNodeId} = ${nodeId}
|
|
1569
|
+
AND ${actionInvocations.dispatchedProvider} = ${providerName}
|
|
1570
|
+
AND ${actionInvocations.status} IN ('pending', 'dispatched', 'invoked')
|
|
1571
|
+
)`;
|
|
1572
|
+
const invocationCompleted = sql `EXISTS (
|
|
1573
|
+
SELECT 1 FROM ${actionInvocations}
|
|
1574
|
+
WHERE ${actionInvocations.workspaceId} = ${workspaceId}
|
|
1575
|
+
AND ${actionInvocations.id} = ${invocation.id}
|
|
1576
|
+
AND ${actionInvocations.dispatchedNodeId} = ${nodeId}
|
|
1577
|
+
AND ${actionInvocations.dispatchedProvider} = ${providerName}
|
|
1578
|
+
AND ${actionInvocations.status} = 'completed'
|
|
1579
|
+
)`;
|
|
1580
|
+
const releaseCanApply = sql `(${generationStillCurrent}) AND (${activeBindingStillCurrent})`;
|
|
1581
|
+
const releasedName = releasedAgentName(agent.name, agent.id);
|
|
1582
|
+
const releasedTokenHash = await sha256Hex(`released:${agent.id}:${randomHex(16)}`);
|
|
1583
|
+
let successIndex = -1;
|
|
1584
|
+
const results = await runAtomicWrites(db, (writeDb) => {
|
|
1585
|
+
const writes = [];
|
|
1586
|
+
// The conflict settlement comes first. In a sequential SQLite batch it
|
|
1587
|
+
// closes the invocation before every mutation below if either the token
|
|
1588
|
+
// generation or node binding changed; on transactional adapters the same
|
|
1589
|
+
// statement ordering and rollback guarantee apply.
|
|
1590
|
+
writes.push(writeDb
|
|
1591
|
+
.update(actionInvocations)
|
|
1592
|
+
.set({
|
|
1593
|
+
status: 'failed',
|
|
1594
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
1595
|
+
completedAt,
|
|
1596
|
+
spawnReservedAt: null,
|
|
1597
|
+
})
|
|
1598
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (${releaseCanApply})`))
|
|
1599
|
+
.returning());
|
|
1600
|
+
writes.push(writeDb
|
|
1601
|
+
.update(nodes)
|
|
1602
|
+
.set({
|
|
1603
|
+
activeAgents: sql `CASE WHEN ${nodes.activeAgents} > 0 THEN ${nodes.activeAgents} - 1 ELSE 0 END`,
|
|
1604
|
+
})
|
|
1605
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId), invocationIsOpen, generationStillCurrent, activeBindingStillCurrent)));
|
|
1606
|
+
writes.push(writeDb
|
|
1607
|
+
.update(agentNodeBindings)
|
|
1608
|
+
.set({ status: 'inactive', updatedAt: completedAt })
|
|
1609
|
+
.where(and(eq(agentNodeBindings.workspaceId, workspaceId), eq(agentNodeBindings.agentId, agent.id), eq(agentNodeBindings.nodeId, nodeId), eq(agentNodeBindings.status, 'active'), invocationIsOpen, generationStillCurrent)));
|
|
1610
|
+
if (input.delete_agent === true) {
|
|
1611
|
+
writes.push(writeDb
|
|
1612
|
+
.delete(channelMembers)
|
|
1613
|
+
.where(and(eq(channelMembers.agentId, agent.id), invocationIsOpen, generationStillCurrent)));
|
|
1614
|
+
writes.push(writeDb
|
|
1615
|
+
.delete(dmParticipants)
|
|
1616
|
+
.where(and(eq(dmParticipants.agentId, agent.id), invocationIsOpen, generationStillCurrent)));
|
|
1617
|
+
}
|
|
1618
|
+
successIndex = writes.length;
|
|
1619
|
+
writes.push(writeDb
|
|
1620
|
+
.update(actionInvocations)
|
|
1621
|
+
.set({
|
|
1622
|
+
output: data.output ?? null,
|
|
1623
|
+
error: null,
|
|
1624
|
+
status: 'completed',
|
|
1625
|
+
completedAt,
|
|
1626
|
+
spawnReservedAt: null,
|
|
1627
|
+
})
|
|
1628
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.dispatchedProvider, providerName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), generationStillCurrent))
|
|
1629
|
+
.returning());
|
|
1630
|
+
const currentAgent = and(eq(agents.workspaceId, workspaceId), eq(agents.id, agent.id), eq(agents.name, name), eq(agents.tokenHash, expectedTokenHash), invocationCompleted);
|
|
1631
|
+
if (input.delete_agent === true) {
|
|
1632
|
+
writes.push(writeDb
|
|
1633
|
+
.update(agents)
|
|
1634
|
+
.set({
|
|
1635
|
+
name: releasedName,
|
|
1636
|
+
handle: `@${releasedName}`,
|
|
1637
|
+
status: RELEASED_AGENT_STATUS,
|
|
1638
|
+
tokenHash: releasedTokenHash,
|
|
1639
|
+
previousTokenHash: null,
|
|
1640
|
+
previousTokenExpiresAt: null,
|
|
1641
|
+
locationType: 'self_connected',
|
|
1642
|
+
locationNodeId: null,
|
|
1643
|
+
lastSeen: completedAt,
|
|
1644
|
+
metadata: sql `json_patch(COALESCE(${agents.metadata}, '{}'), ${JSON.stringify({
|
|
1645
|
+
release: {
|
|
1646
|
+
reason: typeof input.reason === 'string' ? input.reason : null,
|
|
1647
|
+
released_at: completedAt.toISOString(),
|
|
1648
|
+
previous_name: agent.name,
|
|
1649
|
+
},
|
|
1650
|
+
})})`,
|
|
1651
|
+
})
|
|
1652
|
+
.where(currentAgent));
|
|
1653
|
+
}
|
|
1654
|
+
else {
|
|
1655
|
+
const existingMetadata = agent.metadata ?? {};
|
|
1656
|
+
const { spawn: _spawn, cli: _cli, ...restMetadata } = existingMetadata;
|
|
1657
|
+
writes.push(writeDb
|
|
1658
|
+
.update(agents)
|
|
1659
|
+
.set({
|
|
1660
|
+
status: 'offline',
|
|
1661
|
+
locationType: 'self_connected',
|
|
1662
|
+
locationNodeId: null,
|
|
1663
|
+
lastSeen: completedAt,
|
|
1664
|
+
metadata: {
|
|
1665
|
+
...restMetadata,
|
|
1666
|
+
release: {
|
|
1667
|
+
reason: typeof input.reason === 'string' ? input.reason : null,
|
|
1668
|
+
released_at: completedAt.toISOString(),
|
|
1669
|
+
},
|
|
1670
|
+
},
|
|
1671
|
+
})
|
|
1672
|
+
.where(currentAgent));
|
|
1673
|
+
}
|
|
1674
|
+
if (input.delete_agent === true) {
|
|
1675
|
+
// Deleting the implicit node clears action_invocations.dispatched_node_id
|
|
1676
|
+
// via its FK. Do this only after the owned completion and agent CAS have
|
|
1677
|
+
// both succeeded, or their node predicates could no longer match.
|
|
1678
|
+
writes.push(writeDb
|
|
1679
|
+
.delete(nodes)
|
|
1680
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, `node_direct_${agent.id}`), sql `EXISTS (
|
|
1681
|
+
SELECT 1 FROM ${actionInvocations}
|
|
1682
|
+
WHERE ${actionInvocations.workspaceId} = ${workspaceId}
|
|
1683
|
+
AND ${actionInvocations.id} = ${invocation.id}
|
|
1684
|
+
AND ${actionInvocations.status} = 'completed'
|
|
1685
|
+
)`)));
|
|
1686
|
+
}
|
|
1687
|
+
return writes;
|
|
1688
|
+
}, { requireAtomic: true });
|
|
1689
|
+
const completed = results[successIndex];
|
|
1690
|
+
const generationConflict = results[0];
|
|
1691
|
+
const settled = completed[0] ?? generationConflict[0];
|
|
1692
|
+
if (!settled)
|
|
1693
|
+
return null;
|
|
1694
|
+
if (completed[0] && deps) {
|
|
1695
|
+
await emitAgentExitedEffects(deps, workspaceId, {
|
|
1696
|
+
agentId: agent.id,
|
|
1697
|
+
agentName: agent.name,
|
|
1698
|
+
nodeId,
|
|
1699
|
+
invocationId: fleetInvocationId(agent.metadata),
|
|
1700
|
+
reason: 'released',
|
|
1701
|
+
});
|
|
1702
|
+
}
|
|
1703
|
+
return publicInvocation(settled);
|
|
1704
|
+
}
|
|
1234
1705
|
async function applyReleaseCompletionEffect(db, workspaceId, nodeId, invocation, data, deps, options = {}) {
|
|
1235
|
-
if (!
|
|
1706
|
+
if (!isBuiltinReleaseInvocation(invocation) || data.error)
|
|
1236
1707
|
return false;
|
|
1237
1708
|
const input = recordInput(invocation.input);
|
|
1238
1709
|
const name = typeof input.name === 'string' ? input.name : null;
|
|
@@ -1360,13 +1831,137 @@ async function dispatchNodeAttempt(db, workspaceId, invocationId, nodeId, opts)
|
|
|
1360
1831
|
dispatchedNodeId: nodeId,
|
|
1361
1832
|
dispatchedProvider: opts.providerName,
|
|
1362
1833
|
spawnReservedAt: opts.reservationHeld ? new Date() : null,
|
|
1363
|
-
...(opts.
|
|
1834
|
+
...(opts.targetActionId ? { actionId: opts.targetActionId } : {}),
|
|
1364
1835
|
...attemptFields,
|
|
1365
1836
|
})
|
|
1366
|
-
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)
|
|
1837
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedAction ? [
|
|
1838
|
+
eq(actionInvocations.invocationOrigin, 'registered_action'),
|
|
1839
|
+
eq(actionInvocations.actionId, opts.expectedAction.id),
|
|
1840
|
+
sql `EXISTS (
|
|
1841
|
+
SELECT 1 FROM ${actions}
|
|
1842
|
+
WHERE ${actions.workspaceId} = ${workspaceId}
|
|
1843
|
+
AND ${actions.id} = ${opts.expectedAction.id}
|
|
1844
|
+
)`,
|
|
1845
|
+
...(opts.targetActionId || opts.expectedAction.handlerAgentId ? [
|
|
1846
|
+
registeredActionDispatchPredicate({
|
|
1847
|
+
workspaceId,
|
|
1848
|
+
nodeId,
|
|
1849
|
+
providerName: opts.providerName,
|
|
1850
|
+
actionId: opts.targetActionId ?? opts.expectedAction.id,
|
|
1851
|
+
actionName: opts.expectedAction.name,
|
|
1852
|
+
handlerAgentId: opts.expectedAction.handlerAgentId,
|
|
1853
|
+
}),
|
|
1854
|
+
] : []),
|
|
1855
|
+
] : [])))
|
|
1367
1856
|
.returning();
|
|
1368
1857
|
return !!updated;
|
|
1369
1858
|
}
|
|
1859
|
+
function registeredActionDispatchPredicate(args) {
|
|
1860
|
+
const handlerPredicate = args.handlerAgentId
|
|
1861
|
+
? sql `AND ${actions.handlerAgentId} = ${args.handlerAgentId} AND ${actions.handlerNodeId} IS NULL`
|
|
1862
|
+
: sql `AND ${actions.handlerNodeId} = ${args.nodeId}
|
|
1863
|
+
AND COALESCE(${actions.handlerProvider}, ${DEFAULT_PROVIDER_NAME}) = ${args.providerName}`;
|
|
1864
|
+
return sql `EXISTS (
|
|
1865
|
+
SELECT 1 FROM ${actions}
|
|
1866
|
+
WHERE ${actions.workspaceId} = ${args.workspaceId}
|
|
1867
|
+
AND ${actions.id} = ${args.actionId}
|
|
1868
|
+
AND ${actions.name} = ${args.actionName}
|
|
1869
|
+
AND ${actions.isActive} = 1
|
|
1870
|
+
${handlerPredicate}
|
|
1871
|
+
)`;
|
|
1872
|
+
}
|
|
1873
|
+
async function claimRegisteredActionHandoff(db, args) {
|
|
1874
|
+
const claimedActionId = args.targetActionId ?? args.expectedActionId;
|
|
1875
|
+
const stateFields = args.pending
|
|
1876
|
+
? { status: 'pending', dispatchedAt: null, retryAfterAt: args.retryAfterAt ?? null }
|
|
1877
|
+
: dispatchedStateFields({ retryAfterAt: args.retryAfterAt });
|
|
1878
|
+
const attemptFields = args.skipIncrementAttempts
|
|
1879
|
+
? {}
|
|
1880
|
+
: {
|
|
1881
|
+
attemptedNodeIds: sql `json_insert(COALESCE(${actionInvocations.attemptedNodeIds}, '[]'), '$[#]', ${args.nodeId})`,
|
|
1882
|
+
dispatchAttempts: sql `COALESCE(${actionInvocations.dispatchAttempts}, 0) + 1`,
|
|
1883
|
+
};
|
|
1884
|
+
const [claimed] = await db
|
|
1885
|
+
.update(actionInvocations)
|
|
1886
|
+
// This CAS is the durable dispatch linearization point. Claim the exact
|
|
1887
|
+
// action generation, route, and attempt before awaiting the provider. A
|
|
1888
|
+
// retry may hand off between same-name registrations only while it owns the
|
|
1889
|
+
// source generation; a later rescheduler observes the new route instead of
|
|
1890
|
+
// sending the same attempt again.
|
|
1891
|
+
.set({
|
|
1892
|
+
...stateFields,
|
|
1893
|
+
handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${args.nodeId})`,
|
|
1894
|
+
dispatchedNodeId: args.nodeId,
|
|
1895
|
+
dispatchedProvider: args.providerName,
|
|
1896
|
+
spawnReservedAt: args.reservationHeld ? new Date() : null,
|
|
1897
|
+
actionId: claimedActionId,
|
|
1898
|
+
...attemptFields,
|
|
1899
|
+
})
|
|
1900
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.expectedActionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (
|
|
1901
|
+
${actionInvocations.actionId} = ${claimedActionId}
|
|
1902
|
+
AND COALESCE(${actionInvocations.dispatchedNodeId}, '') = ${args.nodeId}
|
|
1903
|
+
AND COALESCE(${actionInvocations.dispatchedProvider}, ${DEFAULT_PROVIDER_NAME}) = ${args.providerName}
|
|
1904
|
+
AND ${actionInvocations.status} IN ('dispatched', 'invoked')
|
|
1905
|
+
)`, sql `EXISTS (
|
|
1906
|
+
SELECT 1 FROM ${actions}
|
|
1907
|
+
WHERE ${actions.workspaceId} = ${args.workspaceId}
|
|
1908
|
+
AND ${actions.id} = ${args.expectedActionId}
|
|
1909
|
+
)`, ...(args.targetActionId ? [registeredActionDispatchPredicate({
|
|
1910
|
+
workspaceId: args.workspaceId,
|
|
1911
|
+
nodeId: args.nodeId,
|
|
1912
|
+
providerName: args.providerName,
|
|
1913
|
+
actionId: args.targetActionId,
|
|
1914
|
+
actionName: args.actionName,
|
|
1915
|
+
})] : [])))
|
|
1916
|
+
.returning({
|
|
1917
|
+
actionId: actionInvocations.actionId,
|
|
1918
|
+
actionName: actionInvocations.actionName,
|
|
1919
|
+
dispatchAttempts: actionInvocations.dispatchAttempts,
|
|
1920
|
+
});
|
|
1921
|
+
return claimed?.actionId ? {
|
|
1922
|
+
actionId: claimed.actionId,
|
|
1923
|
+
actionName: claimed.actionName,
|
|
1924
|
+
dispatchAttempts: claimed.dispatchAttempts,
|
|
1925
|
+
} : null;
|
|
1926
|
+
}
|
|
1927
|
+
async function bindInvocationToRegisteredNodeAction(db, args) {
|
|
1928
|
+
const [bound] = await db
|
|
1929
|
+
.update(actionInvocations)
|
|
1930
|
+
.set({ actionId: args.actionId, invocationOrigin: 'registered_action' })
|
|
1931
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'builtin'), isNull(actionInvocations.actionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), eq(actionInvocations.dispatchAttempts, 0), registeredActionDispatchPredicate(args)))
|
|
1932
|
+
.returning({ id: actionInvocations.id });
|
|
1933
|
+
return !!bound;
|
|
1934
|
+
}
|
|
1935
|
+
async function settleDeletedRegisteredAction(db, workspaceId, invocationId) {
|
|
1936
|
+
// A failed handoff can also mean another dispatcher already advanced the
|
|
1937
|
+
// same invocation to a different live action. Fail only the FK-null state
|
|
1938
|
+
// that proves pruning won; never clobber that concurrent dispatch winner.
|
|
1939
|
+
const [failed] = await db
|
|
1940
|
+
.update(actionInvocations)
|
|
1941
|
+
.set({
|
|
1942
|
+
status: 'failed',
|
|
1943
|
+
error: 'action_deleted',
|
|
1944
|
+
completedAt: new Date(),
|
|
1945
|
+
spawnReservedAt: null,
|
|
1946
|
+
})
|
|
1947
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), isNull(actionInvocations.actionId), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
|
|
1948
|
+
.returning();
|
|
1949
|
+
if (failed)
|
|
1950
|
+
return failed;
|
|
1951
|
+
const [settled] = await db
|
|
1952
|
+
.select()
|
|
1953
|
+
.from(actionInvocations)
|
|
1954
|
+
.where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId)));
|
|
1955
|
+
return settled;
|
|
1956
|
+
}
|
|
1957
|
+
async function settleRegisteredActionSendFailure(db, args) {
|
|
1958
|
+
const [failed] = await db
|
|
1959
|
+
.update(actionInvocations)
|
|
1960
|
+
.set({ status: 'failed', error: 'node_dispatch_unavailable', completedAt: new Date() })
|
|
1961
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.actionId), eq(actionInvocations.dispatchedNodeId, args.nodeId), eq(actionInvocations.dispatchedProvider, args.providerName), eq(actionInvocations.dispatchAttempts, args.dispatchAttempts), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
|
|
1962
|
+
.returning();
|
|
1963
|
+
return failed ?? settleDeletedRegisteredAction(db, args.workspaceId, args.invocationId);
|
|
1964
|
+
}
|
|
1370
1965
|
async function snapshotInvocationHandlerNode(db, workspaceId, invocationId, nodeId) {
|
|
1371
1966
|
const [updated] = await db
|
|
1372
1967
|
.update(actionInvocations)
|
|
@@ -1396,6 +1991,44 @@ export async function markDrainedInvocationDispatched(db, workspaceId, invocatio
|
|
|
1396
1991
|
return !!updated;
|
|
1397
1992
|
}
|
|
1398
1993
|
async function dispatchNodeInvocation(args) {
|
|
1994
|
+
const guardedReleaseHash = args.invocationOrigin === 'builtin' && isReleaseInvocation(args.action)
|
|
1995
|
+
? releaseExpectedTokenHash(args.input)
|
|
1996
|
+
: null;
|
|
1997
|
+
if (guardedReleaseHash) {
|
|
1998
|
+
const name = typeof args.input.name === 'string' ? args.input.name : '';
|
|
1999
|
+
const [current] = await args.db
|
|
2000
|
+
.select({ id: agents.id })
|
|
2001
|
+
.from(agents)
|
|
2002
|
+
.where(and(eq(agents.workspaceId, args.workspaceId), eq(agents.name, name), eq(agents.tokenHash, guardedReleaseHash), sql `EXISTS (
|
|
2003
|
+
SELECT 1 FROM ${agentNodeBindings}
|
|
2004
|
+
WHERE ${agentNodeBindings.workspaceId} = ${args.workspaceId}
|
|
2005
|
+
AND ${agentNodeBindings.agentId} = ${agents.id}
|
|
2006
|
+
AND ${agentNodeBindings.nodeId} = ${args.nodeId}
|
|
2007
|
+
AND ${agentNodeBindings.status} = 'active'
|
|
2008
|
+
)`));
|
|
2009
|
+
if (!current) {
|
|
2010
|
+
await args.db
|
|
2011
|
+
.update(actionInvocations)
|
|
2012
|
+
.set({
|
|
2013
|
+
status: 'failed',
|
|
2014
|
+
error: RELEASE_GENERATION_CONFLICT_CODE,
|
|
2015
|
+
completedAt: new Date(),
|
|
2016
|
+
})
|
|
2017
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
2018
|
+
return { accepted: false, pending: false, sent: false };
|
|
2019
|
+
}
|
|
2020
|
+
if (!args.registry.sendAuthorizedActionToProvider) {
|
|
2021
|
+
await args.db
|
|
2022
|
+
.update(actionInvocations)
|
|
2023
|
+
.set({
|
|
2024
|
+
status: 'failed',
|
|
2025
|
+
error: 'node_dispatch_unavailable',
|
|
2026
|
+
completedAt: new Date(),
|
|
2027
|
+
})
|
|
2028
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
2029
|
+
return { accepted: false, pending: false, sent: false };
|
|
2030
|
+
}
|
|
2031
|
+
}
|
|
1399
2032
|
const frame = {
|
|
1400
2033
|
v: 1,
|
|
1401
2034
|
type: 'action.invoke',
|
|
@@ -1413,6 +2046,32 @@ async function dispatchNodeInvocation(args) {
|
|
|
1413
2046
|
if (!snapshotted)
|
|
1414
2047
|
return { accepted: false, pending: false, sent: false };
|
|
1415
2048
|
const connectedBefore = args.registry.isProviderConnected(args.workspaceId, args.nodeId, args.providerName);
|
|
2049
|
+
const pending = !!args.pending || !connectedBefore;
|
|
2050
|
+
const expectedActionId = args.invocationOrigin === 'registered_action'
|
|
2051
|
+
? (args.expectedActionId ?? args.actionId)
|
|
2052
|
+
: null;
|
|
2053
|
+
let registeredNodeClaim = null;
|
|
2054
|
+
if (args.invocationOrigin === 'registered_action' && !args.agent) {
|
|
2055
|
+
registeredNodeClaim = expectedActionId
|
|
2056
|
+
? await claimRegisteredActionHandoff(args.db, {
|
|
2057
|
+
workspaceId: args.workspaceId,
|
|
2058
|
+
invocationId: args.invocationId,
|
|
2059
|
+
nodeId: args.nodeId,
|
|
2060
|
+
providerName: args.providerName,
|
|
2061
|
+
expectedActionId,
|
|
2062
|
+
targetActionId: args.actionId ?? undefined,
|
|
2063
|
+
actionName: args.action,
|
|
2064
|
+
pending,
|
|
2065
|
+
retryAfterAt: args.retryAfterAt,
|
|
2066
|
+
reservationHeld: args.reservationHeld,
|
|
2067
|
+
skipIncrementAttempts: args.skipIncrementAttempts,
|
|
2068
|
+
})
|
|
2069
|
+
: null;
|
|
2070
|
+
if (!registeredNodeClaim) {
|
|
2071
|
+
const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
|
|
2072
|
+
return { accepted: false, pending: false, sent: false, settled };
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
1416
2075
|
const sent = args.agent && args.actionId
|
|
1417
2076
|
? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
|
|
1418
2077
|
kind: 'agent-action-v1',
|
|
@@ -1421,18 +2080,88 @@ async function dispatchNodeInvocation(args) {
|
|
|
1421
2080
|
handlerAgentId: args.agent.id,
|
|
1422
2081
|
recordAttempt: !args.skipIncrementAttempts,
|
|
1423
2082
|
}) ?? false)
|
|
1424
|
-
:
|
|
1425
|
-
|
|
2083
|
+
: registeredNodeClaim
|
|
2084
|
+
? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
|
|
2085
|
+
kind: 'registered-node-action-v2',
|
|
2086
|
+
invocationId: args.invocationId,
|
|
2087
|
+
actionId: registeredNodeClaim.actionId,
|
|
2088
|
+
dispatchAttempt: registeredNodeClaim.dispatchAttempts,
|
|
2089
|
+
invocationActionName: registeredNodeClaim.actionName,
|
|
2090
|
+
actionName: args.action,
|
|
2091
|
+
}) ?? false)
|
|
2092
|
+
: guardedReleaseHash
|
|
2093
|
+
? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
|
|
2094
|
+
kind: 'release-generation-v1',
|
|
2095
|
+
invocationId: args.invocationId,
|
|
2096
|
+
agentName: typeof args.input.name === 'string' ? args.input.name : '',
|
|
2097
|
+
expectedTokenHash: guardedReleaseHash,
|
|
2098
|
+
}) ?? false)
|
|
2099
|
+
: await args.registry.sendToProvider(args.workspaceId, args.nodeId, args.providerName, frame);
|
|
2100
|
+
if (!sent && guardedReleaseHash) {
|
|
2101
|
+
// An adapter compiled against the older owner-authorization contract may
|
|
2102
|
+
// expose the method but reject the new proof kind. Settle any still-open
|
|
2103
|
+
// row before the caller considers local fallback, so a guarded delete can
|
|
2104
|
+
// never degrade to an unguarded lifecycle mutation.
|
|
2105
|
+
await args.db
|
|
2106
|
+
.update(actionInvocations)
|
|
2107
|
+
.set({
|
|
2108
|
+
status: 'failed',
|
|
2109
|
+
error: 'node_dispatch_unavailable',
|
|
2110
|
+
completedAt: new Date(),
|
|
2111
|
+
})
|
|
2112
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
|
|
2113
|
+
}
|
|
2114
|
+
if (!sent) {
|
|
2115
|
+
if (registeredNodeClaim) {
|
|
2116
|
+
const settled = await settleRegisteredActionSendFailure(args.db, {
|
|
2117
|
+
workspaceId: args.workspaceId,
|
|
2118
|
+
invocationId: args.invocationId,
|
|
2119
|
+
nodeId: args.nodeId,
|
|
2120
|
+
providerName: args.providerName,
|
|
2121
|
+
actionId: registeredNodeClaim.actionId,
|
|
2122
|
+
dispatchAttempts: registeredNodeClaim.dispatchAttempts,
|
|
2123
|
+
});
|
|
2124
|
+
return { accepted: false, pending: false, sent: false, settled };
|
|
2125
|
+
}
|
|
1426
2126
|
return { accepted: false, pending: false, sent: false };
|
|
1427
|
-
|
|
2127
|
+
}
|
|
2128
|
+
// Registered node-action state was committed by the pre-send handoff CAS.
|
|
2129
|
+
// Pruning after provider acceptance may clear the action FK, but it cannot
|
|
2130
|
+
// revoke work the provider already received or prevent route-owned completion.
|
|
2131
|
+
if (registeredNodeClaim) {
|
|
2132
|
+
return { accepted: true, pending, sent: true };
|
|
2133
|
+
}
|
|
2134
|
+
const finalExpectedActionId = expectedActionId;
|
|
1428
2135
|
const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, {
|
|
1429
2136
|
pending,
|
|
1430
2137
|
providerName: args.providerName,
|
|
1431
2138
|
retryAfterAt: args.retryAfterAt,
|
|
1432
2139
|
reservationHeld: args.reservationHeld,
|
|
1433
2140
|
skipIncrementAttempts: args.skipIncrementAttempts || !!args.agent,
|
|
1434
|
-
|
|
2141
|
+
expectedAction: finalExpectedActionId
|
|
2142
|
+
? {
|
|
2143
|
+
id: finalExpectedActionId,
|
|
2144
|
+
name: args.action,
|
|
2145
|
+
...(args.agent ? { handlerAgentId: args.agent.id } : {}),
|
|
2146
|
+
}
|
|
2147
|
+
: undefined,
|
|
2148
|
+
targetActionId: args.agent || args.actionId
|
|
2149
|
+
? finalExpectedActionId ?? undefined
|
|
2150
|
+
: undefined,
|
|
1435
2151
|
});
|
|
2152
|
+
if (!accepted && args.invocationOrigin === 'registered_action') {
|
|
2153
|
+
const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
|
|
2154
|
+
return { accepted: false, pending: false, sent: true, settled };
|
|
2155
|
+
}
|
|
2156
|
+
if (!accepted && guardedReleaseHash) {
|
|
2157
|
+
const [settled] = await args.db
|
|
2158
|
+
.select()
|
|
2159
|
+
.from(actionInvocations)
|
|
2160
|
+
.where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId)));
|
|
2161
|
+
if (settled && !OPEN_INVOCATION_STATUSES.some((status) => status === settled.status)) {
|
|
2162
|
+
return { accepted: false, pending: false, sent: true, settled };
|
|
2163
|
+
}
|
|
2164
|
+
}
|
|
1436
2165
|
return { accepted, pending, sent: true };
|
|
1437
2166
|
}
|
|
1438
2167
|
function attemptedNodeSet(invocation) {
|
|
@@ -1487,12 +2216,16 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
|
|
|
1487
2216
|
const rows = await db
|
|
1488
2217
|
.select({
|
|
1489
2218
|
id: actionInvocations.id,
|
|
1490
|
-
actionId:
|
|
2219
|
+
actionId: actionInvocations.actionId,
|
|
2220
|
+
invocationOrigin: actionInvocations.invocationOrigin,
|
|
1491
2221
|
workspaceId: actionInvocations.workspaceId,
|
|
1492
2222
|
actionName: actionInvocations.actionName,
|
|
1493
2223
|
input: actionInvocations.input,
|
|
2224
|
+
providerAcceptedAttempt: actionInvocations.providerAcceptedAttempt,
|
|
2225
|
+
dispatchAttempts: actionInvocations.dispatchAttempts,
|
|
1494
2226
|
spawnReservedAt: actionInvocations.spawnReservedAt,
|
|
1495
2227
|
dispatchedNodeId: actionInvocations.dispatchedNodeId,
|
|
2228
|
+
dispatchedProvider: actionInvocations.dispatchedProvider,
|
|
1496
2229
|
handlerNodeId: actions.handlerNodeId,
|
|
1497
2230
|
handlerProvider: actions.handlerProvider,
|
|
1498
2231
|
handlerAgentId: agents.id,
|
|
@@ -1510,6 +2243,10 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
|
|
|
1510
2243
|
.orderBy(asc(actionInvocations.createdAt));
|
|
1511
2244
|
let drained = 0;
|
|
1512
2245
|
for (const row of rows) {
|
|
2246
|
+
if (isDeletedRegisteredActionInvocation(row)) {
|
|
2247
|
+
await failOpenInvocationRows(db, workspaceId, [row.id], 'action_deleted');
|
|
2248
|
+
continue;
|
|
2249
|
+
}
|
|
1513
2250
|
const targetAgent = row.handlerAgentId
|
|
1514
2251
|
&& row.handlerAgentName
|
|
1515
2252
|
&& row.handlerAgentLocationType === 'via_node'
|
|
@@ -1532,10 +2269,12 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
|
|
|
1532
2269
|
? row.handlerAgentProvider ?? DEFAULT_PROVIDER_NAME
|
|
1533
2270
|
: row.handlerNodeId
|
|
1534
2271
|
? row.handlerProvider ?? DEFAULT_PROVIDER_NAME
|
|
1535
|
-
:
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
2272
|
+
: isBuiltinReleaseInvocation(row)
|
|
2273
|
+
? row.dispatchedProvider ?? DEFAULT_PROVIDER_NAME
|
|
2274
|
+
: shadowProvider
|
|
2275
|
+
?? (isSpawnInvocation(row.actionName)
|
|
2276
|
+
? (await capacityProviderName(db, workspaceId, nodeId, dispatchActionNameForInvocation(row.actionName, input))) ?? DEFAULT_PROVIDER_NAME
|
|
2277
|
+
: DEFAULT_PROVIDER_NAME);
|
|
1539
2278
|
const nativeSpawn = isSpawnInvocation(row.actionName) && !shadowProvider;
|
|
1540
2279
|
// Skip draining to a provider that is disconnected or whose heartbeat says
|
|
1541
2280
|
// handlers are not live. A missing provider row is a legacy single-socket
|
|
@@ -1577,7 +2316,9 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
|
|
|
1577
2316
|
action: dispatchActionNameForInvocation(row.actionName, input),
|
|
1578
2317
|
input,
|
|
1579
2318
|
agent: targetAgent ? { id: targetAgent.id, name: targetAgent.name } : null,
|
|
1580
|
-
actionId:
|
|
2319
|
+
actionId: row.actionId,
|
|
2320
|
+
expectedActionId: row.actionId,
|
|
2321
|
+
invocationOrigin: row.invocationOrigin,
|
|
1581
2322
|
retryAfterAt: new Date(Date.now() + ACTION_DISPATCH_TIMEOUT_MS),
|
|
1582
2323
|
reservationHeld,
|
|
1583
2324
|
skipIncrementAttempts: row.dispatchedNodeId === nodeId,
|
|
@@ -1599,6 +2340,17 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
|
|
|
1599
2340
|
return drained;
|
|
1600
2341
|
}
|
|
1601
2342
|
export async function rescheduleNodeInvocation(db, registry, invocation, opts = {}) {
|
|
2343
|
+
// Once the exact provider accepted this generation, pruning only clears its
|
|
2344
|
+
// materialized action FK. It must not make timeout, disconnect, or inventory
|
|
2345
|
+
// recovery move the invocation to a same-name replacement and invalidate the
|
|
2346
|
+
// original provider's result.
|
|
2347
|
+
if (isAcceptedDeletedRegisteredActionInvocation(invocation)) {
|
|
2348
|
+
return false;
|
|
2349
|
+
}
|
|
2350
|
+
if (isDeletedRegisteredActionInvocation(invocation)) {
|
|
2351
|
+
await failOpenInvocationRows(db, invocation.workspaceId, [invocation.id], 'action_deleted');
|
|
2352
|
+
return false;
|
|
2353
|
+
}
|
|
1602
2354
|
// Only release capacity the invocation actually holds. A shadowed spawn
|
|
1603
2355
|
// dropped its native reservation at dispatch (spawnReservedAt is null), so it
|
|
1604
2356
|
// must not release capacity that now belongs to another spawn.
|
|
@@ -1633,6 +2385,8 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
|
|
|
1633
2385
|
input: recordInput(invocation.input),
|
|
1634
2386
|
agent: { id: targetAgent.agentId, name: targetAgent.agentName },
|
|
1635
2387
|
actionId: targetAgent.actionId,
|
|
2388
|
+
expectedActionId: invocation.actionId,
|
|
2389
|
+
invocationOrigin: invocation.invocationOrigin,
|
|
1636
2390
|
retryAfterAt: opts.retryAfterAt ?? null,
|
|
1637
2391
|
});
|
|
1638
2392
|
return dispatched.accepted;
|
|
@@ -1640,7 +2394,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
|
|
|
1640
2394
|
// Release invocations carry no actionId, so targetAgentForInvocation() returns
|
|
1641
2395
|
// null. They must never fall through to generic node placement, which could
|
|
1642
2396
|
// route a release to an unrelated node that doesn't own the agent.
|
|
1643
|
-
if (
|
|
2397
|
+
if (isBuiltinReleaseInvocation(invocation)) {
|
|
1644
2398
|
return false;
|
|
1645
2399
|
}
|
|
1646
2400
|
const attempted = attemptedNodeSet(invocation);
|
|
@@ -1677,6 +2431,8 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
|
|
|
1677
2431
|
providerName: target.handlerProvider,
|
|
1678
2432
|
action: actionToSend,
|
|
1679
2433
|
actionId: target.id,
|
|
2434
|
+
expectedActionId: invocation.actionId,
|
|
2435
|
+
invocationOrigin: invocation.invocationOrigin,
|
|
1680
2436
|
input,
|
|
1681
2437
|
queue: target.queue,
|
|
1682
2438
|
reservationHeld: false,
|
|
@@ -1694,14 +2450,23 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
|
|
|
1694
2450
|
? (await capacityProviderName(db, invocation.workspaceId, placement.node.id, actionToSend)) ?? DEFAULT_PROVIDER_NAME
|
|
1695
2451
|
: DEFAULT_PROVIDER_NAME;
|
|
1696
2452
|
if (placement.queued) {
|
|
1697
|
-
|
|
2453
|
+
const expectedActionId = invocation.invocationOrigin === 'registered_action'
|
|
2454
|
+
? invocation.actionId
|
|
2455
|
+
: null;
|
|
2456
|
+
const accepted = await dispatchNodeAttempt(db, invocation.workspaceId, invocation.id, placement.node.id, {
|
|
1698
2457
|
providerName,
|
|
1699
2458
|
pending: true,
|
|
1700
2459
|
retryAfterAt: opts.retryAfterAt ?? null,
|
|
1701
2460
|
reservationHeld: false,
|
|
1702
|
-
|
|
2461
|
+
expectedAction: expectedActionId
|
|
2462
|
+
? { id: expectedActionId, name: actionToSend }
|
|
2463
|
+
: undefined,
|
|
2464
|
+
targetActionId: target?.id,
|
|
1703
2465
|
});
|
|
1704
|
-
|
|
2466
|
+
if (!accepted && invocation.invocationOrigin === 'registered_action') {
|
|
2467
|
+
await settleDeletedRegisteredAction(db, invocation.workspaceId, invocation.id);
|
|
2468
|
+
}
|
|
2469
|
+
return accepted;
|
|
1705
2470
|
}
|
|
1706
2471
|
const dispatched = await dispatchNodeInvocation({
|
|
1707
2472
|
db,
|
|
@@ -1711,7 +2476,9 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
|
|
|
1711
2476
|
nodeId: placement.node.id,
|
|
1712
2477
|
providerName,
|
|
1713
2478
|
action: actionToSend,
|
|
1714
|
-
actionId: target?.id,
|
|
2479
|
+
actionId: target?.id ?? null,
|
|
2480
|
+
expectedActionId: invocation.actionId,
|
|
2481
|
+
invocationOrigin: invocation.invocationOrigin,
|
|
1715
2482
|
input,
|
|
1716
2483
|
reservationHeld,
|
|
1717
2484
|
});
|
|
@@ -1815,6 +2582,8 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
|
|
|
1815
2582
|
.select({
|
|
1816
2583
|
id: actionInvocations.id,
|
|
1817
2584
|
workspaceId: actionInvocations.workspaceId,
|
|
2585
|
+
actionId: actionInvocations.actionId,
|
|
2586
|
+
invocationOrigin: actionInvocations.invocationOrigin,
|
|
1818
2587
|
actionName: actionInvocations.actionName,
|
|
1819
2588
|
callerId: actionInvocations.callerId,
|
|
1820
2589
|
input: actionInvocations.input,
|
|
@@ -1824,6 +2593,7 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
|
|
|
1824
2593
|
durationMs: actionInvocations.durationMs,
|
|
1825
2594
|
dispatchedNodeId: actionInvocations.dispatchedNodeId,
|
|
1826
2595
|
dispatchedAt: actionInvocations.dispatchedAt,
|
|
2596
|
+
providerAcceptedAttempt: actionInvocations.providerAcceptedAttempt,
|
|
1827
2597
|
spawnReservedAt: actionInvocations.spawnReservedAt,
|
|
1828
2598
|
attemptedNodeIds: actionInvocations.attemptedNodeIds,
|
|
1829
2599
|
dispatchAttempts: actionInvocations.dispatchAttempts,
|
|
@@ -1854,6 +2624,12 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
|
|
|
1854
2624
|
// Fall through and fail the invocation if no eligible node exists.
|
|
1855
2625
|
}
|
|
1856
2626
|
}
|
|
2627
|
+
const existingInput = recordInput(existing.input);
|
|
2628
|
+
if (!data.error
|
|
2629
|
+
&& isBuiltinReleaseInvocation(existing)
|
|
2630
|
+
&& existingInput.expected_token_hash !== undefined) {
|
|
2631
|
+
return completeGuardedReleaseNodeInvocation(db, workspaceId, nodeId, providerName, existing, data, deps);
|
|
2632
|
+
}
|
|
1857
2633
|
const [updated] = await db
|
|
1858
2634
|
.update(actionInvocations)
|
|
1859
2635
|
.set({
|