@relaycast/engine 3.1.1 → 4.0.0

package/dist/engine/action.js

@@ -1,14 +1,132 @@
-import { eq, and } from 'drizzle-orm';
-import { actions, actionInvocations, agents } from '../db/schema.js';
+import { and, eq, inArray, lte, sql } from 'drizzle-orm';
+import { actions, actionInvocations, agents, nodes } from '../db/schema.js';
 import { generateId } from './snowflake.js';
 import { codedError } from '../lib/httpError.js';
-export async function registerAction(db, workspaceId, data) {
-    const [agent] = await db
+import { claimSpawnNode, chooseNodeForAction, releaseNodeCapacity, reserveNodeCapacity } from './placement.js';
+const OPEN_INVOCATION_STATUSES = ['pending', 'dispatched', 'invoked'];
+export const ACTION_DISPATCH_TIMEOUT_MS = 30_000;
+const ACTION_RETRY_BACKOFF_MS = 5_000;
+function capabilityName(capability) {
+    if (typeof capability === 'string')
+        return capability;
+    if (capability && typeof capability.name === 'string')
+        return capability.name;
+    return null;
+}
+function isSpawnInvocation(actionName) {
+    return actionName === 'spawn' || actionName.startsWith('spawn:');
+}
+function normalizeAttemptedNodeIds(value) {
+    return Array.isArray(value) ? value.filter((item) => typeof item === 'string') : [];
+}
+function nextRetryAfter(attempts) {
+    const backoff = Math.min(ACTION_RETRY_BACKOFF_MS * 2 ** Math.max(0, attempts - 1), 60_000);
+    return new Date(Date.now() + backoff);
+}
+/**
+ * Field set that moves an invocation into the live `dispatched` state once its
+ * `action.invoke` frame has actually been delivered to the node. Shared by the
+ * live dispatch path (`dispatchNodeAttempt`) and the offline-queue drain path
+ * (`markDrainedInvocationDispatched`) so the dispatch-timeout sweep — which keys
+ * off `dispatchedAt` — and the reschedule path cover drained invocations too.
+ */
+function dispatchedStateFields(opts = {}) {
+    return {
+        status: 'dispatched',
+        dispatchedAt: new Date(),
+        retryAfterAt: opts.retryAfterAt ?? null,
+    };
+}
+function isActionVisibleToCaller(availableTo, callerName) {
+    if (!availableTo || availableTo.length === 0)
+        return true;
+    return !!callerName && availableTo.includes(callerName);
+}
+function requireOneHandler(data) {
+    const hasAgent = !!data.handler_agent;
+    const hasNode = !!data.handler_node;
+    if (hasAgent === hasNode) {
+        throw codedError('Exactly one of handler_agent or handler_node is required', 'invalid_action_handler', 400);
+    }
+}
+function recordInput(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : {};
+}
+function toFleetWireJson(value) {
+    if (value === null)
+        return null;
+    if (typeof value === 'string' || typeof value === 'boolean')
+        return value;
+    if (typeof value === 'number')
+        return Number.isFinite(value) ? value : null;
+    if (Array.isArray(value))
+        return value.map(toFleetWireJson);
+    if (value && typeof value === 'object') {
+        const out = {};
+        for (const [key, nested] of Object.entries(value)) {
+            out[key] = toFleetWireJson(nested);
+        }
+        return out;
+    }
+    return null;
+}
+function publicAction(row) {
+    return {
+        id: row.id,
+        name: row.name,
+        description: row.description,
+        handler_agent: row.handlerAgentName,
+        handler_node: row.handlerNodeName,
+        handler_node_id: row.handlerNodeId,
+        input_schema: row.inputSchema ?? {},
+        output_schema: row.outputSchema ?? {},
+        available_to: row.availableTo ?? null,
+        is_active: row.isActive,
+        created_at: row.createdAt.toISOString(),
+    };
+}
+async function fetchAction(db, workspaceId, actionName) {
+    const [action] = await db
         .select()
-        .from(agents)
-        .where(and(eq(agents.workspaceId, workspaceId), eq(agents.name, data.handler_agent)));
-    if (!agent) {
-        throw codedError(`Agent "${data.handler_agent}" not found`, 'agent_not_found', 404);
+        .from(actions)
+        .where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, actionName), eq(actions.isActive, true)));
+    return action ?? null;
+}
+export async function registerAction(db, workspaceId, data) {
+    requireOneHandler(data);
+    let handlerAgentId = null;
+    let handlerAgentName = null;
+    let handlerNodeId = null;
+    let handlerNodeName = null;
+    if (data.handler_agent) {
+        const [agent] = await db
+            .select()
+            .from(agents)
+            .where(and(eq(agents.workspaceId, workspaceId), eq(agents.name, data.handler_agent)));
+        if (!agent) {
+            throw codedError(`Agent "${data.handler_agent}" not found`, 'agent_not_found', 404);
+        }
+        handlerAgentId = agent.id;
+        handlerAgentName = agent.name;
+    }
+    if (data.handler_node) {
+        const [node] = await db
+            .select()
+            .from(nodes)
+            .where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.name, data.handler_node)));
+        if (!node) {
+            throw codedError(`Node "${data.handler_node}" not found`, 'node_not_found', 404);
+        }
+        const nodeCapabilities = Array.isArray(node.capabilities)
+            ? node.capabilities.map(capabilityName).filter((value) => !!value)
+            : [];
+        if (!nodeCapabilities.includes(data.name)) {
+            throw codedError(`Node "${data.handler_node}" does not provide ${data.name}`, 'capability_mismatch', 409);
+        }
+        handlerNodeId = node.id;
+        handlerNodeName = node.name;
     }
     const id = `act_${generateId()}`;
     const [action] = await db
@@ -18,7 +136,8 @@ export async function registerAction(db, workspaceId, data) {
         workspaceId,
         name: data.name,
         description: data.description,
-        handlerAgentId: agent.id,
+        handlerAgentId,
+        handlerNodeId,
         inputSchema: data.input_schema ?? {},
         outputSchema: data.output_schema ?? {},
         availableTo: data.available_to ?? null,
@@ -28,7 +147,9 @@ export async function registerAction(db, workspaceId, data) {
         id: action.id,
         name: action.name,
         description: action.description,
-        handler_agent: data.handler_agent,
+        handler_agent: handlerAgentName,
+        handler_node: handlerNodeName,
+        handler_node_id: handlerNodeId,
         input_schema: action.inputSchema,
         output_schema: action.outputSchema,
         available_to: action.availableTo ?? null,
@@ -36,19 +157,15 @@ export async function registerAction(db, workspaceId, data) {
         created_at: action.createdAt.toISOString(),
     };
 }
-function isActionVisibleToCaller(availableTo, callerName) {
-    if (!availableTo || availableTo.length === 0)
-        return true;
-    return !!callerName && availableTo.includes(callerName);
-}
 export async function listActions(db, workspaceId, callerName) {
     const rows = await db
         .select({
         id: actions.id,
         name: actions.name,
         description: actions.description,
-        handlerAgentId: actions.handlerAgentId,
         handlerAgentName: agents.name,
+        handlerNodeName: nodes.name,
+        handlerNodeId: actions.handlerNodeId,
         inputSchema: actions.inputSchema,
         outputSchema: actions.outputSchema,
         availableTo: actions.availableTo,
@@ -56,21 +173,12 @@ export async function listActions(db, workspaceId, callerName) {
         createdAt: actions.createdAt,
     })
         .from(actions)
-        .innerJoin(agents, eq(actions.handlerAgentId, agents.id))
+        .leftJoin(agents, eq(actions.handlerAgentId, agents.id))
+        .leftJoin(nodes, eq(actions.handlerNodeId, nodes.id))
         .where(eq(actions.workspaceId, workspaceId));
     return rows
         .filter((r) => isActionVisibleToCaller(r.availableTo ?? null, callerName))
-        .map((r) => ({
-        id: r.id,
-        name: r.name,
-        description: r.description,
-        handler_agent: r.handlerAgentName,
-        input_schema: r.inputSchema,
-        output_schema: r.outputSchema,
-        available_to: r.availableTo ?? null,
-        is_active: r.isActive,
-        created_at: r.createdAt.toISOString(),
-    }));
+        .map(publicAction);
 }
 export async function getAction(db, workspaceId, name, callerName) {
     const [row] = await db
@@ -78,8 +186,9 @@ export async function getAction(db, workspaceId, name, callerName) {
         id: actions.id,
         name: actions.name,
         description: actions.description,
-        handlerAgentId: actions.handlerAgentId,
         handlerAgentName: agents.name,
+        handlerNodeName: nodes.name,
+        handlerNodeId: actions.handlerNodeId,
         inputSchema: actions.inputSchema,
         outputSchema: actions.outputSchema,
         availableTo: actions.availableTo,
@@ -87,21 +196,12 @@ export async function getAction(db, workspaceId, name, callerName) {
         createdAt: actions.createdAt,
     })
         .from(actions)
-        .innerJoin(agents, eq(actions.handlerAgentId, agents.id))
+        .leftJoin(agents, eq(actions.handlerAgentId, agents.id))
+        .leftJoin(nodes, eq(actions.handlerNodeId, nodes.id))
         .where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, name)));
     if (!row || !isActionVisibleToCaller(row.availableTo ?? null, callerName))
         return null;
-    return {
-        id: row.id,
-        name: row.name,
-        description: row.description,
-        handler_agent: row.handlerAgentName,
-        input_schema: row.inputSchema,
-        output_schema: row.outputSchema,
-        available_to: row.availableTo ?? null,
-        is_active: row.isActive,
-        created_at: row.createdAt.toISOString(),
-    };
+    return publicAction(row);
 }
 export async function deleteAction(db, workspaceId, name) {
     const result = await db
@@ -110,44 +210,310 @@ export async function deleteAction(db, workspaceId, name) {
         .returning();
     return result.length > 0;
 }
-export async function invokeAction(db, workspaceId, actionName, data) {
-    const [action] = await db
-        .select()
-        .from(actions)
-        .where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, actionName), eq(actions.isActive, true)));
-    if (!action) {
-        throw codedError(`Action "${actionName}" not found`, 'action_not_found', 404);
-    }
-    // Check availableTo access control — deny if caller is absent OR not in the list
-    if (action.availableTo && action.availableTo.length > 0) {
-        if (!data.caller_name || !action.availableTo.includes(data.caller_name)) {
-            const who = data.caller_name ? `Agent "${data.caller_name}"` : 'Caller';
-            throw codedError(`${who} is not authorized to invoke action "${actionName}"`, 'action_denied', 403);
-        }
-    }
+async function createInvocation(db, workspaceId, action, data) {
     const invocationId = `inv_${generateId()}`;
     const [invocation] = await db
         .insert(actionInvocations)
         .values({
         id: invocationId,
         workspaceId,
-        actionId: action.id,
-        actionName,
+        actionId: action?.id ?? null,
+        actionName: action?.name ?? 'spawn',
         callerId: data.caller_id ?? null,
         callerName: data.caller_name ?? null,
         input: data.input ?? {},
-        status: 'invoked',
+        status: data.status ?? 'pending',
     })
         .returning();
+    return invocation;
+}
+async function dispatchSpawn(args) {
+    if (!args.registry) {
+        throw codedError('Node dispatch is not available', 'node_dispatch_unavailable', 503);
+    }
+    const invocation = await createInvocation(args.db, args.workspaceId, null, {
+        input: args.data.input,
+        caller_id: args.data.caller_id,
+        caller_name: args.data.caller_name,
+    });
+    const placement = await claimSpawnNode(args.db, args.workspaceId, {
+        actionName: 'spawn',
+        input: args.data.input,
+        callerId: args.data.caller_id,
+    });
+    const dispatched = await dispatchNodeInvocation({
+        db: args.db,
+        registry: args.registry,
+        workspaceId: args.workspaceId,
+        invocationId: invocation.id,
+        nodeId: placement.node.id,
+        action: placement.capability,
+        input: recordInput(invocation.input),
+        pending: placement.queued,
+        reservationHeld: !placement.queued,
+    });
+    return {
+        invocation_id: invocation.id,
+        action_name: 'spawn',
+        handler_agent_id: null,
+        handler_node_id: placement.node.id,
+        dispatched_node_id: dispatched.accepted ? placement.node.id : null,
+        input: recordInput(invocation.input),
+        status: dispatched.accepted ? (dispatched.pending ? 'pending' : 'dispatched') : 'pending',
+        created_at: invocation.createdAt.toISOString(),
+    };
+}
+export async function invokeAction(db, workspaceId, actionName, data, options = {}) {
+    const fleetNodesEnabled = options.fleetNodesEnabled ?? true;
+    const action = await fetchAction(db, workspaceId, actionName);
+    if (!action && actionName === 'spawn') {
+        if (!fleetNodesEnabled) {
+            throw codedError('Fleet nodes are disabled for this workspace', 'fleet_nodes_disabled', 404);
+        }
+        return dispatchSpawn({
+            db,
+            registry: options.nodeConnections,
+            workspaceId,
+            data,
+        });
+    }
+    if (!action) {
+        throw codedError(`Action "${actionName}" not found`, 'action_not_found', 404);
+    }
+    // Check availableTo access control — deny if caller is absent OR not in the list
+    if (action.availableTo && action.availableTo.length > 0) {
+        if (!data.caller_name || !action.availableTo.includes(data.caller_name)) {
+            const who = data.caller_name ? `Agent "${data.caller_name}"` : 'Caller';
+            throw codedError(`${who} is not authorized to invoke action "${actionName}"`, 'action_denied', 403);
+        }
+    }
+    if (action.handlerNodeId) {
+        if (!fleetNodesEnabled) {
+            throw codedError('Fleet nodes are disabled for this workspace', 'fleet_nodes_disabled', 404);
+        }
+        if (!options.nodeConnections) {
+            throw codedError('Node dispatch is not available', 'node_dispatch_unavailable', 503);
+        }
+        const invocation = await createInvocation(db, workspaceId, action, {
+            input: data.input,
+            caller_id: data.caller_id,
+            caller_name: data.caller_name,
+        });
+        // Only mark the reservation held when we actually incremented the node's
+        // reserved-capacity counter, so completion/reschedule release stays balanced.
+        // If the reservation can't be taken (node offline / at capacity) the queued
+        // frame reserves later on drain via the spawnReservedAt check.
+        const reservedNode = isSpawnInvocation(action.name)
+            ? await reserveNodeCapacity(db, workspaceId, action.handlerNodeId)
+            : null;
+        const dispatched = await dispatchNodeInvocation({
+            db,
+            registry: options.nodeConnections,
+            workspaceId,
+            invocationId: invocation.id,
+            nodeId: action.handlerNodeId,
+            action: action.name,
+            input: recordInput(invocation.input),
+            reservationHeld: !!reservedNode,
+        });
+        return {
+            invocation_id: invocation.id,
+            action_name: actionName,
+            handler_agent_id: null,
+            handler_node_id: action.handlerNodeId,
+            dispatched_node_id: dispatched.accepted ? action.handlerNodeId : null,
+            input: recordInput(invocation.input),
+            status: dispatched.accepted ? (dispatched.pending ? 'pending' : 'dispatched') : 'pending',
+            created_at: invocation.createdAt.toISOString(),
+        };
+    }
+    if (!action.handlerAgentId) {
+        throw codedError(`Action "${actionName}" has no handler`, 'handler_unavailable', 503);
+    }
+    const invocation = await createInvocation(db, workspaceId, action, {
+        input: data.input,
+        caller_id: data.caller_id,
+        caller_name: data.caller_name,
+        status: 'dispatched',
+    });
     return {
         invocation_id: invocation.id,
         action_name: actionName,
         handler_agent_id: action.handlerAgentId,
-        input: invocation.input,
+        handler_node_id: null,
+        dispatched_node_id: null,
+        input: recordInput(invocation.input),
         status: invocation.status,
         created_at: invocation.createdAt.toISOString(),
     };
 }
+function publicInvocation(row) {
+    return {
+        invocation_id: row.id,
+        action_name: row.actionName,
+        caller_id: row.callerId,
+        caller_name: row.callerName,
+        input: row.input,
+        output: row.output,
+        status: row.status,
+        error: row.error,
+        duration_ms: row.durationMs,
+        dispatched_node_id: row.dispatchedNodeId,
+        dispatched_at: row.dispatchedAt?.toISOString() ?? null,
+        created_at: row.createdAt.toISOString(),
+        completed_at: row.completedAt?.toISOString() ?? null,
+    };
+}
+async function dispatchNodeAttempt(db, workspaceId, invocationId, nodeId, opts = {}) {
+    const stateFields = opts.pending
+        ? { status: 'pending', dispatchedAt: null, retryAfterAt: opts.retryAfterAt ?? null }
+        : dispatchedStateFields({ retryAfterAt: opts.retryAfterAt });
+    const [updated] = await db
+        .update(actionInvocations)
+        .set({
+        ...stateFields,
+        dispatchedNodeId: nodeId,
+        spawnReservedAt: opts.reservationHeld ? new Date() : null,
+        attemptedNodeIds: sql `json_insert(COALESCE(${actionInvocations.attemptedNodeIds}, '[]'), '$[#]', ${nodeId})`,
+        dispatchAttempts: sql `COALESCE(${actionInvocations.dispatchAttempts}, 0) + 1`,
+    })
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
+        .returning();
+    return !!updated;
+}
+/**
+ * Transition a drained offline-queue invocation into the live `dispatched` state
+ * once its queued `action.invoke` frame is actually delivered on node
+ * reconnect/drain. Reuses the same dispatched-state fields as the live dispatch
+ * path (stamping `dispatchedAt` and `retryAfterAt`) so the dispatch-timeout sweep
+ * and reschedule cover drained invocations. This is the SAME attempt that was
+ * queued, so `dispatchAttempts`/`attemptedNodeIds` are intentionally left intact.
+ * Guarded on the invocation still being `pending` on this node so a completion or
+ * reschedule that raced the drain is never clobbered.
+ */
+export async function markDrainedInvocationDispatched(db, workspaceId, invocationId, nodeId) {
+    const [updated] = await db
+        .update(actionInvocations)
+        .set(dispatchedStateFields({ retryAfterAt: new Date(Date.now() + ACTION_DISPATCH_TIMEOUT_MS) }))
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.dispatchedNodeId, nodeId), eq(actionInvocations.status, 'pending')))
+        .returning({ id: actionInvocations.id });
+    return !!updated;
+}
+async function dispatchNodeInvocation(args) {
+    const connectedBefore = args.registry.isNodeConnected(args.workspaceId, args.nodeId);
+    const sent = await args.registry.sendToNode(args.workspaceId, args.nodeId, {
+        v: 1,
+        type: 'action.invoke',
+        invocation_id: args.invocationId,
+        action: args.action,
+        input: toFleetWireJson(args.input),
+    });
+    if (!sent)
+        return { accepted: false, pending: false };
+    const pending = !!args.pending || !connectedBefore;
+    const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, { pending, retryAfterAt: args.retryAfterAt, reservationHeld: args.reservationHeld });
+    return { accepted, pending };
+}
+function attemptedNodeSet(invocation) {
+    return Array.from(new Set([
+        ...normalizeAttemptedNodeIds(invocation.attemptedNodeIds),
+        invocation.dispatchedNodeId,
+    ].filter((nodeId) => !!nodeId)));
+}
+async function selectRetryPlacement(db, invocation, excludeNodeIds) {
+    const input = recordInput(invocation.input);
+    if (isSpawnInvocation(invocation.actionName)) {
+        return claimSpawnNode(db, invocation.workspaceId, {
+            actionName: 'spawn',
+            input,
+            callerId: invocation.callerId,
+            excludeNodeIds,
+        });
+    }
+    return chooseNodeForAction(db, invocation.workspaceId, {
+        actionName: invocation.actionName,
+        input,
+        callerId: invocation.callerId,
+        excludeNodeIds,
+    });
+}
+export async function rescheduleNodeInvocation(db, registry, invocation, opts = {}) {
+    if (invocation.dispatchedNodeId && isSpawnInvocation(invocation.actionName)) {
+        await releaseNodeCapacity(db, invocation.workspaceId, invocation.dispatchedNodeId);
+    }
+    const attempted = attemptedNodeSet(invocation);
+    const current = invocation.dispatchedNodeId ? [invocation.dispatchedNodeId] : [];
+    const input = recordInput(invocation.input);
+    const actionToSend = isSpawnInvocation(invocation.actionName) ? 'spawn' : invocation.actionName;
+    const baseExclude = Array.from(new Set([...attempted, ...current]));
+    const candidates = opts.allowAttemptedFallback ? [baseExclude, []] : [baseExclude];
+    for (const excludeNodeIds of candidates) {
+        try {
+            const placement = await selectRetryPlacement(db, invocation, excludeNodeIds);
+            if (placement.queued) {
+                await dispatchNodeAttempt(db, invocation.workspaceId, invocation.id, placement.node.id, {
+                    pending: true,
+                    retryAfterAt: opts.retryAfterAt ?? null,
+                    reservationHeld: false,
+                });
+                return true;
+            }
+            const dispatched = await dispatchNodeInvocation({
+                db,
+                registry,
+                workspaceId: invocation.workspaceId,
+                invocationId: invocation.id,
+                nodeId: placement.node.id,
+                action: actionToSend,
+                input,
+                reservationHeld: isSpawnInvocation(invocation.actionName) && !placement.queued,
+            });
+            return dispatched.accepted;
+        }
+        catch {
+            // Try the next candidate set.
+        }
+    }
+    await db
+        .update(actionInvocations)
+        .set({
+        status: 'pending',
+        dispatchedNodeId: null,
+        dispatchedAt: null,
+        spawnReservedAt: null,
+        retryAfterAt: opts.retryAfterAt ?? nextRetryAfter(invocation.dispatchAttempts + 1),
+    })
+        .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+    return false;
+}
+export async function rescheduleInvocationsForLostNode(db, registry, workspaceId, nodeId) {
+    const rows = await db
+        .select()
+        .from(actionInvocations)
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.dispatchedNodeId, nodeId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+    let rescheduled = 0;
+    for (const invocation of rows) {
+        try {
+            if (await rescheduleNodeInvocation(db, registry, invocation, { retryAfterAt: nextRetryAfter(invocation.dispatchAttempts + 1) })) {
+                rescheduled++;
+            }
+        }
+        catch {
+            // Keep the invocation pending; another heartbeat/sweep can retry.
+            await db
+                .update(actionInvocations)
+                .set({
+                status: 'pending',
+                dispatchedNodeId: null,
+                dispatchedAt: null,
+                spawnReservedAt: null,
+                retryAfterAt: nextRetryAfter(invocation.dispatchAttempts + 1),
+            })
+                .where(eq(actionInvocations.id, invocation.id));
+        }
+    }
+    return rescheduled;
+}
 export async function completeInvocation(db, workspaceId, actionName, invocationId, data) {
     // Fetch the invocation joined with its action — verify action name matches URL param
     const [existing] = await db
@@ -155,13 +521,20 @@ export async function completeInvocation(db, workspaceId, actionName, invocation
         id: actionInvocations.id,
         status: actionInvocations.status,
         actionName: actionInvocations.actionName,
+        attemptedNodeIds: actionInvocations.attemptedNodeIds,
+        dispatchAttempts: actionInvocations.dispatchAttempts,
+        dispatchedNodeId: actionInvocations.dispatchedNodeId,
         handlerAgentId: actions.handlerAgentId,
+        handlerNodeId: actions.handlerNodeId,
     })
         .from(actionInvocations)
         .leftJoin(actions, eq(actionInvocations.actionId, actions.id))
         .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.actionName, actionName)));
     if (!existing)
         return null;
+    if (existing.handlerNodeId || existing.dispatchedNodeId) {
+        throw codedError('Node-owned invocations must be completed over the node control channel', 'node_owned_invocation', 403);
+    }
     // Agent tokens must belong to the handler agent
     if (data.caller_agent_id && existing.handlerAgentId && data.caller_agent_id !== existing.handlerAgentId) {
         throw codedError('Only the handler agent can complete this invocation', 'forbidden', 403);
@@ -176,21 +549,98 @@ export async function completeInvocation(db, workspaceId, actionName, invocation
         durationMs: data.duration_ms ?? null,
         completedAt: new Date(),
     })
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.actionName, actionName), eq(actionInvocations.status, 'invoked')))
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.actionName, actionName), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
         .returning();
-    // No rows updated means either not found or already completed by a concurrent request
     if (!updated)
         return null;
-    return {
-        invocation_id: updated.id,
-        action_name: updated.actionName,
-        caller_id: updated.callerId,
-        status: updated.status,
-        output: updated.output,
-        error: updated.error,
-        duration_ms: updated.durationMs,
-        completed_at: updated.completedAt?.toISOString() ?? null,
-    };
+    return publicInvocation(updated);
+}
+export async function completeNodeInvocation(db, registry, workspaceId, nodeId, invocationId, data) {
+    const [existing] = await db
+        .select({
+        id: actionInvocations.id,
+        workspaceId: actionInvocations.workspaceId,
+        actionName: actionInvocations.actionName,
+        callerId: actionInvocations.callerId,
+        input: actionInvocations.input,
+        output: actionInvocations.output,
+        status: actionInvocations.status,
+        error: actionInvocations.error,
+        durationMs: actionInvocations.durationMs,
+        dispatchedNodeId: actionInvocations.dispatchedNodeId,
+        dispatchedAt: actionInvocations.dispatchedAt,
+        attemptedNodeIds: actionInvocations.attemptedNodeIds,
+        dispatchAttempts: actionInvocations.dispatchAttempts,
+        createdAt: actionInvocations.createdAt,
+        completedAt: actionInvocations.completedAt,
+    })
+        .from(actionInvocations)
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId)));
+    if (!existing)
+        return null;
+    if (existing.dispatchedNodeId && existing.dispatchedNodeId !== nodeId) {
+        return null;
+    }
+    if (existing.status === 'completed' || existing.status === 'failed') {
+        return null;
+    }
+    if (data.error === 'handler_unavailable') {
+        await db
+            .update(nodes)
+            .set({ handlersLive: false })
+            .where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
+        try {
+            if (await rescheduleNodeInvocation(db, registry, existing, { retryAfterAt: nextRetryAfter(existing.dispatchAttempts + 1) })) {
+                return null;
+            }
+        }
+        catch {
+            // Fall through and fail the invocation if no eligible node exists.
+        }
+    }
+    const [updated] = await db
+        .update(actionInvocations)
+        .set({
+        output: data.output ?? null,
+        error: data.error ?? null,
+        status: data.error ? 'failed' : 'completed',
+        completedAt: new Date(),
+        spawnReservedAt: null,
+    })
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.dispatchedNodeId, nodeId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
+        .returning();
+    if (updated && isSpawnInvocation(updated.actionName) && updated.dispatchedNodeId) {
+        await releaseNodeCapacity(db, workspaceId, updated.dispatchedNodeId);
+    }
+    return updated ? publicInvocation(updated) : null;
+}
+export async function sweepTimedOutInvocations(db, registry, timeoutMs = ACTION_DISPATCH_TIMEOUT_MS) {
+    const now = new Date();
+    const cutoff = new Date(Date.now() - timeoutMs);
+    const rows = await db
+        .select()
+        .from(actionInvocations)
+        .where(and(inArray(actionInvocations.status, ['dispatched']), lte(actionInvocations.dispatchedAt, cutoff)));
+    const pendingRows = await db
+        .select()
+        .from(actionInvocations)
+        .where(and(eq(actionInvocations.status, 'pending'), lte(actionInvocations.retryAfterAt, now)));
+    let rescheduled = 0;
+    for (const invocation of [...rows, ...pendingRows]) {
+        try {
+            const allowFallback = invocation.status === 'pending';
+            if (await rescheduleNodeInvocation(db, registry, invocation, {
+                allowAttemptedFallback: allowFallback,
+                retryAfterAt: allowFallback ? null : nextRetryAfter(invocation.dispatchAttempts + 1),
+            })) {
+                rescheduled++;
+            }
+        }
+        catch {
+            // Leave the invocation for the next sweep.
+        }
+    }
+    return rescheduled;
 }
 export async function getInvocation(db, workspaceId, actionName, invocationId) {
     const [row] = await db
@@ -199,18 +649,6 @@ export async function getInvocation(db, workspaceId, actionName, invocationId) {
         .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.actionName, actionName)));
     if (!row)
         return null;
-    return {
-        invocation_id: row.id,
-        action_name: row.actionName,
-        caller_id: row.callerId,
-        caller_name: row.callerName,
-        input: row.input,
-        output: row.output,
-        status: row.status,
-        error: row.error,
-        duration_ms: row.durationMs,
-        created_at: row.createdAt.toISOString(),
-        completed_at: row.completedAt?.toISOString() ?? null,
-    };
+    return publicInvocation(row);
 }
 //# sourceMappingURL=action.js.map