@relaycast/engine 6.1.0 → 6.3.0

package/dist/engine/action.js

@@ -1,12 +1,23 @@
-import { and, asc, eq, inArray, isNull, lte, or, sql } from 'drizzle-orm';
+import { and, asc, eq, inArray, isNotNull, isNull, lte, or, sql } from 'drizzle-orm';
 import { actions, actionInvocations, agents, agentNodeBindings, nodes } from '../db/schema.js';
 import { generateId } from './snowflake.js';
 import { codedError } from '../lib/httpError.js';
 import { toFleetWireJson } from './deliveryWire.js';
+import { emitAgentExitedEffects, emitInvocationCompletionEffects, fleetInvocationId, } from './invocationCompletion.js';
+import { runAtomic } from '../ports/database.js';
 import { claimSpawnNode, chooseNodeForAction, isNodeLive, releaseNodeCapacity, reserveNodeCapacity } from './placement.js';
 import { DEFAULT_PROVIDER_NAME, capacityProviderName, getProvider, isProviderLive } from './nodeProvider.js';
 const OPEN_INVOCATION_STATUSES = ['pending', 'dispatched', 'invoked'];
 export const ACTION_DISPATCH_TIMEOUT_MS = 30_000;
+/**
+ * How long an agent handler's connection must be CONTINUOUSLY unreachable
+ * before its open invocations are failed with `handler_unavailable`. The clock
+ * starts at the sweep's first unreachable observation and resets when
+ * connectivity recovers — generous enough to ride out a handler restart at any
+ * invocation age, bounded enough that a caller blocked on the invocation gets
+ * a signal instead of an unbounded hang.
+ */
+export const ACTION_HANDLER_UNREACHABLE_TTL_MS = 120_000;
 const ACTION_RETRY_BACKOFF_MS = 5_000;
 const NODE_DRAIN_REQUEUE_RETRY_MS = 5_000;
 function capabilityName(capability) {
@@ -120,7 +131,7 @@ async function fetchNodeAction(db, workspaceId, nodeId, actionName) {
         .where(and(eq(actions.workspaceId, workspaceId), eq(actions.handlerNodeId, nodeId), eq(actions.name, actionName), eq(actions.isActive, true)));
     return action ?? null;
 }
-export async function registerAction(db, workspaceId, data) {
+export async function registerAction(db, workspaceId, data, options = {}) {
     requireOneHandler(data);
     let handlerAgentId = null;
     let handlerAgentName = null;
@@ -155,32 +166,94 @@ export async function registerAction(db, workspaceId, data) {
         handlerNodeName = node.name;
     }
     const id = `act_${generateId()}`;
-    const [action] = await db
-        .insert(actions)
-        .values({
-        id,
-        workspaceId,
-        name: data.name,
+    // Registration is an idempotent assertion (PUT semantics): re-registering an
+    // existing name refreshes the handler pointer, schemas, and visibility so a
+    // reconnecting publisher heals a stale handler instead of surfacing the
+    // unique index as a 500. Provider-owned fields (handlerProvider, isGlobal,
+    // queue) are deliberately not touched — they belong to node-control
+    // registration (materializeProviderActions).
+    const refresh = {
         description: data.description,
         handlerAgentId,
-        handlerNodeId,
         inputSchema: data.input_schema ?? {},
         outputSchema: data.output_schema ?? {},
         availableTo: data.available_to ?? null,
-    })
-        .returning();
+        isActive: true,
+    };
+    let action;
+    let stranded;
+    try {
+        // The read-upsert-strand sequence runs atomically so a concurrent invoke
+        // cannot land between the handler move and the stranded-invocation snapshot
+        // (it either targets the old handler and is swept, or the new handler and
+        // is untouched).
+        ({ action, stranded } = await runAtomic(db, async (tx) => {
+            // Read the row the upsert may refresh so a handler move can be detected.
+            // The agent branch conflicts on the agent-hosted partial index, the node
+            // branch on the per-node index.
+            const [previous] = await tx
+                .select({ id: actions.id, handlerAgentId: actions.handlerAgentId })
+                .from(actions)
+                .where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, data.name), handlerNodeId ? eq(actions.handlerNodeId, handlerNodeId) : isNull(actions.handlerNodeId)));
+            const [row] = await tx
+                .insert(actions)
+                .values({
+                id,
+                workspaceId,
+                name: data.name,
+                description: data.description,
+                handlerAgentId,
+                handlerNodeId,
+                inputSchema: data.input_schema ?? {},
+                outputSchema: data.output_schema ?? {},
+                availableTo: data.available_to ?? null,
+            })
+                .onConflictDoUpdate(handlerNodeId
+                ? { target: [actions.workspaceId, actions.handlerNodeId, actions.name], set: refresh }
+                // Agent-hosted uniqueness is a partial index, so the conflict target
+                // must carry its WHERE clause to match it.
+                : { target: [actions.workspaceId, actions.name], targetWhere: isNull(actions.handlerNodeId), set: refresh })
+                .returning();
+            // A refresh that moves the handler to a DIFFERENT agent strands
+            // invocations already in flight toward the previous handler: completion
+            // authorization follows actions.handlerAgentId (the old handler's
+            // completion would 403) and the timeout sweep would redeliver them to the
+            // new handler, which never received the original dispatch. Terminally
+            // fail them; the callers are told after commit.
+            let strandedRows = [];
+            if (previous && previous.handlerAgentId && previous.handlerAgentId !== handlerAgentId) {
+                const strandedIds = await openInvocationIdsForActions(tx, workspaceId, [previous.id]);
+                strandedRows = await failOpenInvocationRows(tx, workspaceId, strandedIds, 'handler_unavailable');
+            }
+            return { action: row, stranded: strandedRows };
+        }));
+    }
+    catch (err) {
+        // A residual uniqueness race must surface as a conflict, never as a 500
+        // internal_error (which reads as a server outage and invites retries).
+        if (err instanceof Error && /unique constraint failed/i.test(err.message)) {
+            throw codedError(`Action "${data.name}" conflicts with an existing action`, 'action_name_conflict', 409);
+        }
+        throw err;
+    }
+    if (options.completionDeps) {
+        await emitFailedInvocationEffects(options.completionDeps, workspaceId, stranded);
+    }
     return {
-        id: action.id,
-        name: action.name,
-        description: action.description,
-        handler_agent: handlerAgentName,
-        handler_node: handlerNodeName,
-        handler_node_id: handlerNodeId,
-        input_schema: action.inputSchema,
-        output_schema: action.outputSchema,
-        available_to: action.availableTo ?? null,
-        is_active: action.isActive,
-        created_at: action.createdAt.toISOString(),
+        created: action.id === id,
+        action: {
+            id: action.id,
+            name: action.name,
+            description: action.description,
+            handler_agent: handlerAgentName,
+            handler_node: handlerNodeName,
+            handler_node_id: handlerNodeId,
+            input_schema: action.inputSchema,
+            output_schema: action.outputSchema,
+            available_to: action.availableTo ?? null,
+            is_active: action.isActive,
+            created_at: action.createdAt.toISOString(),
+        },
     };
 }
 export async function listActions(db, workspaceId, callerName) {
@@ -229,12 +302,29 @@ export async function getAction(db, workspaceId, name, callerName) {
         return null;
     return publicAction(row);
 }
-export async function deleteAction(db, workspaceId, name) {
-    const result = await db
-        .delete(actions)
-        .where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, name)))
-        .returning();
-    return result.length > 0;
+export async function deleteAction(db, workspaceId, name, options = {}) {
+    // Fail open invocations WITH the delete in one atomic step: actionId is set
+    // null on action delete, which would orphan them outside every
+    // handler-resolution join (the TTL sweep included) so they retry forever and
+    // the caller never hears back. Atomicity closes the window where an invoke
+    // lands between the stranded snapshot and the delete.
+    const { deleted, stranded } = await runAtomic(db, async (tx) => {
+        const rows = await tx
+            .select({ id: actions.id })
+            .from(actions)
+            .where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, name)));
+        const strandedIds = await openInvocationIdsForActions(tx, workspaceId, rows.map((row) => row.id));
+        const strandedRows = await failOpenInvocationRows(tx, workspaceId, strandedIds, 'action_deleted');
+        const result = await tx
+            .delete(actions)
+            .where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, name)))
+            .returning();
+        return { deleted: result.length > 0, stranded: strandedRows };
+    });
+    if (options.completionDeps) {
+        await emitFailedInvocationEffects(options.completionDeps, workspaceId, stranded);
+    }
+    return deleted;
 }
 async function createInvocation(db, workspaceId, action, data) {
     const invocationId = `inv_${generateId()}`;
@@ -253,6 +343,22 @@ async function createInvocation(db, workspaceId, action, data) {
         .returning();
     return invocation;
 }
+/**
+ * Liveness rule for the connection that hosts an agent handler, mirroring the
+ * drain-time rule: the provider socket must be connected, and when a provider
+ * row exists and is explicitly attached its heartbeat must say handlers are
+ * live. A missing provider row (direct node, legacy single-socket broker)
+ * keeps connection state as the available liveness signal.
+ */
+async function isHandlerConnectionLive(db, registry, workspaceId, nodeId, providerName) {
+    if (!registry.isProviderConnected(workspaceId, nodeId, providerName))
+        return false;
+    const provider = await getProvider(db, workspaceId, nodeId, providerName);
+    if (!provider)
+        return true;
+    const explicitlyAttached = registry.isProviderAttached?.(workspaceId, nodeId, providerName) ?? true;
+    return !explicitlyAttached || isProviderLive(provider);
+}
 /** Is a node provider currently invokable — connection up AND heartbeat handlers_live. */
 async function isNodeProviderLive(db, registry, workspaceId, nodeId, providerName) {
     if (!registry.isProviderConnected(workspaceId, nodeId, providerName))
@@ -266,6 +372,78 @@ async function failInvocationForUnavailableProvider(db, workspaceId, invocationI
         .set({ status: 'failed', error: 'handler_unavailable', completedAt: new Date(), spawnReservedAt: null })
         .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
 }
+/**
+ * Terminally fail the given open invocations (no completion emission — call
+ * {@link emitFailedInvocationEffects} with the returned rows after the
+ * enclosing transaction commits). A spawn invocation holding a native node
+ * reservation releases that capacity, mirroring `completeNodeInvocation`.
+ */
+async function failOpenInvocationRows(db, workspaceId, invocationIds, error) {
+    if (invocationIds.length === 0)
+        return [];
+    // Capture reservation state before the update clears spawnReservedAt.
+    const held = await db
+        .select({
+        id: actionInvocations.id,
+        actionName: actionInvocations.actionName,
+        dispatchedNodeId: actionInvocations.dispatchedNodeId,
+        spawnReservedAt: actionInvocations.spawnReservedAt,
+    })
+        .from(actionInvocations)
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.id, invocationIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+    const failed = await db
+        .update(actionInvocations)
+        .set({ status: 'failed', error, completedAt: new Date(), spawnReservedAt: null })
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.id, invocationIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
+        .returning();
+    const failedIds = new Set(failed.map((row) => row.id));
+    for (const row of held) {
+        if (failedIds.has(row.id) && isSpawnInvocation(row.actionName) && row.dispatchedNodeId && row.spawnReservedAt) {
+            await releaseNodeCapacity(db, workspaceId, row.dispatchedNodeId);
+        }
+    }
+    return failed;
+}
+/** Emit `action.failed` to each failed invocation's caller. Best-effort per row. */
+async function emitFailedInvocationEffects(completionDeps, workspaceId, failed) {
+    for (const invocation of failed) {
+        try {
+            await emitInvocationCompletionEffects(completionDeps, workspaceId, {
+                invocation_id: invocation.id,
+                action_name: invocation.actionName,
+                caller_id: invocation.callerId,
+                status: 'failed',
+                output: invocation.output,
+                error: invocation.error,
+            });
+        }
+        catch {
+            // Notification is best-effort; the invocation is already terminally failed.
+        }
+    }
+}
+/**
+ * Terminally fail the given open invocations, emitting `action.failed` to each
+ * caller when completion deps are provided. Used when an invocation can no
+ * longer reach a handler that will answer: the handler pointer moved to a
+ * different agent (re-register takeover), the action was deleted, or the
+ * handler connection stayed unreachable past the TTL.
+ */
+async function failOpenInvocations(db, workspaceId, invocationIds, error, completionDeps) {
+    const failed = await failOpenInvocationRows(db, workspaceId, invocationIds, error);
+    if (completionDeps)
+        await emitFailedInvocationEffects(completionDeps, workspaceId, failed);
+}
+/** Ids of open invocations pointing at any of the given action rows. */
+async function openInvocationIdsForActions(db, workspaceId, actionIds) {
+    if (actionIds.length === 0)
+        return [];
+    const rows = await db
+        .select({ id: actionInvocations.id })
+        .from(actionInvocations)
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.actionId, actionIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+    return rows.map((row) => row.id);
+}
 /**
  * Dispatch an invoke to a specific provider on a node. When the provider is
  * offline the invoke fails fast (the invocation is failed) unless it opted into
@@ -581,11 +759,37 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
     if (!handlerAgent || handlerAgent.locationType !== 'via_node' || !handlerAgent.locationNodeId) {
         throw codedError(`Action "${actionName}" handler is not bound to a node`, 'handler_unavailable', 503);
     }
+    // Fail fast when the handler agent's connection is not live: an invocation
+    // queued toward a dead handler never completes and the caller gets no signal
+    // — it just hangs on the tool call. An action that opted into `queue` keeps
+    // the queue-and-drain semantics (the TTL sweep still bounds it).
+    if (!action.queue) {
+        const live = await isHandlerConnectionLive(db, options.nodeConnections, workspaceId, handlerAgent.locationNodeId, handlerAgent.providerName);
+        if (!live) {
+            throw codedError(`Action "${actionName}" handler "${handlerAgent.name}" has no live connection`, 'handler_unavailable', 503);
+        }
+    }
     const invocation = await createInvocation(db, workspaceId, action, {
         input: data.input,
         caller_id: data.caller_id,
         caller_name: data.caller_name,
     });
+    // Re-validate the handler pointer AFTER the insert. A takeover committing
+    // between action resolution above and the insert misses this invocation in
+    // its stranded snapshot (the row was not visible yet), which would leave it
+    // dispatched toward the old handler whose completions no longer authorize.
+    // The two orderings are now both covered: a takeover before the insert is
+    // caught here; a takeover after the insert sees the row and sweeps it.
+    // (A concurrent DELETE needs no equivalent check — the actionId foreign key
+    // rejects an insert that lands after the delete commits.)
+    const [currentHandler] = await db
+        .select({ handlerAgentId: actions.handlerAgentId })
+        .from(actions)
+        .where(and(eq(actions.workspaceId, workspaceId), eq(actions.id, action.id)));
+    if (!currentHandler || currentHandler.handlerAgentId !== handlerAgent.id) {
+        await failOpenInvocationRows(db, workspaceId, [invocation.id], 'handler_unavailable');
+        throw codedError(`Action "${actionName}" handler changed during invoke`, 'handler_unavailable', 503);
+    }
     const dispatched = await dispatchNodeInvocation({
         db,
         registry: options.nodeConnections,
@@ -625,7 +829,7 @@ function publicInvocation(row) {
         completed_at: row.completedAt?.toISOString() ?? null,
     };
 }
-async function applyReleaseCompletionEffect(db, workspaceId, nodeId, invocation, data) {
+async function applyReleaseCompletionEffect(db, workspaceId, nodeId, invocation, data, deps) {
     if (!isReleaseInvocation(invocation.actionName) || data.error)
         return;
     const input = recordInput(invocation.input);
@@ -648,6 +852,9 @@ async function applyReleaseCompletionEffect(db, workspaceId, nodeId, invocation,
         .returning({ id: agentNodeBindings.id });
     if (!deactivatedBinding)
         return;
+    // Capture exit correlation BEFORE the mutation deletes the row or strips the
+    // spawn/cli metadata, so a durable agent.exited can still be emitted.
+    const exited = { agentId: agent.id, agentName: agent.name, invocationId: fleetInvocationId(agent.metadata) };
     await db
         .update(nodes)
         .set({
@@ -656,28 +863,38 @@ async function applyReleaseCompletionEffect(db, workspaceId, nodeId, invocation,
         .where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
     if (input.delete_agent === true) {
         await db.delete(agents).where(and(eq(agents.workspaceId, workspaceId), eq(agents.id, agent.id)));
-        return;
     }
-    const existingMetadata = agent.metadata ?? {};
-    const { spawn: _spawn, cli: _cli, ...restMetadata } = existingMetadata;
-    await db
-        .update(agents)
-        .set({
-        status: 'offline',
-        // Clear the node location so the agent is no longer routable to the released
-        // node and a repeat release can't re-decrement the node's active count.
-        locationType: 'self_connected',
-        locationNodeId: null,
-        lastSeen: new Date(),
-        metadata: {
-            ...restMetadata,
-            release: {
-                reason: typeof input.reason === 'string' ? input.reason : null,
-                released_at: new Date().toISOString(),
+    else {
+        const existingMetadata = agent.metadata ?? {};
+        const { spawn: _spawn, cli: _cli, ...restMetadata } = existingMetadata;
+        await db
+            .update(agents)
+            .set({
+            status: 'offline',
+            // Clear the node location so the agent is no longer routable to the released
+            // node and a repeat release can't re-decrement the node's active count.
+            locationType: 'self_connected',
+            locationNodeId: null,
+            lastSeen: new Date(),
+            metadata: {
+                ...restMetadata,
+                release: {
+                    reason: typeof input.reason === 'string' ? input.reason : null,
+                    released_at: new Date().toISOString(),
+                },
             },
-        },
-    })
-        .where(and(eq(agents.workspaceId, workspaceId), eq(agents.id, agent.id)));
+        })
+            .where(and(eq(agents.workspaceId, workspaceId), eq(agents.id, agent.id)));
+    }
+    if (deps) {
+        await emitAgentExitedEffects(deps, workspaceId, {
+            agentId: exited.agentId,
+            agentName: exited.agentName,
+            nodeId,
+            invocationId: exited.invocationId,
+            reason: 'released',
+        });
+    }
 }
 async function dispatchNodeAttempt(db, workspaceId, invocationId, nodeId, opts) {
     const stateFields = opts.pending
@@ -840,11 +1057,7 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId) {
         // Skip draining to a provider that is disconnected or whose heartbeat says
         // handlers are not live. A missing provider row is a legacy single-socket
         // node, for which connection state remains the available liveness signal.
-        if (!registry.isProviderConnected(workspaceId, nodeId, providerName))
-            continue;
-        const provider = await getProvider(db, workspaceId, nodeId, providerName);
-        const explicitlyAttached = registry.isProviderAttached?.(workspaceId, nodeId, providerName) ?? true;
-        if (provider && explicitlyAttached && !isProviderLive(provider))
+        if (!(await isHandlerConnectionLive(db, registry, workspaceId, nodeId, providerName)))
             continue;
         let reservedForDrain = false;
         try {
@@ -1112,7 +1325,7 @@ export async function completeInvocation(db, workspaceId, actionName, invocation
         return null;
     return publicInvocation(updated);
 }
-export async function completeNodeInvocation(db, registry, workspaceId, nodeId, providerName, invocationId, data) {
+export async function completeNodeInvocation(db, registry, workspaceId, nodeId, providerName, invocationId, data, deps) {
     const [existing] = await db
         .select({
         id: actionInvocations.id,
@@ -1174,12 +1387,65 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
         await releaseNodeCapacity(db, workspaceId, updated.dispatchedNodeId);
     }
     if (updated) {
-        await applyReleaseCompletionEffect(db, workspaceId, nodeId, existing, data);
+        await applyReleaseCompletionEffect(db, workspaceId, nodeId, existing, data, deps);
     }
     return updated ? publicInvocation(updated) : null;
 }
+/**
+ * Bound the wait on agent-handled invocations: an open invocation whose handler
+ * agent's connection has been CONTINUOUSLY unreachable for the TTL is failed
+ * with `handler_unavailable` — emitting `action.failed` to the caller when
+ * completion deps are provided — instead of staying `pending`/`dispatched`
+ * forever. The first unreachable observation stamps
+ * `handler_unreachable_since`; a recovered connection clears it, so a brief
+ * handler restart never kills an invocation regardless of its age.
+ */
+async function failUnreachableAgentInvocations(db, registry, ttlMs, completionDeps) {
+    const rows = await db
+        .select({
+        id: actionInvocations.id,
+        workspaceId: actionInvocations.workspaceId,
+        handlerUnreachableSince: actionInvocations.handlerUnreachableSince,
+    })
+        .from(actionInvocations)
+        .innerJoin(actions, eq(actionInvocations.actionId, actions.id))
+        .where(and(inArray(actionInvocations.status, ['pending', 'dispatched']), isNull(actions.handlerNodeId), isNotNull(actions.handlerAgentId)));
+    for (const row of rows) {
+        try {
+            // targetAgentForInvocation returns null when the handler agent is gone or
+            // no longer node-bound — both count as unreachable.
+            const target = await targetAgentForInvocation(db, row);
+            const reachable = !!target
+                && await isHandlerConnectionLive(db, registry, row.workspaceId, target.nodeId, target.providerName);
+            if (reachable) {
+                if (row.handlerUnreachableSince) {
+                    await db
+                        .update(actionInvocations)
+                        .set({ handlerUnreachableSince: null })
+                        .where(and(eq(actionInvocations.workspaceId, row.workspaceId), eq(actionInvocations.id, row.id)));
+                }
+                continue;
+            }
+            const since = row.handlerUnreachableSince ?? new Date();
+            if (!row.handlerUnreachableSince) {
+                await db
+                    .update(actionInvocations)
+                    .set({ handlerUnreachableSince: since })
+                    .where(and(eq(actionInvocations.workspaceId, row.workspaceId), eq(actionInvocations.id, row.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+            }
+            if (Date.now() - since.getTime() >= ttlMs) {
+                await failOpenInvocations(db, row.workspaceId, [row.id], 'handler_unavailable', completionDeps);
+            }
+        }
+        catch {
+            // Leave the invocation for the next sweep.
+        }
+    }
+}
 export async function sweepTimedOutInvocations(db, registry, opts = {}) {
-    const timeoutMs = typeof opts === 'number' ? opts : opts?.timeoutMs ?? ACTION_DISPATCH_TIMEOUT_MS;
+    const sweepOpts = typeof opts === 'number' || opts === null ? {} : opts;
+    const timeoutMs = typeof opts === 'number' ? opts : sweepOpts.timeoutMs ?? ACTION_DISPATCH_TIMEOUT_MS;
+    await failUnreachableAgentInvocations(db, registry, sweepOpts.handlerUnreachableTtlMs ?? ACTION_HANDLER_UNREACHABLE_TTL_MS, sweepOpts.completionDeps);
     const now = new Date();
     const cutoff = new Date(Date.now() - timeoutMs);
     const rows = await db