@desplega.ai/agent-swarm 1.20.0 → 1.51.2

package/src/workflows/recovery.ts

@@ -0,0 +1,139 @@
+import {
+  getCompletedStepNodeIds,
+  getDb,
+  getStuckWorkflowRuns,
+  getWorkflow,
+  getWorkflowRun,
+  updateWorkflowRun,
+  updateWorkflowRunStep,
+} from "../be/db";
+import { checkpointStep } from "./checkpoint";
+import { getSuccessors } from "./definition";
+import { findReadyNodes, walkGraph } from "./engine";
+import type { ExecutorRegistry } from "./executors/registry";
+
+/**
+ * Recover incomplete workflow runs on server startup.
+ *
+ * Two cases:
+ * 1. `running` runs — were mid-execution when server died.
+ *    Find completed steps, compute ready nodes, continue walking.
+ * 2. `waiting` runs — were waiting for a task that may have finished while we were down.
+ *    Check if the linked task is done and resume/fail accordingly.
+ */
+export async function recoverIncompleteRuns(registry: ExecutorRegistry): Promise<number> {
+  let recovered = 0;
+
+  // --- Case 1: Running runs that were interrupted mid-execution ---
+  recovered += await recoverRunningRuns(registry);
+
+  // --- Case 2: Waiting runs whose tasks may have finished ---
+  recovered += await recoverWaitingRuns(registry);
+
+  if (recovered > 0) {
+    console.log(`[workflows] Recovered ${recovered} incomplete run(s) on startup`);
+  }
+
+  return recovered;
+}
+
+/**
+ * Resume runs that were in "running" state when the server stopped.
+ * Uses checkpointed step data to find where to continue.
+ */
+async function recoverRunningRuns(registry: ExecutorRegistry): Promise<number> {
+  // Query for all running runs by scanning steps
+  // We need to find runs where status = 'running' and figure out which nodes to resume
+  const runningRunIds = getRunIdsByStatus("running");
+  let recovered = 0;
+
+  for (const runId of runningRunIds) {
+    try {
+      const run = getWorkflowRun(runId);
+      if (!run || run.status !== "running") continue;
+
+      const workflow = getWorkflow(run.workflowId);
+      if (!workflow) continue;
+
+      const completedNodeIds = new Set(getCompletedStepNodeIds(runId));
+      const ctx = (run.context ?? {}) as Record<string, unknown>;
+
+      // Find the next nodes that are ready to execute
+      const readyNodes = findReadyNodes(workflow.definition, completedNodeIds);
+      if (readyNodes.length === 0) {
+        // All nodes completed or nothing is ready — mark as completed
+        updateWorkflowRun(runId, {
+          status: "completed",
+          context: ctx,
+          finishedAt: new Date().toISOString(),
+        });
+      } else {
+        await walkGraph(workflow.definition, runId, ctx, readyNodes, registry, workflow.id);
+      }
+      recovered++;
+    } catch (err) {
+      console.error(`[workflows] Failed to recover running run ${runId}:`, err);
+    }
+  }
+
+  return recovered;
+}
+
+/**
+ * Check waiting runs whose linked tasks may have completed/failed/cancelled
+ * while the server was down.
+ */
+async function recoverWaitingRuns(registry: ExecutorRegistry): Promise<number> {
+  const stuckRuns = getStuckWorkflowRuns();
+  let recovered = 0;
+
+  for (const stuck of stuckRuns) {
+    try {
+      const run = getWorkflowRun(stuck.runId);
+      const workflow = getWorkflow(stuck.workflowId);
+      if (!run || !workflow) continue;
+
+      if (stuck.taskStatus === "completed") {
+        // Task finished while we were down — checkpoint and resume
+        const ctx = (run.context ?? {}) as Record<string, unknown>;
+        const stepOutput = { taskId: stuck.stepId, taskOutput: stuck.taskOutput };
+
+        checkpointStep(stuck.runId, stuck.stepId, stuck.nodeId, { output: stepOutput }, ctx);
+        updateWorkflowRun(stuck.runId, { status: "running" });
+
+        const successors = getSuccessors(workflow.definition, stuck.nodeId, "default");
+        await walkGraph(workflow.definition, stuck.runId, ctx, successors, registry, workflow.id);
+      } else {
+        // Task failed or cancelled — mark run failed
+        const reason =
+          stuck.taskStatus === "failed" ? "Task failed (recovered)" : "Task cancelled (recovered)";
+        const now = new Date().toISOString();
+        updateWorkflowRunStep(stuck.stepId, {
+          status: "failed",
+          error: reason,
+          finishedAt: now,
+        });
+        updateWorkflowRun(stuck.runId, {
+          status: "failed",
+          error: reason,
+          finishedAt: now,
+        });
+      }
+      recovered++;
+    } catch (err) {
+      console.error(`[workflows] Failed to recover waiting run ${stuck.runId}:`, err);
+    }
+  }
+
+  return recovered;
+}
+
+/**
+ * Get run IDs by status. Simple query since there's no dedicated function for this.
+ */
+function getRunIdsByStatus(status: string): string[] {
+  const rows = getDb()
+    .prepare<{ id: string }, [string]>("SELECT id FROM workflow_runs WHERE status = ?")
+    .all(status);
+  return rows.map((r) => r.id);
+}

package/src/workflows/resume.ts

@@ -0,0 +1,229 @@
+import {
+  getCompletedStepNodeIds,
+  getWorkflow,
+  getWorkflowRun,
+  getWorkflowRunStep,
+  getWorkflowRunStepsByRunId,
+  updateWorkflowRun,
+  updateWorkflowRunStep,
+} from "../be/db";
+import { checkpointStep } from "./checkpoint";
+import { findReadyNodes, walkGraph } from "./engine";
+import type { WorkflowEventBus } from "./event-bus";
+import type { ExecutorRegistry } from "./executors/registry";
+
+interface TaskEvent {
+  taskId: string;
+  output?: string;
+  agentId?: string;
+  workflowRunId?: string;
+  workflowRunStepId?: string;
+  failureReason?: string;
+}
+
+/**
+ * Wire up event bus listeners for workflow resume on task lifecycle events.
+ */
+export function setupWorkflowResumeListener(
+  eventBus: WorkflowEventBus,
+  registry: ExecutorRegistry,
+): void {
+  eventBus.on("task.completed", async (data: unknown) => {
+    const event = data as TaskEvent;
+    if (!event.workflowRunId || !event.workflowRunStepId) return;
+    try {
+      await resumeFromTaskCompletion(event, registry);
+    } catch (err) {
+      console.error("[workflows] Resume from task completion failed:", err);
+    }
+  });
+
+  eventBus.on("task.failed", async (data: unknown) => {
+    const event = data as TaskEvent;
+    if (!event.workflowRunId || !event.workflowRunStepId) return;
+    try {
+      await handleTaskFailure(event, event.failureReason ?? "Task failed", registry);
+    } catch (err) {
+      console.error("[workflows] Handle task failure error:", err);
+    }
+  });
+
+  eventBus.on("task.cancelled", async (data: unknown) => {
+    const event = data as TaskEvent;
+    if (!event.workflowRunId || !event.workflowRunStepId) return;
+    try {
+      await handleTaskFailure(event, "Task was cancelled", registry);
+    } catch (err) {
+      console.error("[workflows] Handle task cancellation error:", err);
+    }
+  });
+}
+
+/**
+ * Resume a workflow after a linked task completes.
+ *
+ * 1. Verify run and step are in "waiting" state
+ * 2. Checkpoint step completion with task output
+ * 3. Set run status to "running"
+ * 4. Find successors and continue the graph walk
+ */
+async function resumeFromTaskCompletion(
+  event: TaskEvent,
+  registry: ExecutorRegistry,
+): Promise<void> {
+  const run = getWorkflowRun(event.workflowRunId!);
+  if (!run || (run.status !== "waiting" && run.status !== "running")) return;
+
+  const step = getWorkflowRunStep(event.workflowRunStepId!);
+  if (!step || step.status !== "waiting") return;
+
+  const workflow = getWorkflow(run.workflowId);
+  if (!workflow) return;
+
+  // Checkpoint: atomic step completion + context update
+  const ctx = (run.context ?? {}) as Record<string, unknown>;
+
+  // JSON-parse structured output so downstream nodes can access nested fields
+  let taskOutput: unknown = event.output;
+  if (event.output) {
+    try {
+      const parsed = JSON.parse(event.output);
+      if (typeof parsed === "object" && parsed !== null) {
+        taskOutput = parsed;
+      }
+    } catch {
+      // Not JSON — keep as string (non-structured output tasks)
+    }
+  }
+  const stepOutput = { taskId: event.taskId, taskOutput };
+
+  checkpointStep(run.id, step.id, step.nodeId, { output: stepOutput }, ctx);
+
+  // Set run back to running
+  updateWorkflowRun(run.id, { status: "running" });
+
+  // Use convergence-aware node detection instead of blindly passing successors.
+  // This prevents duplicate step creation for convergence nodes (e.g., fan-out → merge).
+  // findReadyNodes checks ALL predecessors are completed before marking a node ready.
+  const completedNodeIds = new Set(getCompletedStepNodeIds(run.id));
+  const readyNodes = findReadyNodes(workflow.definition, completedNodeIds);
+
+  if (readyNodes.length > 0) {
+    await walkGraph(workflow.definition, run.id, ctx, readyNodes, registry, workflow.id);
+  } else {
+    finalizeOrWait(run.id);
+  }
+}
+
+/**
+ * If no nodes are ready and no steps are still waiting, finalize the run.
+ * Otherwise set it back to waiting for the next task completion.
+ */
+function finalizeOrWait(runId: string): void {
+  const steps = getWorkflowRunStepsByRunId(runId);
+  const hasWaiting = steps.some((s) => s.status === "waiting");
+  if (hasWaiting) {
+    updateWorkflowRun(runId, { status: "waiting" });
+  } else {
+    // All steps done (completed or failed) — finalize the run
+    updateWorkflowRun(runId, {
+      status: "completed",
+      finishedAt: new Date().toISOString(),
+    });
+  }
+}
+
+/**
+ * Handle task failure/cancellation — respects workflow's onNodeFailure config.
+ * 'fail' (default): mark the entire run as failed.
+ * 'continue': treat as completed with error output, let convergence proceed.
+ */
+async function handleTaskFailure(
+  event: TaskEvent,
+  reason: string,
+  registry: ExecutorRegistry,
+): Promise<void> {
+  const run = getWorkflowRun(event.workflowRunId!);
+  if (!run) return;
+
+  const workflow = getWorkflow(run.workflowId);
+  if (!workflow) return;
+
+  const onFailure = workflow.definition.onNodeFailure ?? "fail";
+
+  if (onFailure === "fail") {
+    markRunFailed(event, reason);
+    return;
+  }
+
+  // "continue": treat as completed with error output
+  const step = getWorkflowRunStep(event.workflowRunStepId!);
+  if (!step) return;
+
+  const ctx = (run.context ?? {}) as Record<string, unknown>;
+  const stepOutput = {
+    taskId: event.taskId,
+    taskOutput: `[FAILED: ${reason}] This node failed or was cancelled.`,
+  };
+  checkpointStep(run.id, step.id, step.nodeId, { output: stepOutput }, ctx);
+
+  updateWorkflowRun(run.id, { status: "running" });
+  const completedNodeIds = new Set(getCompletedStepNodeIds(run.id));
+  const readyNodes = findReadyNodes(workflow.definition, completedNodeIds);
+
+  if (readyNodes.length > 0) {
+    await walkGraph(workflow.definition, run.id, ctx, readyNodes, registry, workflow.id);
+  } else {
+    finalizeOrWait(run.id);
+  }
+}
+
+/**
+ * Mark a workflow run as failed when its linked task fails or is cancelled.
+ */
+function markRunFailed(event: TaskEvent, reason: string): void {
+  const now = new Date().toISOString();
+  updateWorkflowRunStep(event.workflowRunStepId!, {
+    status: "failed",
+    error: reason,
+    finishedAt: now,
+  });
+  updateWorkflowRun(event.workflowRunId!, {
+    status: "failed",
+    error: reason,
+    finishedAt: now,
+  });
+}
+
+/**
+ * Retry a failed workflow run from its failed step.
+ */
+export async function retryFailedRun(runId: string, registry: ExecutorRegistry): Promise<void> {
+  const run = getWorkflowRun(runId);
+  if (!run || run.status !== "failed") throw new Error("Run is not in failed state");
+
+  const workflow = getWorkflow(run.workflowId);
+  if (!workflow) throw new Error("Workflow not found");
+
+  // Find the failed step
+  const steps = getWorkflowRunStepsByRunId(runId);
+  const failedStep = steps.find((s) => s.status === "failed");
+  if (!failedStep) throw new Error("No failed step found");
+
+  // Reset step and run
+  updateWorkflowRunStep(failedStep.id, { status: "pending", error: undefined });
+  const ctx = (run.context ?? {}) as Record<string, unknown>;
+  updateWorkflowRun(runId, { status: "running", error: undefined, context: ctx });
+
+  // Resume from the failed node — use findReadyNodes for convergence safety
+  const completedNodeIds = new Set(getCompletedStepNodeIds(runId));
+  const readyNodes = findReadyNodes(workflow.definition, completedNodeIds);
+  const failedNode = workflow.definition.nodes.find((n) => n.id === failedStep.nodeId);
+  if (!failedNode) throw new Error(`Node ${failedStep.nodeId} not found in workflow definition`);
+
+  // Include the failed node if it's not already in ready nodes
+  const nodesToRun = readyNodes.some((n) => n.id === failedNode.id)
+    ? readyNodes
+    : [failedNode, ...readyNodes];
+  await walkGraph(workflow.definition, runId, ctx, nodesToRun, registry, workflow.id);
+}

package/src/workflows/retry-poller.ts

@@ -0,0 +1,216 @@
+import {
+  getRetryableSteps,
+  getWorkflow,
+  getWorkflowRun,
+  updateWorkflowRun,
+  updateWorkflowRunStep,
+} from "../be/db";
+import type { RetryPolicy } from "../types";
+import { checkpointStep, checkpointStepFailure } from "./checkpoint";
+import { getSuccessors } from "./definition";
+import { walkGraph } from "./engine";
+import type { ExecutorRegistry } from "./executors/registry";
+import { deepInterpolate } from "./template";
+import { runStepValidation } from "./validation";
+
+let pollerTimeout: ReturnType<typeof setTimeout> | null = null;
+
+/**
+ * Start the retry poller.
+ *
+ * Uses setTimeout chaining (not setInterval) to prevent overlap —
+ * the next tick is scheduled only after the current one completes.
+ */
+export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000): void {
+  if (pollerTimeout !== null) return; // Already running
+
+  async function poll(): Promise<void> {
+    try {
+      const retryableSteps = getRetryableSteps();
+
+      for (const step of retryableSteps) {
+        try {
+          const run = getWorkflowRun(step.runId);
+          if (!run) continue;
+
+          const workflow = getWorkflow(run.workflowId);
+          if (!workflow) continue;
+
+          // Find the node definition for this step
+          const node = workflow.definition.nodes.find((n) => n.id === step.nodeId);
+          if (!node) continue;
+
+          console.log(
+            `[workflows] Retrying step ${step.nodeId} (attempt ${step.retryCount}) for run ${step.runId}`,
+          );
+
+          // If the run was failed (due to this step), set it back to running
+          if (run.status === "failed") {
+            updateWorkflowRun(run.id, {
+              status: "running",
+              error: undefined,
+            });
+          }
+
+          // Clear the retry marker so this step isn't picked up again
+          updateWorkflowRunStep(step.id, {
+            status: "running",
+            error: undefined,
+            nextRetryAt: undefined,
+          });
+
+          const ctx = (run.context ?? {}) as Record<string, unknown>;
+
+          // Deep-interpolate config
+          const { value: interpolatedValue } = deepInterpolate(node.config, ctx);
+          const interpolatedConfig = interpolatedValue as Record<string, unknown>;
+
+          // Get executor and re-run
+          const executor = registry.get(node.type);
+          const meta = {
+            runId: run.id,
+            stepId: step.id,
+            nodeId: step.nodeId,
+            workflowId: workflow.id,
+            dryRun: false,
+          };
+
+          try {
+            const result = await executor.run({
+              config: interpolatedConfig,
+              context: ctx,
+              meta,
+            });
+
+            if (result.status === "failed") {
+              // Re-failed — use the EXISTING retryCount from the step.
+              // checkpointStepFailure handles marking run as failed if no retries left,
+              // or setting nextRetryAt for the next poll cycle.
+              const retryPolicy = node.retry || executor.retryPolicy;
+              checkpointStepFailure(
+                run.id,
+                step.id,
+                result.error || "Retry failed",
+                step.retryCount,
+                retryPolicy,
+              );
+            } else {
+              // Success! Re-run validation if configured before checkpointing.
+              if (node.validation) {
+                const validationResult = await runStepValidation(
+                  registry,
+                  node,
+                  result.output,
+                  ctx,
+                  meta,
+                );
+
+                if (validationResult.outcome === "halt") {
+                  checkpointStepFailure(
+                    run.id,
+                    step.id,
+                    "Validation failed (mustPass)",
+                    step.retryCount,
+                  );
+                  continue;
+                }
+
+                if (validationResult.outcome === "retry") {
+                  // Append validation context to history
+                  if (validationResult.retryContext) {
+                    const historyKey = `${node.id}_validations`;
+                    const existing = (ctx[historyKey] as unknown[]) || [];
+                    ctx[historyKey] = [...existing, validationResult.retryContext];
+                  }
+                  const retryPolicy = node.validation.retry || node.retry;
+                  checkpointStepFailure(
+                    run.id,
+                    step.id,
+                    "Validation failed, retrying",
+                    step.retryCount,
+                    retryPolicy,
+                  );
+                  continue;
+                }
+              }
+
+              // Validation passed (or no validation) — checkpoint and continue
+              checkpointStep(run.id, step.id, step.nodeId, result, ctx);
+
+              const port = result.nextPort || "default";
+              const successors = getSuccessors(workflow.definition, step.nodeId, port);
+              if (successors.length > 0) {
+                await walkGraph(
+                  workflow.definition,
+                  run.id,
+                  ctx,
+                  successors,
+                  registry,
+                  workflow.id,
+                );
+              } else {
+                // No successors — check if run is complete
+                updateWorkflowRun(run.id, {
+                  status: "completed",
+                  context: ctx,
+                  finishedAt: new Date().toISOString(),
+                });
+              }
+            }
+          } catch (err) {
+            // Execution threw — treat as failure
+            const errorMsg = err instanceof Error ? err.message : String(err);
+            const retryPolicy = node.retry || executor.retryPolicy;
+            checkpointStepFailure(run.id, step.id, errorMsg, step.retryCount, retryPolicy);
+          }
+        } catch (err) {
+          console.error(`[workflows] Retry failed for step ${step.id}:`, err);
+        }
+      }
+    } catch (err) {
+      console.error("[workflows] Retry poller error:", err);
+    }
+
+    // Schedule next tick after completion
+    pollerTimeout = setTimeout(poll, intervalMs);
+  }
+
+  // Start the first tick
+  pollerTimeout = setTimeout(poll, intervalMs);
+}
+
+/**
+ * Stop the retry poller (for clean shutdown).
+ */
+export function stopRetryPoller(): void {
+  if (pollerTimeout !== null) {
+    clearTimeout(pollerTimeout);
+    pollerTimeout = null;
+  }
+}
+
+/**
+ * Calculate retry delay based on policy and attempt number.
+ */
+export function calculateDelay(policy: RetryPolicy, attempt: number): number {
+  let delay: number;
+
+  switch (policy.strategy) {
+    case "exponential": {
+      // Exponential with full jitter
+      const base = policy.baseDelayMs * 2 ** attempt;
+      delay = Math.random() * Math.min(base, policy.maxDelayMs);
+      break;
+    }
+    case "linear":
+      delay = policy.baseDelayMs * (attempt + 1);
+      break;
+    case "static":
+      delay = policy.baseDelayMs;
+      break;
+    default:
+      delay = policy.baseDelayMs;
+  }
+
+  return Math.min(delay, policy.maxDelayMs);
+}

package/src/workflows/template.ts

@@ -0,0 +1,74 @@
+/**
+ * Replace {{path.to.value}} tokens in a template string
+ * with values from the context object.
+ *
+ * Returns the interpolated result and a list of any unresolved token paths.
+ */
+
+export interface InterpolateResult {
+  result: string;
+  unresolved: string[];
+}
+
+export function interpolate(template: string, ctx: Record<string, unknown>): InterpolateResult {
+  const unresolved: string[] = [];
+  const result = template.replace(/\{\{([^}]+)\}\}/g, (_match, path: string) => {
+    const keys = path.trim().split(".");
+    let value: unknown = ctx;
+    for (const key of keys) {
+      if (value == null || typeof value !== "object") {
+        unresolved.push(path.trim());
+        return "";
+      }
+      value = (value as Record<string, unknown>)[key];
+    }
+    if (value == null) {
+      unresolved.push(path.trim());
+      return "";
+    }
+    return typeof value === "object" ? safeStringify(value) : String(value);
+  });
+  return { result, unresolved };
+}
+
+/**
+ * Circular-reference-safe JSON.stringify for interpolation.
+ * Returns "[Circular]" instead of throwing on circular structures.
+ */
+function safeStringify(value: unknown): string {
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return "[Circular]";
+  }
+}
+
+/**
+ * Deep-interpolate an arbitrary value tree (objects, arrays, strings).
+ * Non-string leaves are passed through unchanged.
+ */
+export function deepInterpolate(
+  value: unknown,
+  ctx: Record<string, unknown>,
+): { value: unknown; unresolved: string[] } {
+  const allUnresolved: string[] = [];
+
+  function walk(v: unknown): unknown {
+    if (typeof v === "string") {
+      const { result, unresolved } = interpolate(v, ctx);
+      allUnresolved.push(...unresolved);
+      return result;
+    }
+    if (Array.isArray(v)) return v.map(walk);
+    if (v != null && typeof v === "object") {
+      const out: Record<string, unknown> = {};
+      for (const [k, val] of Object.entries(v)) {
+        out[k] = walk(val);
+      }
+      return out;
+    }
+    return v;
+  }
+
+  return { value: walk(value), unresolved: allUnresolved };
+}