@desplega.ai/agent-swarm 1.20.0 → 1.49.0

package/src/workflows/resume.ts

@@ -0,0 +1,143 @@
+import {
+  getWorkflow,
+  getWorkflowRun,
+  getWorkflowRunStep,
+  getWorkflowRunStepsByRunId,
+  updateWorkflowRun,
+  updateWorkflowRunStep,
+} from "../be/db";
+import { checkpointStep } from "./checkpoint";
+import { getSuccessors } from "./definition";
+import { walkGraph } from "./engine";
+import type { WorkflowEventBus } from "./event-bus";
+import type { ExecutorRegistry } from "./executors/registry";
+
+interface TaskEvent {
+  taskId: string;
+  output?: string;
+  agentId?: string;
+  workflowRunId?: string;
+  workflowRunStepId?: string;
+  failureReason?: string;
+}
+
+/**
+ * Wire up event bus listeners for workflow resume on task lifecycle events.
+ */
+export function setupWorkflowResumeListener(
+  eventBus: WorkflowEventBus,
+  registry: ExecutorRegistry,
+): void {
+  eventBus.on("task.completed", async (data: unknown) => {
+    const event = data as TaskEvent;
+    if (!event.workflowRunId || !event.workflowRunStepId) return;
+    try {
+      await resumeFromTaskCompletion(event, registry);
+    } catch (err) {
+      console.error("[workflows] Resume from task completion failed:", err);
+    }
+  });
+
+  eventBus.on("task.failed", async (data: unknown) => {
+    const event = data as TaskEvent;
+    if (!event.workflowRunId || !event.workflowRunStepId) return;
+    markRunFailed(event, event.failureReason ?? "Task failed");
+  });
+
+  eventBus.on("task.cancelled", async (data: unknown) => {
+    const event = data as TaskEvent;
+    if (!event.workflowRunId || !event.workflowRunStepId) return;
+    markRunFailed(event, "Task was cancelled");
+  });
+}
+
+/**
+ * Resume a workflow after a linked task completes.
+ *
+ * 1. Verify run and step are in "waiting" state
+ * 2. Checkpoint step completion with task output
+ * 3. Set run status to "running"
+ * 4. Find successors and continue the graph walk
+ */
+async function resumeFromTaskCompletion(
+  event: TaskEvent,
+  registry: ExecutorRegistry,
+): Promise<void> {
+  const run = getWorkflowRun(event.workflowRunId!);
+  if (!run || run.status !== "waiting") return;
+
+  const step = getWorkflowRunStep(event.workflowRunStepId!);
+  if (!step || step.status !== "waiting") return;
+
+  const workflow = getWorkflow(run.workflowId);
+  if (!workflow) return;
+
+  // Checkpoint: atomic step completion + context update
+  const ctx = (run.context ?? {}) as Record<string, unknown>;
+
+  // JSON-parse structured output so downstream nodes can access nested fields
+  let taskOutput: unknown = event.output;
+  if (event.output) {
+    try {
+      const parsed = JSON.parse(event.output);
+      if (typeof parsed === "object" && parsed !== null) {
+        taskOutput = parsed;
+      }
+    } catch {
+      // Not JSON — keep as string (non-structured output tasks)
+    }
+  }
+  const stepOutput = { taskId: event.taskId, taskOutput };
+
+  checkpointStep(run.id, step.id, step.nodeId, { output: stepOutput }, ctx);
+
+  // Set run back to running
+  updateWorkflowRun(run.id, { status: "running" });
+
+  // Find successors and continue DAG walk
+  const successors = getSuccessors(workflow.definition, step.nodeId, "default");
+  await walkGraph(workflow.definition, run.id, ctx, successors, registry, workflow.id);
+}
+
+/**
+ * Mark a workflow run as failed when its linked task fails or is cancelled.
+ */
+function markRunFailed(event: TaskEvent, reason: string): void {
+  const now = new Date().toISOString();
+  updateWorkflowRunStep(event.workflowRunStepId!, {
+    status: "failed",
+    error: reason,
+    finishedAt: now,
+  });
+  updateWorkflowRun(event.workflowRunId!, {
+    status: "failed",
+    error: reason,
+    finishedAt: now,
+  });
+}
+
+/**
+ * Retry a failed workflow run from its failed step.
+ */
+export async function retryFailedRun(runId: string, registry: ExecutorRegistry): Promise<void> {
+  const run = getWorkflowRun(runId);
+  if (!run || run.status !== "failed") throw new Error("Run is not in failed state");
+
+  const workflow = getWorkflow(run.workflowId);
+  if (!workflow) throw new Error("Workflow not found");
+
+  // Find the failed step
+  const steps = getWorkflowRunStepsByRunId(runId);
+  const failedStep = steps.find((s) => s.status === "failed");
+  if (!failedStep) throw new Error("No failed step found");
+
+  // Reset step and run
+  updateWorkflowRunStep(failedStep.id, { status: "pending", error: undefined });
+  const ctx = (run.context ?? {}) as Record<string, unknown>;
+  updateWorkflowRun(runId, { status: "running", error: undefined, context: ctx });
+
+  // Resume from the failed node
+  const node = workflow.definition.nodes.find((n) => n.id === failedStep.nodeId);
+  if (!node) throw new Error(`Node ${failedStep.nodeId} not found in workflow definition`);
+  await walkGraph(workflow.definition, runId, ctx, [node], registry, workflow.id);
+}

package/src/workflows/retry-poller.ts

@@ -0,0 +1,216 @@
+import {
+  getRetryableSteps,
+  getWorkflow,
+  getWorkflowRun,
+  updateWorkflowRun,
+  updateWorkflowRunStep,
+} from "../be/db";
+import type { RetryPolicy } from "../types";
+import { checkpointStep, checkpointStepFailure } from "./checkpoint";
+import { getSuccessors } from "./definition";
+import { walkGraph } from "./engine";
+import type { ExecutorRegistry } from "./executors/registry";
+import { deepInterpolate } from "./template";
+import { runStepValidation } from "./validation";
+
+let pollerTimeout: ReturnType<typeof setTimeout> | null = null;
+
+/**
+ * Start the retry poller.
+ *
+ * Uses setTimeout chaining (not setInterval) to prevent overlap —
+ * the next tick is scheduled only after the current one completes.
+ */
+export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000): void {
+  if (pollerTimeout !== null) return; // Already running
+
+  async function poll(): Promise<void> {
+    try {
+      const retryableSteps = getRetryableSteps();
+
+      for (const step of retryableSteps) {
+        try {
+          const run = getWorkflowRun(step.runId);
+          if (!run) continue;
+
+          const workflow = getWorkflow(run.workflowId);
+          if (!workflow) continue;
+
+          // Find the node definition for this step
+          const node = workflow.definition.nodes.find((n) => n.id === step.nodeId);
+          if (!node) continue;
+
+          console.log(
+            `[workflows] Retrying step ${step.nodeId} (attempt ${step.retryCount}) for run ${step.runId}`,
+          );
+
+          // If the run was failed (due to this step), set it back to running
+          if (run.status === "failed") {
+            updateWorkflowRun(run.id, {
+              status: "running",
+              error: undefined,
+            });
+          }
+
+          // Clear the retry marker so this step isn't picked up again
+          updateWorkflowRunStep(step.id, {
+            status: "running",
+            error: undefined,
+            nextRetryAt: undefined,
+          });
+
+          const ctx = (run.context ?? {}) as Record<string, unknown>;
+
+          // Deep-interpolate config
+          const { value: interpolatedValue } = deepInterpolate(node.config, ctx);
+          const interpolatedConfig = interpolatedValue as Record<string, unknown>;
+
+          // Get executor and re-run
+          const executor = registry.get(node.type);
+          const meta = {
+            runId: run.id,
+            stepId: step.id,
+            nodeId: step.nodeId,
+            workflowId: workflow.id,
+            dryRun: false,
+          };
+
+          try {
+            const result = await executor.run({
+              config: interpolatedConfig,
+              context: ctx,
+              meta,
+            });
+
+            if (result.status === "failed") {
+              // Re-failed — use the EXISTING retryCount from the step.
+              // checkpointStepFailure handles marking run as failed if no retries left,
+              // or setting nextRetryAt for the next poll cycle.
+              const retryPolicy = node.retry || executor.retryPolicy;
+              checkpointStepFailure(
+                run.id,
+                step.id,
+                result.error || "Retry failed",
+                step.retryCount,
+                retryPolicy,
+              );
+            } else {
+              // Success! Re-run validation if configured before checkpointing.
+              if (node.validation) {
+                const validationResult = await runStepValidation(
+                  registry,
+                  node,
+                  result.output,
+                  ctx,
+                  meta,
+                );
+
+                if (validationResult.outcome === "halt") {
+                  checkpointStepFailure(
+                    run.id,
+                    step.id,
+                    "Validation failed (mustPass)",
+                    step.retryCount,
+                  );
+                  continue;
+                }
+
+                if (validationResult.outcome === "retry") {
+                  // Append validation context to history
+                  if (validationResult.retryContext) {
+                    const historyKey = `${node.id}_validations`;
+                    const existing = (ctx[historyKey] as unknown[]) || [];
+                    ctx[historyKey] = [...existing, validationResult.retryContext];
+                  }
+                  const retryPolicy = node.validation.retry || node.retry;
+                  checkpointStepFailure(
+                    run.id,
+                    step.id,
+                    "Validation failed, retrying",
+                    step.retryCount,
+                    retryPolicy,
+                  );
+                  continue;
+                }
+              }
+
+              // Validation passed (or no validation) — checkpoint and continue
+              checkpointStep(run.id, step.id, step.nodeId, result, ctx);
+
+              const port = result.nextPort || "default";
+              const successors = getSuccessors(workflow.definition, step.nodeId, port);
+              if (successors.length > 0) {
+                await walkGraph(
+                  workflow.definition,
+                  run.id,
+                  ctx,
+                  successors,
+                  registry,
+                  workflow.id,
+                );
+              } else {
+                // No successors — check if run is complete
+                updateWorkflowRun(run.id, {
+                  status: "completed",
+                  context: ctx,
+                  finishedAt: new Date().toISOString(),
+                });
+              }
+            }
+          } catch (err) {
+            // Execution threw — treat as failure
+            const errorMsg = err instanceof Error ? err.message : String(err);
+            const retryPolicy = node.retry || executor.retryPolicy;
+            checkpointStepFailure(run.id, step.id, errorMsg, step.retryCount, retryPolicy);
+          }
+        } catch (err) {
+          console.error(`[workflows] Retry failed for step ${step.id}:`, err);
+        }
+      }
+    } catch (err) {
+      console.error("[workflows] Retry poller error:", err);
+    }
+
+    // Schedule next tick after completion
+    pollerTimeout = setTimeout(poll, intervalMs);
+  }
+
+  // Start the first tick
+  pollerTimeout = setTimeout(poll, intervalMs);
+}
+
+/**
+ * Stop the retry poller (for clean shutdown).
+ */
+export function stopRetryPoller(): void {
+  if (pollerTimeout !== null) {
+    clearTimeout(pollerTimeout);
+    pollerTimeout = null;
+  }
+}
+
+/**
+ * Calculate retry delay based on policy and attempt number.
+ */
+export function calculateDelay(policy: RetryPolicy, attempt: number): number {
+  let delay: number;
+
+  switch (policy.strategy) {
+    case "exponential": {
+      // Exponential with full jitter
+      const base = policy.baseDelayMs * 2 ** attempt;
+      delay = Math.random() * Math.min(base, policy.maxDelayMs);
+      break;
+    }
+    case "linear":
+      delay = policy.baseDelayMs * (attempt + 1);
+      break;
+    case "static":
+      delay = policy.baseDelayMs;
+      break;
+    default:
+      delay = policy.baseDelayMs;
+  }
+
+  return Math.min(delay, policy.maxDelayMs);
+}

package/src/workflows/template.ts

@@ -0,0 +1,74 @@
+/**
+ * Replace {{path.to.value}} tokens in a template string
+ * with values from the context object.
+ *
+ * Returns the interpolated result and a list of any unresolved token paths.
+ */
+
+export interface InterpolateResult {
+  result: string;
+  unresolved: string[];
+}
+
+export function interpolate(template: string, ctx: Record<string, unknown>): InterpolateResult {
+  const unresolved: string[] = [];
+  const result = template.replace(/\{\{([^}]+)\}\}/g, (_match, path: string) => {
+    const keys = path.trim().split(".");
+    let value: unknown = ctx;
+    for (const key of keys) {
+      if (value == null || typeof value !== "object") {
+        unresolved.push(path.trim());
+        return "";
+      }
+      value = (value as Record<string, unknown>)[key];
+    }
+    if (value == null) {
+      unresolved.push(path.trim());
+      return "";
+    }
+    return typeof value === "object" ? safeStringify(value) : String(value);
+  });
+  return { result, unresolved };
+}
+
+/**
+ * Circular-reference-safe JSON.stringify for interpolation.
+ * Returns "[Circular]" instead of throwing on circular structures.
+ */
+function safeStringify(value: unknown): string {
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return "[Circular]";
+  }
+}
+
+/**
+ * Deep-interpolate an arbitrary value tree (objects, arrays, strings).
+ * Non-string leaves are passed through unchanged.
+ */
+export function deepInterpolate(
+  value: unknown,
+  ctx: Record<string, unknown>,
+): { value: unknown; unresolved: string[] } {
+  const allUnresolved: string[] = [];
+
+  function walk(v: unknown): unknown {
+    if (typeof v === "string") {
+      const { result, unresolved } = interpolate(v, ctx);
+      allUnresolved.push(...unresolved);
+      return result;
+    }
+    if (Array.isArray(v)) return v.map(walk);
+    if (v != null && typeof v === "object") {
+      const out: Record<string, unknown> = {};
+      for (const [k, val] of Object.entries(v)) {
+        out[k] = walk(val);
+      }
+      return out;
+    }
+    return v;
+  }
+
+  return { value: walk(value), unresolved: allUnresolved };
+}

package/src/workflows/templates.ts

@@ -0,0 +1,86 @@
+import type { WorkflowDefinition, WorkflowTemplate } from "../types";
+
+/**
+ * Validate that all required template variables are provided.
+ * Returns a list of missing required variable names.
+ */
+export function validateTemplateVariables(
+  template: WorkflowTemplate,
+  provided: Record<string, unknown>,
+): string[] {
+  const missing: string[] = [];
+  for (const variable of template.variables) {
+    if (variable.required && !(variable.name in provided)) {
+      // Check if there's a default value
+      if (variable.default === undefined) {
+        missing.push(variable.name);
+      }
+    }
+  }
+  return missing;
+}
+
+/**
+ * Instantiate a workflow template with the given variables.
+ *
+ * 1. Validates all required variables are provided
+ * 2. Deep-clones the template definition
+ * 3. Replaces `{{variable}}` placeholders in all string fields
+ * 4. Returns a valid WorkflowDefinition
+ */
+export function instantiateTemplate(
+  template: WorkflowTemplate,
+  variables: Record<string, unknown>,
+): WorkflowDefinition {
+  // Validate required variables
+  const missing = validateTemplateVariables(template, variables);
+  if (missing.length > 0) {
+    throw new Error(`Missing required template variables: ${missing.join(", ")}`);
+  }
+
+  // Build resolved variables (apply defaults for missing optional vars)
+  const resolved: Record<string, unknown> = {};
+  for (const variable of template.variables) {
+    if (variable.name in variables) {
+      resolved[variable.name] = variables[variable.name];
+    } else if (variable.default !== undefined) {
+      resolved[variable.name] = variable.default;
+    }
+  }
+
+  // Deep-clone the definition and replace placeholders
+  const cloned = JSON.parse(JSON.stringify(template.definition)) as WorkflowDefinition;
+  return replaceVariables(cloned, resolved);
+}
+
+/**
+ * Deep-replace `{{variableName}}` placeholders in all string fields of an object.
+ */
+function replaceVariables<T>(obj: T, variables: Record<string, unknown>): T {
+  if (typeof obj === "string") {
+    return replaceStringPlaceholders(obj, variables) as T;
+  }
+  if (Array.isArray(obj)) {
+    return obj.map((item) => replaceVariables(item, variables)) as T;
+  }
+  if (obj !== null && typeof obj === "object") {
+    const result: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = replaceVariables(value, variables);
+    }
+    return result as T;
+  }
+  return obj;
+}
+
+/**
+ * Replace `{{variableName}}` patterns in a string with variable values.
+ */
+function replaceStringPlaceholders(template: string, variables: Record<string, unknown>): string {
+  return template.replace(/\{\{(\w+)\}\}/g, (_match, name: string) => {
+    const value = variables[name];
+    if (value === undefined) return _match; // Leave unresolved placeholders as-is
+    if (typeof value === "object" && value !== null) return JSON.stringify(value);
+    return String(value);
+  });
+}

package/src/workflows/triggers.ts

@@ -0,0 +1,95 @@
+import crypto from "node:crypto";
+import { getWorkflow } from "../be/db";
+import type { TriggerConfig } from "../types";
+import { startWorkflowExecution } from "./engine";
+import type { ExecutorRegistry } from "./executors/registry";
+
+/**
+ * Handle an incoming webhook trigger for a workflow.
+ *
+ * 1. Loads the workflow and finds a webhook trigger in `triggers[]`
+ * 2. If `hmacSecret` is set, verifies HMAC-SHA256 signature
+ * 3. Starts the workflow execution with the webhook payload
+ */
+export async function handleWebhookTrigger(
+  workflowId: string,
+  payload: unknown,
+  signature: string | undefined,
+  signatureHeader: string | undefined,
+  registry: ExecutorRegistry,
+): Promise<{ runId: string }> {
+  const workflow = getWorkflow(workflowId);
+  if (!workflow) {
+    throw new WebhookError("Workflow not found", 404);
+  }
+
+  if (!workflow.enabled) {
+    throw new WebhookError("Workflow is disabled", 400);
+  }
+
+  // Find webhook trigger in triggers[]
+  const webhookTrigger = workflow.triggers.find((t: TriggerConfig) => t.type === "webhook");
+
+  // If the workflow has a webhook trigger with an hmacSecret, verify the signature
+  if (webhookTrigger && webhookTrigger.type === "webhook" && webhookTrigger.hmacSecret) {
+    if (!signature && !signatureHeader) {
+      throw new WebhookError("Missing signature", 401);
+    }
+
+    const rawSignature = signatureHeader || signature || "";
+    const isValid = verifyHmacSignature(
+      webhookTrigger.hmacSecret,
+      typeof payload === "string" ? payload : JSON.stringify(payload),
+      rawSignature,
+    );
+
+    if (!isValid) {
+      throw new WebhookError("Invalid signature", 401);
+    }
+  }
+
+  const runId = await startWorkflowExecution(workflow, payload, registry);
+  return { runId };
+}
+
+/**
+ * Verify HMAC-SHA256 signature.
+ * Supports both `sha256=<hex>` format and raw hex.
+ */
+export function verifyHmacSignature(
+  secret: string,
+  body: string,
+  providedSignature: string,
+): boolean {
+  const hmac = crypto.createHmac("sha256", secret);
+  hmac.update(body);
+  const expectedHex = hmac.digest("hex");
+
+  // Support "sha256=<hex>" format (GitHub-style)
+  const normalizedProvided = providedSignature.startsWith("sha256=")
+    ? providedSignature.slice(7)
+    : providedSignature;
+
+  // Constant-time comparison
+  try {
+    return crypto.timingSafeEqual(
+      Buffer.from(normalizedProvided, "hex"),
+      Buffer.from(expectedHex, "hex"),
+    );
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Error class for webhook-specific errors with HTTP status codes.
+ */
+export class WebhookError extends Error {
+  constructor(
+    message: string,
+    public readonly statusCode: number,
+  ) {
+    super(message);
+    this.name = "WebhookError";
+  }
+}