@desplega.ai/agent-swarm 2.5.4 → 2.7.0

package/src/be/db/workflow-runs.ts

@@ -1,12 +1,161 @@
-import { TERMINAL_TASK_STATUSES, type WorkflowRunStatus } from "../../types";
+import {
+  TERMINAL_TASK_STATUSES,
+  type WorkflowRun,
+  type WorkflowRunStatus,
+  type WorkflowRunStep,
+  type WorkflowRunStepSlim,
+  type WorkflowRunStepStatus,
+  type WorkflowRunSummary,
+} from "../../types";
 import { scrubSecrets } from "../../utils/secret-scrubber";
+import { normalizeDate, normalizeDateRequired } from "../date-utils";
 import { scrubJsonValue } from "../scrub-json";
+import { defineWorkflowPayload, type WorkflowPayloadView } from "../workflow-replay";
 import { getDbClient } from "./runtime";
 
 // Parent/child links between workflow runs (migration 203): a `sub-workflow`
 // step starts a child run that records the step in `parentStepId`, and the
 // step's `retryCount` at that time in `parentStepAttempt` (migration 240).
 
+export type WorkflowRunRow = {
+  id: string;
+  workflowId: string;
+  status: string;
+  triggerType: string;
+  triggerData: string | null;
+  /** Absent from list rows: see `WORKFLOW_RUN_SUMMARY_COLUMNS`. */
+  context?: string | null;
+  /** Sealed exact `context` (migration 201). Absent from list rows. */
+  context_replay?: string | null;
+  error: string | null;
+  created_by: string | null;
+  startedAt: string;
+  lastUpdatedAt: string;
+  finishedAt: string | null;
+  /** The `sub-workflow` step that started this run (migration 203). */
+  parentStepId?: string | null;
+  /** That step's attempt (`retryCount`) when it started this run (migration 240). */
+  parentStepAttempt?: number | null;
+};
+
+/**
+ * `view` picks the `context` copy: `replay` (default) is the exact sealed value
+ * the engine resumes from; `display` is the scrubbed column for API, MCP and UI
+ * responses. See `src/be/workflow-replay.ts`.
+ */
+export function rowToWorkflowRun(
+  row: WorkflowRunRow,
+  view: WorkflowPayloadView = "replay",
+): WorkflowRun {
+  const run: WorkflowRun = {
+    id: row.id,
+    workflowId: row.workflowId,
+    status: row.status as WorkflowRunStatus,
+    triggerData: row.triggerData ? JSON.parse(row.triggerData) : undefined,
+    context: undefined,
+    error: row.error ?? undefined,
+    createdBy: row.created_by ?? undefined,
+    startedAt: normalizeDateRequired(row.startedAt),
+    lastUpdatedAt: normalizeDateRequired(row.lastUpdatedAt),
+    finishedAt: normalizeDate(row.finishedAt) ?? undefined,
+    parentStepId: row.parentStepId ?? undefined,
+    parentStepAttempt: row.parentStepId ? (row.parentStepAttempt ?? 0) : undefined,
+  };
+  return defineWorkflowPayload(run, "context", row.context, row.context_replay, view, {
+    runId: row.id,
+    target: "its context",
+  });
+}
+
+// Everything but `context`. `triggerData` stays: it is small, and stored
+// scripts read it off list rows (dedupe by trigger, failure reports). The
+// parent link and attempt stay so a list row identifies a child's invocation.
+export const WORKFLOW_RUN_SUMMARY_COLUMNS =
+  "id, workflowId, status, triggerType, triggerData, error, created_by, startedAt, lastUpdatedAt, finishedAt, parentStepId, parentStepAttempt";
+
+export type WorkflowRunStepRow = {
+  id: string;
+  runId: string;
+  nodeId: string;
+  nodeType: string;
+  status: string;
+  /** Absent from slim rows: see `WORKFLOW_RUN_STEP_SLIM_COLUMNS`. */
+  input?: string | null;
+  output: string | null;
+  /** Sealed exact `output` (migration 201). Absent from slim rows. */
+  output_replay?: string | null;
+  error: string | null;
+  startedAt: string;
+  finishedAt: string | null;
+  retryCount: number;
+  maxRetries: number;
+  nextRetryAt: string | null;
+  idempotencyKey: string | null;
+  diagnostics: string | null;
+  nextPort: string | null;
+};
+
+/** `view` picks the `output` copy, as in `rowToWorkflowRun`. */
+export function rowToWorkflowRunStep(
+  row: WorkflowRunStepRow,
+  view: WorkflowPayloadView = "replay",
+): WorkflowRunStep {
+  const step: WorkflowRunStep = {
+    id: row.id,
+    runId: row.runId,
+    nodeId: row.nodeId,
+    nodeType: row.nodeType,
+    status: row.status as WorkflowRunStepStatus,
+    input: row.input ? JSON.parse(row.input) : undefined,
+    output: undefined,
+    error: row.error ?? undefined,
+    startedAt: normalizeDateRequired(row.startedAt),
+    finishedAt: normalizeDate(row.finishedAt) ?? undefined,
+    retryCount: row.retryCount,
+    maxRetries: row.maxRetries,
+    nextRetryAt: normalizeDate(row.nextRetryAt) ?? undefined,
+    idempotencyKey: row.idempotencyKey ?? undefined,
+    diagnostics: row.diagnostics ?? undefined,
+    nextPort: row.nextPort ?? undefined,
+  };
+  return defineWorkflowPayload(step, "output", row.output, row.output_replay, view, {
+    runId: row.runId,
+    target: `the output of step ${row.id} (node ${row.nodeId})`,
+  });
+}
+
+// Everything but `input` (a context snapshot per step, so loops grow it with
+// every iteration) and `output_replay` (the display view never opens it).
+const WORKFLOW_RUN_STEP_SLIM_COLUMNS =
+  "id, runId, nodeId, nodeType, status, output, error, startedAt, finishedAt, retryCount, maxRetries, nextRetryAt, idempotencyKey, diagnostics, nextPort";
+
+/**
+ * `getWorkflowRun` without `context`, in the display view. Backs
+ * `GET /api/workflow-runs/{id}?fields=slim`: the large column is neither read
+ * nor parsed.
+ */
+export async function getWorkflowRunSummary(id: string): Promise<WorkflowRunSummary | null> {
+  const row = await getDbClient().get<WorkflowRunRow>(
+    `SELECT ${WORKFLOW_RUN_SUMMARY_COLUMNS} FROM workflow_runs WHERE id = ?`,
+    [id],
+  );
+  return row ? rowToWorkflowRun(row, "display") : null;
+}
+
+/**
+ * `getWorkflowRunStepsByRunId` without `input`, in the display view. Same
+ * order, including the rowid tie-break for steps that share `startedAt`.
+ */
+export async function getWorkflowRunStepsSlimByRunId(
+  runId: string,
+): Promise<WorkflowRunStepSlim[]> {
+  const rows = await getDbClient().query<WorkflowRunStepRow>(
+    `SELECT ${WORKFLOW_RUN_STEP_SLIM_COLUMNS} FROM workflow_run_steps WHERE runId = ? ORDER BY startedAt ASC, rowid ASC`,
+    [runId],
+  );
+  return rows.map((row) => rowToWorkflowRunStep(row, "display"));
+}
+
 const TERMINAL_STATUSES = new Set<WorkflowRunStatus>([
   "completed",
   "failed",
@@ -242,6 +391,39 @@ export async function workflowRunTreeHasOpenWork(runIds: string[]): Promise<bool
   return open;
 }
 
+export type WorkflowRunTree = {
+  rootRunId: string;
+  /** Root first, in `getWorkflowRunTreeIds` order. List rows: no `context`. */
+  runs: WorkflowRun[];
+  hasOpenWork: boolean;
+};
+
+/**
+ * The run and every run below it, with whether any of them still has open
+ * work (see `workflowRunTreeHasOpenWork`). One read transaction, so `runs` and
+ * `hasOpenWork` come from the same snapshot. Null when the root is missing.
+ * Throws WorkflowRunTreeError on a cycle or a chain past the depth bound.
+ */
+export async function getWorkflowRunTree(rootRunId: string): Promise<WorkflowRunTree | null> {
+  return getDbClient().transaction(
+    async () => {
+      if ((await getWorkflowRunStatus(rootRunId)) === null) return null;
+      const ids = await getWorkflowRunTreeIds(rootRunId);
+      const byId = new Map<string, WorkflowRun>();
+      await forEachChunk(ids, async (chunk, placeholders) => {
+        const rows = await getDbClient().query<WorkflowRunRow>(
+          `SELECT ${WORKFLOW_RUN_SUMMARY_COLUMNS} FROM workflow_runs WHERE id IN (${placeholders})`,
+          chunk,
+        );
+        for (const row of rows) byId.set(row.id, rowToWorkflowRun(row, "display"));
+      });
+      const runs = ids.flatMap((id) => byId.get(id) ?? []);
+      return { rootRunId, runs, hasOpenWork: await workflowRunTreeHasOpenWork(ids) };
+    },
+    { readOnly: true },
+  );
+}
+
 /**
  * Cancel the pending, running and waiting steps of the given runs. The
  * migration-140 trigger closes each cancelled step's pending approval.
@@ -283,7 +465,8 @@ export async function disarmWorkflowTreeRetries(
 /**
  * Mark the given runs that are still running or waiting `cancelled`, without
  * the per-run `workflow.child.finished` wake-up: it is for runs whose parent
- * step the same cancel closes. A terminal run keeps its status and reason.
+ * step the same cancel closes. Each run it cancels still emits
+ * `workflow.run.finished`. A terminal run keeps its status and reason.
  */
 export async function cancelLiveWorkflowTreeRuns(
   runIds: string[],
@@ -291,12 +474,14 @@ export async function cancelLiveWorkflowTreeRuns(
   finishedAt: string,
 ): Promise<void> {
   await forEachChunk(runIds, async (chunk, placeholders) => {
-    await getDbClient().run(
+    const cancelled = await getDbClient().query<{ id: string; workflowId: string }>(
       `UPDATE workflow_runs
           SET status = 'cancelled', error = ?, finishedAt = ?, lastUpdatedAt = ?
-        WHERE id IN (${placeholders}) AND status IN ('running', 'waiting')`,
+        WHERE id IN (${placeholders}) AND status IN ('running', 'waiting')
+        RETURNING id, workflowId`,
       [scrubSecrets(reason), finishedAt, finishedAt, ...chunk],
     );
+    for (const run of cancelled) emitRunFinished(run.id, run.workflowId, "cancelled");
   });
 }
 
@@ -461,14 +646,42 @@ export function emitChildRunFinished(
 }
 
 function emitAfterCommit(childRunId: string, parentStepId: string): void {
+  emitBusEventAfterCommit("workflow.child.finished", { childRunId, parentStepId });
+}
+
+/** Bus event every run emits once when it turns terminal. */
+export const WORKFLOW_RUN_FINISHED_EVENT = "workflow.run.finished";
+
+export function isTerminalWorkflowRunStatus(status: WorkflowRunStatus): boolean {
+  return TERMINAL_STATUSES.has(status);
+}
+
+/**
+ * Announce after commit that a run turned terminal, top-level and child runs
+ * alike, so an event-mode `wait` can resume when any run ends. Callers emit
+ * only when a live run (running or waiting) turns terminal, never on a later
+ * update of a run that is already terminal. The id is `runId`, never `workflowRunId` or
+ * `_runId`: those would put the finished run in the waiting run's own `run`
+ * scope, so waiters use scope `global` and filter on `runId`.
+ */
+export function emitRunFinished(
+  runId: string,
+  workflowId: string,
+  status: WorkflowRunStatus,
+): void {
+  if (!TERMINAL_STATUSES.has(status)) return;
+  emitBusEventAfterCommit(WORKFLOW_RUN_FINISHED_EVENT, { runId, workflowId, status });
+}
+
+function emitBusEventAfterCommit(event: string, payload: Record<string, unknown>): void {
   getDbClient().afterCommit(() => {
     import("../../workflows/event-bus")
       .then(({ workflowEventBus }) => {
-        workflowEventBus.emit("workflow.child.finished", { childRunId, parentStepId });
+        workflowEventBus.emit(event, payload);
       })
       .catch((err) =>
         console.error(
-          "[db] workflow.child.finished event not emitted:",
+          `[db] ${event} event not emitted:`,
           scrubSecrets(err instanceof Error ? err.message : String(err)),
         ),
       );

package/src/be/db-retention.ts

@@ -1,6 +1,7 @@
 import { AsyncLocalStorage } from "node:async_hooks";
 import type { Attributes, DbRetentionSweepMetric, SpanStatus, SwarmSpan } from "../otel";
 import { recordDbRetentionStatement, recordDbRetentionSweep, startSpan, withSpan } from "../otel";
+import { TERMINAL_SCRIPT_RUN_STATUSES } from "../types";
 import { isEnvFlagEnabled } from "../utils/env-flag";
 import { scrubSecrets } from "../utils/secret-scrubber";
 import { getDbClient } from "./db";
@@ -23,7 +24,11 @@ const YIELD_MS = 25;
  *
  * Two policies exist:
  *  - `age`: the env value is a number of days; rows whose `timeColumn` is older
- *    than that horizon are deleted.
+ *    than that horizon are deleted. Optional narrowing: `terminalStatuses`
+ *    deletes only rows whose `statusColumn` is in that list, `keepWhenSet`
+ *    keeps every row where that column is non-NULL, and
+ *    `cutoffFormat: "sqliteDatetime"` compares against a `YYYY-MM-DD HH:MM:SS`
+ *    cutoff for a column that mixes that format with ISO strings.
  *  - `keepLatest`: the env value is a count; within each `partitionBy` group,
  *    only the newest `rankBy` rows are kept. The newest row of a group is
  *    never deleted, because the smallest accepted count is 1.
@@ -65,6 +70,29 @@ export const DB_RETENTION_TABLES = [
     initialBatchSize: 50,
     maxBatchSize: 100,
   },
+  {
+    // Every inline script run stores its full source, args and output. Only
+    // terminal runs are eligible: a running or paused durable run must survive
+    // to be resumed. Runs with an idempotencyKey are kept, so a retried launch
+    // still gets its 409 instead of executing twice. The status filter, the
+    // time range and the key exclusion all resolve on indexes
+    // (idx_script_runs_status_startedAt, idx_script_runs_idempotencyKey).
+    // Workflow runs write `datetime('now')` (space separator) while inline runs
+    // write ISO strings, hence the sqliteDatetime cutoff. script_run_journal
+    // rows go with their run through ON DELETE CASCADE on idx_srj_runId.
+    // Rows average ~30 KB and reach ~1.3 MB, so batches stay small.
+    table: "script_runs",
+    policy: "age",
+    timeColumn: "startedAt",
+    cutoffFormat: "sqliteDatetime",
+    statusColumn: "status",
+    terminalStatuses: TERMINAL_SCRIPT_RUN_STATUSES,
+    keepWhenSet: "idempotencyKey",
+    envKey: "SCRIPT_RUNS_RETENTION_DAYS",
+    metricsKey: "scriptRuns",
+    initialBatchSize: 100,
+    maxBatchSize: 500,
+  },
 ] as const satisfies ReadonlyArray<
   {
     table: string;
@@ -73,7 +101,14 @@ export const DB_RETENTION_TABLES = [
     initialBatchSize?: number;
     maxBatchSize?: number;
   } & (
-    | { policy: "age"; timeColumn: string }
+    | {
+        policy: "age";
+        timeColumn: string;
+        cutoffFormat?: "sqliteDatetime";
+        statusColumn?: string;
+        terminalStatuses?: readonly string[];
+        keepWhenSet?: string;
+      }
     | { policy: "keepLatest"; partitionBy: string; rankBy: string }
   )
 >;
@@ -326,13 +361,33 @@ export function sweepStatements(table: RetentionTable, value: number, now: Date)
       params: [value],
     };
   }
-  const cutoff = new Date(now.getTime() - value * DAY_MS).toISOString();
+  const iso = new Date(now.getTime() - value * DAY_MS).toISOString();
+  // Text comparison of the two formats: ' ' sorts before 'T', so against a
+  // space-separated cutoff an ISO row on the cutoff's own date is kept, never
+  // deleted early. It is swept on the next day's ticks instead.
+  const cutoff = "cutoffFormat" in table ? `${iso.slice(0, 10)} ${iso.slice(11, 19)}` : iso;
+  const conditions: string[] = [];
+  const params: Array<string | number> = [];
+  if ("terminalStatuses" in table) {
+    conditions.push(
+      `${table.statusColumn} IN (${table.terminalStatuses.map(() => "?").join(", ")})`,
+    );
+    params.push(...table.terminalStatuses);
+  }
+  conditions.push(`${table.timeColumn} < ?`);
+  params.push(cutoff);
+  if ("keepWhenSet" in table) {
+    conditions.push(
+      `rowid NOT IN (SELECT rowid FROM ${table.table} WHERE ${table.keepWhenSet} IS NOT NULL)`,
+    );
+  }
+  const where = conditions.join(" AND ");
   return {
-    backlogSql: `SELECT COUNT(*) AS n FROM ${table.table} WHERE ${table.timeColumn} < ?`,
+    backlogSql: `SELECT COUNT(*) AS n FROM ${table.table} WHERE ${where}`,
     deleteSql: `DELETE FROM ${table.table} WHERE rowid IN (
-         SELECT rowid FROM ${table.table} WHERE ${table.timeColumn} < ? ORDER BY ${table.timeColumn} LIMIT ?
+         SELECT rowid FROM ${table.table} WHERE ${where} ORDER BY ${table.timeColumn} LIMIT ?
        )`,
-    params: [cutoff],
+    params,
   };
 }