@desplega.ai/agent-swarm 2.5.3 → 2.5.4

package/src/be/db/approvals.ts

@@ -162,3 +162,51 @@ export async function getPendingApprovalsForTasks(taskIds: string[]): Promise<
     taskIds,
   );
 }
+
+/** The claim key of one Slack thread's cancellation post: `<channel>:<messageTs>`. */
+export function approvalCancellationNotificationKey(target: string, messageTs: string): string {
+  return `${target}:${messageTs}`;
+}
+
+/**
+ * Ids of the cancelled requests of the given runs with a Slack cancellation
+ * post not yet delivered (never claimed, or a claim a failed post released).
+ * A delivered post never comes back, so a repeat cancel of a settled tree
+ * retries only what failed. One query per 500 runs.
+ */
+export async function listUndeliveredCancelledApprovalRequestIds(
+  runIds: string[],
+): Promise<string[]> {
+  const ids: string[] = [];
+  for (let i = 0; i < runIds.length; i += 500) {
+    const chunk = runIds.slice(i, i + 500);
+    const rows = await getDbClient().query<{
+      id: string;
+      notificationChannels: string;
+      claims: string | null;
+    }>(
+      `SELECT id, notificationChannels, cancellationNotificationClaims AS claims
+         FROM approval_requests
+        WHERE workflowRunId IN (${chunk.map(() => "?").join(", ")})
+          AND status = 'cancelled' AND notificationChannels IS NOT NULL`,
+      chunk,
+    );
+    for (const row of rows) {
+      const claims = row.claims ? (JSON.parse(row.claims) as Record<string, unknown>) : {};
+      const channels = JSON.parse(row.notificationChannels) as Array<{
+        channel?: string;
+        target?: string;
+        messageTs?: string;
+      }>;
+      const undelivered = channels.some(
+        (n) =>
+          n.channel === "slack" &&
+          n.target &&
+          n.messageTs &&
+          claims[approvalCancellationNotificationKey(n.target, n.messageTs)] !== "delivered",
+      );
+      if (undelivered) ids.push(row.id);
+    }
+  }
+  return ids;
+}

package/src/be/db/tasks/write.ts

@@ -657,7 +657,15 @@ export async function overwriteTerminalTaskResultText(
   return row ? rowToAgentTask(row) : task;
 }
 
-export async function cancelTask(id: string, reason?: string): Promise<AgentTask | null> {
+/**
+ * `cascadeDependents: false` skips the dependents sweep for a caller that
+ * already knows no task depends on this one (a bulk cancel looks once).
+ */
+export async function cancelTask(
+  id: string,
+  reason?: string,
+  options: { cascadeDependents?: boolean } = {},
+): Promise<AgentTask | null> {
   const oldTask = await getTaskById(id);
   if (!oldTask) return null;
 
@@ -736,10 +744,12 @@ export async function cancelTask(id: string, reason?: string): Promise<AgentTask
       );
     }
 
-    try {
-      await dependencies.cascadeFailDependents(id, "cancelled");
-    } catch (err) {
-      console.error("[cancelTask] cascade-fail dependents error:", err);
+    if (options.cascadeDependents !== false) {
+      try {
+        await dependencies.cascadeFailDependents(id, "cancelled");
+      } catch (err) {
+        console.error("[cancelTask] cascade-fail dependents error:", err);
+      }
     }
   }
 

package/src/be/db/workflow-idempotency.ts

@@ -0,0 +1,183 @@
+import { getCurrentRequestUserId } from "../../utils/request-auth-context";
+import { openSealedJson, sealJson } from "../sealed-json";
+import { getDbClient } from "./runtime";
+
+/**
+ * Server-side `Idempotency-Key` records for workflow create and trigger.
+ *
+ * A record is written in the transaction that accepts the operation (the
+ * workflow insert, or the run insert with its initial state), so it exists if
+ * and only if the operation committed. Readers look it up before re-running
+ * any check that can change between attempts (enabled, cooldown, readiness,
+ * trigger schema) and replay the stored response instead.
+ */
+
+export type WorkflowIdempotencyOperation = "workflow.create" | "workflow.trigger";
+
+/** Who may replay a record: one caller, one operation, one target, one key. */
+export interface WorkflowIdempotencyScope {
+  /** Stable caller identity, e.g. `user:<id>`; never a credential value. */
+  principal: string;
+  operation: WorkflowIdempotencyOperation;
+  /** The triggered workflow id; empty for create. */
+  targetId: string;
+  /** The client's `Idempotency-Key` value. Stored only as a hash. */
+  key: string;
+}
+
+/** A claim for one request: its scope plus the fingerprint of what it asked for. */
+export interface WorkflowIdempotencyClaim extends WorkflowIdempotencyScope {
+  fingerprint: string;
+}
+
+export interface AcceptedWorkflowOperation {
+  fingerprint: string;
+  status: number;
+  /** The original response body; null once the workflow it names was deleted. */
+  body: unknown;
+  resourceDeletedAt: string | null;
+}
+
+/**
+ * The key was accepted for another request while this one was getting ready.
+ * Thrown inside the accepting transaction, so this request commits nothing;
+ * the caller looks the record up again and answers from it.
+ */
+export class IdempotencyKeyTakenError extends Error {
+  constructor() {
+    super("Idempotency-Key was accepted by a concurrent request");
+    this.name = "IdempotencyKeyTakenError";
+  }
+}
+
+type AcceptedRow = {
+  requestFingerprint: string;
+  responseStatus: number;
+  responseBody: string | null;
+  resourceDeletedAt: string | null;
+};
+
+function sha256(text: string): string {
+  const hasher = new Bun.CryptoHasher("sha256");
+  hasher.update(text);
+  return hasher.digest("hex");
+}
+
+function sortKeys(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(sortKeys);
+  if (!value || typeof value !== "object") return value;
+  // Code-unit order, not localeCompare: the fingerprint must not depend on the
+  // server's locale.
+  return Object.fromEntries(
+    Object.entries(value as Record<string, unknown>)
+      .filter(([, nested]) => nested !== undefined)
+      .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
+      .map(([key, nested]) => [key, sortKeys(nested)]),
+  );
+}
+
+/**
+ * Fingerprint of an accepted request body. Object key order does not matter,
+ * array order does, and every value counts as sent (nothing is redacted first,
+ * so two requests that differ only in a secret never match). Only the hash is
+ * stored.
+ */
+export function workflowRequestFingerprint(
+  operation: WorkflowIdempotencyOperation,
+  body: unknown,
+): string {
+  return sha256(`${operation}\n${JSON.stringify(sortKeys(body ?? null))}`);
+}
+
+const SCOPE_WHERE = "principal = ? AND operation = ? AND targetId = ? AND keyHash = ?";
+
+function scopeParams(scope: WorkflowIdempotencyScope): string[] {
+  return [scope.principal, scope.operation, scope.targetId, sha256(scope.key)];
+}
+
+export async function findAcceptedWorkflowOperation(
+  scope: WorkflowIdempotencyScope,
+): Promise<AcceptedWorkflowOperation | null> {
+  const row = await getDbClient().get<AcceptedRow>(
+    `SELECT requestFingerprint, responseStatus, responseBody, resourceDeletedAt
+       FROM workflow_idempotency_keys WHERE ${SCOPE_WHERE}`,
+    scopeParams(scope),
+  );
+  if (!row) return null;
+  return {
+    fingerprint: row.requestFingerprint,
+    status: row.responseStatus,
+    body: row.responseBody === null ? null : openSealedJson(row.responseBody),
+    resourceDeletedAt: row.resourceDeletedAt,
+  };
+}
+
+/**
+ * Throw `IdempotencyKeyTakenError` when the scope already has a record. Inside
+ * the accepting transaction, call it before writing a resource whose insert
+ * could collide with the concurrent winner's (a unique workflow name), so the
+ * loser replays instead of failing on that constraint.
+ */
+export async function assertWorkflowIdempotencyKeyUnused(
+  scope: WorkflowIdempotencyScope,
+): Promise<void> {
+  const taken = await getDbClient().get(
+    `SELECT 1 FROM workflow_idempotency_keys WHERE ${SCOPE_WHERE}`,
+    scopeParams(scope),
+  );
+  if (taken) throw new IdempotencyKeyTakenError();
+}
+
+/**
+ * Record the accepted operation. Call it inside the transaction that creates
+ * the resource: a rollback drops both. Throws `IdempotencyKeyTakenError` when
+ * the scope already has a record (a concurrent request committed first); the
+ * unique index backs this across processes.
+ */
+export async function recordAcceptedWorkflowOperation(
+  claim: WorkflowIdempotencyClaim,
+  accepted: { workflowId: string; runId?: string; status: number; body: unknown },
+): Promise<void> {
+  await assertWorkflowIdempotencyKeyUnused(claim);
+  const actor = getCurrentRequestUserId() ?? null;
+  try {
+    await getDbClient().run(
+      `INSERT INTO workflow_idempotency_keys
+         (id, principal, operation, targetId, keyHash, requestFingerprint, workflowId, runId,
+          responseStatus, responseBody, createdAt, created_by, updated_by)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        crypto.randomUUID(),
+        ...scopeParams(claim),
+        claim.fingerprint,
+        accepted.workflowId,
+        accepted.runId ?? null,
+        accepted.status,
+        sealJson(accepted.body),
+        new Date().toISOString(),
+        actor,
+        actor,
+      ],
+    );
+  } catch (err) {
+    if (err instanceof Error && err.message.includes("UNIQUE constraint failed")) {
+      throw new IdempotencyKeyTakenError();
+    }
+    throw err;
+  }
+}
+
+/**
+ * Turn every record naming `workflowId` into a tombstone: the stored response
+ * (which may carry the deleted workflow's definition) is cleared and the key
+ * stays taken, so a retry is told the resource is gone instead of creating or
+ * triggering anything again. Runs in the delete's transaction.
+ */
+export async function tombstoneWorkflowIdempotencyKeys(workflowId: string): Promise<void> {
+  await getDbClient().run(
+    `UPDATE workflow_idempotency_keys
+        SET responseBody = NULL, resourceDeletedAt = ?, updated_by = ?
+      WHERE workflowId = ? AND resourceDeletedAt IS NULL`,
+    [new Date().toISOString(), getCurrentRequestUserId() ?? null, workflowId],
+  );
+}

package/src/be/db/workflow-runs.ts

@@ -1,4 +1,4 @@
-import type { WorkflowRunStatus } from "../../types";
+import { TERMINAL_TASK_STATUSES, type WorkflowRunStatus } from "../../types";
 import { scrubSecrets } from "../../utils/secret-scrubber";
 import { scrubJsonValue } from "../scrub-json";
 import { getDbClient } from "./runtime";
@@ -14,6 +14,292 @@ const TERMINAL_STATUSES = new Set<WorkflowRunStatus>([
   "skipped",
 ]);
 
+/**
+ * Deepest run tree the lineage guard and the cascade cancel walk. Twice the
+ * sub-workflow nesting cap (MAX_SUB_WORKFLOW_DEPTH, 16), so any tree the
+ * engine admits fits; a deeper chain is corrupt and fails closed.
+ */
+export const WORKFLOW_RUN_TREE_MAX_DEPTH = 32;
+
+/** Thrown when a run's stored parent links cannot be walked: a cycle, or a chain past the depth bound. */
+export class WorkflowRunTreeError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "WorkflowRunTreeError";
+  }
+}
+
+/**
+ * Thrown when work is created for a workflow step that may no longer run: the
+ * step is not `running`, its run is not live, or the run's lineage is blocked
+ * (see `getRunLineageBlock`).
+ */
+export class WorkflowStepNotExecutableError extends Error {
+  constructor(stepId: string, reason: string) {
+    super(`Workflow step ${stepId} may not start new work: ${reason}`);
+    this.name = "WorkflowStepNotExecutableError";
+  }
+}
+
+/**
+ * Why the run and its ancestors forbid new engine work, or null when they
+ * allow it. Blocked when the run is missing, the run or an ancestor is
+ * `cancelled`, a stored parent link points at a missing step or run, the
+ * links form a cycle, or the chain is deeper than WORKFLOW_RUN_TREE_MAX_DEPTH.
+ * A completed or failed ancestor does not block: a child of an earlier
+ * attempt may outlive its parent's run. The run's own live status is left to
+ * the caller, since the retry claim also revives a `failed` run.
+ *
+ * Call it inside the transaction that writes the new work, so a cascade
+ * cancel either commits first and is seen here or sees the new row itself.
+ */
+export async function getRunLineageBlock(runId: string): Promise<string | null> {
+  const rows = await getDbClient().query<{
+    id: string;
+    status: WorkflowRunStatus;
+    parentStepId: string | null;
+    depth: number;
+  }>(
+    `WITH RECURSIVE lineage(id, status, parentStepId, depth) AS (
+       SELECT id, status, parentStepId, 0 FROM workflow_runs WHERE id = ?
+       UNION ALL
+       SELECT r.id, r.status, r.parentStepId, l.depth + 1
+         FROM lineage l
+         JOIN workflow_run_steps s ON s.id = l.parentStepId
+         JOIN workflow_runs r ON r.id = s.runId
+        WHERE l.depth < ?
+     )
+     SELECT id, status, parentStepId, depth FROM lineage ORDER BY depth`,
+    [runId, WORKFLOW_RUN_TREE_MAX_DEPTH],
+  );
+  if (rows.length === 0) return `Workflow run ${runId} is missing`;
+  const seen = new Set<string>();
+  for (const row of rows) {
+    if (seen.has(row.id)) return `Workflow run ${runId} has a cycle in its parent links`;
+    seen.add(row.id);
+    if (row.status === "cancelled") {
+      return row.id === runId
+        ? `Workflow run ${runId} is cancelled`
+        : `Ancestor workflow run ${row.id} is cancelled`;
+    }
+  }
+  const top = rows[rows.length - 1]!;
+  if (top.parentStepId == null) return null;
+  if (top.depth >= WORKFLOW_RUN_TREE_MAX_DEPTH) {
+    return `Workflow run ${runId} is nested deeper than ${WORKFLOW_RUN_TREE_MAX_DEPTH} levels`;
+  }
+  return `Parent step ${top.parentStepId} of workflow run ${top.id} is missing`;
+}
+
+/**
+ * The run and every run below it, root first: children of every attempt of
+ * every step, through terminal runs too, since a run that ended can still have
+ * a live child of an earlier attempt. The tree comes from stored links only
+ * (child.parentStepId → step → run). There is no size limit: the engine admits
+ * any number of children per run, and a tree it admitted must stay
+ * cancellable. Throws WorkflowRunTreeError on a cycle or past `maxDepth`, so
+ * the caller's transaction rolls back whole.
+ */
+export async function getWorkflowRunTreeIds(
+  rootId: string,
+  bounds: { maxDepth?: number } = {},
+): Promise<string[]> {
+  const maxDepth = bounds.maxDepth ?? WORKFLOW_RUN_TREE_MAX_DEPTH;
+  const ids = [rootId];
+  const seen = new Set(ids);
+  // One level per pass, so a cycle stops at its first repeated run instead of
+  // unrolling down to the depth bound.
+  let level = [rootId];
+  for (let depth = 1; level.length > 0; depth++) {
+    const next: string[] = [];
+    for (let i = 0; i < level.length; i += TREE_QUERY_CHUNK) {
+      const chunk = level.slice(i, i + TREE_QUERY_CHUNK);
+      const rows = await getDbClient().query<{ id: string }>(
+        `SELECT c.id
+           FROM workflow_run_steps s
+           JOIN workflow_runs c ON c.parentStepId = s.id
+          WHERE s.runId IN (${chunk.map(() => "?").join(", ")})`,
+        chunk,
+      );
+      for (const { id } of rows) {
+        // Each run has one parent link, so reaching a run twice means a cycle.
+        if (seen.has(id)) {
+          throw new WorkflowRunTreeError(`Workflow run tree of ${rootId} has a cycle at run ${id}`);
+        }
+        seen.add(id);
+        next.push(id);
+      }
+    }
+    if (next.length > 0 && depth > maxDepth) {
+      throw new WorkflowRunTreeError(
+        `Workflow run tree of ${rootId} is nested deeper than ${maxDepth} levels`,
+      );
+    }
+    ids.push(...next);
+    level = next;
+  }
+  return ids;
+}
+
+/** Parent runs per children query, well under SQLite's bound-parameter limit. */
+const TREE_QUERY_CHUNK = 500;
+
+/** A run's status alone, without opening its sealed context. Null when the run is missing. */
+export async function getWorkflowRunStatus(runId: string): Promise<WorkflowRunStatus | null> {
+  const row = await getDbClient().get<{ status: WorkflowRunStatus }>(
+    "SELECT status FROM workflow_runs WHERE id = ?",
+    [runId],
+  );
+  return row?.status ?? null;
+}
+
+/** Run `fn` over `ids` in chunks small enough for one statement's bound parameters. */
+async function forEachChunk(
+  ids: string[],
+  fn: (chunk: string[], placeholders: string) => Promise<void>,
+): Promise<void> {
+  for (let i = 0; i < ids.length; i += TREE_QUERY_CHUNK) {
+    const chunk = ids.slice(i, i + TREE_QUERY_CHUNK);
+    await fn(chunk, chunk.map(() => "?").join(", "));
+  }
+}
+
+/**
+ * Ids of the non-terminal tasks of the given runs, including tasks of earlier
+ * attempts: a retry detaches a task from its step, but the run link stays.
+ */
+export async function getOpenWorkflowTreeTaskIds(runIds: string[]): Promise<string[]> {
+  const ids: string[] = [];
+  const terminal = TERMINAL_TASK_STATUSES.map(() => "?").join(", ");
+  await forEachChunk(runIds, async (chunk, placeholders) => {
+    const rows = await getDbClient().query<{ id: string }>(
+      `SELECT id FROM agent_tasks
+        WHERE workflowRunId IN (${placeholders}) AND status NOT IN (${terminal})`,
+      [...chunk, ...TERMINAL_TASK_STATUSES],
+    );
+    for (const row of rows) ids.push(row.id);
+  });
+  return ids;
+}
+
+/**
+ * Ids of tasks that a non-terminal task lists in `dependsOn`: the only
+ * dependents a cancel can fail. One scan over open tasks, so a cancel of many
+ * tasks looks for dependents once instead of once per task.
+ */
+export async function getTaskIdsWithDependents(): Promise<Set<string>> {
+  const rows = await getDbClient().query<{ id: string }>(
+    `SELECT DISTINCT dep.value AS id
+       FROM agent_tasks t, json_each(t.dependsOn) AS dep
+      WHERE t.status NOT IN (${TERMINAL_TASK_STATUSES.map(() => "?").join(", ")})
+        AND t.dependsOn IS NOT NULL AND t.dependsOn != '[]'`,
+    [...TERMINAL_TASK_STATUSES],
+  );
+  return new Set(rows.map((row) => row.id));
+}
+
+/**
+ * True when any of the given runs still has work a cancel would close: a live
+ * run, a pending, running or waiting step, a queued retry, an open task or a
+ * pending approval. Reads only, with a LIMIT 1 probe per chunk.
+ */
+export async function workflowRunTreeHasOpenWork(runIds: string[]): Promise<boolean> {
+  const terminal = TERMINAL_TASK_STATUSES.map(() => "?").join(", ");
+  const probes: Array<{ sql: (ids: string) => string; params: string[] }> = [
+    {
+      sql: (ids) =>
+        `SELECT 1 FROM workflow_runs WHERE id IN (${ids}) AND status IN ('running', 'waiting') LIMIT 1`,
+      params: [],
+    },
+    {
+      sql: (ids) => `SELECT 1 FROM workflow_run_steps
+        WHERE runId IN (${ids})
+          AND (status IN ('pending', 'running', 'waiting')
+               OR (status = 'failed' AND nextRetryAt IS NOT NULL))
+        LIMIT 1`,
+      params: [],
+    },
+    {
+      sql: (ids) =>
+        `SELECT 1 FROM agent_tasks WHERE workflowRunId IN (${ids}) AND status NOT IN (${terminal}) LIMIT 1`,
+      params: [...TERMINAL_TASK_STATUSES],
+    },
+    {
+      sql: (ids) =>
+        `SELECT 1 FROM approval_requests WHERE workflowRunId IN (${ids}) AND status = 'pending' LIMIT 1`,
+      params: [],
+    },
+  ];
+  let open = false;
+  await forEachChunk(runIds, async (chunk, placeholders) => {
+    for (const probe of probes) {
+      if (open) return;
+      if (await getDbClient().get(probe.sql(placeholders), [...chunk, ...probe.params])) {
+        open = true;
+      }
+    }
+  });
+  return open;
+}
+
+/**
+ * Cancel the pending, running and waiting steps of the given runs. The
+ * migration-140 trigger closes each cancelled step's pending approval.
+ */
+export async function cancelOpenWorkflowTreeSteps(
+  runIds: string[],
+  reason: string,
+  finishedAt: string,
+): Promise<void> {
+  await forEachChunk(runIds, async (chunk, placeholders) => {
+    await getDbClient().run(
+      `UPDATE workflow_run_steps
+          SET status = 'cancelled', error = ?, finishedAt = ?
+        WHERE runId IN (${placeholders}) AND status IN ('pending', 'running', 'waiting')`,
+      [scrubSecrets(reason), finishedAt, ...chunk],
+    );
+  });
+}
+
+/**
+ * Drop the pending retries of the failed steps of the given runs. The step
+ * keeps its status, error, output and retryCount; only the queued next
+ * attempt goes.
+ */
+export async function disarmWorkflowTreeRetries(
+  runIds: string[],
+  finishedAt: string,
+): Promise<void> {
+  await forEachChunk(runIds, async (chunk, placeholders) => {
+    await getDbClient().run(
+      `UPDATE workflow_run_steps
+          SET nextRetryAt = NULL, finishedAt = COALESCE(finishedAt, ?)
+        WHERE runId IN (${placeholders}) AND status = 'failed' AND nextRetryAt IS NOT NULL`,
+      [finishedAt, ...chunk],
+    );
+  });
+}
+
+/**
+ * Mark the given runs that are still running or waiting `cancelled`, without
+ * the per-run `workflow.child.finished` wake-up: it is for runs whose parent
+ * step the same cancel closes. A terminal run keeps its status and reason.
+ */
+export async function cancelLiveWorkflowTreeRuns(
+  runIds: string[],
+  reason: string,
+  finishedAt: string,
+): Promise<void> {
+  await forEachChunk(runIds, async (chunk, placeholders) => {
+    await getDbClient().run(
+      `UPDATE workflow_runs
+          SET status = 'cancelled', error = ?, finishedAt = ?, lastUpdatedAt = ?
+        WHERE id IN (${placeholders}) AND status IN ('running', 'waiting')`,
+      [scrubSecrets(reason), finishedAt, finishedAt, ...chunk],
+    );
+  });
+}
+
 /** The id of the child run a `sub-workflow` step started on `attempt`, if any. */
 export async function getChildWorkflowRunId(
   parentStepId: string,
@@ -28,23 +314,48 @@ export async function getChildWorkflowRunId(
 
 /**
  * True while `attempt` of the step is executing in a live run: the step is
- * `running` with that `retryCount`. A child run is created only then, so a
- * cancelled run, a step another attempt moved on, or a step already parked
- * never gets a new child.
+ * `running` with that `retryCount` and no ancestor run is cancelled. A child
+ * run is created only then, so a cancelled run or tree, a step another attempt
+ * moved on, or a step already parked never gets a new child.
  */
 export async function isParentStepAttemptCurrent(
   parentStepId: string,
   attempt: number,
 ): Promise<boolean> {
-  const row = await getDbClient().get<{ found: number }>(
-    `SELECT 1 AS found
+  const row = await getDbClient().get<{ runId: string }>(
+    `SELECT r.id AS runId
        FROM workflow_run_steps s
        JOIN workflow_runs r ON r.id = s.runId
       WHERE s.id = ? AND s.status = 'running' AND s.retryCount = ?
         AND r.status IN ('running', 'waiting')`,
     [parentStepId, attempt],
   );
-  return row != null;
+  return row != null && (await getRunLineageBlock(row.runId)) == null;
+}
+
+/**
+ * Why `stepId` may not start new work (a task, an approval, a child run), or
+ * null when it may: the step is `running` in a live run whose lineage is not
+ * blocked. Call it inside the transaction that inserts the work.
+ */
+export async function getStepExecutionBlock(stepId: string): Promise<string | null> {
+  const row = await getDbClient().get<{
+    stepStatus: string;
+    runId: string;
+    runStatus: WorkflowRunStatus;
+  }>(
+    `SELECT s.status AS stepStatus, r.id AS runId, r.status AS runStatus
+       FROM workflow_run_steps s
+       JOIN workflow_runs r ON r.id = s.runId
+      WHERE s.id = ?`,
+    [stepId],
+  );
+  if (!row) return "the step or its run is missing";
+  if (row.runStatus !== "running" && row.runStatus !== "waiting") {
+    return `its run is ${row.runStatus}`;
+  }
+  if (row.stepStatus !== "running") return `the step is ${row.stepStatus}`;
+  return getRunLineageBlock(row.runId);
 }
 
 /**