@desplega.ai/agent-swarm 2.5.2 → 2.5.4

package/src/workflows/approval-cancel.ts

@@ -4,16 +4,15 @@ import {
   getApprovalRequestById,
   getDbClient,
   getTaskById,
-  getWorkflowRun,
-  listCancelledApprovalRequestsForRun,
   resolveApprovalRequest,
 } from "../be/db";
+import { getWorkflowRunStatus } from "../be/db/workflow-runs";
 import { can, type RbacPrincipal } from "../rbac";
 import {
   createApprovalFollowUpTask,
   postApprovalCancellationUpdates,
 } from "./approval-notifications";
-import { cancelWorkflowRunRows } from "./resume";
+import { cancelWorkflowRunRows, postTreeApprovalCancellations } from "./resume";
 
 export type CancelApprovalRequestResult =
   | { ok: true; request: ApprovalRequest; alreadyCancelled: boolean; runCancelled: boolean }
@@ -126,25 +125,28 @@ export async function cancelApprovalRequestAndRun(
     if (!request) return null;
 
     // The request is cancelled first so it keeps this reason; the run cancel
-    // then skips it because it is no longer pending.
+    // then skips it because it is no longer pending. The run's child tree is
+    // cancelled with it.
     let runCancelled = false;
+    let treeRunIds: string[] = [];
     const runId = request.workflowRunId;
     if (runId) {
-      const run = await getWorkflowRun(runId);
-      if (run && LIVE_RUN_STATUSES.has(run.status)) {
-        runCancelled = await cancelWorkflowRunRows(runId, data.reason);
+      const status = await getWorkflowRunStatus(runId);
+      if (status && LIVE_RUN_STATUSES.has(status)) {
+        const result = await cancelWorkflowRunRows(runId, data.reason);
+        runCancelled = result.cancelled;
+        treeRunIds = result.runIds;
       }
     }
 
     const slackReason = data.slackReason ?? data.reason;
     client.afterCommit(async () => {
       await postApprovalCancellationUpdates([request], slackReason);
-      if (runCancelled && runId) {
-        // Other requests the run cancel closed get the run's reason.
-        const others = (await listCancelledApprovalRequestsForRun(runId)).filter(
-          (row) => row.id !== request.id,
-        );
-        await postApprovalCancellationUpdates(others, data.reason);
+      if (runCancelled) {
+        // Other requests the tree cancel closed get the run's reason.
+        await postTreeApprovalCancellations(treeRunIds, data.reason, {
+          exceptRequestId: request.id,
+        });
       }
     });
     return { request, runCancelled };

package/src/workflows/approval-notifications.ts

@@ -1,3 +1,4 @@
+import { formatConfigResult } from "../be/approval-config-question";
 import {
   type ApprovalRequest,
   claimApprovalCancellationNotification,
@@ -8,6 +9,7 @@ import {
   releaseApprovalCancellationNotificationClaim,
   updateApprovalRequestNotifications,
 } from "../be/db";
+import { approvalCancellationNotificationKey } from "../be/db/approvals";
 import { resolveTemplate } from "../prompts/resolver";
 import { getSlackApp } from "../slack/app";
 import { resolveSlackConversationId } from "../slack/dm";
@@ -34,11 +36,15 @@ function escapeSlackMrkdwn(text: string): string {
  * question stays answerable on the dashboard page the card's button opens.
  */
 export function buildSlackQuestionsSummary(
-  questions: ReadonlyArray<{ label: string }>,
+  questions: ReadonlyArray<{ label?: string; type?: string }>,
   timeoutText = "",
 ): string {
   const header = "*Questions:*\n";
-  const lines = questions.map((q) => `• ${escapeSlackMrkdwn(q.label.replace(/\s+/g, " ").trim())}`);
+  // A config question is only ever answered on the dashboard, never in Slack.
+  const lines = questions.map(
+    (q) =>
+      `• ${escapeSlackMrkdwn((q.label ?? "").replace(/\s+/g, " ").trim())}${q.type === "config" ? " (set on dashboard)" : ""}`,
+  );
   const fits = (shown: string[], omitted: number) => {
     const tail =
       omitted > 0 ? `\n_…and ${omitted} more. Answer all of them on the dashboard._` : "";
@@ -71,7 +77,7 @@ export interface ApprovalNotificationChannel {
 export interface ApprovalNotificationsInput {
   requestId: string;
   title: string;
-  questions: ReadonlyArray<{ label: string }>;
+  questions: ReadonlyArray<{ label?: string; type?: string }>;
   timeoutSeconds?: number;
   notifications: ReadonlyArray<ApprovalNotificationChannel>;
 }
@@ -258,7 +264,10 @@ export async function postApprovalCancellationUpdates(
 
     for (const notification of slackThreads) {
       if (notification.channel !== "slack" || !notification.messageTs) continue;
-      const notificationKey = `${notification.target}:${notification.messageTs}`;
+      const notificationKey = approvalCancellationNotificationKey(
+        notification.target,
+        notification.messageTs,
+      );
       const claim = await claimApprovalCancellationNotification(approval.id, notificationKey);
       if (!claim) continue;
       try {
@@ -355,6 +364,8 @@ function formatResponses(
       let answerText: string;
       if (answer == null) {
         answerText = "(no answer)";
+      } else if (q.type === "config") {
+        answerText = formatConfigResult(answer) ?? "(unreadable config result)";
       } else if (q.type === "approval") {
         const a = answer as { approved?: boolean; comment?: string };
         answerText = a.approved ? "Approved" : "Rejected";

package/src/workflows/checkpoint.ts

@@ -1,8 +1,32 @@
 import { getDbClient, getWorkflowRun, updateWorkflowRun, updateWorkflowRunStep } from "../be/db";
+import { getRunLineageBlock } from "../be/db/workflow-runs";
 import type { RetryPolicy } from "../types";
 
+/**
+ * The execution a checkpoint belongs to: the step attempt (`retryCount`) that
+ * claimed the row. A fenced checkpoint writes only while that attempt still
+ * owns the row: the step is `running` with that `retryCount` in a `running` or
+ * `waiting` run with no cancelled ancestor. A cancel, a newer attempt or
+ * another settle that committed meanwhile wins, and the stale execution
+ * writes nothing.
+ */
+export type StepFence = { retryCount: number };
+
+async function holdsFence(runId: string, stepId: string, fence: StepFence): Promise<boolean> {
+  const row = await getDbClient().get<{ found: number }>(
+    `SELECT 1 AS found
+       FROM workflow_run_steps s
+       JOIN workflow_runs r ON r.id = s.runId
+      WHERE s.id = ? AND s.runId = ? AND s.status = 'running' AND s.retryCount = ?
+        AND r.status IN ('running', 'waiting')`,
+    [stepId, runId, fence.retryCount],
+  );
+  return row != null && (await getRunLineageBlock(runId)) == null;
+}
+
 /**
  * Checkpoint a successful step — atomic DB write of step result + run context.
+ * Returns false, writing nothing, when `fence` no longer holds.
  */
 export async function checkpointStep(
   runId: string,
@@ -10,8 +34,10 @@ export async function checkpointStep(
   nodeId: string,
   result: { output?: unknown; nextPort?: string },
   ctx: Record<string, unknown>,
-): Promise<void> {
-  await getDbClient().transaction(async () => {
+  options: { fence?: StepFence } = {},
+): Promise<boolean> {
+  return getDbClient().transaction(async () => {
+    if (options.fence && !(await holdsFence(runId, stepId, options.fence))) return false;
     await updateWorkflowRunStep(stepId, {
       status: "completed",
       output: result.output,
@@ -24,11 +50,14 @@ export async function checkpointStep(
     await updateWorkflowRun(runId, {
       context: ctx,
     });
+    return true;
   });
 }
 
 /**
  * Checkpoint a step failure — marks step failed, calculates retry if applicable.
+ * With `fence`, a stale execution writes nothing and gets `stale: true`.
+ * `output` is kept on the failed step for observability, in the same write.
  */
 export async function checkpointStepFailure(
   runId: string,
@@ -36,49 +65,56 @@ export async function checkpointStepFailure(
   error: string,
   retryCount: number,
   retryPolicy?: RetryPolicy,
-  options?: { markRunFailed?: boolean },
-): Promise<{ shouldRetry: boolean }> {
+  options?: { markRunFailed?: boolean; fence?: StepFence; output?: unknown },
+): Promise<{ shouldRetry: boolean; stale?: true }> {
   const now = new Date().toISOString();
 
-  if (retryPolicy && retryCount < retryPolicy.maxRetries) {
-    const delay = calculateBackoff(retryPolicy, retryCount);
-    const nextRetryAt = new Date(Date.now() + delay).toISOString();
+  return getDbClient().transaction(async () => {
+    if (options?.fence && !(await holdsFence(runId, stepId, options.fence))) {
+      return { shouldRetry: false, stale: true as const };
+    }
+
+    if (retryPolicy && retryCount < retryPolicy.maxRetries) {
+      const delay = calculateBackoff(retryPolicy, retryCount);
+      const nextRetryAt = new Date(Date.now() + delay).toISOString();
+
+      await updateWorkflowRunStep(stepId, {
+        status: "failed",
+        error,
+        output: options?.output,
+        retryCount: retryCount + 1,
+        maxRetries: retryPolicy.maxRetries,
+        nextRetryAt,
+      });
+
+      return { shouldRetry: true };
+    }
 
+    // No retries left — mark step failed, and optionally the run too
+    // Clear nextRetryAt so the poller stops picking this step up
     await updateWorkflowRunStep(stepId, {
       status: "failed",
       error,
-      retryCount: retryCount + 1,
-      maxRetries: retryPolicy.maxRetries,
-      nextRetryAt,
+      output: options?.output,
+      finishedAt: now,
+      nextRetryAt: null,
     });
 
-    return { shouldRetry: true };
-  }
-
-  // No retries left — mark step failed, and optionally the run too
-  // Clear nextRetryAt so the poller stops picking this step up
-  await updateWorkflowRunStep(stepId, {
-    status: "failed",
-    error,
-    finishedAt: now,
-    nextRetryAt: null,
-  });
-
-  const markRunFailed = options?.markRunFailed ?? true;
-  if (markRunFailed) {
-    // A cancel or finalization that committed while the step ran wins.
-    await getDbClient().transaction(async () => {
+    const markRunFailed = options?.markRunFailed ?? true;
+    if (markRunFailed) {
+      // A cancel or finalization that committed while the step ran wins.
       const run = await getWorkflowRun(runId);
-      if (run?.status !== "running" && run?.status !== "waiting") return;
-      await updateWorkflowRun(runId, {
-        status: "failed",
-        error: `Step failed: ${error}`,
-        finishedAt: now,
-      });
-    });
-  }
+      if (run?.status === "running" || run?.status === "waiting") {
+        await updateWorkflowRun(runId, {
+          status: "failed",
+          error: `Step failed: ${error}`,
+          finishedAt: now,
+        });
+      }
+    }
 
-  return { shouldRetry: false };
+    return { shouldRetry: false };
+  });
 }
 
 /**
@@ -95,6 +131,7 @@ export async function checkpointStepWaiting(
   options: { keepRunStatus?: boolean } = {},
 ): Promise<boolean> {
   return getDbClient().transaction(async () => {
+    if (await getRunLineageBlock(runId)) return false;
     const step = await getDbClient().get<{ id: string }>(
       `UPDATE workflow_run_steps
          SET status = 'waiting'