@desplega.ai/agent-swarm 1.165.0 → 1.166.0

package/src/workflows/resume.ts

@@ -17,6 +17,7 @@ import {
   updateWorkflowRun,
   updateWorkflowRunStep,
 } from "../be/db";
+import type { WaitStateRow } from "../types";
 import { scrubSecrets } from "../utils/secret-scrubber";
 import {
   type ApprovalSlackClient,
@@ -26,7 +27,7 @@ import { shapeApprovalResolution } from "./approval-resolution";
 import { loadCompletedStepRouting } from "./completed-step-routing";
 import { FAILED_TASK_OUTPUT_PREFIX } from "./constants";
 import { getNextTargets } from "./definition";
-import { findReadyNodes, hasRunningStep, walkGraph } from "./engine";
+import { failRunOnUnreadableReplay, findReadyNodes, hasRunningStep, walkGraph } from "./engine";
 import type { WorkflowEventBus } from "./event-bus";
 import { workflowEventBus } from "./event-bus";
 import type { ExecutorRegistry } from "./executors/registry";
@@ -127,6 +128,23 @@ export function setupWorkflowResumeListener(
   };
 }
 
+/**
+ * Run one live resume of a waiting step. A sealed replay copy that cannot be
+ * opened fails the step and its run now, instead of logging and leaving the
+ * run `waiting` until a recovery sweep. Any other error rethrows.
+ */
+async function failClosedOnUnreadableReplay(
+  runId: string,
+  stepId: string,
+  resume: () => Promise<void>,
+): Promise<void> {
+  try {
+    await resume();
+  } catch (err) {
+    if (!(await failRunOnUnreadableReplay(runId, err, stepId))) throw err;
+  }
+}
+
 /**
  * Resume a workflow after a linked task completes.
  *
@@ -138,6 +156,15 @@ export function setupWorkflowResumeListener(
 async function resumeFromTaskCompletion(
   event: TaskEvent,
   registry: ExecutorRegistry,
+): Promise<void> {
+  await failClosedOnUnreadableReplay(event.workflowRunId!, event.workflowRunStepId!, () =>
+    resumeFromTaskCompletionUnguarded(event, registry),
+  );
+}
+
+async function resumeFromTaskCompletionUnguarded(
+  event: TaskEvent,
+  registry: ExecutorRegistry,
 ): Promise<void> {
   const run = await getWorkflowRun(event.workflowRunId!);
   if (!run || (run.status !== "waiting" && run.status !== "running")) return;
@@ -245,6 +272,17 @@ async function handleTaskFailure(
   reason: string,
   registry: ExecutorRegistry,
   options: { retryable: boolean },
+): Promise<void> {
+  await failClosedOnUnreadableReplay(event.workflowRunId!, event.workflowRunStepId!, () =>
+    handleTaskFailureUnguarded(event, reason, registry, options),
+  );
+}
+
+async function handleTaskFailureUnguarded(
+  event: TaskEvent,
+  reason: string,
+  registry: ExecutorRegistry,
+  options: { retryable: boolean },
 ): Promise<void> {
   const run = await getWorkflowRun(event.workflowRunId!);
   if (!run || (run.status !== "waiting" && run.status !== "running")) return;
@@ -541,6 +579,15 @@ export async function cancelWorkflowRunStep(
 async function resumeFromApprovalResolution(
   event: ApprovalEvent,
   registry: ExecutorRegistry,
+): Promise<void> {
+  await failClosedOnUnreadableReplay(event.workflowRunId!, event.workflowRunStepId!, () =>
+    resumeFromApprovalResolutionUnguarded(event, registry),
+  );
+}
+
+async function resumeFromApprovalResolutionUnguarded(
+  event: ApprovalEvent,
+  registry: ExecutorRegistry,
 ): Promise<void> {
   const run = await getWorkflowRun(event.workflowRunId!);
   if (!run || (run.status !== "waiting" && run.status !== "running")) return;
@@ -551,8 +598,6 @@ async function resumeFromApprovalResolution(
   const workflow = await getWorkflow(run.workflowId);
   if (!workflow) return;
 
-  const ctx = (run.context ?? {}) as Record<string, unknown>;
-
   if (event.status === "cancelled") {
     console.warn(
       `[workflows] approval ${event.requestId} is cancelled; step ${event.workflowRunStepId} stays waiting for the run cancel path`,
@@ -560,6 +605,10 @@ async function resumeFromApprovalResolution(
     return;
   }
 
+  // Opens the sealed replay context: after the cancelled bail-out, which leaves
+  // the step to the run cancel path.
+  const ctx = (run.context ?? {}) as Record<string, unknown>;
+
   // Output and port for the approval status. A step parked by an executor other
   // than human-in-the-loop shapes its own (see shapeApprovalResolution).
   const { output: stepOutput, nextPort } = shapeApprovalResolution(
@@ -643,7 +692,17 @@ export async function resumeWaitState(
   if (!result.updated || !result.row) return;
 
   const waitRow = result.row;
+  await failClosedOnUnreadableReplay(waitRow.workflowRunId, waitRow.workflowRunStepId, () =>
+    resumeClaimedWait(waitRow, status, cappedPayload, registry),
+  );
+}
 
+async function resumeClaimedWait(
+  waitRow: WaitStateRow,
+  status: "fired" | "timeout",
+  cappedPayload: unknown,
+  registry: ExecutorRegistry,
+): Promise<void> {
   // 2. Load the surrounding run + step. If anything has moved on (cancelled,
   // failed, retried, etc.), stay quiet.
   const run = await getWorkflowRun(waitRow.workflowRunId);

package/src/workflows/retry-poller.ts

@@ -11,6 +11,7 @@ import { checkpointStep, checkpointStepFailure, checkpointStepWaiting } from "./
 import { getSuccessors, resolveValidationPort } from "./definition";
 import {
   buildNodeInterpolationCtx,
+  failRunOnUnreadableReplay,
   holdWorkflowRun,
   interpolateNodeConfig,
   rehydrateCompletedStepOutputs,
@@ -200,11 +201,13 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
             }
           } catch (err) {
             // Execution threw — treat as failure
+            if (await failRunOnUnreadableReplay(run.id, err, step.id)) continue;
             const errorMsg = err instanceof Error ? err.message : String(err);
             const retryPolicy = node.retry || executor.retryPolicy;
             await checkpointStepFailure(run.id, step.id, errorMsg, step.retryCount, retryPolicy);
           }
         } catch (err) {
+          if (await failRunOnUnreadableReplay(step.runId, err, step.id)) continue;
           console.error(`[workflows] Retry failed for step ${step.id}:`, err);
         } finally {
           release?.();

package/dist/boot-scrub-logs-246yn076.js

@@ -1,99 +0,0 @@
-import {
-  getDbClient,
-  init_db
-} from "./cli-qdv8kgz7.js";
-import"./cli-9s3h836g.js";
-import"./cli-3qhyh31j.js";
-import"./cli-3a4m1aws.js";
-import"./cli-7mm2r1kp.js";
-import"./cli-g5erf0zt.js";
-import"./cli-dmtrxqhg.js";
-import"./cli-tadfgd0t.js";
-import"./cli-ywybs5as.js";
-import"./cli-gd4favx9.js";
-import"./cli-agamgm0p.js";
-import"./cli-5c2j5yvx.js";
-import"./cli-1a2j50m0.js";
-import"./cli-f9w1semm.js";
-import"./cli-ej6gbxfb.js";
-import"./cli-zsh093hk.js";
-import"./cli-sd0mxa3c.js";
-import"./cli-w5e1d1vv.js";
-import"./cli-1kzfvd1z.js";
-import"./cli-zf8jjwex.js";
-import {
-  init_secret_scrubber,
-  scrubSecrets
-} from "./cli-h1spv8v4.js";
-import"./cli-bc05qbss.js";
-
-// src/be/boot-scrub-logs.ts
-init_secret_scrubber();
-init_db();
-var SCRUB_KEY = "boot-scrub-logs-v2";
-var CURSOR_KEY = "boot-scrub-logs-v2-cursor";
-var BATCH_SIZE = 200;
-var yieldTick = () => new Promise((r) => setTimeout(r, 5));
-async function runBootScrubLogs() {
-  const done = await getDbClient().get("SELECT key FROM seed_state WHERE kind = ? AND key = ?", ["maintenance", SCRUB_KEY]);
-  if (done)
-    return;
-  const savedCursor = (await getDbClient().get("SELECT seededHash FROM seed_state WHERE kind = ? AND key = ?", ["maintenance", CURSOR_KEY]))?.seededHash ?? "";
-  const lastProcessedId = savedCursor || "";
-  const totalRemaining = (await getDbClient().get(`SELECT COUNT(*) as count FROM session_logs
-         WHERE id > ?
-           AND (content LIKE '%lin!_oauth!_%' ESCAPE '!'
-             OR content LIKE '%lin!_api!_%' ESCAPE '!'
-             OR content LIKE '%npm!_%' ESCAPE '!'
-             OR content LIKE '%ATATT%')`, [lastProcessedId]))?.count ?? 0;
-  if (totalRemaining === 0) {
-    await markDone();
-    return;
-  }
-  console.log(`[boot-scrub-logs] starting: ${totalRemaining} candidate rows remaining` + (lastProcessedId ? ` (resuming from cursor ${lastProcessedId.slice(0, 8)}…)` : ""));
-  const saveCursorSql = `INSERT INTO seed_state (kind, key, seededHash, seededAt)
-     VALUES ('maintenance', '${CURSOR_KEY}', ?, datetime('now'))
-     ON CONFLICT (kind, key) DO UPDATE SET seededHash = ?, seededAt = datetime('now')`;
-  let scrubbed = 0;
-  let scanned = 0;
-  let cursor = lastProcessedId;
-  for (;; ) {
-    const rows = await getDbClient().query(`SELECT id, content FROM session_logs
-       WHERE id > ?
-         AND (content LIKE '%lin!_oauth!_%' ESCAPE '!'
-           OR content LIKE '%lin!_api!_%' ESCAPE '!'
-           OR content LIKE '%npm!_%' ESCAPE '!'
-           OR content LIKE '%ATATT%')
-       ORDER BY id ASC
-       LIMIT ${BATCH_SIZE}`, [cursor]);
-    if (rows.length === 0)
-      break;
-    const batchLastId = rows[rows.length - 1].id;
-    await getDbClient().transaction(async (tx) => {
-      for (const row of rows) {
-        const cleaned = scrubSecrets(row.content);
-        if (cleaned !== row.content) {
-          await tx.run("UPDATE session_logs SET content = ? WHERE id = ?", [cleaned, row.id]);
-          scrubbed++;
-        }
-      }
-      await tx.run(saveCursorSql, [batchLastId, batchLastId]);
-    });
-    scanned += rows.length;
-    cursor = batchLastId;
-    await yieldTick();
-  }
-  await markDone();
-  await getDbClient().run("DELETE FROM seed_state WHERE kind = 'maintenance' AND key = ?", [
-    CURSOR_KEY
-  ]);
-  console.log(`[boot-scrub-logs] complete: scanned=${scanned} scrubbed=${scrubbed}`);
-}
-async function markDone() {
-  await getDbClient().run(`INSERT INTO seed_state (kind, key, seededHash, seededAt)
-     VALUES ('maintenance', ?, 'done', datetime('now'))
-     ON CONFLICT (kind, key) DO UPDATE SET seededHash = 'done', seededAt = datetime('now')`, [SCRUB_KEY]);
-}
-export {
-  runBootScrubLogs
-};