@hasna/loops 0.4.5 → 0.4.6

package/dist/sdk/index.js

@@ -1491,11 +1491,14 @@ function ensurePrivateStorePath(file) {
 class Store {
   db;
   rootDir;
+  memoryRootDir;
   constructor(path) {
     const file = path ?? dbPath();
     if (file !== ":memory:")
       ensurePrivateStorePath(file);
     this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) : dirname2(file);
+    if (file === ":memory:")
+      this.memoryRootDir = this.rootDir;
     this.db = new Database(file);
     this.db.exec("PRAGMA foreign_keys = ON;");
     this.db.exec("PRAGMA busy_timeout = 5000;");
@@ -1948,9 +1951,12 @@ class Store {
     const rows = this.db.query("SELECT * FROM loops WHERE name = ? ORDER BY created_at DESC LIMIT 2").all(idOrName);
     if (rows.length === 0)
       throw new LoopNotFoundError(idOrName);
-    if (rows.length > 1)
+    if (rows.length === 1)
+      return rowToLoop(rows[0]);
+    const active = this.db.query("SELECT * FROM loops WHERE name = ? AND archived_at IS NULL ORDER BY created_at DESC LIMIT 2").all(idOrName);
+    if (active.length !== 1)
       throw new AmbiguousNameError(idOrName);
-    return rowToLoop(rows[0]);
+    return rowToLoop(active[0]);
   }
   requireLoop(idOrName) {
     return this.getLoop(idOrName) ?? this.findLoopByName(idOrName) ?? (() => {
@@ -2291,6 +2297,7 @@ class Store {
     return this.transact(() => {
       const loop = this.requireLoop(idOrName);
       this.setWorkflowWorkItemsForLoop(loop.id, "cancelled", "loop deleted", nowIso());
+      this.db.query("DELETE FROM loop_runs WHERE loop_id = ?").run(loop.id);
       const res = this.db.query("DELETE FROM loops WHERE id = ?").run(loop.id);
       return res.changes > 0;
     });
@@ -2775,7 +2782,10 @@ class Store {
         return rowToGoal(row);
     }
     if (context.sourceType && context.sourceId) {
-      const row = this.db.query("SELECT * FROM goals WHERE source_type = ? AND source_id = ? ORDER BY created_at DESC LIMIT 1").get(context.sourceType, context.sourceId);
+      const terminalPlaceholders = GOAL_TERMINAL.map(() => "?").join(", ");
+      const row = this.db.query(`SELECT * FROM goals
+           WHERE source_type = ? AND source_id = ? AND status NOT IN (${terminalPlaceholders})
+           ORDER BY created_at DESC LIMIT 1`).get(context.sourceType, context.sourceId, ...GOAL_TERMINAL);
       if (row)
         return rowToGoal(row);
     }
@@ -3195,6 +3205,41 @@ class Store {
       throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
     return run;
   }
+  recordWorkflowStepProgress(workflowRunId, stepId, progress, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs
+           SET stdout=COALESCE($stdout, stdout),
+               stderr=COALESCE($stderr, stderr),
+               updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $stdout: progress.stdout === undefined ? null : scrubbedOrNull(progress.stdout),
+        $stderr: progress.stderr === undefined ? null : scrubbedOrNull(progress.stderr),
+        $updated: now,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: now
+      });
+      if (res.changes === 1) {
+        this.appendWorkflowEvent(workflowRunId, "step_progress", stepId, progress.payload);
+      }
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
+    const run = this.getWorkflowStepRun(workflowRunId, stepId);
+    if (!run)
+      throw new Error(`workflow step run not found after progress update: ${workflowRunId}/${stepId}`);
+    return run;
+  }
   recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
     return this.transact(() => {
       const now = nowIso();
@@ -3632,7 +3677,8 @@ class Store {
                  AND ($daemonLeaseId IS NULL OR EXISTS (
                    SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
                  ))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, claim_token=NULL, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
-               duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
+               duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
+               WHERE id=$id AND status='running'`).run(params);
       const run = this.getRun(id);
       if (!run)
         throw new Error(`run not found after finalize: ${id}`);
@@ -4160,12 +4206,18 @@ class Store {
   }
   close() {
     this.db.close();
+    if (this.memoryRootDir) {
+      try {
+        rmSync2(this.memoryRootDir, { recursive: true, force: true });
+      } catch {}
+      this.memoryRootDir = undefined;
+    }
   }
 }
 // package.json
 var package_default = {
   name: "@hasna/loops",
-  version: "0.4.5",
+  version: "0.4.6",
   description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
   type: "module",
   main: "dist/index.js",
@@ -4623,10 +4675,7 @@ async function stopDaemon(opts = {}) {
   const store = new Store(join4(dirname3(path), "loops.db"));
   try {
     const lease = store.getDaemonLease();
-    if (!lease || lease.pid !== state.pid || new Date(lease.expiresAt).getTime() <= Date.now()) {
-      removePid(path);
-      return { wasRunning: false, stopped: false, forced: false, pid: state.pid };
-    }
+    const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
     try {
       process.kill(state.pid, "SIGTERM");
     } catch {
@@ -4646,11 +4695,14 @@ async function stopDaemon(opts = {}) {
     } catch {}
     await sleep(150);
     removePid(path);
-    const ownedRuns = store.listRuns({ status: "running", limit: 1000 }).filter((run) => run.claimedBy !== undefined && run.claimedBy.endsWith(`:${lease.id}`));
-    const reapedPgids = await reapProcessGroups(ownedRuns.map(toReapableProcess), {
-      sleep,
-      graceMs: opts.reapGraceMs
-    });
+    let reapedPgids = [];
+    if (leaseVerified && lease) {
+      const ownedRuns = store.listRuns({ status: "running", limit: 1000 }).filter((run) => run.claimedBy !== undefined && run.claimedBy.endsWith(`:${lease.id}`));
+      reapedPgids = await reapProcessGroups(ownedRuns.map(toReapableProcess), {
+        sleep,
+        graceMs: opts.reapGraceMs
+      });
+    }
     return { wasRunning: true, stopped: !isAlive(state.pid, record?.startedAt), forced: true, pid: state.pid, reapedPgids };
   } finally {
     store.close();
@@ -4984,7 +5036,12 @@ function notifySpawn(pid, opts) {
   if (!pid)
     return;
   opts.onSpawn?.(pid);
-  opts.onSpawnProcess?.({ pid, pgid: pid, processStartedAt: nowIso() });
+  const startedMs = processStartTimeMs(pid);
+  opts.onSpawnProcess?.({
+    pid,
+    pgid: pid,
+    processStartedAt: startedMs !== undefined ? new Date(startedMs).toISOString() : nowIso()
+  });
 }
 function shellQuote(value) {
   return `'${value.replace(/'/g, `'\\''`)}'`;
@@ -5339,6 +5396,13 @@ function numberField(value, key) {
 function codewithAgentStatus(readJson) {
   return stringField(recordField(readJson, "agent"), "status") ?? stringField(recordField(readJson, "statusSnapshot"), "status");
 }
+function codewithAgentLastEventSeq(readJson) {
+  const agentSeq = numberField(recordField(readJson, "agent"), "lastEventSeq");
+  const snapshotSeq = numberField(recordField(readJson, "statusSnapshot"), "lastEventSeq");
+  if (agentSeq !== undefined && snapshotSeq !== undefined)
+    return Math.max(agentSeq, snapshotSeq);
+  return agentSeq ?? snapshotSeq;
+}
 function codewithAgentEvidence(startJson, readJson, logsJson) {
   const agent = recordField(readJson, "agent") ?? recordField(startJson, "agent");
   const statusSnapshot = recordField(readJson, "statusSnapshot");
@@ -5369,6 +5433,21 @@ function codewithAgentEvidence(startJson, readJson, logsJson) {
     events
   }, null, 2);
 }
+function codewithAgentProgress(readJson) {
+  const agent = recordField(readJson, "agent");
+  const statusSnapshot = recordField(readJson, "statusSnapshot");
+  return {
+    provider: "codewith",
+    agentId: stringField(agent, "agentId"),
+    status: stringField(agent, "status") ?? stringField(statusSnapshot, "status"),
+    summary: stringField(statusSnapshot, "summary"),
+    statusReason: stringField(agent, "statusReason"),
+    threadId: stringField(agent, "threadId"),
+    rolloutPath: stringField(agent, "rolloutPath"),
+    pid: numberField(agent, "pid"),
+    lastEventSeq: codewithAgentLastEventSeq(readJson)
+  };
+}
 function sleep(ms) {
   return new Promise((resolve3) => setTimeout(resolve3, ms));
 }
@@ -5403,6 +5482,7 @@ async function executeCodewithDurableAgent(spec, metadata, opts, env, startedAt)
   if (!agentId) {
     return failureResult(startedAt, "codewith agent start did not return agent.agentId", { stdout: start.stdout, stderr: stderr.value() });
   }
+  opts.onAgentProgress?.(codewithAgentProgress(startJson));
   const stopAgent = async () => {
     await spawnCapture(spec.command, codewithAgentControlArgs(target, "stop", agentId), {
       cwd: spec.cwd,
@@ -5445,10 +5525,15 @@ async function executeCodewithDurableAgent(spec, metadata, opts, env, startedAt)
       });
     }
     const status = codewithAgentStatus(lastReadJson);
-    const fingerprint = JSON.stringify({ status, snapshot: recordField(lastReadJson, "statusSnapshot") ?? null });
+    const fingerprint = JSON.stringify({
+      status,
+      agentLastEventSeq: numberField(recordField(lastReadJson, "agent"), "lastEventSeq"),
+      snapshot: recordField(lastReadJson, "statusSnapshot") ?? null
+    });
     if (fingerprint !== lastFingerprint) {
       lastFingerprint = fingerprint;
       lastProgressAt = Date.now();
+      opts.onAgentProgress?.(codewithAgentProgress(lastReadJson));
     }
     if (status === "completed" || status === "failed" || status === "cancelled") {
       const logs = await spawnCapture(spec.command, codewithAgentControlArgs(target, "logs", agentId), {
@@ -6376,7 +6461,7 @@ ${failure.evidence.stderr}` : undefined
 
 `);
   const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
-  const tags = ["bug", "openloops", "loop-health", failure.classification];
+  const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
   const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
   return {
     title,
@@ -6391,7 +6476,7 @@ ${failure.evidence.stderr}` : undefined
       comment: ["todos", "comment", "<task-id>", description]
     },
     futureNativeUpsert: {
-      command: "todos upsert",
+      command: "todos task upsert",
       fields: {
         title,
         description,
@@ -7595,173 +7680,213 @@ async function executeWorkflow(store, workflow, opts = {}) {
   if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
     return workflowResult(run, run.status, startedAt, run.finishedAt ?? nowIso(), store.listWorkflowStepRuns(run.id), run.error);
   }
+  const resumedRunningSteps = store.listWorkflowStepRuns(run.id).filter((step) => step.status === "running");
+  if (resumedRunningSteps.length > 0 && resumedRunningSteps.every((step) => step.pid !== undefined)) {
+    try {
+      store.recoverWorkflowRun(run.id, "workflow run resumed after lease takeover");
+    } catch {}
+  }
   const ordered = workflowExecutionOrder(workflow);
   const byId = new Map(workflow.steps.map((step) => [step.id, step]));
   let blockingError;
   let terminalStatus = "succeeded";
-  for (const step of ordered) {
-    if (store.isWorkflowRunTerminal(run.id)) {
-      terminalStatus = store.requireWorkflowRun(run.id).status;
-      blockingError = "workflow run was cancelled";
-      break;
-    }
-    const pendingTimeout = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
-    if (pendingTimeout) {
-      terminalStatus = "timed_out";
-      blockingError = pendingTimeout;
-      break;
-    }
-    const existing = store.getWorkflowStepRun(run.id, step.id);
-    if (existing?.status === "succeeded" || existing?.status === "skipped" || existing?.status === "cancelled")
-      continue;
-    const blockedBy = (step.dependsOn ?? []).find((dependencyId) => {
-      const dependencyRun = store.getWorkflowStepRun(run.id, dependencyId);
-      const dependencyStep = byId.get(dependencyId);
-      if (dependencyRun?.status === "succeeded")
-        return false;
-      return !dependencyStep?.continueOnFailure;
-    });
-    if (blockedBy) {
-      opts.beforePersist?.();
-      if (isBlockedStepRun(store.getWorkflowStepRun(run.id, blockedBy))) {
-        store.skipWorkflowStepRun(run.id, step.id, `${BLOCKED_STEP_ERROR_PREFIX} upstream step ${blockedBy} was blocked`, {
-          daemonLeaseId: opts.daemonLeaseId
-        });
+  try {
+    for (const step of ordered) {
+      if (store.isWorkflowRunTerminal(run.id)) {
+        terminalStatus = store.requireWorkflowRun(run.id).status;
+        blockingError = "workflow run was cancelled";
+        break;
+      }
+      const pendingTimeout = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
+      if (pendingTimeout) {
+        terminalStatus = "timed_out";
+        blockingError = pendingTimeout;
+        break;
+      }
+      const existing = store.getWorkflowStepRun(run.id, step.id);
+      if (existing?.status === "succeeded" || existing?.status === "skipped" || existing?.status === "cancelled")
+        continue;
+      const blockedBy = (step.dependsOn ?? []).find((dependencyId) => {
+        const dependencyRun = store.getWorkflowStepRun(run.id, dependencyId);
+        const dependencyStep = byId.get(dependencyId);
+        if (dependencyRun?.status === "succeeded")
+          return false;
+        return !dependencyStep?.continueOnFailure;
+      });
+      if (blockedBy) {
+        opts.beforePersist?.();
+        if (isBlockedStepRun(store.getWorkflowStepRun(run.id, blockedBy))) {
+          store.skipWorkflowStepRun(run.id, step.id, `${BLOCKED_STEP_ERROR_PREFIX} upstream step ${blockedBy} was blocked`, {
+            daemonLeaseId: opts.daemonLeaseId
+          });
+          continue;
+        }
+        store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
+        blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
+        terminalStatus = "failed";
         continue;
       }
-      store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
-      blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
-      terminalStatus = "failed";
-      continue;
-    }
-    opts.beforePersist?.();
-    const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
-    if (startedStep.status !== "running") {
-      terminalStatus = "failed";
-      blockingError = `step ${step.id} could not start because workflow is no longer running`;
-      break;
-    }
-    const stepContext = goalExecutionContext({
-      loop: opts.loop,
-      loopRun: opts.loopRun,
-      scheduledFor: opts.scheduledFor,
-      workflow,
-      workflowRunId: run.id,
-      workflowStepId: step.id
-    });
-    let result;
-    const controller = new AbortController;
-    const externalAbort = () => controller.abort();
-    if (opts.signal?.aborted)
-      controller.abort();
-    opts.signal?.addEventListener("abort", externalAbort, { once: true });
-    const cancelTimer = setInterval(() => {
-      if (store.getWorkflowRun(run.id)?.status === "cancelled")
+      opts.beforePersist?.();
+      const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
+      if (startedStep.status !== "running") {
+        terminalStatus = "failed";
+        blockingError = `step ${step.id} could not start because workflow is no longer running`;
+        break;
+      }
+      const stepContext = goalExecutionContext({
+        loop: opts.loop,
+        loopRun: opts.loopRun,
+        scheduledFor: opts.scheduledFor,
+        workflow,
+        workflowRunId: run.id,
+        workflowStepId: step.id
+      });
+      let result;
+      const controller = new AbortController;
+      const externalAbort = () => controller.abort();
+      if (opts.signal?.aborted)
         controller.abort();
-    }, opts.cancelPollMs ?? 500);
-    cancelTimer.unref();
-    try {
-      if (step.goal) {
-        result = await runGoal(store, step.goal, {
-          ...opts,
-          model: opts.goalModel,
-          target: targetWithStepAccount(step),
-          signal: controller.signal,
-          context: stepContext
-        });
-      } else {
-        result = await executeTarget(targetWithStepAccount(step, opts.goalNodePrompt), executionMetadata(stepContext), {
-          ...opts,
-          machine: opts.machine ?? opts.loop?.machine,
-          signal: controller.signal,
-          onSpawn: (pid) => {
-            opts.beforePersist?.();
-            store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
-            opts.onSpawn?.(pid);
-          }
+      opts.signal?.addEventListener("abort", externalAbort, { once: true });
+      const cancelTimer = setInterval(() => {
+        if (store.getWorkflowRun(run.id)?.status === "cancelled")
+          controller.abort();
+      }, opts.cancelPollMs ?? 500);
+      cancelTimer.unref();
+      try {
+        if (step.goal) {
+          result = await runGoal(store, step.goal, {
+            ...opts,
+            model: opts.goalModel,
+            target: targetWithStepAccount(step),
+            signal: controller.signal,
+            context: stepContext
+          });
+        } else {
+          result = await executeTarget(targetWithStepAccount(step, opts.goalNodePrompt), executionMetadata(stepContext), {
+            ...opts,
+            machine: opts.machine ?? opts.loop?.machine,
+            signal: controller.signal,
+            onAgentProgress: (progress) => {
+              const stdout = JSON.stringify({ agentProgress: progress }, null, 2);
+              opts.beforePersist?.();
+              store.recordWorkflowStepProgress(run.id, step.id, {
+                stdout,
+                payload: progress
+              }, {
+                daemonLeaseId: opts.daemonLeaseId
+              });
+              opts.onAgentProgress?.(progress);
+            },
+            onSpawn: (pid) => {
+              opts.beforePersist?.();
+              store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
+              opts.onSpawn?.(pid);
+            }
+          });
+        }
+      } catch (error) {
+        const finishedAt2 = nowIso();
+        result = {
+          status: "failed",
+          stdout: "",
+          stderr: "",
+          error: error instanceof Error ? error.message : String(error),
+          startedAt: startedStep.startedAt ?? finishedAt2,
+          finishedAt: finishedAt2,
+          durationMs: new Date(finishedAt2).getTime() - new Date(startedStep.startedAt ?? finishedAt2).getTime()
+        };
+      } finally {
+        clearInterval(cancelTimer);
+        opts.signal?.removeEventListener("abort", externalAbort);
+      }
+      const timeoutMessage = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
+      if (timeoutMessage && result.status === "failed") {
+        result = { ...result, status: "timed_out", error: timeoutMessage };
+      }
+      if (store.isWorkflowRunTerminal(run.id)) {
+        terminalStatus = store.requireWorkflowRun(run.id).status;
+        blockingError = "workflow run was cancelled";
+        break;
+      }
+      opts.beforePersist?.();
+      const blockedExit = result.status === "failed" && result.exitCode !== undefined && blockedExitCodesForStep(step).includes(result.exitCode);
+      if (blockedExit) {
+        store.finalizeWorkflowStepRun(run.id, step.id, {
+          status: "skipped",
+          finishedAt: result.finishedAt,
+          durationMs: result.durationMs,
+          stdout: result.stdout,
+          stderr: result.stderr,
+          exitCode: result.exitCode,
+          error: `${BLOCKED_STEP_ERROR_PREFIX} step ${step.id} exited with blocked exit code ${result.exitCode}`
+        }, {
+          daemonLeaseId: opts.daemonLeaseId
         });
+        continue;
       }
-    } catch (error) {
-      const finishedAt2 = nowIso();
-      result = {
-        status: "failed",
-        stdout: "",
-        stderr: "",
-        error: error instanceof Error ? error.message : String(error),
-        startedAt: startedStep.startedAt ?? finishedAt2,
-        finishedAt: finishedAt2,
-        durationMs: new Date(finishedAt2).getTime() - new Date(startedStep.startedAt ?? finishedAt2).getTime()
-      };
-    } finally {
-      clearInterval(cancelTimer);
-      opts.signal?.removeEventListener("abort", externalAbort);
-    }
-    const timeoutMessage = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
-    if (timeoutMessage && result.status === "failed") {
-      result = { ...result, status: "timed_out", error: timeoutMessage };
-    }
-    if (store.isWorkflowRunTerminal(run.id)) {
-      terminalStatus = store.requireWorkflowRun(run.id).status;
-      blockingError = "workflow run was cancelled";
-      break;
-    }
-    opts.beforePersist?.();
-    const blockedExit = result.status === "failed" && result.exitCode !== undefined && blockedExitCodesForStep(step).includes(result.exitCode);
-    if (blockedExit) {
       store.finalizeWorkflowStepRun(run.id, step.id, {
-        status: "skipped",
+        status: result.status,
         finishedAt: result.finishedAt,
         durationMs: result.durationMs,
         stdout: result.stdout,
         stderr: result.stderr,
         exitCode: result.exitCode,
-        error: `${BLOCKED_STEP_ERROR_PREFIX} step ${step.id} exited with blocked exit code ${result.exitCode}`
+        error: result.error
       }, {
         daemonLeaseId: opts.daemonLeaseId
       });
-      continue;
+      if (result.status !== "succeeded" && !step.continueOnFailure) {
+        terminalStatus = result.status;
+        blockingError = `step ${step.id} ${result.status}${result.error ? `: ${result.error}` : ""}`;
+        break;
+      }
     }
-    store.finalizeWorkflowStepRun(run.id, step.id, {
-      status: result.status,
-      finishedAt: result.finishedAt,
-      durationMs: result.durationMs,
-      stdout: result.stdout,
-      stderr: result.stderr,
-      exitCode: result.exitCode,
-      error: result.error
+    if (terminalStatus !== "succeeded") {
+      for (const step of ordered) {
+        const existing = store.getWorkflowStepRun(run.id, step.id);
+        if (existing?.status === "pending" || existing?.status === "running") {
+          store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
+            daemonLeaseId: opts.daemonLeaseId
+          });
+        }
+      }
+    }
+    const finishedAt = nowIso();
+    if (store.isWorkflowRunTerminal(run.id)) {
+      const terminalRun = store.requireWorkflowRun(run.id);
+      return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, store.listWorkflowStepRuns(run.id), terminalRun.error ?? blockingError);
+    }
+    opts.beforePersist?.();
+    const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
+      finishedAt,
+      durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
+      error: blockingError
     }, {
       daemonLeaseId: opts.daemonLeaseId
     });
-    if (result.status !== "succeeded" && !step.continueOnFailure) {
-      terminalStatus = result.status;
-      blockingError = `step ${step.id} ${result.status}${result.error ? `: ${result.error}` : ""}`;
-      break;
-    }
-  }
-  if (terminalStatus !== "succeeded") {
-    for (const step of ordered) {
-      const existing = store.getWorkflowStepRun(run.id, step.id);
-      if (existing?.status === "pending" || existing?.status === "running") {
-        store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
+    return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, store.listWorkflowStepRuns(run.id), blockingError);
+  } catch (error) {
+    if (opts.daemonLeaseId && error instanceof Error && error.message === "daemon lease lost")
+      throw error;
+    if (error instanceof Error && error.message.includes("workflow step is not claimable"))
+      throw error;
+    const message = error instanceof Error ? error.message : String(error);
+    const finishedAt = nowIso();
+    try {
+      if (!store.isWorkflowRunTerminal(run.id)) {
+        store.finalizeWorkflowRun(run.id, "failed", {
+          finishedAt,
+          durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
+          error: message
+        }, {
           daemonLeaseId: opts.daemonLeaseId
         });
       }
-    }
-  }
-  const finishedAt = nowIso();
-  if (store.isWorkflowRunTerminal(run.id)) {
-    const terminalRun = store.requireWorkflowRun(run.id);
-    return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, store.listWorkflowStepRuns(run.id), terminalRun.error ?? blockingError);
+    } catch {}
+    const current = store.getWorkflowRun(run.id) ?? run;
+    const resultStatus = current.status === "running" ? "failed" : current.status;
+    return workflowResult(current, resultStatus, startedAt, finishedAt, store.listWorkflowStepRuns(run.id), current.error ?? message);
   }
-  opts.beforePersist?.();
-  const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
-    finishedAt,
-    durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
-    error: blockingError
-  }, {
-    daemonLeaseId: opts.daemonLeaseId
-  });
-  return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, store.listWorkflowStepRuns(run.id), blockingError);
 }
 function preflightWorkflow(workflow, opts = {}) {
   return workflowExecutionOrder(workflow).map((step) => {
@@ -8132,6 +8257,25 @@ function advanceOptions(deps) {
     onRun: deps.onRun
   };
 }
+var TERMINAL_RUN_STATUSES2 = new Set([
+  "succeeded",
+  "failed",
+  "timed_out",
+  "abandoned",
+  "skipped"
+]);
+function repairWedgedTerminalSlot(deps, loop, scheduledFor, now) {
+  const existing = deps.store.getRunBySlot(loop.id, scheduledFor);
+  if (!existing || !TERMINAL_RUN_STATUSES2.has(existing.status))
+    return;
+  try {
+    advanceLoop(deps.store, loop, existing, new Date(existing.finishedAt ?? now), existing.status === "succeeded", advanceOptions(deps));
+  } catch (error) {
+    if (deps.daemonLeaseId && isDaemonLeaseLost(error))
+      return;
+    throw error;
+  }
+}
 async function runSlot(deps, loop, scheduledFor) {
   const now = deps.now?.() ?? new Date;
   deps.beforeRun?.(loop, scheduledFor);
@@ -8158,8 +8302,10 @@ async function runSlot(deps, loop, scheduledFor) {
       return;
     throw error;
   }
-  if (!claim)
+  if (!claim) {
+    repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
     return;
+  }
   deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
   deps.onRun?.(claim.run);
   const finalRun = await executeClaimedRun({
@@ -8205,8 +8351,10 @@ function claimSlot(deps, loop, scheduledFor) {
       return;
     throw error;
   }
-  if (!claim)
+  if (!claim) {
+    repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
     return;
+  }
   deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
   deps.onRun?.(claim.run);
   return claim;
@@ -8323,13 +8471,19 @@ class LoopsClient {
     return this.store.requireLoop(idOrName);
   }
   pause(idOrName) {
-    return this.store.updateLoop(this.get(idOrName).id, { status: "paused" });
+    return this.store.updateLoop(this.store.requireUniqueLoop(idOrName).id, { status: "paused" });
   }
   resume(idOrName) {
-    return this.store.updateLoop(this.get(idOrName).id, { status: "active" });
+    const loop = this.store.requireUniqueLoop(idOrName);
+    let nextRunAt = loop.nextRunAt;
+    if (!nextRunAt) {
+      const now = new Date;
+      nextRunAt = computeNextAfter(loop.schedule, now, now);
+    }
+    return this.store.updateLoop(loop.id, { status: "active", nextRunAt });
   }
   stop(idOrName) {
-    return this.store.updateLoop(this.get(idOrName).id, { status: "stopped", nextRunAt: undefined });
+    return this.store.updateLoop(this.store.requireUniqueLoop(idOrName).id, { status: "stopped", nextRunAt: undefined });
   }
   archive(idOrName) {
     return this.store.archiveLoop(idOrName);
@@ -8338,7 +8492,7 @@ class LoopsClient {
     return this.store.unarchiveLoop(idOrName);
   }
   delete(idOrName) {
-    return this.store.deleteLoop(idOrName);
+    return this.store.deleteLoop(this.store.requireUniqueLoop(idOrName).id);
   }
   runs(idOrName, filters = {}) {
     let loopId;
@@ -8370,7 +8524,7 @@ class LoopsClient {
     return tick({ store: this.store, runnerId: this.runnerId });
   }
   async runNow(idOrName) {
-    const result = await runLoopNow({ store: this.store, idOrName, runnerId: this.runnerId });
+    const result = await runLoopNow({ store: this.store, idOrName: this.store.requireUniqueLoop(idOrName).id, runnerId: this.runnerId });
     return result.run;
   }
   exportBundle(opts = {}) {