@hasna/loops 0.3.2 → 0.3.3

package/dist/cli/index.js

@@ -615,6 +615,13 @@ class Store {
     } catch {}
     this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
   }
+  assertDaemonLeaseFence(opts = {}, now = nowIso()) {
+    if (!opts.daemonLeaseId)
+      return;
+    const row = this.db.query("SELECT id FROM daemon_lease WHERE id = ? AND expires_at > ?").get(opts.daemonLeaseId, now);
+    if (!row)
+      throw new Error("daemon lease lost");
+  }
   createLoop(input, from = new Date) {
     const now = nowIso();
     const loop = {
@@ -684,21 +691,31 @@ class Store {
          ORDER BY next_run_at ASC`).all(now.toISOString());
     return rows.map(rowToLoop);
   }
-  updateLoop(id, patch) {
+  updateLoop(id, patch, opts = {}) {
     const current = this.getLoop(id);
     if (!current)
       throw new Error(`loop not found: ${id}`);
-    const merged = { ...current, ...patch, updatedAt: nowIso() };
+    const updated = (opts.now ?? new Date).toISOString();
+    const merged = { ...current, ...patch, updatedAt: updated };
     this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
-         expires_at=$expiresAt, updated_at=$updated WHERE id=$id`).run({
+         expires_at=$expiresAt, updated_at=$updated
+         WHERE id=$id
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
       $id: id,
       $status: merged.status,
       $nextRun: merged.nextRunAt ?? null,
       $retrySlot: merged.retryScheduledFor ?? null,
       $expiresAt: merged.expiresAt ?? null,
-      $updated: merged.updatedAt
+      $updated: merged.updatedAt,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: updated
     });
-    return merged;
+    const after = this.getLoop(id);
+    if (!after)
+      throw new Error(`loop not found after update: ${id}`);
+    return after;
   }
   deleteLoop(idOrName) {
     const loop = this.requireLoop(idOrName);
@@ -762,11 +779,14 @@ class Store {
     const now = nowIso();
     if (input.idempotencyKey) {
       const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
-      if (existing)
+      if (existing) {
+        this.assertDaemonLeaseFence(input);
         return rowToWorkflowRun(existing);
+      }
     }
     this.db.exec("BEGIN IMMEDIATE");
     try {
+      this.assertDaemonLeaseFence(input, now);
       if (input.idempotencyKey) {
         const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
         if (existing) {
@@ -865,31 +885,60 @@ class Store {
     const run = this.getWorkflowRun(workflowRunId);
     return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
   }
-  startWorkflowStepRun(workflowRunId, stepId) {
-    const now = nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs
-         SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
-          pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId
-           AND step_id=$stepId
-           AND status IN ('pending', 'failed', 'timed_out')
-           AND EXISTS (
-             SELECT 1 FROM workflow_runs
-             WHERE id=$workflowRunId AND status='running'
-           )`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
-    const run = this.getWorkflowStepRun(workflowRunId, stepId);
-    if (!run)
-      throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
-    if (res.changes !== 1) {
-      throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
+  startWorkflowStepRun(workflowRunId, stepId, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs
+           SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
+            pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId
+             AND step_id=$stepId
+             AND status IN ('pending', 'failed', 'timed_out')
+             AND EXISTS (
+               SELECT 1 FROM workflow_runs
+               WHERE id=$workflowRunId AND status='running'
+             )
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $started: now,
+        $updated: now,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: now
+      });
+      const run = this.getWorkflowStepRun(workflowRunId, stepId);
+      if (!run)
+        throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
+      if (res.changes !== 1) {
+        throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
+      }
+      this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
+      this.db.exec("COMMIT");
+      return run;
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
     }
-    this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
-    return run;
   }
-  markWorkflowStepPid(workflowRunId, stepId, pid) {
-    const now = nowIso();
+  markWorkflowStepPid(workflowRunId, stepId, pid, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
     this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
+      $workflowRunId: workflowRunId,
+      $stepId: stepId,
+      $pid: pid,
+      $updated: now,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: now
+    });
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
@@ -917,60 +966,110 @@ class Store {
       recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
     };
   }
-  finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
+  finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
-         pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
-      $workflowRunId: workflowRunId,
-      $stepId: stepId,
-      $status: patch.status,
-      $finished: finishedAt,
-      $exitCode: patch.exitCode ?? null,
-      $durationMs: patch.durationMs ?? null,
-      $stdout: patch.stdout ?? null,
-      $stderr: patch.stderr ?? null,
-      $error: patch.error ?? null,
-      $updated: finishedAt
-    });
-    if (res.changes === 1) {
-      this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
-        exitCode: patch.exitCode,
-        error: patch.error
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
+           pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $status: patch.status,
+        $finished: finishedAt,
+        $exitCode: patch.exitCode ?? null,
+        $durationMs: patch.durationMs ?? null,
+        $stdout: patch.stdout ?? null,
+        $stderr: patch.stderr ?? null,
+        $error: patch.error ?? null,
+        $updated: finishedAt,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: (opts.now ?? new Date(finishedAt)).toISOString()
       });
+      if (res.changes === 1) {
+        this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
+          exitCode: patch.exitCode,
+          error: patch.error
+        });
+      }
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
     }
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
     return run;
   }
-  skipWorkflowStepRun(workflowRunId, stepId, reason) {
-    const now = nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
-    if (res.changes === 1)
-      this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
+  skipWorkflowStepRun(workflowRunId, stepId, reason, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $finished: now,
+        $error: reason,
+        $updated: now,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: now
+      });
+      if (res.changes === 1)
+        this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
     return run;
   }
-  finalizeWorkflowRun(workflowRunId, status, patch = {}) {
+  finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
-         WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
-      $id: workflowRunId,
-      $status: status,
-      $finished: finishedAt,
-      $durationMs: patch.durationMs ?? null,
-      $error: patch.error ?? null,
-      $updated: finishedAt
-    });
+    let changed = false;
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
+           WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $id: workflowRunId,
+        $status: status,
+        $finished: finishedAt,
+        $durationMs: patch.durationMs ?? null,
+        $error: patch.error ?? null,
+        $updated: finishedAt,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: (opts.now ?? new Date(finishedAt)).toISOString()
+      });
+      changed = res.changes === 1;
+      if (changed)
+        this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     const run = this.getWorkflowRun(workflowRunId);
     if (!run)
       throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
-    if (res.changes === 1)
-      this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
     return run;
   }
   cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
@@ -1024,10 +1123,24 @@ class Store {
     const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
     return (row?.count ?? 0) > 0;
   }
-  markRunPid(id, pid, claimedBy) {
-    const now = nowIso();
+  markRunPid(id, pid, claimedBy, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
     const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
-             WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({
+      $id: id,
+      $pid: pid,
+      $updated: now,
+      $claimedBy: claimedBy,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: now
+    }) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
+             WHERE id=$id AND status='running'
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({ $id: id, $pid: pid, $updated: now, $daemonLeaseId: opts.daemonLeaseId ?? null, $now: now });
     if (res.changes !== 1)
       return;
     return this.getRun(id);
@@ -1042,7 +1155,7 @@ class Store {
            AND wsr.pid IS NOT NULL`).all(loopRunId);
     return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
   }
-  createSkippedRun(loop, scheduledFor, reason) {
+  createSkippedRun(loop, scheduledFor, reason, opts = {}) {
     const now = nowIso();
     const run = {
       id: genId(),
@@ -1056,21 +1169,31 @@ class Store {
       createdAt: now,
       updatedAt: now
     };
-    this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
-          claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
-         VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
-          NULL, NULL, $error, $created, $updated)`).run({
-      $id: run.id,
-      $loopId: run.loopId,
-      $loopName: run.loopName,
-      $scheduledFor: run.scheduledFor,
-      $attempt: run.attempt,
-      $status: run.status,
-      $finished: run.finishedAt ?? null,
-      $error: run.error ?? null,
-      $created: run.createdAt,
-      $updated: run.updatedAt
-    });
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      this.assertDaemonLeaseFence(opts, now);
+      this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
+            claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
+           VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
+            NULL, NULL, $error, $created, $updated)`).run({
+        $id: run.id,
+        $loopId: run.loopId,
+        $loopName: run.loopName,
+        $scheduledFor: run.scheduledFor,
+        $attempt: run.attempt,
+        $status: run.status,
+        $finished: run.finishedAt ?? null,
+        $error: run.error ?? null,
+        $created: run.createdAt,
+        $updated: run.updatedAt
+      });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     return this.getRunBySlot(loop.id, scheduledFor) ?? run;
   }
   getRun(id) {
@@ -1081,11 +1204,20 @@ class Store {
     const row = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND scheduled_for = ?").get(loopId, scheduledFor);
     return row ? rowToRun(row) : undefined;
   }
-  claimRun(loop, scheduledFor, runnerId, now = new Date) {
+  nextRetryableRun(loopId, maxAttempts, afterScheduledFor) {
+    const row = afterScheduledFor ? this.db.query(`SELECT * FROM loop_runs
+             WHERE loop_id = ? AND scheduled_for > ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
+             ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, afterScheduledFor, maxAttempts) : this.db.query(`SELECT * FROM loop_runs
+             WHERE loop_id = ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
+             ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, maxAttempts);
+    return row ? rowToRun(row) : undefined;
+  }
+  claimRun(loop, scheduledFor, runnerId, now = new Date, opts = {}) {
     const startedAt = now.toISOString();
     const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
     this.db.exec("BEGIN IMMEDIATE");
     try {
+      this.assertDaemonLeaseFence(opts, startedAt);
       const existing = this.getRunBySlot(loop.id, scheduledFor);
       if (existing) {
         if (existing.status === "running") {
@@ -1180,11 +1312,15 @@ class Store {
       $error: patch.error ?? null,
       $updated: finishedAt,
       $claimedBy: opts.claimedBy ?? null,
-      $now: (opts.now ?? new Date).toISOString()
+      $now: (opts.now ?? new Date).toISOString(),
+      $daemonLeaseId: opts.daemonLeaseId ?? null
     };
     const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
              duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
-             WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
              duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
     const run = this.getRun(id);
     if (!run)
@@ -1193,10 +1329,20 @@ class Store {
       return run;
     return run;
   }
-  heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
+  heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
     const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
     const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
-         WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $claimedBy: claimedBy, $expires: expiresAt, $updated: now.toISOString() });
+         WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
+      $id: id,
+      $claimedBy: claimedBy,
+      $expires: expiresAt,
+      $updated: now.toISOString(),
+      $now: now.toISOString(),
+      $daemonLeaseId: opts.daemonLeaseId ?? null
+    });
     if (res.changes !== 1)
       return;
     return this.getRun(id);
@@ -1215,7 +1361,7 @@ class Store {
     }
     return rows.map(rowToRun);
   }
-  recoverExpiredRunLeases(now = new Date) {
+  recoverExpiredRunLeases(now = new Date, opts = {}) {
     const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
     const recovered = [];
     for (const row of rows) {
@@ -1224,20 +1370,63 @@ class Store {
       if (this.hasLiveWorkflowStepProcesses(row.id))
         continue;
       const finished = now.toISOString();
-      this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
-           error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
-      const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
-      for (const workflowRow of workflowRows) {
-        this.db.query(`UPDATE workflow_runs
-             SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
-             WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
-        this.db.query(`UPDATE workflow_step_runs
-             SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
-             WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
-        this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
-          error: "parent loop run lease expired before completion",
-          loopRunId: row.id
+      this.db.exec("BEGIN IMMEDIATE");
+      try {
+        const res = this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
+             error='run lease expired before completion', updated_at=$updated
+             WHERE id=$id AND status='running' AND lease_expires_at <= $now
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({
+          $id: row.id,
+          $finished: finished,
+          $updated: finished,
+          $now: finished,
+          $daemonLeaseId: opts.daemonLeaseId ?? null
         });
+        if (res.changes !== 1) {
+          this.db.exec("COMMIT");
+          continue;
+        }
+        const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
+        for (const workflowRow of workflowRows) {
+          const workflowRes = this.db.query(`UPDATE workflow_runs
+               SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
+               WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
+                 AND ($daemonLeaseId IS NULL OR EXISTS (
+                   SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+                 ))`).run({
+            $id: workflowRow.id,
+            $finished: finished,
+            $updated: finished,
+            $now: finished,
+            $daemonLeaseId: opts.daemonLeaseId ?? null
+          });
+          if (workflowRes.changes !== 1)
+            continue;
+          this.db.query(`UPDATE workflow_step_runs
+               SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
+               WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')
+                 AND ($daemonLeaseId IS NULL OR EXISTS (
+                   SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+                 ))`).run({
+            $workflowRunId: workflowRow.id,
+            $finished: finished,
+            $updated: finished,
+            $now: finished,
+            $daemonLeaseId: opts.daemonLeaseId ?? null
+          });
+          this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
+            error: "parent loop run lease expired before completion",
+            loopRunId: row.id
+          });
+        }
+        this.db.exec("COMMIT");
+      } catch (error) {
+        try {
+          this.db.exec("ROLLBACK");
+        } catch {}
+        throw error;
       }
       const run = this.getRun(row.id);
       if (run)
@@ -1245,11 +1434,14 @@ class Store {
     }
     return recovered;
   }
-  expireLoops(now = new Date) {
+  expireLoops(now = new Date, opts = {}) {
     const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
     const expired = [];
-    for (const row of rows)
-      expired.push(this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }));
+    for (const row of rows) {
+      const updated = this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }, opts);
+      if (updated.status === "expired")
+        expired.push(updated);
+    }
     return expired;
   }
   countLoops(status) {
@@ -1292,7 +1484,7 @@ class Store {
   }
   heartbeatDaemonLease(id, ttlMs, now = new Date) {
     const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
-    const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString() });
+    const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id AND expires_at > $now`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString(), $now: now.toISOString() });
     if (res.changes !== 1)
       return;
     return this.getDaemonLease();
@@ -1315,11 +1507,14 @@ import { Command } from "commander";
 
 // src/lib/format.ts
 var TEXT_OUTPUT_LIMIT = 32 * 1024;
-function redact(value, visible = 80) {
+var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
+function redact(value, visible = 0) {
   if (!value)
     return value;
   if (value.length <= visible)
     return value;
+  if (visible <= 0)
+    return `[redacted ${value.length} chars]`;
   return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
 }
 function truncateTextOutput(value) {
@@ -1328,6 +1523,21 @@ function truncateTextOutput(value) {
   return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
 [truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
 }
+function redactSensitivePayload(value, key) {
+  if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
+    if (typeof value === "string")
+      return redact(value);
+    if (value === undefined || value === null)
+      return value;
+    return "[redacted]";
+  }
+  if (Array.isArray(value))
+    return value.map((item) => redactSensitivePayload(item));
+  if (value && typeof value === "object") {
+    return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
+  }
+  return value;
+}
 function textOutputBlocks(value, opts = {}) {
   const indent = opts.indent ?? "";
   const nested = `${indent}  `;
@@ -1359,6 +1569,14 @@ function publicRun(run, showOutput = false) {
     stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined
   };
 }
+function publicExecutorResult(result, showOutput = false) {
+  return {
+    ...result,
+    stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
+    stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
+    error: redact(result.error)
+  };
+}
 function publicWorkflow(workflow) {
   return {
     ...workflow,
@@ -1369,17 +1587,18 @@ function publicWorkflow(workflow) {
   };
 }
 function publicWorkflowRun(run) {
-  return { ...run };
+  return { ...run, error: redact(run.error) };
 }
 function publicWorkflowStepRun(run, showOutput = false) {
   return {
     ...run,
     stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
-    stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined
+    stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
+    error: redact(run.error)
   };
 }
 function publicWorkflowEvent(event) {
-  return { ...event };
+  return { ...event, payload: redactSensitivePayload(event.payload) };
 }
 
 // src/lib/executor.ts
@@ -1615,7 +1834,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "cursor":
       args.push("-p");
@@ -1623,7 +1842,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "codewith":
       args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : [], "--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
@@ -1635,7 +1854,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "codex":
       args.push("exec", "--json", "--ephemeral", "--ask-for-approval", "never", "--sandbox", "workspace-write");
@@ -1645,7 +1864,7 @@ function agentArgs(target) {
         args.push("--cd", target.cwd);
       if (target.model)
         args.push("--model", target.model);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "aicopilot":
       args.push("run", "--format", "json");
@@ -1657,7 +1876,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "opencode":
       args.push("run", "--format", "json");
@@ -1669,7 +1888,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
   }
 }
@@ -1694,7 +1913,8 @@ function commandSpec(target) {
     cwd: agentTarget.cwd,
     timeoutMs: agentTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS,
     account: agentTarget.account,
-    accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider)
+    accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider),
+    stdin: agentTarget.prompt
   };
 }
 function executionEnv(spec, metadata, opts) {
@@ -1763,10 +1983,17 @@ async function executeTarget(target, metadata = {}, opts = {}) {
     env,
     shell: spec.shell ?? false,
     detached: true,
-    stdio: ["ignore", "pipe", "pipe"]
+    stdio: spec.stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"]
   });
   if (child.pid)
     opts.onSpawn?.(child.pid);
+  if (spec.stdin !== undefined && child.stdin) {
+    child.stdin.on("error", (err) => {
+      if (err.code !== "EPIPE")
+        error = err.message;
+    });
+    child.stdin.end(spec.stdin);
+  }
   const abortHandler = () => {
     error = "cancelled";
     if (child.pid)
@@ -1775,10 +2002,10 @@ async function executeTarget(target, metadata = {}, opts = {}) {
   if (opts.signal?.aborted)
     abortHandler();
   opts.signal?.addEventListener("abort", abortHandler, { once: true });
-  child.stdout.on("data", (chunk) => {
+  child.stdout?.on("data", (chunk) => {
     stdout = appendBounded(stdout, chunk, maxOutputBytes);
   });
-  child.stderr.on("data", (chunk) => {
+  child.stderr?.on("data", (chunk) => {
     stderr = appendBounded(stderr, chunk, maxOutputBytes);
   });
   const timer = setTimeout(() => {
@@ -1877,7 +2104,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
     loop: opts.loop,
     loopRun: opts.loopRun,
     scheduledFor: opts.scheduledFor,
-    idempotencyKey: opts.idempotencyKey
+    idempotencyKey: opts.idempotencyKey,
+    daemonLeaseId: opts.daemonLeaseId
   });
   const startedAt = run.startedAt ?? nowIso();
   if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
@@ -1911,12 +2139,14 @@ async function executeWorkflow(store, workflow, opts = {}) {
       return !dependencyStep?.continueOnFailure;
     });
     if (blockedBy) {
-      store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`);
+      opts.beforePersist?.();
+      store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
       blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
       terminalStatus = "failed";
       continue;
     }
-    const startedStep = store.startWorkflowStepRun(run.id, step.id);
+    opts.beforePersist?.();
+    const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
     if (startedStep.status !== "running") {
       terminalStatus = "failed";
       blockingError = `step ${step.id} could not start because workflow is no longer running`;
@@ -1948,7 +2178,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
         ...opts,
         signal: controller.signal,
         onSpawn: (pid) => {
-          store.markWorkflowStepPid(run.id, step.id, pid);
+          opts.beforePersist?.();
+          store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
           opts.onSpawn?.(pid);
         }
       });
@@ -1976,6 +2207,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
       blockingError = "workflow run was cancelled";
       break;
     }
+    opts.beforePersist?.();
     store.finalizeWorkflowStepRun(run.id, step.id, {
       status: result.status,
       finishedAt: result.finishedAt,
@@ -1984,6 +2216,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
       stderr: result.stderr,
       exitCode: result.exitCode,
       error: result.error
+    }, {
+      daemonLeaseId: opts.daemonLeaseId
     });
     if (result.status !== "succeeded" && !step.continueOnFailure) {
       terminalStatus = result.status;
@@ -1995,7 +2229,9 @@ async function executeWorkflow(store, workflow, opts = {}) {
     for (const step of ordered) {
       const existing = store.getWorkflowStepRun(run.id, step.id);
       if (existing?.status === "pending" || existing?.status === "running") {
-        store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run");
+        store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
+          daemonLeaseId: opts.daemonLeaseId
+        });
       }
     }
   }
@@ -2005,10 +2241,13 @@ async function executeWorkflow(store, workflow, opts = {}) {
     const steps2 = store.listWorkflowStepRuns(run.id);
     return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
   }
+  opts.beforePersist?.();
   const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
     finishedAt,
     durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
     error: blockingError
+  }, {
+    daemonLeaseId: opts.daemonLeaseId
   });
   const steps = store.listWorkflowStepRuns(run.id);
   return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
@@ -2056,52 +2295,81 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
 
 // src/lib/scheduler.ts
 function manualRunScheduledFor(loop, now = new Date) {
-  if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
+  if (loop.status === "active" && loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
     return loop.retryScheduledFor ?? loop.nextRunAt;
   }
   return now.toISOString();
 }
 function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
+  if (loop.status !== "active")
+    return false;
   if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
     return false;
   return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
 }
+function manualRunSource(loop, scheduledFor, now = new Date) {
+  if (loop.status !== "active")
+    return "ad_hoc";
+  if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
+    return "ad_hoc";
+  if (loop.retryScheduledFor && scheduledFor === loop.retryScheduledFor)
+    return "retry_slot";
+  return "due_slot";
+}
 function nextAfterRetry(loop, now) {
   return new Date(now.getTime() + loop.retryDelayMs).toISOString();
 }
-function advanceLoop(store, loop, run, finishedAt, succeeded) {
+function isDaemonLeaseLost(error) {
+  return error instanceof Error && error.message === "daemon lease lost";
+}
+function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
   if (run.status === "running")
     return;
   const current = store.getLoop(loop.id);
   if (!current || current.status !== "active")
     return;
-  const shouldRetry = !succeeded && run.attempt < loop.maxAttempts;
+  if (current.retryScheduledFor && current.retryScheduledFor !== run.scheduledFor)
+    return;
+  const shouldRetry = !succeeded && run.attempt < current.maxAttempts;
   if (shouldRetry) {
-    store.updateLoop(loop.id, {
+    store.updateLoop(current.id, {
       status: "active",
-      nextRunAt: nextAfterRetry(loop, finishedAt),
+      nextRunAt: nextAfterRetry(current, finishedAt),
       retryScheduledFor: run.scheduledFor
-    });
+    }, { daemonLeaseId: opts.daemonLeaseId });
     return;
   }
-  const nextRunAt = computeNextAfter(loop.schedule, new Date(run.scheduledFor), finishedAt);
-  store.updateLoop(loop.id, {
+  const deferredRetry = store.nextRetryableRun(current.id, current.maxAttempts, run.scheduledFor);
+  if (deferredRetry) {
+    store.updateLoop(current.id, {
+      status: "active",
+      nextRunAt: nextAfterRetry(current, finishedAt),
+      retryScheduledFor: deferredRetry.scheduledFor
+    }, { daemonLeaseId: opts.daemonLeaseId });
+    return;
+  }
+  const nextRunAt = computeNextAfter(current.schedule, new Date(run.scheduledFor), finishedAt);
+  store.updateLoop(current.id, {
     status: nextRunAt ? "active" : "stopped",
     nextRunAt,
     retryScheduledFor: undefined
-  });
+  }, { daemonLeaseId: opts.daemonLeaseId });
 }
 async function executeClaimedRun(deps) {
   let heartbeat;
   const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
   heartbeat = setInterval(() => {
-    deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs);
+    deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs, new Date, {
+      daemonLeaseId: deps.daemonLeaseId
+    });
   }, heartbeatEveryMs);
   heartbeat.unref();
   try {
     const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
-      onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId)
+      daemonLeaseId: deps.daemonLeaseId,
+      onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
     })))(deps.loop, deps.run);
+    deps.beforeFinalize?.(deps.loop, deps.run);
     return deps.store.finalizeRun(deps.run.id, {
       status: result.status,
       finishedAt: result.finishedAt,
@@ -2113,10 +2381,16 @@ async function executeClaimedRun(deps) {
       pid: result.pid
     }, {
       claimedBy: deps.runnerId,
+      daemonLeaseId: deps.daemonLeaseId,
       now: deps.now?.() ?? new Date(result.finishedAt)
     });
   } catch (err) {
     deps.onError?.(deps.loop, err);
+    try {
+      deps.beforeFinalize?.(deps.loop, deps.run);
+    } catch {
+      return deps.store.getRun(deps.run.id) ?? deps.run;
+    }
     const finishedAt = new Date;
     return deps.store.finalizeRun(deps.run.id, {
       status: "failed",
@@ -2127,6 +2401,7 @@ async function executeClaimedRun(deps) {
       error: err instanceof Error ? err.message : String(err)
     }, {
       claimedBy: deps.runnerId,
+      daemonLeaseId: deps.daemonLeaseId,
       now: deps.now?.() ?? finishedAt
     });
   } finally {
@@ -2136,15 +2411,33 @@ async function executeClaimedRun(deps) {
 }
 async function runSlot(deps, loop, scheduledFor) {
   const now = deps.now?.() ?? new Date;
+  deps.beforeRun?.(loop, scheduledFor);
   if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
-    const skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active");
-    advanceLoop(deps.store, loop, skipped, now, true);
+    let skipped;
+    try {
+      skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
+        daemonLeaseId: deps.daemonLeaseId
+      });
+    } catch (error) {
+      if (deps.daemonLeaseId && isDaemonLeaseLost(error))
+        return;
+      throw error;
+    }
+    advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
     deps.onRun?.(skipped);
     return skipped;
   }
-  const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
+  let claim;
+  try {
+    claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
+  } catch (error) {
+    if (deps.daemonLeaseId && isDaemonLeaseLost(error))
+      return;
+    throw error;
+  }
   if (!claim)
     return;
+  deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
   deps.onRun?.(claim.run);
   const finalRun = await executeClaimedRun({
     store: deps.store,
@@ -2153,21 +2446,42 @@ async function runSlot(deps, loop, scheduledFor) {
     run: claim.run,
     now: deps.now,
     execute: deps.execute,
+    beforeFinalize: deps.beforeFinalize,
+    daemonLeaseId: deps.daemonLeaseId,
     onError: deps.onError
   });
-  advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
+  advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", { daemonLeaseId: deps.daemonLeaseId });
   deps.onRun?.(finalRun);
   return finalRun;
 }
 async function tick(deps) {
   const now = deps.now?.() ?? new Date;
-  const recovered = deps.store.recoverExpiredRunLeases(now);
+  const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
+  const recoveredByLoop = new Map;
   for (const run of recovered) {
-    const loop = deps.store.getLoop(run.loopId);
-    if (loop)
-      advanceLoop(deps.store, loop, run, new Date(run.finishedAt ?? now), false);
+    recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
+  }
+  for (const runs of recoveredByLoop.values()) {
+    const loop = deps.store.getLoop(runs[0].loopId);
+    if (!loop)
+      continue;
+    const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
+    if (retryable) {
+      advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
+        daemonLeaseId: deps.daemonLeaseId
+      });
+      continue;
+    }
+    for (const run of runs) {
+      const current = deps.store.getLoop(run.loopId);
+      if (current) {
+        advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
+          daemonLeaseId: deps.daemonLeaseId
+        });
+      }
+    }
   }
-  const expired = deps.store.expireLoops(now);
+  const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
   const claimed = [];
   const completed = [];
   const skipped = [];
@@ -2388,8 +2702,10 @@ async function runDaemon(opts = {}) {
         const result = await tick({
           store,
           runnerId,
+          daemonLeaseId: leaseId,
+          beforeRun: () => ensureLease(),
           execute: async (loop, run) => {
-            const heartbeatMs = Math.max(1000, Math.floor(leaseTtlMs / 3));
+            const heartbeatMs = Math.max(25, Math.min(1000, intervalMs, Math.floor(leaseTtlMs / 10)));
             const timer = setInterval(() => {
               try {
                 ensureLease();
@@ -2401,8 +2717,14 @@ async function runDaemon(opts = {}) {
             try {
               const result2 = await executeLoopTarget(store, loop, run, {
                 signal: runAbort.signal,
-                onSpawn: (pid) => store.markRunPid(run.id, pid, runnerId)
+                beforePersist: () => ensureLease(),
+                daemonLeaseId: leaseId,
+                onSpawn: (pid) => {
+                  ensureLease();
+                  store.markRunPid(run.id, pid, runnerId, { daemonLeaseId: leaseId });
+                }
               });
+              ensureLease();
               if (leaseLost)
                 throw new Error("daemon lease lost during run");
               return result2;
@@ -2410,6 +2732,7 @@ async function runDaemon(opts = {}) {
               clearInterval(timer);
             }
           },
+          beforeFinalize: () => ensureLease(),
           onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
         });
         const changed = result.completed.length + result.skipped.length + result.recovered.length + result.expired.length;
@@ -2615,7 +2938,7 @@ function runDoctor(store) {
 
 // src/cli/index.ts
 var program = new Command;
-program.name("loops").description("Persistent local loops for commands and headless coding agents").version("0.3.2");
+program.name("loops").description("Persistent local loops for commands and headless coding agents").version("0.3.3");
 program.option("-j, --json", "print JSON");
 function isJson() {
   return Boolean(program.opts().json);
@@ -2834,7 +3157,7 @@ workflows.command("inspect <runId>").description("show a workflow run with steps
     const events = store.listWorkflowEvents(run.id);
     const value = {
       workflowRun: publicWorkflowRun(run),
-      steps: steps.map((step) => publicWorkflowStepRun(step, isJson())),
+      steps: steps.map((step) => publicWorkflowStepRun(step)),
       events: events.map(publicWorkflowEvent)
     };
     if (isJson())
@@ -2842,7 +3165,8 @@ workflows.command("inspect <runId>").description("show a workflow run with steps
     else {
       console.log(`${run.id}  ${run.status}  ${run.workflowName}`);
       for (const step of steps) {
-        console.log(`  ${String(step.sequence).padStart(2, "0")}  ${step.status.padEnd(10)}  ${step.stepId}  ${step.error ?? ""}`);
+        const publicStep = publicWorkflowStepRun(step);
+        console.log(`  ${String(step.sequence).padStart(2, "0")}  ${step.status.padEnd(10)}  ${step.stepId}  ${publicStep.error ?? ""}`);
       }
       console.log(`  events=${events.length}`);
     }
@@ -2858,7 +3182,7 @@ workflows.command("run <idOrName>").option("--show-output", "show step stdout/st
     const run = store.listWorkflowRuns({ workflowId: workflow.id, limit: 1 })[0];
     const steps = run ? store.listWorkflowStepRuns(run.id) : [];
     const value = {
-      result,
+      result: publicExecutorResult(result),
       workflowRun: run ? publicWorkflowRun(run) : undefined,
       steps: steps.map((step) => publicWorkflowStepRun(step, opts.showOutput))
     };
@@ -2867,7 +3191,8 @@ workflows.command("run <idOrName>").option("--show-output", "show step stdout/st
     else {
       console.log(`${run?.id ?? workflow.id} ${result.status}`);
       for (const step of steps) {
-        console.log(`  ${String(step.sequence).padStart(2, "0")}  ${step.status.padEnd(10)}  ${step.stepId}  ${step.error ?? ""}`);
+        const publicStep = publicWorkflowStepRun(step, opts.showOutput);
+        console.log(`  ${String(step.sequence).padStart(2, "0")}  ${step.status.padEnd(10)}  ${step.stepId}  ${publicStep.error ?? ""}`);
         if (opts.showOutput)
           printTextOutput(step);
       }
@@ -3006,16 +3331,27 @@ program.command("run-now <idOrName>").option("--show-output", "show stdout/stder
     const loop = store.requireLoop(idOrName);
     const runnerId = `manual:${process.pid}`;
     const now = new Date;
-    const scheduledFor = manualRunScheduledFor(loop, now);
-    const shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
-    const claim = store.claimRun(loop, scheduledFor, runnerId, now);
+    let scheduledFor = manualRunScheduledFor(loop, now);
+    let source = manualRunSource(loop, scheduledFor, now);
+    let shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
+    let claim = store.claimRun(loop, scheduledFor, runnerId, now);
+    if (!claim && shouldAdvance) {
+      const existing = store.getRunBySlot(loop.id, scheduledFor);
+      if (existing && existing.status !== "running") {
+        scheduledFor = now.toISOString();
+        source = "ad_hoc";
+        shouldAdvance = false;
+        claim = store.claimRun(loop, scheduledFor, runnerId, now);
+      }
+    }
     if (!claim)
       throw new Error("could not claim manual run");
     const run = await executeClaimedRun({ store, runnerId, loop: claim.loop, run: claim.run });
     if (shouldAdvance) {
       advanceLoop(store, claim.loop, run, new Date(run.finishedAt ?? new Date), run.status === "succeeded");
     }
-    print(publicRun(run, opts.showOutput), `${run.id} ${run.status}`);
+    const value = { ...publicRun(run, opts.showOutput), runNow: { source, advancesLoop: shouldAdvance } };
+    print(value, `${run.id} ${run.status} source=${source} slot=${run.scheduledFor}`);
     if (!isJson() && opts.showOutput)
       printTextOutput(run);
     if (run.status !== "succeeded")