@hasna/loops 0.3.2 → 0.3.3

package/dist/index.js

@@ -613,6 +613,13 @@ class Store {
     } catch {}
     this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
   }
+  assertDaemonLeaseFence(opts = {}, now = nowIso()) {
+    if (!opts.daemonLeaseId)
+      return;
+    const row = this.db.query("SELECT id FROM daemon_lease WHERE id = ? AND expires_at > ?").get(opts.daemonLeaseId, now);
+    if (!row)
+      throw new Error("daemon lease lost");
+  }
   createLoop(input, from = new Date) {
     const now = nowIso();
     const loop = {
@@ -682,21 +689,31 @@ class Store {
          ORDER BY next_run_at ASC`).all(now.toISOString());
     return rows.map(rowToLoop);
   }
-  updateLoop(id, patch) {
+  updateLoop(id, patch, opts = {}) {
     const current = this.getLoop(id);
     if (!current)
       throw new Error(`loop not found: ${id}`);
-    const merged = { ...current, ...patch, updatedAt: nowIso() };
+    const updated = (opts.now ?? new Date).toISOString();
+    const merged = { ...current, ...patch, updatedAt: updated };
     this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
-         expires_at=$expiresAt, updated_at=$updated WHERE id=$id`).run({
+         expires_at=$expiresAt, updated_at=$updated
+         WHERE id=$id
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
       $id: id,
       $status: merged.status,
       $nextRun: merged.nextRunAt ?? null,
       $retrySlot: merged.retryScheduledFor ?? null,
       $expiresAt: merged.expiresAt ?? null,
-      $updated: merged.updatedAt
+      $updated: merged.updatedAt,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: updated
     });
-    return merged;
+    const after = this.getLoop(id);
+    if (!after)
+      throw new Error(`loop not found after update: ${id}`);
+    return after;
   }
   deleteLoop(idOrName) {
     const loop = this.requireLoop(idOrName);
@@ -760,11 +777,14 @@ class Store {
     const now = nowIso();
     if (input.idempotencyKey) {
       const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
-      if (existing)
+      if (existing) {
+        this.assertDaemonLeaseFence(input);
         return rowToWorkflowRun(existing);
+      }
     }
     this.db.exec("BEGIN IMMEDIATE");
     try {
+      this.assertDaemonLeaseFence(input, now);
       if (input.idempotencyKey) {
         const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
         if (existing) {
@@ -863,31 +883,60 @@ class Store {
     const run = this.getWorkflowRun(workflowRunId);
     return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
   }
-  startWorkflowStepRun(workflowRunId, stepId) {
-    const now = nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs
-         SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
-          pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId
-           AND step_id=$stepId
-           AND status IN ('pending', 'failed', 'timed_out')
-           AND EXISTS (
-             SELECT 1 FROM workflow_runs
-             WHERE id=$workflowRunId AND status='running'
-           )`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
-    const run = this.getWorkflowStepRun(workflowRunId, stepId);
-    if (!run)
-      throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
-    if (res.changes !== 1) {
-      throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
+  startWorkflowStepRun(workflowRunId, stepId, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs
+           SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
+            pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId
+             AND step_id=$stepId
+             AND status IN ('pending', 'failed', 'timed_out')
+             AND EXISTS (
+               SELECT 1 FROM workflow_runs
+               WHERE id=$workflowRunId AND status='running'
+             )
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $started: now,
+        $updated: now,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: now
+      });
+      const run = this.getWorkflowStepRun(workflowRunId, stepId);
+      if (!run)
+        throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
+      if (res.changes !== 1) {
+        throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
+      }
+      this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
+      this.db.exec("COMMIT");
+      return run;
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
     }
-    this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
-    return run;
   }
-  markWorkflowStepPid(workflowRunId, stepId, pid) {
-    const now = nowIso();
+  markWorkflowStepPid(workflowRunId, stepId, pid, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
     this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
+      $workflowRunId: workflowRunId,
+      $stepId: stepId,
+      $pid: pid,
+      $updated: now,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: now
+    });
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
@@ -915,60 +964,110 @@ class Store {
       recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
     };
   }
-  finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
+  finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
-         pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
-      $workflowRunId: workflowRunId,
-      $stepId: stepId,
-      $status: patch.status,
-      $finished: finishedAt,
-      $exitCode: patch.exitCode ?? null,
-      $durationMs: patch.durationMs ?? null,
-      $stdout: patch.stdout ?? null,
-      $stderr: patch.stderr ?? null,
-      $error: patch.error ?? null,
-      $updated: finishedAt
-    });
-    if (res.changes === 1) {
-      this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
-        exitCode: patch.exitCode,
-        error: patch.error
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
+           pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $status: patch.status,
+        $finished: finishedAt,
+        $exitCode: patch.exitCode ?? null,
+        $durationMs: patch.durationMs ?? null,
+        $stdout: patch.stdout ?? null,
+        $stderr: patch.stderr ?? null,
+        $error: patch.error ?? null,
+        $updated: finishedAt,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: (opts.now ?? new Date(finishedAt)).toISOString()
       });
+      if (res.changes === 1) {
+        this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
+          exitCode: patch.exitCode,
+          error: patch.error
+        });
+      }
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
     }
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
     return run;
   }
-  skipWorkflowStepRun(workflowRunId, stepId, reason) {
-    const now = nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
-    if (res.changes === 1)
-      this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
+  skipWorkflowStepRun(workflowRunId, stepId, reason, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $finished: now,
+        $error: reason,
+        $updated: now,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: now
+      });
+      if (res.changes === 1)
+        this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
     return run;
   }
-  finalizeWorkflowRun(workflowRunId, status, patch = {}) {
+  finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
-         WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
-      $id: workflowRunId,
-      $status: status,
-      $finished: finishedAt,
-      $durationMs: patch.durationMs ?? null,
-      $error: patch.error ?? null,
-      $updated: finishedAt
-    });
+    let changed = false;
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
+           WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $id: workflowRunId,
+        $status: status,
+        $finished: finishedAt,
+        $durationMs: patch.durationMs ?? null,
+        $error: patch.error ?? null,
+        $updated: finishedAt,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: (opts.now ?? new Date(finishedAt)).toISOString()
+      });
+      changed = res.changes === 1;
+      if (changed)
+        this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     const run = this.getWorkflowRun(workflowRunId);
     if (!run)
       throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
-    if (res.changes === 1)
-      this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
     return run;
   }
   cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
@@ -1022,10 +1121,24 @@ class Store {
     const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
     return (row?.count ?? 0) > 0;
   }
-  markRunPid(id, pid, claimedBy) {
-    const now = nowIso();
+  markRunPid(id, pid, claimedBy, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
     const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
-             WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({
+      $id: id,
+      $pid: pid,
+      $updated: now,
+      $claimedBy: claimedBy,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: now
+    }) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
+             WHERE id=$id AND status='running'
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({ $id: id, $pid: pid, $updated: now, $daemonLeaseId: opts.daemonLeaseId ?? null, $now: now });
     if (res.changes !== 1)
       return;
     return this.getRun(id);
@@ -1040,7 +1153,7 @@ class Store {
            AND wsr.pid IS NOT NULL`).all(loopRunId);
     return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
   }
-  createSkippedRun(loop, scheduledFor, reason) {
+  createSkippedRun(loop, scheduledFor, reason, opts = {}) {
     const now = nowIso();
     const run = {
       id: genId(),
@@ -1054,21 +1167,31 @@ class Store {
       createdAt: now,
       updatedAt: now
     };
-    this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
-          claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
-         VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
-          NULL, NULL, $error, $created, $updated)`).run({
-      $id: run.id,
-      $loopId: run.loopId,
-      $loopName: run.loopName,
-      $scheduledFor: run.scheduledFor,
-      $attempt: run.attempt,
-      $status: run.status,
-      $finished: run.finishedAt ?? null,
-      $error: run.error ?? null,
-      $created: run.createdAt,
-      $updated: run.updatedAt
-    });
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      this.assertDaemonLeaseFence(opts, now);
+      this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
+            claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
+           VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
+            NULL, NULL, $error, $created, $updated)`).run({
+        $id: run.id,
+        $loopId: run.loopId,
+        $loopName: run.loopName,
+        $scheduledFor: run.scheduledFor,
+        $attempt: run.attempt,
+        $status: run.status,
+        $finished: run.finishedAt ?? null,
+        $error: run.error ?? null,
+        $created: run.createdAt,
+        $updated: run.updatedAt
+      });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     return this.getRunBySlot(loop.id, scheduledFor) ?? run;
   }
   getRun(id) {
@@ -1079,11 +1202,20 @@ class Store {
     const row = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND scheduled_for = ?").get(loopId, scheduledFor);
     return row ? rowToRun(row) : undefined;
   }
-  claimRun(loop, scheduledFor, runnerId, now = new Date) {
+  nextRetryableRun(loopId, maxAttempts, afterScheduledFor) {
+    const row = afterScheduledFor ? this.db.query(`SELECT * FROM loop_runs
+             WHERE loop_id = ? AND scheduled_for > ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
+             ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, afterScheduledFor, maxAttempts) : this.db.query(`SELECT * FROM loop_runs
+             WHERE loop_id = ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
+             ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, maxAttempts);
+    return row ? rowToRun(row) : undefined;
+  }
+  claimRun(loop, scheduledFor, runnerId, now = new Date, opts = {}) {
     const startedAt = now.toISOString();
     const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
     this.db.exec("BEGIN IMMEDIATE");
     try {
+      this.assertDaemonLeaseFence(opts, startedAt);
       const existing = this.getRunBySlot(loop.id, scheduledFor);
       if (existing) {
         if (existing.status === "running") {
@@ -1178,11 +1310,15 @@ class Store {
       $error: patch.error ?? null,
       $updated: finishedAt,
       $claimedBy: opts.claimedBy ?? null,
-      $now: (opts.now ?? new Date).toISOString()
+      $now: (opts.now ?? new Date).toISOString(),
+      $daemonLeaseId: opts.daemonLeaseId ?? null
     };
     const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
              duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
-             WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
              duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
     const run = this.getRun(id);
     if (!run)
@@ -1191,10 +1327,20 @@ class Store {
       return run;
     return run;
   }
-  heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
+  heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
     const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
     const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
-         WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $claimedBy: claimedBy, $expires: expiresAt, $updated: now.toISOString() });
+         WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
+      $id: id,
+      $claimedBy: claimedBy,
+      $expires: expiresAt,
+      $updated: now.toISOString(),
+      $now: now.toISOString(),
+      $daemonLeaseId: opts.daemonLeaseId ?? null
+    });
     if (res.changes !== 1)
       return;
     return this.getRun(id);
@@ -1213,7 +1359,7 @@ class Store {
     }
     return rows.map(rowToRun);
   }
-  recoverExpiredRunLeases(now = new Date) {
+  recoverExpiredRunLeases(now = new Date, opts = {}) {
     const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
     const recovered = [];
     for (const row of rows) {
@@ -1222,20 +1368,63 @@ class Store {
       if (this.hasLiveWorkflowStepProcesses(row.id))
         continue;
       const finished = now.toISOString();
-      this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
-           error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
-      const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
-      for (const workflowRow of workflowRows) {
-        this.db.query(`UPDATE workflow_runs
-             SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
-             WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
-        this.db.query(`UPDATE workflow_step_runs
-             SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
-             WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
-        this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
-          error: "parent loop run lease expired before completion",
-          loopRunId: row.id
+      this.db.exec("BEGIN IMMEDIATE");
+      try {
+        const res = this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
+             error='run lease expired before completion', updated_at=$updated
+             WHERE id=$id AND status='running' AND lease_expires_at <= $now
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({
+          $id: row.id,
+          $finished: finished,
+          $updated: finished,
+          $now: finished,
+          $daemonLeaseId: opts.daemonLeaseId ?? null
         });
+        if (res.changes !== 1) {
+          this.db.exec("COMMIT");
+          continue;
+        }
+        const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
+        for (const workflowRow of workflowRows) {
+          const workflowRes = this.db.query(`UPDATE workflow_runs
+               SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
+               WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
+                 AND ($daemonLeaseId IS NULL OR EXISTS (
+                   SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+                 ))`).run({
+            $id: workflowRow.id,
+            $finished: finished,
+            $updated: finished,
+            $now: finished,
+            $daemonLeaseId: opts.daemonLeaseId ?? null
+          });
+          if (workflowRes.changes !== 1)
+            continue;
+          this.db.query(`UPDATE workflow_step_runs
+               SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
+               WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')
+                 AND ($daemonLeaseId IS NULL OR EXISTS (
+                   SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+                 ))`).run({
+            $workflowRunId: workflowRow.id,
+            $finished: finished,
+            $updated: finished,
+            $now: finished,
+            $daemonLeaseId: opts.daemonLeaseId ?? null
+          });
+          this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
+            error: "parent loop run lease expired before completion",
+            loopRunId: row.id
+          });
+        }
+        this.db.exec("COMMIT");
+      } catch (error) {
+        try {
+          this.db.exec("ROLLBACK");
+        } catch {}
+        throw error;
       }
       const run = this.getRun(row.id);
       if (run)
@@ -1243,11 +1432,14 @@ class Store {
     }
     return recovered;
   }
-  expireLoops(now = new Date) {
+  expireLoops(now = new Date, opts = {}) {
     const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
     const expired = [];
-    for (const row of rows)
-      expired.push(this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }));
+    for (const row of rows) {
+      const updated = this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }, opts);
+      if (updated.status === "expired")
+        expired.push(updated);
+    }
     return expired;
   }
   countLoops(status) {
@@ -1290,7 +1482,7 @@ class Store {
   }
   heartbeatDaemonLease(id, ttlMs, now = new Date) {
     const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
-    const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString() });
+    const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id AND expires_at > $now`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString(), $now: now.toISOString() });
     if (res.changes !== 1)
       return;
     return this.getDaemonLease();
@@ -1540,7 +1732,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "cursor":
       args.push("-p");
@@ -1548,7 +1740,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "codewith":
       args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : [], "--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
@@ -1560,7 +1752,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "codex":
       args.push("exec", "--json", "--ephemeral", "--ask-for-approval", "never", "--sandbox", "workspace-write");
@@ -1570,7 +1762,7 @@ function agentArgs(target) {
         args.push("--cd", target.cwd);
       if (target.model)
         args.push("--model", target.model);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "aicopilot":
       args.push("run", "--format", "json");
@@ -1582,7 +1774,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "opencode":
       args.push("run", "--format", "json");
@@ -1594,7 +1786,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
   }
 }
@@ -1619,7 +1811,8 @@ function commandSpec(target) {
     cwd: agentTarget.cwd,
     timeoutMs: agentTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS,
     account: agentTarget.account,
-    accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider)
+    accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider),
+    stdin: agentTarget.prompt
   };
 }
 function executionEnv(spec, metadata, opts) {
@@ -1688,10 +1881,17 @@ async function executeTarget(target, metadata = {}, opts = {}) {
     env,
     shell: spec.shell ?? false,
     detached: true,
-    stdio: ["ignore", "pipe", "pipe"]
+    stdio: spec.stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"]
   });
   if (child.pid)
     opts.onSpawn?.(child.pid);
+  if (spec.stdin !== undefined && child.stdin) {
+    child.stdin.on("error", (err) => {
+      if (err.code !== "EPIPE")
+        error = err.message;
+    });
+    child.stdin.end(spec.stdin);
+  }
   const abortHandler = () => {
     error = "cancelled";
     if (child.pid)
@@ -1700,10 +1900,10 @@ async function executeTarget(target, metadata = {}, opts = {}) {
   if (opts.signal?.aborted)
     abortHandler();
   opts.signal?.addEventListener("abort", abortHandler, { once: true });
-  child.stdout.on("data", (chunk) => {
+  child.stdout?.on("data", (chunk) => {
     stdout = appendBounded(stdout, chunk, maxOutputBytes);
   });
-  child.stderr.on("data", (chunk) => {
+  child.stderr?.on("data", (chunk) => {
     stderr = appendBounded(stderr, chunk, maxOutputBytes);
   });
   const timer = setTimeout(() => {
@@ -1802,7 +2002,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
     loop: opts.loop,
     loopRun: opts.loopRun,
     scheduledFor: opts.scheduledFor,
-    idempotencyKey: opts.idempotencyKey
+    idempotencyKey: opts.idempotencyKey,
+    daemonLeaseId: opts.daemonLeaseId
   });
   const startedAt = run.startedAt ?? nowIso();
   if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
@@ -1836,12 +2037,14 @@ async function executeWorkflow(store, workflow, opts = {}) {
       return !dependencyStep?.continueOnFailure;
     });
     if (blockedBy) {
-      store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`);
+      opts.beforePersist?.();
+      store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
       blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
       terminalStatus = "failed";
       continue;
     }
-    const startedStep = store.startWorkflowStepRun(run.id, step.id);
+    opts.beforePersist?.();
+    const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
     if (startedStep.status !== "running") {
       terminalStatus = "failed";
       blockingError = `step ${step.id} could not start because workflow is no longer running`;
@@ -1873,7 +2076,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
         ...opts,
         signal: controller.signal,
         onSpawn: (pid) => {
-          store.markWorkflowStepPid(run.id, step.id, pid);
+          opts.beforePersist?.();
+          store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
           opts.onSpawn?.(pid);
         }
       });
@@ -1901,6 +2105,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
       blockingError = "workflow run was cancelled";
       break;
     }
+    opts.beforePersist?.();
     store.finalizeWorkflowStepRun(run.id, step.id, {
       status: result.status,
       finishedAt: result.finishedAt,
@@ -1909,6 +2114,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
       stderr: result.stderr,
       exitCode: result.exitCode,
       error: result.error
+    }, {
+      daemonLeaseId: opts.daemonLeaseId
     });
     if (result.status !== "succeeded" && !step.continueOnFailure) {
       terminalStatus = result.status;
@@ -1920,7 +2127,9 @@ async function executeWorkflow(store, workflow, opts = {}) {
     for (const step of ordered) {
       const existing = store.getWorkflowStepRun(run.id, step.id);
       if (existing?.status === "pending" || existing?.status === "running") {
-        store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run");
+        store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
+          daemonLeaseId: opts.daemonLeaseId
+        });
       }
     }
   }
@@ -1930,10 +2139,13 @@ async function executeWorkflow(store, workflow, opts = {}) {
     const steps2 = store.listWorkflowStepRuns(run.id);
     return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
   }
+  opts.beforePersist?.();
   const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
     finishedAt,
     durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
     error: blockingError
+  }, {
+    daemonLeaseId: opts.daemonLeaseId
   });
   const steps = store.listWorkflowStepRuns(run.id);
   return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
@@ -1981,52 +2193,81 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
 
 // src/lib/scheduler.ts
 function manualRunScheduledFor(loop, now = new Date) {
-  if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
+  if (loop.status === "active" && loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
     return loop.retryScheduledFor ?? loop.nextRunAt;
   }
   return now.toISOString();
 }
 function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
+  if (loop.status !== "active")
+    return false;
   if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
     return false;
   return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
 }
+function manualRunSource(loop, scheduledFor, now = new Date) {
+  if (loop.status !== "active")
+    return "ad_hoc";
+  if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
+    return "ad_hoc";
+  if (loop.retryScheduledFor && scheduledFor === loop.retryScheduledFor)
+    return "retry_slot";
+  return "due_slot";
+}
 function nextAfterRetry(loop, now) {
   return new Date(now.getTime() + loop.retryDelayMs).toISOString();
 }
-function advanceLoop(store, loop, run, finishedAt, succeeded) {
+function isDaemonLeaseLost(error) {
+  return error instanceof Error && error.message === "daemon lease lost";
+}
+function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
   if (run.status === "running")
     return;
   const current = store.getLoop(loop.id);
   if (!current || current.status !== "active")
     return;
-  const shouldRetry = !succeeded && run.attempt < loop.maxAttempts;
+  if (current.retryScheduledFor && current.retryScheduledFor !== run.scheduledFor)
+    return;
+  const shouldRetry = !succeeded && run.attempt < current.maxAttempts;
   if (shouldRetry) {
-    store.updateLoop(loop.id, {
+    store.updateLoop(current.id, {
       status: "active",
-      nextRunAt: nextAfterRetry(loop, finishedAt),
+      nextRunAt: nextAfterRetry(current, finishedAt),
       retryScheduledFor: run.scheduledFor
-    });
+    }, { daemonLeaseId: opts.daemonLeaseId });
+    return;
+  }
+  const deferredRetry = store.nextRetryableRun(current.id, current.maxAttempts, run.scheduledFor);
+  if (deferredRetry) {
+    store.updateLoop(current.id, {
+      status: "active",
+      nextRunAt: nextAfterRetry(current, finishedAt),
+      retryScheduledFor: deferredRetry.scheduledFor
+    }, { daemonLeaseId: opts.daemonLeaseId });
     return;
   }
-  const nextRunAt = computeNextAfter(loop.schedule, new Date(run.scheduledFor), finishedAt);
-  store.updateLoop(loop.id, {
+  const nextRunAt = computeNextAfter(current.schedule, new Date(run.scheduledFor), finishedAt);
+  store.updateLoop(current.id, {
     status: nextRunAt ? "active" : "stopped",
     nextRunAt,
     retryScheduledFor: undefined
-  });
+  }, { daemonLeaseId: opts.daemonLeaseId });
 }
 async function executeClaimedRun(deps) {
   let heartbeat;
   const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
   heartbeat = setInterval(() => {
-    deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs);
+    deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs, new Date, {
+      daemonLeaseId: deps.daemonLeaseId
+    });
   }, heartbeatEveryMs);
   heartbeat.unref();
   try {
     const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
-      onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId)
+      daemonLeaseId: deps.daemonLeaseId,
+      onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
     })))(deps.loop, deps.run);
+    deps.beforeFinalize?.(deps.loop, deps.run);
     return deps.store.finalizeRun(deps.run.id, {
       status: result.status,
       finishedAt: result.finishedAt,
@@ -2038,10 +2279,16 @@ async function executeClaimedRun(deps) {
       pid: result.pid
     }, {
       claimedBy: deps.runnerId,
+      daemonLeaseId: deps.daemonLeaseId,
       now: deps.now?.() ?? new Date(result.finishedAt)
     });
   } catch (err) {
     deps.onError?.(deps.loop, err);
+    try {
+      deps.beforeFinalize?.(deps.loop, deps.run);
+    } catch {
+      return deps.store.getRun(deps.run.id) ?? deps.run;
+    }
     const finishedAt = new Date;
     return deps.store.finalizeRun(deps.run.id, {
       status: "failed",
@@ -2052,6 +2299,7 @@ async function executeClaimedRun(deps) {
       error: err instanceof Error ? err.message : String(err)
     }, {
       claimedBy: deps.runnerId,
+      daemonLeaseId: deps.daemonLeaseId,
       now: deps.now?.() ?? finishedAt
     });
   } finally {
@@ -2061,15 +2309,33 @@ async function executeClaimedRun(deps) {
 }
 async function runSlot(deps, loop, scheduledFor) {
   const now = deps.now?.() ?? new Date;
+  deps.beforeRun?.(loop, scheduledFor);
   if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
-    const skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active");
-    advanceLoop(deps.store, loop, skipped, now, true);
+    let skipped;
+    try {
+      skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
+        daemonLeaseId: deps.daemonLeaseId
+      });
+    } catch (error) {
+      if (deps.daemonLeaseId && isDaemonLeaseLost(error))
+        return;
+      throw error;
+    }
+    advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
     deps.onRun?.(skipped);
     return skipped;
   }
-  const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
+  let claim;
+  try {
+    claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
+  } catch (error) {
+    if (deps.daemonLeaseId && isDaemonLeaseLost(error))
+      return;
+    throw error;
+  }
   if (!claim)
     return;
+  deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
   deps.onRun?.(claim.run);
   const finalRun = await executeClaimedRun({
     store: deps.store,
@@ -2078,21 +2344,42 @@ async function runSlot(deps, loop, scheduledFor) {
     run: claim.run,
     now: deps.now,
     execute: deps.execute,
+    beforeFinalize: deps.beforeFinalize,
+    daemonLeaseId: deps.daemonLeaseId,
     onError: deps.onError
   });
-  advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
+  advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", { daemonLeaseId: deps.daemonLeaseId });
   deps.onRun?.(finalRun);
   return finalRun;
 }
 async function tick(deps) {
   const now = deps.now?.() ?? new Date;
-  const recovered = deps.store.recoverExpiredRunLeases(now);
+  const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
+  const recoveredByLoop = new Map;
   for (const run of recovered) {
-    const loop = deps.store.getLoop(run.loopId);
-    if (loop)
-      advanceLoop(deps.store, loop, run, new Date(run.finishedAt ?? now), false);
+    recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
   }
-  const expired = deps.store.expireLoops(now);
+  for (const runs of recoveredByLoop.values()) {
+    const loop = deps.store.getLoop(runs[0].loopId);
+    if (!loop)
+      continue;
+    const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
+    if (retryable) {
+      advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
+        daemonLeaseId: deps.daemonLeaseId
+      });
+      continue;
+    }
+    for (const run of runs) {
+      const current = deps.store.getLoop(run.loopId);
+      if (current) {
+        advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
+          daemonLeaseId: deps.daemonLeaseId
+        });
+      }
+    }
+  }
+  const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
   const claimed = [];
   const completed = [];
   const skipped = [];
@@ -2158,9 +2445,17 @@ class LoopsClient {
   async runNow(idOrName) {
     const loop = this.get(idOrName);
     const now = new Date;
-    const scheduledFor = manualRunScheduledFor(loop, now);
-    const shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
-    const claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
+    let scheduledFor = manualRunScheduledFor(loop, now);
+    let shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
+    let claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
+    if (!claim && shouldAdvance) {
+      const existing = this.store.getRunBySlot(loop.id, scheduledFor);
+      if (existing && existing.status !== "running") {
+        scheduledFor = now.toISOString();
+        shouldAdvance = false;
+        claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
+      }
+    }
     if (!claim)
       throw new Error(`could not claim manual run for ${idOrName}`);
     const run = await executeClaimedRun({ store: this.store, runnerId: this.runnerId, loop: claim.loop, run: claim.run });