@hasna/loops 0.3.2 → 0.3.4

package/dist/lib/store.js

@@ -345,6 +345,7 @@ function rowToLoop(row) {
     status: row.status,
     schedule: JSON.parse(row.schedule_json),
     target: JSON.parse(row.target_json),
+    machine: row.machine_json ? JSON.parse(row.machine_json) : undefined,
     nextRunAt: row.next_run_at ?? undefined,
     retryScheduledFor: row.retry_scheduled_for ?? undefined,
     catchUp: row.catch_up,
@@ -487,6 +488,7 @@ class Store {
         status TEXT NOT NULL,
         schedule_json TEXT NOT NULL,
         target_json TEXT NOT NULL,
+        machine_json TEXT,
         next_run_at TEXT,
         retry_scheduled_for TEXT,
         catch_up TEXT NOT NULL,
@@ -608,10 +610,21 @@ class Store {
       );
       CREATE INDEX IF NOT EXISTS idx_workflow_events_run_sequence ON workflow_events(workflow_run_id, sequence);
     `);
+    try {
+      this.db.query("ALTER TABLE loops ADD COLUMN machine_json TEXT").run();
+    } catch {}
     try {
       this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
     } catch {}
     this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
+    this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
+  }
+  assertDaemonLeaseFence(opts = {}, now = nowIso()) {
+    if (!opts.daemonLeaseId)
+      return;
+    const row = this.db.query("SELECT id FROM daemon_lease WHERE id = ? AND expires_at > ?").get(opts.daemonLeaseId, now);
+    if (!row)
+      throw new Error("daemon lease lost");
   }
   createLoop(input, from = new Date) {
     const now = nowIso();
@@ -622,6 +635,7 @@ class Store {
       status: "active",
       schedule: input.schedule,
       target: input.target,
+      machine: input.machine,
       nextRunAt: initialNextRun(input.schedule, from),
       catchUp: input.catchUp ?? "latest",
       catchUpLimit: input.catchUpLimit ?? 50,
@@ -633,9 +647,9 @@ class Store {
       createdAt: now,
       updatedAt: now
     };
-    this.db.query(`INSERT INTO loops (id, name, description, status, schedule_json, target_json, next_run_at, retry_scheduled_for,
+    this.db.query(`INSERT INTO loops (id, name, description, status, schedule_json, target_json, machine_json, next_run_at, retry_scheduled_for,
           catch_up, catch_up_limit, overlap, max_attempts, retry_delay_ms, lease_ms, expires_at, created_at, updated_at)
-         VALUES ($id, $name, $description, $status, $schedule, $target, $nextRun, NULL, $catchUp, $catchUpLimit,
+         VALUES ($id, $name, $description, $status, $schedule, $target, $machine, $nextRun, NULL, $catchUp, $catchUpLimit,
           $overlap, $maxAttempts, $retryDelay, $leaseMs, $expiresAt, $created, $updated)`).run({
       $id: loop.id,
       $name: loop.name,
@@ -643,6 +657,7 @@ class Store {
       $status: loop.status,
       $schedule: JSON.stringify(loop.schedule),
       $target: JSON.stringify(loop.target),
+      $machine: loop.machine ? JSON.stringify(loop.machine) : null,
       $nextRun: loop.nextRunAt ?? null,
       $catchUp: loop.catchUp,
       $catchUpLimit: loop.catchUpLimit,
@@ -682,21 +697,31 @@ class Store {
          ORDER BY next_run_at ASC`).all(now.toISOString());
     return rows.map(rowToLoop);
   }
-  updateLoop(id, patch) {
+  updateLoop(id, patch, opts = {}) {
     const current = this.getLoop(id);
     if (!current)
       throw new Error(`loop not found: ${id}`);
-    const merged = { ...current, ...patch, updatedAt: nowIso() };
+    const updated = (opts.now ?? new Date).toISOString();
+    const merged = { ...current, ...patch, updatedAt: updated };
     this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
-         expires_at=$expiresAt, updated_at=$updated WHERE id=$id`).run({
+         expires_at=$expiresAt, updated_at=$updated
+         WHERE id=$id
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
       $id: id,
       $status: merged.status,
       $nextRun: merged.nextRunAt ?? null,
       $retrySlot: merged.retryScheduledFor ?? null,
       $expiresAt: merged.expiresAt ?? null,
-      $updated: merged.updatedAt
+      $updated: merged.updatedAt,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: updated
     });
-    return merged;
+    const after = this.getLoop(id);
+    if (!after)
+      throw new Error(`loop not found after update: ${id}`);
+    return after;
   }
   deleteLoop(idOrName) {
     const loop = this.requireLoop(idOrName);
@@ -760,11 +785,14 @@ class Store {
     const now = nowIso();
     if (input.idempotencyKey) {
       const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
-      if (existing)
+      if (existing) {
+        this.assertDaemonLeaseFence(input);
         return rowToWorkflowRun(existing);
+      }
     }
     this.db.exec("BEGIN IMMEDIATE");
     try {
+      this.assertDaemonLeaseFence(input, now);
       if (input.idempotencyKey) {
         const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
         if (existing) {
@@ -863,31 +891,60 @@ class Store {
     const run = this.getWorkflowRun(workflowRunId);
     return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
   }
-  startWorkflowStepRun(workflowRunId, stepId) {
-    const now = nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs
-         SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
-          pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId
-           AND step_id=$stepId
-           AND status IN ('pending', 'failed', 'timed_out')
-           AND EXISTS (
-             SELECT 1 FROM workflow_runs
-             WHERE id=$workflowRunId AND status='running'
-           )`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
-    const run = this.getWorkflowStepRun(workflowRunId, stepId);
-    if (!run)
-      throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
-    if (res.changes !== 1) {
-      throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
+  startWorkflowStepRun(workflowRunId, stepId, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs
+           SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
+            pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId
+             AND step_id=$stepId
+             AND status IN ('pending', 'failed', 'timed_out')
+             AND EXISTS (
+               SELECT 1 FROM workflow_runs
+               WHERE id=$workflowRunId AND status='running'
+             )
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $started: now,
+        $updated: now,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: now
+      });
+      const run = this.getWorkflowStepRun(workflowRunId, stepId);
+      if (!run)
+        throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
+      if (res.changes !== 1) {
+        throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
+      }
+      this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
+      this.db.exec("COMMIT");
+      return run;
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
     }
-    this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
-    return run;
   }
-  markWorkflowStepPid(workflowRunId, stepId, pid) {
-    const now = nowIso();
+  markWorkflowStepPid(workflowRunId, stepId, pid, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
     this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
+      $workflowRunId: workflowRunId,
+      $stepId: stepId,
+      $pid: pid,
+      $updated: now,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: now
+    });
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
@@ -915,60 +972,110 @@ class Store {
       recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
     };
   }
-  finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
+  finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
-         pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
-      $workflowRunId: workflowRunId,
-      $stepId: stepId,
-      $status: patch.status,
-      $finished: finishedAt,
-      $exitCode: patch.exitCode ?? null,
-      $durationMs: patch.durationMs ?? null,
-      $stdout: patch.stdout ?? null,
-      $stderr: patch.stderr ?? null,
-      $error: patch.error ?? null,
-      $updated: finishedAt
-    });
-    if (res.changes === 1) {
-      this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
-        exitCode: patch.exitCode,
-        error: patch.error
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
+           pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $status: patch.status,
+        $finished: finishedAt,
+        $exitCode: patch.exitCode ?? null,
+        $durationMs: patch.durationMs ?? null,
+        $stdout: patch.stdout ?? null,
+        $stderr: patch.stderr ?? null,
+        $error: patch.error ?? null,
+        $updated: finishedAt,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: (opts.now ?? new Date(finishedAt)).toISOString()
       });
+      if (res.changes === 1) {
+        this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
+          exitCode: patch.exitCode,
+          error: patch.error
+        });
+      }
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
     }
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
     return run;
   }
-  skipWorkflowStepRun(workflowRunId, stepId, reason) {
-    const now = nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
-    if (res.changes === 1)
-      this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
+  skipWorkflowStepRun(workflowRunId, stepId, reason, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $finished: now,
+        $error: reason,
+        $updated: now,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: now
+      });
+      if (res.changes === 1)
+        this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
     return run;
   }
-  finalizeWorkflowRun(workflowRunId, status, patch = {}) {
+  finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
-         WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
-      $id: workflowRunId,
-      $status: status,
-      $finished: finishedAt,
-      $durationMs: patch.durationMs ?? null,
-      $error: patch.error ?? null,
-      $updated: finishedAt
-    });
+    let changed = false;
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
+           WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $id: workflowRunId,
+        $status: status,
+        $finished: finishedAt,
+        $durationMs: patch.durationMs ?? null,
+        $error: patch.error ?? null,
+        $updated: finishedAt,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: (opts.now ?? new Date(finishedAt)).toISOString()
+      });
+      changed = res.changes === 1;
+      if (changed)
+        this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     const run = this.getWorkflowRun(workflowRunId);
     if (!run)
       throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
-    if (res.changes === 1)
-      this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
     return run;
   }
   cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
@@ -1022,10 +1129,24 @@ class Store {
     const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
     return (row?.count ?? 0) > 0;
   }
-  markRunPid(id, pid, claimedBy) {
-    const now = nowIso();
+  markRunPid(id, pid, claimedBy, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
     const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
-             WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({
+      $id: id,
+      $pid: pid,
+      $updated: now,
+      $claimedBy: claimedBy,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: now
+    }) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
+             WHERE id=$id AND status='running'
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({ $id: id, $pid: pid, $updated: now, $daemonLeaseId: opts.daemonLeaseId ?? null, $now: now });
     if (res.changes !== 1)
       return;
     return this.getRun(id);
@@ -1040,7 +1161,7 @@ class Store {
            AND wsr.pid IS NOT NULL`).all(loopRunId);
     return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
   }
-  createSkippedRun(loop, scheduledFor, reason) {
+  createSkippedRun(loop, scheduledFor, reason, opts = {}) {
     const now = nowIso();
     const run = {
       id: genId(),
@@ -1054,21 +1175,31 @@ class Store {
       createdAt: now,
       updatedAt: now
     };
-    this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
-          claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
-         VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
-          NULL, NULL, $error, $created, $updated)`).run({
-      $id: run.id,
-      $loopId: run.loopId,
-      $loopName: run.loopName,
-      $scheduledFor: run.scheduledFor,
-      $attempt: run.attempt,
-      $status: run.status,
-      $finished: run.finishedAt ?? null,
-      $error: run.error ?? null,
-      $created: run.createdAt,
-      $updated: run.updatedAt
-    });
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      this.assertDaemonLeaseFence(opts, now);
+      this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
+            claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
+           VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
+            NULL, NULL, $error, $created, $updated)`).run({
+        $id: run.id,
+        $loopId: run.loopId,
+        $loopName: run.loopName,
+        $scheduledFor: run.scheduledFor,
+        $attempt: run.attempt,
+        $status: run.status,
+        $finished: run.finishedAt ?? null,
+        $error: run.error ?? null,
+        $created: run.createdAt,
+        $updated: run.updatedAt
+      });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     return this.getRunBySlot(loop.id, scheduledFor) ?? run;
   }
   getRun(id) {
@@ -1079,11 +1210,20 @@ class Store {
     const row = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND scheduled_for = ?").get(loopId, scheduledFor);
     return row ? rowToRun(row) : undefined;
   }
-  claimRun(loop, scheduledFor, runnerId, now = new Date) {
+  nextRetryableRun(loopId, maxAttempts, afterScheduledFor) {
+    const row = afterScheduledFor ? this.db.query(`SELECT * FROM loop_runs
+             WHERE loop_id = ? AND scheduled_for > ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
+             ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, afterScheduledFor, maxAttempts) : this.db.query(`SELECT * FROM loop_runs
+             WHERE loop_id = ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
+             ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, maxAttempts);
+    return row ? rowToRun(row) : undefined;
+  }
+  claimRun(loop, scheduledFor, runnerId, now = new Date, opts = {}) {
     const startedAt = now.toISOString();
     const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
     this.db.exec("BEGIN IMMEDIATE");
     try {
+      this.assertDaemonLeaseFence(opts, startedAt);
       const existing = this.getRunBySlot(loop.id, scheduledFor);
       if (existing) {
         if (existing.status === "running") {
@@ -1178,11 +1318,15 @@ class Store {
       $error: patch.error ?? null,
       $updated: finishedAt,
       $claimedBy: opts.claimedBy ?? null,
-      $now: (opts.now ?? new Date).toISOString()
+      $now: (opts.now ?? new Date).toISOString(),
+      $daemonLeaseId: opts.daemonLeaseId ?? null
     };
     const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
              duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
-             WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
              duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
     const run = this.getRun(id);
     if (!run)
@@ -1191,10 +1335,20 @@ class Store {
       return run;
     return run;
   }
-  heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
+  heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
     const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
     const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
-         WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $claimedBy: claimedBy, $expires: expiresAt, $updated: now.toISOString() });
+         WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
+      $id: id,
+      $claimedBy: claimedBy,
+      $expires: expiresAt,
+      $updated: now.toISOString(),
+      $now: now.toISOString(),
+      $daemonLeaseId: opts.daemonLeaseId ?? null
+    });
     if (res.changes !== 1)
       return;
     return this.getRun(id);
@@ -1213,7 +1367,7 @@ class Store {
     }
     return rows.map(rowToRun);
   }
-  recoverExpiredRunLeases(now = new Date) {
+  recoverExpiredRunLeases(now = new Date, opts = {}) {
     const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
     const recovered = [];
     for (const row of rows) {
@@ -1222,20 +1376,63 @@ class Store {
       if (this.hasLiveWorkflowStepProcesses(row.id))
         continue;
       const finished = now.toISOString();
-      this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
-           error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
-      const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
-      for (const workflowRow of workflowRows) {
-        this.db.query(`UPDATE workflow_runs
-             SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
-             WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
-        this.db.query(`UPDATE workflow_step_runs
-             SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
-             WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
-        this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
-          error: "parent loop run lease expired before completion",
-          loopRunId: row.id
+      this.db.exec("BEGIN IMMEDIATE");
+      try {
+        const res = this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
+             error='run lease expired before completion', updated_at=$updated
+             WHERE id=$id AND status='running' AND lease_expires_at <= $now
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({
+          $id: row.id,
+          $finished: finished,
+          $updated: finished,
+          $now: finished,
+          $daemonLeaseId: opts.daemonLeaseId ?? null
         });
+        if (res.changes !== 1) {
+          this.db.exec("COMMIT");
+          continue;
+        }
+        const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
+        for (const workflowRow of workflowRows) {
+          const workflowRes = this.db.query(`UPDATE workflow_runs
+               SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
+               WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
+                 AND ($daemonLeaseId IS NULL OR EXISTS (
+                   SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+                 ))`).run({
+            $id: workflowRow.id,
+            $finished: finished,
+            $updated: finished,
+            $now: finished,
+            $daemonLeaseId: opts.daemonLeaseId ?? null
+          });
+          if (workflowRes.changes !== 1)
+            continue;
+          this.db.query(`UPDATE workflow_step_runs
+               SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
+               WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')
+                 AND ($daemonLeaseId IS NULL OR EXISTS (
+                   SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+                 ))`).run({
+            $workflowRunId: workflowRow.id,
+            $finished: finished,
+            $updated: finished,
+            $now: finished,
+            $daemonLeaseId: opts.daemonLeaseId ?? null
+          });
+          this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
+            error: "parent loop run lease expired before completion",
+            loopRunId: row.id
+          });
+        }
+        this.db.exec("COMMIT");
+      } catch (error) {
+        try {
+          this.db.exec("ROLLBACK");
+        } catch {}
+        throw error;
       }
       const run = this.getRun(row.id);
       if (run)
@@ -1243,11 +1440,14 @@ class Store {
     }
     return recovered;
   }
-  expireLoops(now = new Date) {
+  expireLoops(now = new Date, opts = {}) {
     const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
     const expired = [];
-    for (const row of rows)
-      expired.push(this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }));
+    for (const row of rows) {
+      const updated = this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }, opts);
+      if (updated.status === "expired")
+        expired.push(updated);
+    }
     return expired;
   }
   countLoops(status) {
@@ -1290,7 +1490,7 @@ class Store {
   }
   heartbeatDaemonLease(id, ttlMs, now = new Date) {
     const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
-    const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString() });
+    const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id AND expires_at > $now`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString(), $now: now.toISOString() });
     if (res.changes !== 1)
       return;
     return this.getDaemonLease();