@hasna/loops 0.3.2 → 0.3.4

package/dist/sdk/index.js

@@ -345,6 +345,7 @@ function rowToLoop(row) {
     status: row.status,
     schedule: JSON.parse(row.schedule_json),
     target: JSON.parse(row.target_json),
+    machine: row.machine_json ? JSON.parse(row.machine_json) : undefined,
     nextRunAt: row.next_run_at ?? undefined,
     retryScheduledFor: row.retry_scheduled_for ?? undefined,
     catchUp: row.catch_up,
@@ -487,6 +488,7 @@ class Store {
         status TEXT NOT NULL,
         schedule_json TEXT NOT NULL,
         target_json TEXT NOT NULL,
+        machine_json TEXT,
         next_run_at TEXT,
         retry_scheduled_for TEXT,
         catch_up TEXT NOT NULL,
@@ -608,10 +610,21 @@ class Store {
       );
       CREATE INDEX IF NOT EXISTS idx_workflow_events_run_sequence ON workflow_events(workflow_run_id, sequence);
     `);
+    try {
+      this.db.query("ALTER TABLE loops ADD COLUMN machine_json TEXT").run();
+    } catch {}
     try {
       this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
     } catch {}
     this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
+    this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
+  }
+  assertDaemonLeaseFence(opts = {}, now = nowIso()) {
+    if (!opts.daemonLeaseId)
+      return;
+    const row = this.db.query("SELECT id FROM daemon_lease WHERE id = ? AND expires_at > ?").get(opts.daemonLeaseId, now);
+    if (!row)
+      throw new Error("daemon lease lost");
   }
   createLoop(input, from = new Date) {
     const now = nowIso();
@@ -622,6 +635,7 @@ class Store {
       status: "active",
       schedule: input.schedule,
       target: input.target,
+      machine: input.machine,
       nextRunAt: initialNextRun(input.schedule, from),
       catchUp: input.catchUp ?? "latest",
       catchUpLimit: input.catchUpLimit ?? 50,
@@ -633,9 +647,9 @@ class Store {
       createdAt: now,
       updatedAt: now
     };
-    this.db.query(`INSERT INTO loops (id, name, description, status, schedule_json, target_json, next_run_at, retry_scheduled_for,
+    this.db.query(`INSERT INTO loops (id, name, description, status, schedule_json, target_json, machine_json, next_run_at, retry_scheduled_for,
           catch_up, catch_up_limit, overlap, max_attempts, retry_delay_ms, lease_ms, expires_at, created_at, updated_at)
-         VALUES ($id, $name, $description, $status, $schedule, $target, $nextRun, NULL, $catchUp, $catchUpLimit,
+         VALUES ($id, $name, $description, $status, $schedule, $target, $machine, $nextRun, NULL, $catchUp, $catchUpLimit,
           $overlap, $maxAttempts, $retryDelay, $leaseMs, $expiresAt, $created, $updated)`).run({
       $id: loop.id,
       $name: loop.name,
@@ -643,6 +657,7 @@ class Store {
       $status: loop.status,
       $schedule: JSON.stringify(loop.schedule),
       $target: JSON.stringify(loop.target),
+      $machine: loop.machine ? JSON.stringify(loop.machine) : null,
       $nextRun: loop.nextRunAt ?? null,
       $catchUp: loop.catchUp,
       $catchUpLimit: loop.catchUpLimit,
@@ -682,21 +697,31 @@ class Store {
          ORDER BY next_run_at ASC`).all(now.toISOString());
     return rows.map(rowToLoop);
   }
-  updateLoop(id, patch) {
+  updateLoop(id, patch, opts = {}) {
     const current = this.getLoop(id);
     if (!current)
       throw new Error(`loop not found: ${id}`);
-    const merged = { ...current, ...patch, updatedAt: nowIso() };
+    const updated = (opts.now ?? new Date).toISOString();
+    const merged = { ...current, ...patch, updatedAt: updated };
     this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
-         expires_at=$expiresAt, updated_at=$updated WHERE id=$id`).run({
+         expires_at=$expiresAt, updated_at=$updated
+         WHERE id=$id
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
       $id: id,
       $status: merged.status,
       $nextRun: merged.nextRunAt ?? null,
       $retrySlot: merged.retryScheduledFor ?? null,
       $expiresAt: merged.expiresAt ?? null,
-      $updated: merged.updatedAt
+      $updated: merged.updatedAt,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: updated
     });
-    return merged;
+    const after = this.getLoop(id);
+    if (!after)
+      throw new Error(`loop not found after update: ${id}`);
+    return after;
   }
   deleteLoop(idOrName) {
     const loop = this.requireLoop(idOrName);
@@ -760,11 +785,14 @@ class Store {
     const now = nowIso();
     if (input.idempotencyKey) {
       const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
-      if (existing)
+      if (existing) {
+        this.assertDaemonLeaseFence(input);
         return rowToWorkflowRun(existing);
+      }
     }
     this.db.exec("BEGIN IMMEDIATE");
     try {
+      this.assertDaemonLeaseFence(input, now);
       if (input.idempotencyKey) {
         const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
         if (existing) {
@@ -863,31 +891,60 @@ class Store {
     const run = this.getWorkflowRun(workflowRunId);
     return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
   }
-  startWorkflowStepRun(workflowRunId, stepId) {
-    const now = nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs
-         SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
-          pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId
-           AND step_id=$stepId
-           AND status IN ('pending', 'failed', 'timed_out')
-           AND EXISTS (
-             SELECT 1 FROM workflow_runs
-             WHERE id=$workflowRunId AND status='running'
-           )`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
-    const run = this.getWorkflowStepRun(workflowRunId, stepId);
-    if (!run)
-      throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
-    if (res.changes !== 1) {
-      throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
+  startWorkflowStepRun(workflowRunId, stepId, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs
+           SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
+            pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId
+             AND step_id=$stepId
+             AND status IN ('pending', 'failed', 'timed_out')
+             AND EXISTS (
+               SELECT 1 FROM workflow_runs
+               WHERE id=$workflowRunId AND status='running'
+             )
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $started: now,
+        $updated: now,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: now
+      });
+      const run = this.getWorkflowStepRun(workflowRunId, stepId);
+      if (!run)
+        throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
+      if (res.changes !== 1) {
+        throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
+      }
+      this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
+      this.db.exec("COMMIT");
+      return run;
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
     }
-    this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
-    return run;
   }
-  markWorkflowStepPid(workflowRunId, stepId, pid) {
-    const now = nowIso();
+  markWorkflowStepPid(workflowRunId, stepId, pid, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
     this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
+      $workflowRunId: workflowRunId,
+      $stepId: stepId,
+      $pid: pid,
+      $updated: now,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: now
+    });
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
@@ -915,60 +972,110 @@ class Store {
       recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
     };
   }
-  finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
+  finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
-         pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
-      $workflowRunId: workflowRunId,
-      $stepId: stepId,
-      $status: patch.status,
-      $finished: finishedAt,
-      $exitCode: patch.exitCode ?? null,
-      $durationMs: patch.durationMs ?? null,
-      $stdout: patch.stdout ?? null,
-      $stderr: patch.stderr ?? null,
-      $error: patch.error ?? null,
-      $updated: finishedAt
-    });
-    if (res.changes === 1) {
-      this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
-        exitCode: patch.exitCode,
-        error: patch.error
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
+           pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $status: patch.status,
+        $finished: finishedAt,
+        $exitCode: patch.exitCode ?? null,
+        $durationMs: patch.durationMs ?? null,
+        $stdout: patch.stdout ?? null,
+        $stderr: patch.stderr ?? null,
+        $error: patch.error ?? null,
+        $updated: finishedAt,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: (opts.now ?? new Date(finishedAt)).toISOString()
       });
+      if (res.changes === 1) {
+        this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
+          exitCode: patch.exitCode,
+          error: patch.error
+        });
+      }
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
     }
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
     return run;
   }
-  skipWorkflowStepRun(workflowRunId, stepId, reason) {
-    const now = nowIso();
-    const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
-    if (res.changes === 1)
-      this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
+  skipWorkflowStepRun(workflowRunId, stepId, reason, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
+           WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $workflowRunId: workflowRunId,
+        $stepId: stepId,
+        $finished: now,
+        $error: reason,
+        $updated: now,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: now
+      });
+      if (res.changes === 1)
+        this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
     return run;
   }
-  finalizeWorkflowRun(workflowRunId, status, patch = {}) {
+  finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
-         WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
-      $id: workflowRunId,
-      $status: status,
-      $finished: finishedAt,
-      $durationMs: patch.durationMs ?? null,
-      $error: patch.error ?? null,
-      $updated: finishedAt
-    });
+    let changed = false;
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
+           WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
+             AND ($daemonLeaseId IS NULL OR EXISTS (
+               SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+             ))`).run({
+        $id: workflowRunId,
+        $status: status,
+        $finished: finishedAt,
+        $durationMs: patch.durationMs ?? null,
+        $error: patch.error ?? null,
+        $updated: finishedAt,
+        $daemonLeaseId: opts.daemonLeaseId ?? null,
+        $now: (opts.now ?? new Date(finishedAt)).toISOString()
+      });
+      changed = res.changes === 1;
+      if (changed)
+        this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     const run = this.getWorkflowRun(workflowRunId);
     if (!run)
       throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
-    if (res.changes === 1)
-      this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
     return run;
   }
   cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
@@ -1022,10 +1129,24 @@ class Store {
     const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
     return (row?.count ?? 0) > 0;
   }
-  markRunPid(id, pid, claimedBy) {
-    const now = nowIso();
+  markRunPid(id, pid, claimedBy, opts = {}) {
+    const now = (opts.now ?? new Date).toISOString();
     const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
-             WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({
+      $id: id,
+      $pid: pid,
+      $updated: now,
+      $claimedBy: claimedBy,
+      $daemonLeaseId: opts.daemonLeaseId ?? null,
+      $now: now
+    }) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
+             WHERE id=$id AND status='running'
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({ $id: id, $pid: pid, $updated: now, $daemonLeaseId: opts.daemonLeaseId ?? null, $now: now });
     if (res.changes !== 1)
       return;
     return this.getRun(id);
@@ -1040,7 +1161,7 @@ class Store {
            AND wsr.pid IS NOT NULL`).all(loopRunId);
     return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
   }
-  createSkippedRun(loop, scheduledFor, reason) {
+  createSkippedRun(loop, scheduledFor, reason, opts = {}) {
     const now = nowIso();
     const run = {
       id: genId(),
@@ -1054,21 +1175,31 @@ class Store {
       createdAt: now,
       updatedAt: now
     };
-    this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
-          claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
-         VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
-          NULL, NULL, $error, $created, $updated)`).run({
-      $id: run.id,
-      $loopId: run.loopId,
-      $loopName: run.loopName,
-      $scheduledFor: run.scheduledFor,
-      $attempt: run.attempt,
-      $status: run.status,
-      $finished: run.finishedAt ?? null,
-      $error: run.error ?? null,
-      $created: run.createdAt,
-      $updated: run.updatedAt
-    });
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      this.assertDaemonLeaseFence(opts, now);
+      this.db.query(`INSERT OR IGNORE INTO loop_runs (id, loop_id, loop_name, scheduled_for, attempt, status, started_at, finished_at,
+            claimed_by, lease_expires_at, pid, exit_code, duration_ms, stdout, stderr, error, created_at, updated_at)
+           VALUES ($id, $loopId, $loopName, $scheduledFor, $attempt, $status, NULL, $finished, NULL, NULL, NULL, NULL, NULL,
+            NULL, NULL, $error, $created, $updated)`).run({
+        $id: run.id,
+        $loopId: run.loopId,
+        $loopName: run.loopName,
+        $scheduledFor: run.scheduledFor,
+        $attempt: run.attempt,
+        $status: run.status,
+        $finished: run.finishedAt ?? null,
+        $error: run.error ?? null,
+        $created: run.createdAt,
+        $updated: run.updatedAt
+      });
+      this.db.exec("COMMIT");
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
     return this.getRunBySlot(loop.id, scheduledFor) ?? run;
   }
   getRun(id) {
@@ -1079,11 +1210,20 @@ class Store {
     const row = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND scheduled_for = ?").get(loopId, scheduledFor);
     return row ? rowToRun(row) : undefined;
   }
-  claimRun(loop, scheduledFor, runnerId, now = new Date) {
+  nextRetryableRun(loopId, maxAttempts, afterScheduledFor) {
+    const row = afterScheduledFor ? this.db.query(`SELECT * FROM loop_runs
+             WHERE loop_id = ? AND scheduled_for > ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
+             ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, afterScheduledFor, maxAttempts) : this.db.query(`SELECT * FROM loop_runs
+             WHERE loop_id = ? AND status IN ('failed', 'timed_out', 'abandoned') AND attempt < ?
+             ORDER BY scheduled_for ASC LIMIT 1`).get(loopId, maxAttempts);
+    return row ? rowToRun(row) : undefined;
+  }
+  claimRun(loop, scheduledFor, runnerId, now = new Date, opts = {}) {
     const startedAt = now.toISOString();
     const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
     this.db.exec("BEGIN IMMEDIATE");
     try {
+      this.assertDaemonLeaseFence(opts, startedAt);
       const existing = this.getRunBySlot(loop.id, scheduledFor);
       if (existing) {
         if (existing.status === "running") {
@@ -1178,11 +1318,15 @@ class Store {
       $error: patch.error ?? null,
       $updated: finishedAt,
       $claimedBy: opts.claimedBy ?? null,
-      $now: (opts.now ?? new Date).toISOString()
+      $now: (opts.now ?? new Date).toISOString(),
+      $daemonLeaseId: opts.daemonLeaseId ?? null
     };
     const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
              duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
-             WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
              duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
     const run = this.getRun(id);
     if (!run)
@@ -1191,10 +1335,20 @@ class Store {
       return run;
     return run;
   }
-  heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
+  heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
     const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
     const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
-         WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $claimedBy: claimedBy, $expires: expiresAt, $updated: now.toISOString() });
+         WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
+           AND ($daemonLeaseId IS NULL OR EXISTS (
+             SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+           ))`).run({
+      $id: id,
+      $claimedBy: claimedBy,
+      $expires: expiresAt,
+      $updated: now.toISOString(),
+      $now: now.toISOString(),
+      $daemonLeaseId: opts.daemonLeaseId ?? null
+    });
     if (res.changes !== 1)
       return;
     return this.getRun(id);
@@ -1213,7 +1367,7 @@ class Store {
     }
     return rows.map(rowToRun);
   }
-  recoverExpiredRunLeases(now = new Date) {
+  recoverExpiredRunLeases(now = new Date, opts = {}) {
     const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
     const recovered = [];
     for (const row of rows) {
@@ -1222,20 +1376,63 @@ class Store {
       if (this.hasLiveWorkflowStepProcesses(row.id))
         continue;
       const finished = now.toISOString();
-      this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
-           error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
-      const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
-      for (const workflowRow of workflowRows) {
-        this.db.query(`UPDATE workflow_runs
-             SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
-             WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
-        this.db.query(`UPDATE workflow_step_runs
-             SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
-             WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
-        this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
-          error: "parent loop run lease expired before completion",
-          loopRunId: row.id
+      this.db.exec("BEGIN IMMEDIATE");
+      try {
+        const res = this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
+             error='run lease expired before completion', updated_at=$updated
+             WHERE id=$id AND status='running' AND lease_expires_at <= $now
+               AND ($daemonLeaseId IS NULL OR EXISTS (
+                 SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+               ))`).run({
+          $id: row.id,
+          $finished: finished,
+          $updated: finished,
+          $now: finished,
+          $daemonLeaseId: opts.daemonLeaseId ?? null
         });
+        if (res.changes !== 1) {
+          this.db.exec("COMMIT");
+          continue;
+        }
+        const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
+        for (const workflowRow of workflowRows) {
+          const workflowRes = this.db.query(`UPDATE workflow_runs
+               SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
+               WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
+                 AND ($daemonLeaseId IS NULL OR EXISTS (
+                   SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+                 ))`).run({
+            $id: workflowRow.id,
+            $finished: finished,
+            $updated: finished,
+            $now: finished,
+            $daemonLeaseId: opts.daemonLeaseId ?? null
+          });
+          if (workflowRes.changes !== 1)
+            continue;
+          this.db.query(`UPDATE workflow_step_runs
+               SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
+               WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')
+                 AND ($daemonLeaseId IS NULL OR EXISTS (
+                   SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
+                 ))`).run({
+            $workflowRunId: workflowRow.id,
+            $finished: finished,
+            $updated: finished,
+            $now: finished,
+            $daemonLeaseId: opts.daemonLeaseId ?? null
+          });
+          this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
+            error: "parent loop run lease expired before completion",
+            loopRunId: row.id
+          });
+        }
+        this.db.exec("COMMIT");
+      } catch (error) {
+        try {
+          this.db.exec("ROLLBACK");
+        } catch {}
+        throw error;
       }
       const run = this.getRun(row.id);
       if (run)
@@ -1243,11 +1440,14 @@ class Store {
     }
     return recovered;
   }
-  expireLoops(now = new Date) {
+  expireLoops(now = new Date, opts = {}) {
     const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
     const expired = [];
-    for (const row of rows)
-      expired.push(this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }));
+    for (const row of rows) {
+      const updated = this.updateLoop(row.id, { status: "expired", nextRunAt: undefined }, opts);
+      if (updated.status === "expired")
+        expired.push(updated);
+    }
     return expired;
   }
   countLoops(status) {
@@ -1290,7 +1490,7 @@ class Store {
   }
   heartbeatDaemonLease(id, ttlMs, now = new Date) {
     const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
-    const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString() });
+    const res = this.db.query(`UPDATE daemon_lease SET heartbeat_at=$heartbeat, expires_at=$expires, updated_at=$updated WHERE id=$id AND expires_at > $now`).run({ $id: id, $heartbeat: now.toISOString(), $expires: expiresAt, $updated: now.toISOString(), $now: now.toISOString() });
     if (res.changes !== 1)
       return;
     return this.getDaemonLease();
@@ -1308,8 +1508,9 @@ class Store {
 }
 
 // src/lib/executor.ts
-import { spawn } from "child_process";
+import { spawn, spawnSync as spawnSync2 } from "child_process";
 import { once } from "events";
+import { resolveMachineCommand } from "@hasna/machines/consumer";
 
 // src/lib/accounts.ts
 import { spawnSync } from "child_process";
@@ -1466,6 +1667,59 @@ function commandNotFoundMessage(command, env = process.env) {
   return `Executable not found in PATH: ${command}. Effective PATH=${env.PATH || "(empty)"}`;
 }
 
+// src/lib/machines.ts
+import {
+  discoverMachineTopology,
+  resolveMachineRoute
+} from "@hasna/machines/consumer";
+function compact(value) {
+  const text = value?.trim();
+  return text ? text : undefined;
+}
+function entryToSummary(entry, topology) {
+  return {
+    id: entry.machine_id,
+    hostname: compact(entry.hostname),
+    platform: compact(entry.platform),
+    user: compact(entry.user),
+    workspacePath: compact(entry.workspace_path),
+    route: entry.ssh.route,
+    local: entry.machine_id === topology.local_machine_id || entry.ssh.route === "local",
+    heartbeatStatus: entry.heartbeat_status,
+    tailscaleOnline: entry.tailscale.online,
+    tags: entry.tags
+  };
+}
+function machineFromRoute(route, topology) {
+  if (!route.ok || !route.machine_id) {
+    throw new Error(`OpenMachines route not found for machine: ${route.requested_machine_id}`);
+  }
+  const entry = topology.machines.find((machine) => machine.machine_id === route.machine_id);
+  return {
+    id: route.machine_id,
+    requestedId: route.requested_machine_id !== route.machine_id ? route.requested_machine_id : undefined,
+    route: route.route,
+    local: route.local,
+    confidence: route.confidence,
+    workspacePath: compact(entry?.workspace_path),
+    resolvedAt: route.generated_at,
+    packageVersion: route.package.version,
+    warnings: route.warnings.length ? route.warnings : undefined
+  };
+}
+function listOpenMachines() {
+  const topology = discoverMachineTopology();
+  return topology.machines.map((entry) => entryToSummary(entry, topology));
+}
+function resolveLoopMachine(machineId) {
+  const topology = discoverMachineTopology();
+  const route = resolveMachineRoute(machineId, { topology });
+  return machineFromRoute(route, topology);
+}
+function refreshLoopMachine(machine) {
+  return resolveLoopMachine(machine.id);
+}
+
 // src/lib/executor.ts
 var DEFAULT_TIMEOUT_MS = 30 * 60000;
 var DEFAULT_MAX_OUTPUT_BYTES = 256 * 1024;
@@ -1486,6 +1740,23 @@ var AUTH_ENV_KEYS = [
   "XDG_STATE_HOME",
   "XDG_CACHE_HOME"
 ];
+var TRANSPORT_ENV_KEYS = new Set([
+  "BUN_INSTALL",
+  "HOME",
+  "LANG",
+  "LANGUAGE",
+  "LOGNAME",
+  "PATH",
+  "SHELL",
+  "SSH_AGENT_PID",
+  "SSH_AUTH_SOCK",
+  "TERM",
+  "TMP",
+  "TMPDIR",
+  "TEMP",
+  "USER",
+  "XDG_RUNTIME_DIR"
+]);
 function appendBounded(current, chunk, maxBytes) {
   const next = current + chunk.toString("utf8");
   if (Buffer.byteLength(next, "utf8") <= maxBytes)
@@ -1512,6 +1783,29 @@ function killProcessGroup(pid) {
     }
   }, 2000).unref();
 }
+function shellQuote(value) {
+  return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+function metadataEnv(metadata) {
+  const env = {};
+  if (metadata.loopId)
+    env.LOOPS_LOOP_ID = metadata.loopId;
+  if (metadata.loopName)
+    env.LOOPS_LOOP_NAME = metadata.loopName;
+  if (metadata.runId)
+    env.LOOPS_RUN_ID = metadata.runId;
+  if (metadata.scheduledFor)
+    env.LOOPS_SCHEDULED_FOR = metadata.scheduledFor;
+  if (metadata.workflowId)
+    env.LOOPS_WORKFLOW_ID = metadata.workflowId;
+  if (metadata.workflowName)
+    env.LOOPS_WORKFLOW_NAME = metadata.workflowName;
+  if (metadata.workflowRunId)
+    env.LOOPS_WORKFLOW_RUN_ID = metadata.workflowRunId;
+  if (metadata.workflowStepId)
+    env.LOOPS_WORKFLOW_STEP_ID = metadata.workflowStepId;
+  return env;
+}
 function providerCommand(provider) {
   switch (provider) {
     case "claude":
@@ -1540,7 +1834,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "cursor":
       args.push("-p");
@@ -1548,7 +1842,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "codewith":
       args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : [], "--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", "workspace-write", "--skip-git-repo-check");
@@ -1560,7 +1854,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "codex":
       args.push("exec", "--json", "--ephemeral", "--ask-for-approval", "never", "--sandbox", "workspace-write");
@@ -1570,7 +1864,7 @@ function agentArgs(target) {
         args.push("--cd", target.cwd);
       if (target.model)
         args.push("--model", target.model);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "aicopilot":
       args.push("run", "--format", "json");
@@ -1582,7 +1876,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
     case "opencode":
       args.push("run", "--format", "json");
@@ -1594,7 +1888,7 @@ function agentArgs(target) {
         args.push("--model", target.model);
       if (target.agent)
         args.push("--agent", target.agent);
-      args.push(...target.extraArgs ?? [], target.prompt);
+      args.push(...target.extraArgs ?? []);
       return args;
   }
 }
@@ -1619,7 +1913,8 @@ function commandSpec(target) {
     cwd: agentTarget.cwd,
     timeoutMs: agentTarget.timeoutMs ?? DEFAULT_TIMEOUT_MS,
     account: agentTarget.account,
-    accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider)
+    accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider),
+    stdin: agentTarget.prompt
   };
 }
 function executionEnv(spec, metadata, opts) {
@@ -1632,26 +1927,213 @@ function executionEnv(spec, metadata, opts) {
   }
   Object.assign(env, spec.env ?? {});
   env.PATH = normalizeExecutionPath(env);
-  if (metadata.loopId)
-    env.LOOPS_LOOP_ID = metadata.loopId;
-  if (metadata.loopName)
-    env.LOOPS_LOOP_NAME = metadata.loopName;
-  if (metadata.runId)
-    env.LOOPS_RUN_ID = metadata.runId;
-  if (metadata.scheduledFor)
-    env.LOOPS_SCHEDULED_FOR = metadata.scheduledFor;
-  if (metadata.workflowId)
-    env.LOOPS_WORKFLOW_ID = metadata.workflowId;
-  if (metadata.workflowName)
-    env.LOOPS_WORKFLOW_NAME = metadata.workflowName;
-  if (metadata.workflowRunId)
-    env.LOOPS_WORKFLOW_RUN_ID = metadata.workflowRunId;
-  if (metadata.workflowStepId)
-    env.LOOPS_WORKFLOW_STEP_ID = metadata.workflowStepId;
+  Object.assign(env, metadataEnv(metadata));
   return env;
 }
+function resolvedMachine(opts) {
+  if (!opts.machine)
+    return;
+  return (opts.machineResolver ?? refreshLoopMachine)(opts.machine);
+}
+function commandForShell(spec) {
+  if (!spec.args.length)
+    return spec.command;
+  return [spec.command, ...spec.args.map(shellQuote)].join(" ");
+}
+function hereDoc(value) {
+  let delimiter2 = `__OPENLOOPS_STDIN_${Math.random().toString(36).slice(2).toUpperCase()}__`;
+  while (value.split(/\r?\n/).includes(delimiter2)) {
+    delimiter2 = `__OPENLOOPS_STDIN_${Math.random().toString(36).slice(2).toUpperCase()}__`;
+  }
+  return [`cat > "$__OPENLOOPS_STDIN" <<'${delimiter2}'`, value, delimiter2];
+}
+function remoteBootstrapLines(spec, metadata) {
+  const lines = [
+    "set -e",
+    'export PATH="$HOME/.local/bin:$HOME/.bun/bin:$HOME/.cargo/bin:$HOME/.npm-global/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin${PATH:+:$PATH}"'
+  ];
+  if (spec.cwd)
+    lines.push(`cd ${shellQuote(spec.cwd)}`);
+  if (spec.account) {
+    if (!spec.accountTool)
+      throw new Error("account.tool is required when no provider tool can be inferred");
+    lines.push("if ! command -v accounts >/dev/null 2>&1; then echo 'accounts CLI is not available on remote machine' >&2; exit 127; fi", `unset ${AUTH_ENV_KEYS.join(" ")}`, `eval "$(accounts env ${shellQuote(spec.account.profile)} --tool ${shellQuote(spec.accountTool)})"`, `export LOOPS_ACCOUNT_PROFILE=${shellQuote(spec.account.profile)}`, `export LOOPS_ACCOUNT_TOOL=${shellQuote(spec.accountTool)}`);
+  }
+  for (const [key, value] of Object.entries({ ...metadataEnv(metadata), ...spec.env ?? {} })) {
+    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key))
+      continue;
+    lines.push(`export ${key}=${shellQuote(value)}`);
+  }
+  return lines;
+}
+function remoteScript(spec, metadata) {
+  const lines = remoteBootstrapLines(spec, metadata);
+  let stdinRedirect = "";
+  if (spec.stdin !== undefined) {
+    lines.push('__OPENLOOPS_STDIN="$(mktemp -t openloops-stdin.XXXXXX)"', `trap 'rm -f "$__OPENLOOPS_STDIN"' EXIT`);
+    lines.push(...hereDoc(spec.stdin));
+    stdinRedirect = ' < "$__OPENLOOPS_STDIN"';
+  }
+  const invocation = spec.shell ? `sh -lc ${shellQuote(commandForShell(spec))}${stdinRedirect}` : `${[spec.command, ...spec.args].map(shellQuote).join(" ")}${stdinRedirect}`;
+  lines.push(invocation);
+  return `${lines.join(`
+`)}
+`;
+}
+function remotePreflightScript(spec, metadata) {
+  return [
+    ...remoteBootstrapLines(spec, metadata),
+    "command -v bash >/dev/null 2>&1",
+    `command -v ${shellQuote(spec.shell ? "sh" : spec.command)} >/dev/null 2>&1`
+  ].join(`
+`);
+}
+function transportEnv(opts) {
+  const source = opts.env ?? process.env;
+  const env = {};
+  for (const [key, value] of Object.entries(source)) {
+    if (value === undefined)
+      continue;
+    if (TRANSPORT_ENV_KEYS.has(key) || key.startsWith("LC_"))
+      env[key] = value;
+  }
+  env.PATH = normalizeExecutionPath(env);
+  return env;
+}
+function preflightRemoteSpec(spec, machine, metadata, opts) {
+  const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
+  const result = spawnSync2(plan.command, plan.args, {
+    encoding: "utf8",
+    env: transportEnv(opts),
+    input: remotePreflightScript(spec, metadata),
+    stdio: ["pipe", "pipe", "pipe"],
+    timeout: 15000
+  });
+  if (result.error)
+    throw new Error(`remote preflight failed on ${machine.id}: ${result.error.message}`);
+  if ((result.status ?? 1) !== 0) {
+    const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
+    throw new Error(`remote preflight failed on ${machine.id}${detail ? `: ${detail}` : ""}`);
+  }
+}
+async function executeRemoteSpec(spec, machine, metadata, opts) {
+  const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
+  const startedAt = nowIso();
+  let stdout = "";
+  let stderr = "";
+  let timedOut = false;
+  let exitCode;
+  let error;
+  let plan;
+  let script;
+  try {
+    plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
+    script = remoteScript(spec, metadata);
+  } catch (err) {
+    return {
+      status: "failed",
+      stdout: "",
+      stderr: "",
+      error: err instanceof Error ? err.message : String(err),
+      startedAt,
+      finishedAt: nowIso(),
+      durationMs: 0
+    };
+  }
+  const child = spawn(plan.command, plan.args, {
+    env: transportEnv(opts),
+    detached: true,
+    stdio: ["pipe", "pipe", "pipe"]
+  });
+  if (child.pid)
+    opts.onSpawn?.(child.pid);
+  child.stdin?.on("error", (err) => {
+    if (err.code !== "EPIPE")
+      error = err.message;
+  });
+  child.stdin?.end(script);
+  const abortHandler = () => {
+    error = "cancelled";
+    if (child.pid)
+      killProcessGroup(child.pid);
+  };
+  if (opts.signal?.aborted)
+    abortHandler();
+  opts.signal?.addEventListener("abort", abortHandler, { once: true });
+  child.stdout?.on("data", (chunk) => {
+    stdout = appendBounded(stdout, chunk, maxOutputBytes);
+  });
+  child.stderr?.on("data", (chunk) => {
+    stderr = appendBounded(stderr, chunk, maxOutputBytes);
+  });
+  const timer = setTimeout(() => {
+    timedOut = true;
+    if (child.pid)
+      killProcessGroup(child.pid);
+  }, spec.timeoutMs);
+  timer.unref();
+  try {
+    const [code, signal] = await once(child, "exit");
+    if (typeof code === "number")
+      exitCode = code;
+    if (signal)
+      error = `terminated by ${signal}`;
+  } catch (err) {
+    error = err instanceof Error ? err.message : String(err);
+  } finally {
+    clearTimeout(timer);
+    opts.signal?.removeEventListener("abort", abortHandler);
+  }
+  const finishedAt = nowIso();
+  const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
+  if (timedOut) {
+    return {
+      status: "timed_out",
+      exitCode,
+      stdout,
+      stderr,
+      error: `timed out after ${spec.timeoutMs}ms`,
+      pid: child.pid,
+      startedAt,
+      finishedAt,
+      durationMs
+    };
+  }
+  if (error || exitCode !== 0) {
+    return {
+      status: "failed",
+      exitCode,
+      stdout,
+      stderr,
+      error: error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`,
+      pid: child.pid,
+      startedAt,
+      finishedAt,
+      durationMs
+    };
+  }
+  return {
+    status: "succeeded",
+    exitCode,
+    stdout,
+    stderr,
+    pid: child.pid,
+    startedAt,
+    finishedAt,
+    durationMs
+  };
+}
 function preflightTarget(target, metadata = {}, opts = {}) {
   const spec = commandSpec(target);
+  const machine = resolvedMachine(opts);
+  if (machine && !machine.local) {
+    preflightRemoteSpec(spec, machine, metadata, opts);
+    return {
+      command: spec.command,
+      accountProfile: spec.account?.profile,
+      accountTool: spec.accountTool
+    };
+  }
   const env = executionEnv(spec, metadata, opts);
   if (!spec.shell && !executableExists(spec.command, env)) {
     throw new Error(commandNotFoundMessage(spec.command, env));
@@ -1664,6 +2146,9 @@ function preflightTarget(target, metadata = {}, opts = {}) {
 }
 async function executeTarget(target, metadata = {}, opts = {}) {
   const spec = commandSpec(target);
+  const machine = resolvedMachine(opts);
+  if (machine && !machine.local)
+    return executeRemoteSpec(spec, machine, metadata, opts);
   const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
   const startedAt = nowIso();
   let stdout = "";
@@ -1688,10 +2173,17 @@ async function executeTarget(target, metadata = {}, opts = {}) {
     env,
     shell: spec.shell ?? false,
     detached: true,
-    stdio: ["ignore", "pipe", "pipe"]
+    stdio: spec.stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"]
   });
   if (child.pid)
     opts.onSpawn?.(child.pid);
+  if (spec.stdin !== undefined && child.stdin) {
+    child.stdin.on("error", (err) => {
+      if (err.code !== "EPIPE")
+        error = err.message;
+    });
+    child.stdin.end(spec.stdin);
+  }
   const abortHandler = () => {
     error = "cancelled";
     if (child.pid)
@@ -1700,10 +2192,10 @@ async function executeTarget(target, metadata = {}, opts = {}) {
   if (opts.signal?.aborted)
     abortHandler();
   opts.signal?.addEventListener("abort", abortHandler, { once: true });
-  child.stdout.on("data", (chunk) => {
+  child.stdout?.on("data", (chunk) => {
     stdout = appendBounded(stdout, chunk, maxOutputBytes);
   });
-  child.stderr.on("data", (chunk) => {
+  child.stderr?.on("data", (chunk) => {
     stderr = appendBounded(stderr, chunk, maxOutputBytes);
   });
   const timer = setTimeout(() => {
@@ -1772,7 +2264,7 @@ async function executeLoop(loop, run, opts = {}) {
     loopName: loop.name,
     runId: run.id,
     scheduledFor: run.scheduledFor
-  }, opts);
+  }, { ...opts, machine: opts.machine ?? loop.machine });
 }
 
 // src/lib/workflow-runner.ts
@@ -1802,7 +2294,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
     loop: opts.loop,
     loopRun: opts.loopRun,
     scheduledFor: opts.scheduledFor,
-    idempotencyKey: opts.idempotencyKey
+    idempotencyKey: opts.idempotencyKey,
+    daemonLeaseId: opts.daemonLeaseId
   });
   const startedAt = run.startedAt ?? nowIso();
   if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
@@ -1836,12 +2329,14 @@ async function executeWorkflow(store, workflow, opts = {}) {
       return !dependencyStep?.continueOnFailure;
     });
     if (blockedBy) {
-      store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`);
+      opts.beforePersist?.();
+      store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
       blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
       terminalStatus = "failed";
       continue;
     }
-    const startedStep = store.startWorkflowStepRun(run.id, step.id);
+    opts.beforePersist?.();
+    const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
     if (startedStep.status !== "running") {
       terminalStatus = "failed";
       blockingError = `step ${step.id} could not start because workflow is no longer running`;
@@ -1871,9 +2366,11 @@ async function executeWorkflow(store, workflow, opts = {}) {
     try {
       result = await executeTarget(targetWithStepAccount(step), metadata, {
         ...opts,
+        machine: opts.machine ?? opts.loop?.machine,
         signal: controller.signal,
         onSpawn: (pid) => {
-          store.markWorkflowStepPid(run.id, step.id, pid);
+          opts.beforePersist?.();
+          store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
           opts.onSpawn?.(pid);
         }
       });
@@ -1901,6 +2398,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
       blockingError = "workflow run was cancelled";
       break;
     }
+    opts.beforePersist?.();
     store.finalizeWorkflowStepRun(run.id, step.id, {
       status: result.status,
       finishedAt: result.finishedAt,
@@ -1909,6 +2407,8 @@ async function executeWorkflow(store, workflow, opts = {}) {
       stderr: result.stderr,
       exitCode: result.exitCode,
       error: result.error
+    }, {
+      daemonLeaseId: opts.daemonLeaseId
     });
     if (result.status !== "succeeded" && !step.continueOnFailure) {
       terminalStatus = result.status;
@@ -1920,7 +2420,9 @@ async function executeWorkflow(store, workflow, opts = {}) {
     for (const step of ordered) {
       const existing = store.getWorkflowStepRun(run.id, step.id);
       if (existing?.status === "pending" || existing?.status === "running") {
-        store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run");
+        store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
+          daemonLeaseId: opts.daemonLeaseId
+        });
       }
     }
   }
@@ -1930,10 +2432,13 @@ async function executeWorkflow(store, workflow, opts = {}) {
     const steps2 = store.listWorkflowStepRuns(run.id);
     return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
   }
+  opts.beforePersist?.();
   const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
     finishedAt,
     durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
     error: blockingError
+  }, {
+    daemonLeaseId: opts.daemonLeaseId
   });
   const steps = store.listWorkflowStepRuns(run.id);
   return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
@@ -1981,52 +2486,81 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
 
 // src/lib/scheduler.ts
 function manualRunScheduledFor(loop, now = new Date) {
-  if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
+  if (loop.status === "active" && loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
     return loop.retryScheduledFor ?? loop.nextRunAt;
   }
   return now.toISOString();
 }
 function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
+  if (loop.status !== "active")
+    return false;
   if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
     return false;
   return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
 }
+function manualRunSource(loop, scheduledFor, now = new Date) {
+  if (loop.status !== "active")
+    return "ad_hoc";
+  if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
+    return "ad_hoc";
+  if (loop.retryScheduledFor && scheduledFor === loop.retryScheduledFor)
+    return "retry_slot";
+  return "due_slot";
+}
 function nextAfterRetry(loop, now) {
   return new Date(now.getTime() + loop.retryDelayMs).toISOString();
 }
-function advanceLoop(store, loop, run, finishedAt, succeeded) {
+function isDaemonLeaseLost(error) {
+  return error instanceof Error && error.message === "daemon lease lost";
+}
+function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
   if (run.status === "running")
     return;
   const current = store.getLoop(loop.id);
   if (!current || current.status !== "active")
     return;
-  const shouldRetry = !succeeded && run.attempt < loop.maxAttempts;
+  if (current.retryScheduledFor && current.retryScheduledFor !== run.scheduledFor)
+    return;
+  const shouldRetry = !succeeded && run.attempt < current.maxAttempts;
   if (shouldRetry) {
-    store.updateLoop(loop.id, {
+    store.updateLoop(current.id, {
       status: "active",
-      nextRunAt: nextAfterRetry(loop, finishedAt),
+      nextRunAt: nextAfterRetry(current, finishedAt),
       retryScheduledFor: run.scheduledFor
-    });
+    }, { daemonLeaseId: opts.daemonLeaseId });
     return;
   }
-  const nextRunAt = computeNextAfter(loop.schedule, new Date(run.scheduledFor), finishedAt);
-  store.updateLoop(loop.id, {
+  const deferredRetry = store.nextRetryableRun(current.id, current.maxAttempts, run.scheduledFor);
+  if (deferredRetry) {
+    store.updateLoop(current.id, {
+      status: "active",
+      nextRunAt: nextAfterRetry(current, finishedAt),
+      retryScheduledFor: deferredRetry.scheduledFor
+    }, { daemonLeaseId: opts.daemonLeaseId });
+    return;
+  }
+  const nextRunAt = computeNextAfter(current.schedule, new Date(run.scheduledFor), finishedAt);
+  store.updateLoop(current.id, {
     status: nextRunAt ? "active" : "stopped",
     nextRunAt,
     retryScheduledFor: undefined
-  });
+  }, { daemonLeaseId: opts.daemonLeaseId });
 }
 async function executeClaimedRun(deps) {
   let heartbeat;
   const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
   heartbeat = setInterval(() => {
-    deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs);
+    deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs, new Date, {
+      daemonLeaseId: deps.daemonLeaseId
+    });
   }, heartbeatEveryMs);
   heartbeat.unref();
   try {
     const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
-      onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId)
+      daemonLeaseId: deps.daemonLeaseId,
+      onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
     })))(deps.loop, deps.run);
+    deps.beforeFinalize?.(deps.loop, deps.run);
     return deps.store.finalizeRun(deps.run.id, {
       status: result.status,
       finishedAt: result.finishedAt,
@@ -2038,10 +2572,16 @@ async function executeClaimedRun(deps) {
       pid: result.pid
     }, {
       claimedBy: deps.runnerId,
+      daemonLeaseId: deps.daemonLeaseId,
       now: deps.now?.() ?? new Date(result.finishedAt)
     });
   } catch (err) {
     deps.onError?.(deps.loop, err);
+    try {
+      deps.beforeFinalize?.(deps.loop, deps.run);
+    } catch {
+      return deps.store.getRun(deps.run.id) ?? deps.run;
+    }
     const finishedAt = new Date;
     return deps.store.finalizeRun(deps.run.id, {
       status: "failed",
@@ -2052,6 +2592,7 @@ async function executeClaimedRun(deps) {
       error: err instanceof Error ? err.message : String(err)
     }, {
       claimedBy: deps.runnerId,
+      daemonLeaseId: deps.daemonLeaseId,
       now: deps.now?.() ?? finishedAt
     });
   } finally {
@@ -2061,15 +2602,33 @@ async function executeClaimedRun(deps) {
 }
 async function runSlot(deps, loop, scheduledFor) {
   const now = deps.now?.() ?? new Date;
+  deps.beforeRun?.(loop, scheduledFor);
   if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
-    const skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active");
-    advanceLoop(deps.store, loop, skipped, now, true);
+    let skipped;
+    try {
+      skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
+        daemonLeaseId: deps.daemonLeaseId
+      });
+    } catch (error) {
+      if (deps.daemonLeaseId && isDaemonLeaseLost(error))
+        return;
+      throw error;
+    }
+    advanceLoop(deps.store, loop, skipped, now, true, { daemonLeaseId: deps.daemonLeaseId });
     deps.onRun?.(skipped);
     return skipped;
   }
-  const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
+  let claim;
+  try {
+    claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
+  } catch (error) {
+    if (deps.daemonLeaseId && isDaemonLeaseLost(error))
+      return;
+    throw error;
+  }
   if (!claim)
     return;
+  deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
   deps.onRun?.(claim.run);
   const finalRun = await executeClaimedRun({
     store: deps.store,
@@ -2078,21 +2637,42 @@ async function runSlot(deps, loop, scheduledFor) {
     run: claim.run,
     now: deps.now,
     execute: deps.execute,
+    beforeFinalize: deps.beforeFinalize,
+    daemonLeaseId: deps.daemonLeaseId,
     onError: deps.onError
   });
-  advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
+  advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", { daemonLeaseId: deps.daemonLeaseId });
   deps.onRun?.(finalRun);
   return finalRun;
 }
 async function tick(deps) {
   const now = deps.now?.() ?? new Date;
-  const recovered = deps.store.recoverExpiredRunLeases(now);
+  const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
+  const recoveredByLoop = new Map;
   for (const run of recovered) {
-    const loop = deps.store.getLoop(run.loopId);
-    if (loop)
-      advanceLoop(deps.store, loop, run, new Date(run.finishedAt ?? now), false);
+    recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
+  }
+  for (const runs of recoveredByLoop.values()) {
+    const loop = deps.store.getLoop(runs[0].loopId);
+    if (!loop)
+      continue;
+    const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
+    if (retryable) {
+      advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
+        daemonLeaseId: deps.daemonLeaseId
+      });
+      continue;
+    }
+    for (const run of runs) {
+      const current = deps.store.getLoop(run.loopId);
+      if (current) {
+        advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
+          daemonLeaseId: deps.daemonLeaseId
+        });
+      }
+    }
   }
-  const expired = deps.store.expireLoops(now);
+  const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
   const claimed = [];
   const completed = [];
   const skipped = [];
@@ -2158,9 +2738,17 @@ class LoopsClient {
   async runNow(idOrName) {
     const loop = this.get(idOrName);
     const now = new Date;
-    const scheduledFor = manualRunScheduledFor(loop, now);
-    const shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
-    const claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
+    let scheduledFor = manualRunScheduledFor(loop, now);
+    let shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
+    let claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
+    if (!claim && shouldAdvance) {
+      const existing = this.store.getRunBySlot(loop.id, scheduledFor);
+      if (existing && existing.status !== "running") {
+        scheduledFor = now.toISOString();
+        shouldAdvance = false;
+        claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
+      }
+    }
     if (!claim)
       throw new Error(`could not claim manual run for ${idOrName}`);
     const run = await executeClaimedRun({ store: this.store, runnerId: this.runnerId, loop: claim.loop, run: claim.run });