@hasna/loops 0.2.0 → 0.3.0

package/dist/index.js

@@ -247,6 +247,9 @@ function validateTarget(value, label) {
   assertObject(value, label);
   if (value.type === "command") {
     assertString(value.command, `${label}.command`);
+    if (value.shell !== true && /\s/.test(value.command.trim())) {
+      throw new Error(`${label}.command must be an executable without spaces when shell is false; put flags in args or set shell true`);
+    }
     return value;
   }
   if (value.type === "agent") {
@@ -412,6 +415,7 @@ function rowToWorkflowStepRun(row) {
     startedAt: row.started_at ?? undefined,
     finishedAt: row.finished_at ?? undefined,
     exitCode: row.exit_code ?? undefined,
+    pid: row.pid ?? undefined,
     durationMs: row.duration_ms ?? undefined,
     stdout: row.stdout ?? undefined,
     stderr: row.stderr ?? undefined,
@@ -433,6 +437,14 @@ function rowToWorkflowEvent(row) {
     createdAt: row.created_at
   };
 }
+function isProcessAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
 function rowToLease(row) {
   return {
     id: row.id,
@@ -565,6 +577,7 @@ class Store {
         started_at TEXT,
         finished_at TEXT,
         exit_code INTEGER,
+        pid INTEGER,
         duration_ms INTEGER,
         stdout TEXT,
         stderr TEXT,
@@ -590,6 +603,9 @@ class Store {
       );
       CREATE INDEX IF NOT EXISTS idx_workflow_events_run_sequence ON workflow_events(workflow_run_id, sequence);
     `);
+    try {
+      this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
+    } catch {}
     this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
   }
   createLoop(input, from = new Date) {
@@ -770,8 +786,8 @@ class Store {
       input.workflow.steps.forEach((step, sequence) => {
         const account = step.account ?? step.target.account;
         this.db.query(`INSERT INTO workflow_step_runs (id, workflow_run_id, step_id, sequence, status, started_at, finished_at,
-              exit_code, duration_ms, stdout, stderr, error, account_profile, account_tool, created_at, updated_at)
-             VALUES ($id, $workflowRunId, $stepId, $sequence, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+              exit_code, pid, duration_ms, stdout, stderr, error, account_profile, account_tool, created_at, updated_at)
+             VALUES ($id, $workflowRunId, $stepId, $sequence, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
               $accountProfile, $accountTool, $created, $updated)`).run({
           $id: genId(),
           $workflowRunId: runId,
@@ -812,6 +828,12 @@ class Store {
     const row = this.db.query("SELECT * FROM workflow_runs WHERE id = ?").get(id);
     return row ? rowToWorkflowRun(row) : undefined;
   }
+  requireWorkflowRun(id) {
+    const run = this.getWorkflowRun(id);
+    if (!run)
+      throw new Error(`workflow run not found: ${id}`);
+    return run;
+  }
   listWorkflowRuns(opts = {}) {
     const limit = opts.limit ?? 100;
     let rows;
@@ -832,23 +854,67 @@ class Store {
     const row = this.db.query("SELECT * FROM workflow_step_runs WHERE workflow_run_id = ? AND step_id = ?").get(workflowRunId, stepId);
     return row ? rowToWorkflowStepRun(row) : undefined;
   }
+  isWorkflowRunTerminal(workflowRunId) {
+    const run = this.getWorkflowRun(workflowRunId);
+    return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
+  }
   startWorkflowStepRun(workflowRunId, stepId) {
     const now = nowIso();
-    this.db.query(`UPDATE workflow_step_runs
+    const res = this.db.query(`UPDATE workflow_step_runs
          SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
-          stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running', 'failed', 'timed_out')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
-    this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
+          pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId
+           AND step_id=$stepId
+           AND status IN ('pending', 'failed', 'timed_out')
+           AND EXISTS (
+             SELECT 1 FROM workflow_runs
+             WHERE id=$workflowRunId AND status='running'
+           )`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
+    if (res.changes !== 1) {
+      throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
+    }
+    this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
     return run;
   }
+  markWorkflowStepPid(workflowRunId, stepId, pid) {
+    const now = nowIso();
+    this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
+    const run = this.getWorkflowStepRun(workflowRunId, stepId);
+    if (!run)
+      throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
+    return run;
+  }
+  recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
+    const now = nowIso();
+    const before = this.listWorkflowStepRuns(workflowRunId).filter((step) => step.status === "running");
+    const live = before.filter((step) => step.pid !== undefined && isProcessAlive(step.pid));
+    if (live.length > 0) {
+      throw new Error(`cannot recover workflow run while step processes are still alive: ${live.map((step) => `${step.stepId} pid=${step.pid}`).join(", ")}`);
+    }
+    this.db.query(`UPDATE workflow_step_runs
+         SET status='pending', started_at=NULL, finished_at=NULL, exit_code=NULL, pid=NULL, duration_ms=NULL,
+          stdout=NULL, stderr=NULL, error=$reason, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND status='running'`).run({ $workflowRunId: workflowRunId, $reason: reason, $updated: now });
+    if (before.length > 0) {
+      this.appendWorkflowEvent(workflowRunId, "recovered", undefined, {
+        reason,
+        recoveredSteps: before.map((step) => step.stepId)
+      });
+    }
+    return {
+      run: this.requireWorkflowRun(workflowRunId),
+      recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
+    };
+  }
   finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
-         stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId`).run({
+    const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
+         pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
       $workflowRunId: workflowRunId,
       $stepId: stepId,
       $status: patch.status,
@@ -860,10 +926,12 @@ class Store {
       $error: patch.error ?? null,
       $updated: finishedAt
     });
-    this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
-      exitCode: patch.exitCode,
-      error: patch.error
-    });
+    if (res.changes === 1) {
+      this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
+        exitCode: patch.exitCode,
+        error: patch.error
+      });
+    }
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
@@ -871,9 +939,10 @@ class Store {
   }
   skipWorkflowStepRun(workflowRunId, stepId, reason) {
     const now = nowIso();
-    this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
-    this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
+    const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
+    if (res.changes === 1)
+      this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
@@ -881,8 +950,8 @@ class Store {
   }
   finalizeWorkflowRun(workflowRunId, status, patch = {}) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
-         WHERE id=$id`).run({
+    const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
+         WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
       $id: workflowRunId,
       $status: status,
       $finished: finishedAt,
@@ -890,12 +959,36 @@ class Store {
       $error: patch.error ?? null,
       $updated: finishedAt
     });
-    this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
     const run = this.getWorkflowRun(workflowRunId);
     if (!run)
       throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
+    if (res.changes === 1)
+      this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
     return run;
   }
+  cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
+    const now = nowIso();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const run = this.requireWorkflowRun(workflowRunId);
+      if (!["succeeded", "failed", "timed_out", "cancelled"].includes(run.status)) {
+        this.db.query(`UPDATE workflow_runs
+             SET status='cancelled', finished_at=$finished, error=$reason, updated_at=$updated
+             WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRunId, $finished: now, $reason: reason, $updated: now });
+        this.db.query(`UPDATE workflow_step_runs
+             SET status='cancelled', finished_at=$finished, pid=NULL, error=$reason, updated_at=$updated
+             WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $finished: now, $reason: reason, $updated: now });
+        this.appendWorkflowEvent(workflowRunId, "cancelled", undefined, { reason });
+      }
+      this.db.exec("COMMIT");
+      return this.requireWorkflowRun(workflowRunId);
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
+  }
   appendWorkflowEvent(workflowRunId, eventType, stepId, payload) {
     const now = nowIso();
     const current = this.db.query("SELECT MAX(sequence) AS sequence FROM workflow_events WHERE workflow_run_id = ?").get(workflowRunId);
@@ -924,6 +1017,24 @@ class Store {
     const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
     return (row?.count ?? 0) > 0;
   }
+  markRunPid(id, pid, claimedBy) {
+    const now = nowIso();
+    const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
+    if (res.changes !== 1)
+      return;
+    return this.getRun(id);
+  }
+  hasLiveWorkflowStepProcesses(loopRunId) {
+    const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid
+         FROM workflow_runs wr
+         JOIN workflow_step_runs wsr ON wsr.workflow_run_id = wr.id
+         WHERE wr.loop_run_id = ?
+           AND wr.status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
+           AND wsr.status = 'running'
+           AND wsr.pid IS NOT NULL`).all(loopRunId);
+    return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
+  }
   createSkippedRun(loop, scheduledFor, reason) {
     const now = nowIso();
     const run = {
@@ -971,6 +1082,14 @@ class Store {
       const existing = this.getRunBySlot(loop.id, scheduledFor);
       if (existing) {
         if (existing.status === "running") {
+          if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && existing.pid && isProcessAlive(existing.pid)) {
+            this.db.exec("COMMIT");
+            return;
+          }
+          if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && this.hasLiveWorkflowStepProcesses(existing.id)) {
+            this.db.exec("COMMIT");
+            return;
+          }
           const res3 = this.db.query(`UPDATE loop_runs SET status='running', started_at=$started, finished_at=NULL,
                claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
                duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
@@ -1093,8 +1212,26 @@ class Store {
     const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
     const recovered = [];
     for (const row of rows) {
+      if (row.pid && isProcessAlive(row.pid))
+        continue;
+      if (this.hasLiveWorkflowStepProcesses(row.id))
+        continue;
+      const finished = now.toISOString();
       this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
-           error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: now.toISOString(), $updated: now.toISOString() });
+           error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
+      const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
+      for (const workflowRow of workflowRows) {
+        this.db.query(`UPDATE workflow_runs
+             SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
+             WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
+        this.db.query(`UPDATE workflow_step_runs
+             SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
+             WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
+        this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
+          error: "parent loop run lease expired before completion",
+          loopRunId: row.id
+        });
+      }
       const run = this.getRun(row.id);
       if (run)
         recovered.push(run);
@@ -1166,8 +1303,9 @@ class Store {
 }
 
 // src/lib/executor.ts
-import { spawn } from "child_process";
+import { spawn, spawnSync as spawnSync2 } from "child_process";
 import { once } from "events";
+import { existsSync as existsSync2 } from "fs";
 
 // src/lib/accounts.ts
 import { spawnSync } from "child_process";
@@ -1215,6 +1353,25 @@ function primaryAccountDir(output) {
   }
   return;
 }
+function accountDirEnvVar(tool) {
+  switch (tool) {
+    case "claude":
+      return "CLAUDE_CONFIG_DIR";
+    case "codex":
+    case "codex-app":
+      return "CODEX_HOME";
+    case "cursor":
+      return "CURSOR_CONFIG_DIR";
+    case "opencode":
+      return "OPENCODE_CONFIG_DIR";
+    case "codewith":
+      return "CODEWITH_HOME";
+    case "aicopilot":
+      return "AICOPILOT_CONFIG_DIR";
+    default:
+      return;
+  }
+}
 function resolveAccountEnv(account, toolHint, env) {
   if (!account)
     return {};
@@ -1233,13 +1390,14 @@ function resolveAccountEnv(account, toolHint, env) {
     const stderr = result.stderr.trim();
     throw new Error(`accounts env failed for ${account.profile}/${tool}${stderr ? `: ${stderr}` : ""}`);
   }
-  const profileDir = primaryAccountDir(result.stdout);
+  const accountEnv = parseAccountExportLines(result.stdout);
+  const profileDir = (accountDirEnvVar(tool) ? accountEnv[accountDirEnvVar(tool)] : undefined) ?? primaryAccountDir(result.stdout);
   if (!profileDir)
     throw new Error(`accounts env returned no profile directory for ${account.profile}/${tool}`);
   if (!existsSync(profileDir))
     throw new Error(`account profile directory does not exist for ${account.profile}/${tool}: ${profileDir}`);
   return {
-    ...parseAccountExportLines(result.stdout),
+    ...accountEnv,
     LOOPS_ACCOUNT_PROFILE: account.profile,
     LOOPS_ACCOUNT_TOOL: tool
   };
@@ -1428,6 +1586,27 @@ function executionEnv(spec, metadata, opts) {
     env.LOOPS_WORKFLOW_STEP_ID = metadata.workflowStepId;
   return env;
 }
+function commandExists(command, env) {
+  if (command.includes("/") && existsSync2(command))
+    return true;
+  const result = spawnSync2("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], {
+    env,
+    stdio: "ignore"
+  });
+  return (result.status ?? 1) === 0;
+}
+function preflightTarget(target, metadata = {}, opts = {}) {
+  const spec = commandSpec(target);
+  const env = executionEnv(spec, metadata, opts);
+  if (!spec.shell && !commandExists(spec.command, env)) {
+    throw new Error(`Executable not found in PATH: ${spec.command}`);
+  }
+  return {
+    command: spec.command,
+    accountProfile: spec.account?.profile,
+    accountTool: spec.accountTool
+  };
+}
 async function executeTarget(target, metadata = {}, opts = {}) {
   const spec = commandSpec(target);
   const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
@@ -1438,6 +1617,17 @@ async function executeTarget(target, metadata = {}, opts = {}) {
   let exitCode;
   let error;
   const env = executionEnv(spec, metadata, opts);
+  if (!spec.shell && !commandExists(spec.command, env)) {
+    return {
+      status: "failed",
+      stdout: "",
+      stderr: "",
+      error: `Executable not found in PATH: ${spec.command}`,
+      startedAt,
+      finishedAt: nowIso(),
+      durationMs: 0
+    };
+  }
   const child = spawn(spec.command, spec.args, {
     cwd: spec.cwd,
     env,
@@ -1445,6 +1635,16 @@ async function executeTarget(target, metadata = {}, opts = {}) {
     detached: true,
     stdio: ["ignore", "pipe", "pipe"]
   });
+  if (child.pid)
+    opts.onSpawn?.(child.pid);
+  const abortHandler = () => {
+    error = "cancelled";
+    if (child.pid)
+      killProcessGroup(child.pid);
+  };
+  if (opts.signal?.aborted)
+    abortHandler();
+  opts.signal?.addEventListener("abort", abortHandler, { once: true });
   child.stdout.on("data", (chunk) => {
     stdout = appendBounded(stdout, chunk, maxOutputBytes);
   });
@@ -1467,6 +1667,7 @@ async function executeTarget(target, metadata = {}, opts = {}) {
     error = err instanceof Error ? err.message : String(err);
   } finally {
     clearTimeout(timer);
+    opts.signal?.removeEventListener("abort", abortHandler);
   }
   const finishedAt = nowIso();
   const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
@@ -1522,14 +1723,16 @@ async function executeLoop(loop, run, opts = {}) {
 // src/lib/workflow-runner.ts
 function targetWithStepAccount(step) {
   const account = step.account ?? step.target.account;
-  if (!account)
+  const timeoutMs = step.timeoutMs ?? step.target.timeoutMs;
+  if (!account && timeoutMs === step.target.timeoutMs)
     return step.target;
-  return { ...step.target, account };
+  return { ...step.target, account, timeoutMs };
 }
 function workflowResult(workflowRun, status, startedAt, finishedAt, stdout, error) {
+  const executorStatus = status === "succeeded" ? "succeeded" : status === "timed_out" ? "timed_out" : "failed";
   return {
-    status,
-    exitCode: status === "succeeded" ? 0 : 1,
+    status: executorStatus,
+    exitCode: executorStatus === "succeeded" ? 0 : 1,
     stdout,
     stderr: "",
     error,
@@ -1547,7 +1750,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
     idempotencyKey: opts.idempotencyKey
   });
   const startedAt = run.startedAt ?? nowIso();
-  if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out") {
+  if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
     const steps2 = store.listWorkflowStepRuns(run.id);
     return workflowResult(run, run.status, startedAt, run.finishedAt ?? nowIso(), JSON.stringify({ workflowRun: run, steps: steps2 }, null, 2), run.error);
   }
@@ -1556,8 +1759,19 @@ async function executeWorkflow(store, workflow, opts = {}) {
   let blockingError;
   let terminalStatus = "succeeded";
   for (const step of ordered) {
+    if (store.isWorkflowRunTerminal(run.id)) {
+      terminalStatus = store.requireWorkflowRun(run.id).status;
+      blockingError = "workflow run was cancelled";
+      break;
+    }
+    const pendingTimeout = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
+    if (pendingTimeout) {
+      terminalStatus = "timed_out";
+      blockingError = pendingTimeout;
+      break;
+    }
     const existing = store.getWorkflowStepRun(run.id, step.id);
-    if (existing?.status === "succeeded" || existing?.status === "skipped")
+    if (existing?.status === "succeeded" || existing?.status === "skipped" || existing?.status === "cancelled")
       continue;
     const blockedBy = (step.dependsOn ?? []).find((dependencyId) => {
       const dependencyRun = store.getWorkflowStepRun(run.id, dependencyId);
@@ -1572,8 +1786,13 @@ async function executeWorkflow(store, workflow, opts = {}) {
       terminalStatus = "failed";
       continue;
     }
-    store.startWorkflowStepRun(run.id, step.id);
-    const result = await executeTarget(targetWithStepAccount(step), {
+    const startedStep = store.startWorkflowStepRun(run.id, step.id);
+    if (startedStep.status !== "running") {
+      terminalStatus = "failed";
+      blockingError = `step ${step.id} could not start because workflow is no longer running`;
+      break;
+    }
+    const metadata = {
       loopId: opts.loop?.id,
       loopName: opts.loop?.name,
       runId: opts.loopRun?.id,
@@ -1582,7 +1801,51 @@ async function executeWorkflow(store, workflow, opts = {}) {
       workflowName: workflow.name,
       workflowRunId: run.id,
       workflowStepId: step.id
-    }, opts);
+    };
+    let result;
+    const controller = new AbortController;
+    const externalAbort = () => controller.abort();
+    if (opts.signal?.aborted)
+      controller.abort();
+    opts.signal?.addEventListener("abort", externalAbort, { once: true });
+    const cancelTimer = setInterval(() => {
+      if (store.getWorkflowRun(run.id)?.status === "cancelled")
+        controller.abort();
+    }, opts.cancelPollMs ?? 500);
+    cancelTimer.unref();
+    try {
+      result = await executeTarget(targetWithStepAccount(step), metadata, {
+        ...opts,
+        signal: controller.signal,
+        onSpawn: (pid) => {
+          store.markWorkflowStepPid(run.id, step.id, pid);
+          opts.onSpawn?.(pid);
+        }
+      });
+    } catch (error) {
+      const finishedAt2 = nowIso();
+      result = {
+        status: "failed",
+        stdout: "",
+        stderr: "",
+        error: error instanceof Error ? error.message : String(error),
+        startedAt: startedStep.startedAt ?? finishedAt2,
+        finishedAt: finishedAt2,
+        durationMs: new Date(finishedAt2).getTime() - new Date(startedStep.startedAt ?? finishedAt2).getTime()
+      };
+    } finally {
+      clearInterval(cancelTimer);
+      opts.signal?.removeEventListener("abort", externalAbort);
+    }
+    const timeoutMessage = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
+    if (timeoutMessage && result.status === "failed") {
+      result = { ...result, status: "timed_out", error: timeoutMessage };
+    }
+    if (store.isWorkflowRunTerminal(run.id)) {
+      terminalStatus = store.requireWorkflowRun(run.id).status;
+      blockingError = "workflow run was cancelled";
+      break;
+    }
     store.finalizeWorkflowStepRun(run.id, step.id, {
       status: result.status,
       finishedAt: result.finishedAt,
@@ -1607,6 +1870,11 @@ async function executeWorkflow(store, workflow, opts = {}) {
     }
   }
   const finishedAt = nowIso();
+  if (store.isWorkflowRunTerminal(run.id)) {
+    const terminalRun = store.requireWorkflowRun(run.id);
+    const steps2 = store.listWorkflowStepRuns(run.id);
+    return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
+  }
   const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
     finishedAt,
     durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
@@ -1615,20 +1883,59 @@ async function executeWorkflow(store, workflow, opts = {}) {
   const steps = store.listWorkflowStepRuns(run.id);
   return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
 }
+function preflightWorkflow(workflow, opts = {}) {
+  return workflowExecutionOrder(workflow).map((step) => preflightTarget(targetWithStepAccount(step), {
+    workflowId: workflow.id,
+    workflowName: workflow.name,
+    workflowStepId: step.id
+  }, opts));
+}
 async function executeLoopTarget(store, loop, run, opts = {}) {
   if (loop.target.type !== "workflow")
     return executeLoop(loop, run, opts);
   const workflow = store.requireWorkflow(loop.target.workflowId);
-  return executeWorkflow(store, workflow, {
-    ...opts,
-    loop,
-    loopRun: run,
-    scheduledFor: run.scheduledFor,
-    idempotencyKey: `${loop.id}:${run.scheduledFor}`
-  });
+  const controller = loop.target.timeoutMs ? new AbortController : undefined;
+  let workflowTimedOut = false;
+  const externalAbort = () => controller?.abort();
+  if (controller && opts.signal?.aborted)
+    controller.abort();
+  if (controller)
+    opts.signal?.addEventListener("abort", externalAbort, { once: true });
+  const timer = controller ? setTimeout(() => {
+    workflowTimedOut = true;
+    controller.abort();
+  }, loop.target.timeoutMs) : undefined;
+  timer?.unref();
+  try {
+    return await executeWorkflow(store, workflow, {
+      ...opts,
+      signal: controller?.signal ?? opts.signal,
+      signalTimeoutMessage: () => workflowTimedOut && loop.target.type === "workflow" ? `workflow timed out after ${loop.target.timeoutMs}ms` : undefined,
+      loop,
+      loopRun: run,
+      scheduledFor: run.scheduledFor,
+      idempotencyKey: `${loop.id}:${run.scheduledFor}:attempt:${run.attempt}`
+    });
+  } finally {
+    if (timer)
+      clearTimeout(timer);
+    if (controller)
+      opts.signal?.removeEventListener("abort", externalAbort);
+  }
 }
 
 // src/lib/scheduler.ts
+function manualRunScheduledFor(loop, now = new Date) {
+  if (loop.nextRunAt && new Date(loop.nextRunAt).getTime() <= now.getTime()) {
+    return loop.retryScheduledFor ?? loop.nextRunAt;
+  }
+  return now.toISOString();
+}
+function shouldAdvanceManualRun(loop, scheduledFor, now = new Date) {
+  if (!loop.nextRunAt || new Date(loop.nextRunAt).getTime() > now.getTime())
+    return false;
+  return scheduledFor === (loop.retryScheduledFor ?? loop.nextRunAt);
+}
 function nextAfterRetry(loop, now) {
   return new Date(now.getTime() + loop.retryDelayMs).toISOString();
 }
@@ -1654,27 +1961,18 @@ function advanceLoop(store, loop, run, finishedAt, succeeded) {
     retryScheduledFor: undefined
   });
 }
-async function runSlot(deps, loop, scheduledFor) {
-  const now = deps.now?.() ?? new Date;
-  if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
-    const skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active");
-    advanceLoop(deps.store, loop, skipped, now, true);
-    deps.onRun?.(skipped);
-    return skipped;
-  }
-  const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
-  if (!claim)
-    return;
-  deps.onRun?.(claim.run);
+async function executeClaimedRun(deps) {
   let heartbeat;
-  const heartbeatEveryMs = Math.max(1000, Math.min(60000, Math.floor(claim.loop.leaseMs / 3)));
+  const heartbeatEveryMs = Math.max(10, Math.min(60000, Math.floor(deps.loop.leaseMs / 3)));
   heartbeat = setInterval(() => {
-    deps.store.heartbeatRunLease(claim.run.id, deps.runnerId, claim.loop.leaseMs);
+    deps.store.heartbeatRunLease(deps.run.id, deps.runnerId, deps.loop.leaseMs);
   }, heartbeatEveryMs);
   heartbeat.unref();
   try {
-    const result = await (deps.execute ?? ((loop2, run) => executeLoopTarget(deps.store, loop2, run)))(claim.loop, claim.run);
-    const finalRun = deps.store.finalizeRun(claim.run.id, {
+    const result = await (deps.execute ?? ((loop, run) => executeLoopTarget(deps.store, loop, run, {
+      onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId)
+    })))(deps.loop, deps.run);
+    return deps.store.finalizeRun(deps.run.id, {
       status: result.status,
       finishedAt: result.finishedAt,
       durationMs: result.durationMs,
@@ -1687,16 +1985,13 @@ async function runSlot(deps, loop, scheduledFor) {
       claimedBy: deps.runnerId,
       now: deps.now?.() ?? new Date(result.finishedAt)
     });
-    advanceLoop(deps.store, claim.loop, finalRun, new Date(result.finishedAt), finalRun.status === "succeeded");
-    deps.onRun?.(finalRun);
-    return finalRun;
   } catch (err) {
-    deps.onError?.(claim.loop, err);
+    deps.onError?.(deps.loop, err);
     const finishedAt = new Date;
-    const finalRun = deps.store.finalizeRun(claim.run.id, {
+    return deps.store.finalizeRun(deps.run.id, {
       status: "failed",
       finishedAt: finishedAt.toISOString(),
-      durationMs: finishedAt.getTime() - new Date(claim.run.startedAt ?? claim.run.createdAt).getTime(),
+      durationMs: finishedAt.getTime() - new Date(deps.run.startedAt ?? deps.run.createdAt).getTime(),
       stdout: "",
       stderr: "",
       error: err instanceof Error ? err.message : String(err)
@@ -1704,14 +1999,36 @@ async function runSlot(deps, loop, scheduledFor) {
       claimedBy: deps.runnerId,
       now: deps.now?.() ?? finishedAt
     });
-    advanceLoop(deps.store, claim.loop, finalRun, finishedAt, false);
-    deps.onRun?.(finalRun);
-    return finalRun;
   } finally {
     if (heartbeat)
       clearInterval(heartbeat);
   }
 }
+async function runSlot(deps, loop, scheduledFor) {
+  const now = deps.now?.() ?? new Date;
+  if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
+    const skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active");
+    advanceLoop(deps.store, loop, skipped, now, true);
+    deps.onRun?.(skipped);
+    return skipped;
+  }
+  const claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now);
+  if (!claim)
+    return;
+  deps.onRun?.(claim.run);
+  const finalRun = await executeClaimedRun({
+    store: deps.store,
+    runnerId: deps.runnerId,
+    loop: claim.loop,
+    run: claim.run,
+    now: deps.now,
+    execute: deps.execute,
+    onError: deps.onError
+  });
+  advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded");
+  deps.onRun?.(finalRun);
+  return finalRun;
+}
 async function tick(deps) {
   const now = deps.now?.() ?? new Date;
   const recovered = deps.store.recoverExpiredRunLeases(now);
@@ -1785,23 +2102,17 @@ class LoopsClient {
   }
   async runNow(idOrName) {
     const loop = this.get(idOrName);
-    const scheduledFor = new Date().toISOString();
-    const claim = this.store.claimRun(loop, scheduledFor, this.runnerId);
+    const now = new Date;
+    const scheduledFor = manualRunScheduledFor(loop, now);
+    const shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
+    const claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
     if (!claim)
       throw new Error(`could not claim manual run for ${idOrName}`);
-    const result = await executeLoopTarget(this.store, loop, claim.run);
-    return this.store.finalizeRun(claim.run.id, {
-      status: result.status,
-      finishedAt: result.finishedAt,
-      durationMs: result.durationMs,
-      stdout: result.stdout,
-      stderr: result.stderr,
-      exitCode: result.exitCode,
-      error: result.error,
-      pid: result.pid
-    }, {
-      claimedBy: claim.run.claimedBy
-    });
+    const run = await executeClaimedRun({ store: this.store, runnerId: this.runnerId, loop: claim.loop, run: claim.run });
+    if (shouldAdvance) {
+      advanceLoop(this.store, claim.loop, run, new Date(run.finishedAt ?? new Date), run.status === "succeeded");
+    }
+    return run;
   }
   close() {
     if (this.ownStore)
@@ -1811,10 +2122,214 @@ class LoopsClient {
 function loops(opts = {}) {
   return new LoopsClient(opts);
 }
+// src/lib/doctor.ts
+import { spawnSync as spawnSync3 } from "child_process";
+import { accessSync, constants } from "fs";
+
+// src/daemon/control.ts
+import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync, rmSync, writeFileSync } from "fs";
+import { hostname } from "os";
+import { dirname as dirname2 } from "path";
+
+// src/daemon/loop.ts
+function realSleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function runLoop(opts) {
+  const sleep = opts.sleep ?? realSleep;
+  const sliceMs = opts.sliceMs ?? 200;
+  while (!opts.shouldStop()) {
+    try {
+      await opts.tickFn();
+    } catch (err) {
+      opts.onTickError?.(err);
+    }
+    let waited = 0;
+    while (waited < opts.intervalMs && !opts.shouldStop()) {
+      const chunk = Math.min(sliceMs, opts.intervalMs - waited);
+      await sleep(chunk);
+      waited += chunk;
+    }
+  }
+}
+
+// src/daemon/control.ts
+function readPid(path = pidFilePath()) {
+  if (!existsSync3(path))
+    return;
+  try {
+    const pid = Number(readFileSync(path, "utf8").trim());
+    return Number.isInteger(pid) && pid > 0 ? pid : undefined;
+  } catch {
+    return;
+  }
+}
+function writePid(pid = process.pid, path = pidFilePath()) {
+  mkdirSync3(dirname2(path), { recursive: true, mode: 448 });
+  writeFileSync(path, String(pid));
+}
+function removePid(path = pidFilePath()) {
+  rmSync(path, { force: true });
+}
+function isAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return err.code === "EPERM";
+  }
+}
+function isDaemonRunning(path = pidFilePath()) {
+  const pid = readPid(path);
+  if (!pid)
+    return { running: false, stale: false };
+  if (isAlive(pid))
+    return { running: true, stale: false, pid };
+  return { running: false, stale: true, pid };
+}
+function daemonStatus(store, path = pidFilePath()) {
+  return {
+    ...isDaemonRunning(path),
+    lease: store.getDaemonLease(),
+    host: hostname(),
+    loops: {
+      total: store.countLoops(),
+      active: store.countLoops("active"),
+      paused: store.countLoops("paused"),
+      stopped: store.countLoops("stopped"),
+      expired: store.countLoops("expired")
+    },
+    runs: {
+      total: store.countRuns(),
+      running: store.countRuns("running"),
+      failed: store.countRuns("failed"),
+      succeeded: store.countRuns("succeeded"),
+      abandoned: store.countRuns("abandoned")
+    },
+    logPath: daemonLogPath()
+  };
+}
+async function stopDaemon(opts = {}) {
+  const path = opts.path ?? pidFilePath();
+  const state = isDaemonRunning(path);
+  if (state.stale) {
+    removePid(path);
+    return { wasRunning: false, stopped: false, forced: false, pid: state.pid };
+  }
+  if (!state.running || !state.pid)
+    return { wasRunning: false, stopped: false, forced: false };
+  const store = new Store;
+  try {
+    const lease = store.getDaemonLease();
+    if (!lease || lease.pid !== state.pid || new Date(lease.expiresAt).getTime() <= Date.now()) {
+      removePid(path);
+      return { wasRunning: false, stopped: false, forced: false, pid: state.pid };
+    }
+  } finally {
+    store.close();
+  }
+  const sleep = opts.sleep ?? realSleep;
+  try {
+    process.kill(state.pid, "SIGTERM");
+  } catch {
+    removePid(path);
+    return { wasRunning: true, stopped: true, forced: false, pid: state.pid };
+  }
+  const steps = Math.max(1, Math.ceil((opts.timeoutMs ?? 6000) / 100));
+  for (let i = 0;i < steps; i++) {
+    await sleep(100);
+    if (!isAlive(state.pid)) {
+      removePid(path);
+      return { wasRunning: true, stopped: true, forced: false, pid: state.pid };
+    }
+  }
+  try {
+    process.kill(state.pid, "SIGKILL");
+  } catch {}
+  await sleep(150);
+  removePid(path);
+  return { wasRunning: true, stopped: !isAlive(state.pid), forced: true, pid: state.pid };
+}
+
+// src/lib/doctor.ts
+var PROVIDER_COMMANDS = [
+  "claude",
+  "cursor-agent",
+  "codewith",
+  "aicopilot",
+  "opencode",
+  "codex"
+];
+function hasCommand(command) {
+  const result = spawnSync3("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
+  return (result.status ?? 1) === 0;
+}
+function commandVersion(command) {
+  const result = spawnSync3(command, ["--version"], {
+    encoding: "utf8",
+    stdio: ["ignore", "pipe", "pipe"]
+  });
+  if ((result.status ?? 1) !== 0)
+    return;
+  return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
+}
+function runDoctor(store) {
+  const checks = [];
+  try {
+    const dir = ensureDataDir();
+    accessSync(dir, constants.R_OK | constants.W_OK);
+    checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
+  } catch (error) {
+    checks.push({
+      id: "data-dir",
+      status: "fail",
+      message: "data directory is not writable",
+      detail: error instanceof Error ? error.message : String(error)
+    });
+  }
+  const bunVersion = commandVersion("bun");
+  checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
+  const accountsVersion = commandVersion("accounts");
+  checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
+  for (const command of PROVIDER_COMMANDS) {
+    checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
+  }
+  const status = daemonStatus(store);
+  checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
+  const failedRuns = store.countRuns("failed");
+  checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
+  for (const loop of store.listLoops({ status: "active" })) {
+    try {
+      if (loop.target.type === "workflow") {
+        const workflow = store.requireWorkflow(loop.target.workflowId);
+        for (const step of workflowExecutionOrder(workflow)) {
+          preflightTarget({ ...step.target, account: step.account ?? step.target.account, timeoutMs: step.timeoutMs ?? step.target.timeoutMs }, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id });
+        }
+      } else {
+        preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name });
+      }
+      checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
+    } catch (error) {
+      checks.push({
+        id: `loop:${loop.id}:preflight`,
+        status: "warn",
+        message: `active loop target preflight failed: ${loop.name}`,
+        detail: error instanceof Error ? error.message : String(error)
+      });
+    }
+  }
+  return {
+    ok: checks.every((check) => check.status !== "fail"),
+    checks
+  };
+}
 export {
   workflowExecutionOrder,
   workflowBodyFromJson,
   tick,
+  runDoctor,
+  preflightWorkflow,
+  preflightTarget,
   parseDuration,
   parseCron,
   nextCronRun,