@hasna/loops 0.1.0 → 0.3.0

package/dist/lib/store.js

@@ -234,6 +234,103 @@ function parseDuration(input) {
   return Math.round(value * multiplier);
 }
 
+// src/lib/workflow-spec.ts
+function assertObject(value, label) {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    throw new Error(`${label} must be an object`);
+}
+function assertString(value, label) {
+  if (typeof value !== "string" || value.trim() === "")
+    throw new Error(`${label} must be a non-empty string`);
+}
+function validateTarget(value, label) {
+  assertObject(value, label);
+  if (value.type === "command") {
+    assertString(value.command, `${label}.command`);
+    if (value.shell !== true && /\s/.test(value.command.trim())) {
+      throw new Error(`${label}.command must be an executable without spaces when shell is false; put flags in args or set shell true`);
+    }
+    return value;
+  }
+  if (value.type === "agent") {
+    assertString(value.provider, `${label}.provider`);
+    assertString(value.prompt, `${label}.prompt`);
+    const providers = ["claude", "cursor", "codewith", "aicopilot", "opencode", "codex"];
+    if (!providers.includes(value.provider))
+      throw new Error(`${label}.provider must be one of ${providers.join(", ")}`);
+    return value;
+  }
+  throw new Error(`${label}.type must be command or agent`);
+}
+function normalizeCreateWorkflowInput(input) {
+  assertString(input.name, "workflow.name");
+  if (!Array.isArray(input.steps) || input.steps.length === 0)
+    throw new Error("workflow.steps must contain at least one step");
+  const seen = new Set;
+  const steps = input.steps.map((step, index) => {
+    assertObject(step, `workflow.steps[${index}]`);
+    assertString(step.id, `workflow.steps[${index}].id`);
+    if (seen.has(step.id))
+      throw new Error(`duplicate workflow step id: ${step.id}`);
+    seen.add(step.id);
+    return {
+      ...step,
+      id: step.id,
+      target: validateTarget(step.target, `workflow.steps[${index}].target`),
+      dependsOn: step.dependsOn ?? [],
+      continueOnFailure: step.continueOnFailure ?? false
+    };
+  });
+  for (const step of steps) {
+    for (const dependency of step.dependsOn ?? []) {
+      if (!seen.has(dependency))
+        throw new Error(`step ${step.id} depends on missing step ${dependency}`);
+      if (dependency === step.id)
+        throw new Error(`step ${step.id} cannot depend on itself`);
+    }
+  }
+  workflowExecutionOrder({ steps });
+  return { ...input, name: input.name.trim(), version: input.version ?? 1, steps };
+}
+function workflowExecutionOrder(workflow) {
+  const byId = new Map(workflow.steps.map((step) => [step.id, step]));
+  const visiting = new Set;
+  const visited = new Set;
+  const order = [];
+  function visit(step) {
+    if (visited.has(step.id))
+      return;
+    if (visiting.has(step.id))
+      throw new Error(`workflow dependency cycle includes step ${step.id}`);
+    visiting.add(step.id);
+    for (const dependencyId of step.dependsOn ?? []) {
+      const dependency = byId.get(dependencyId);
+      if (!dependency)
+        throw new Error(`step ${step.id} depends on missing step ${dependencyId}`);
+      visit(dependency);
+    }
+    visiting.delete(step.id);
+    visited.add(step.id);
+    order.push(step);
+  }
+  for (const step of workflow.steps)
+    visit(step);
+  return order;
+}
+function workflowBodyFromJson(value, fallbackName) {
+  assertObject(value, "workflow file");
+  const rawName = fallbackName ?? value.name;
+  assertString(rawName, "workflow.name");
+  if (!Array.isArray(value.steps))
+    throw new Error("workflow.steps must be an array");
+  return normalizeCreateWorkflowInput({
+    name: rawName,
+    description: typeof value.description === "string" ? value.description : undefined,
+    version: typeof value.version === "number" ? value.version : undefined,
+    steps: value.steps
+  });
+}
+
 // src/lib/store.ts
 function rowToLoop(row) {
   return {
@@ -278,6 +375,76 @@ function rowToRun(row) {
     updatedAt: row.updated_at
   };
 }
+function rowToWorkflow(row) {
+  return {
+    id: row.id,
+    name: row.name,
+    description: row.description ?? undefined,
+    version: row.version,
+    status: row.status,
+    steps: JSON.parse(row.steps_json),
+    createdAt: row.created_at,
+    updatedAt: row.updated_at
+  };
+}
+function rowToWorkflowRun(row) {
+  return {
+    id: row.id,
+    workflowId: row.workflow_id,
+    workflowName: row.workflow_name,
+    loopId: row.loop_id ?? undefined,
+    loopRunId: row.loop_run_id ?? undefined,
+    scheduledFor: row.scheduled_for ?? undefined,
+    idempotencyKey: row.idempotency_key ?? undefined,
+    status: row.status,
+    startedAt: row.started_at ?? undefined,
+    finishedAt: row.finished_at ?? undefined,
+    durationMs: row.duration_ms ?? undefined,
+    error: row.error ?? undefined,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at
+  };
+}
+function rowToWorkflowStepRun(row) {
+  return {
+    id: row.id,
+    workflowRunId: row.workflow_run_id,
+    stepId: row.step_id,
+    sequence: row.sequence,
+    status: row.status,
+    startedAt: row.started_at ?? undefined,
+    finishedAt: row.finished_at ?? undefined,
+    exitCode: row.exit_code ?? undefined,
+    pid: row.pid ?? undefined,
+    durationMs: row.duration_ms ?? undefined,
+    stdout: row.stdout ?? undefined,
+    stderr: row.stderr ?? undefined,
+    error: row.error ?? undefined,
+    accountProfile: row.account_profile ?? undefined,
+    accountTool: row.account_tool ?? undefined,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at
+  };
+}
+function rowToWorkflowEvent(row) {
+  return {
+    id: row.id,
+    workflowRunId: row.workflow_run_id,
+    sequence: row.sequence,
+    eventType: row.event_type,
+    stepId: row.step_id ?? undefined,
+    payload: row.payload_json ? JSON.parse(row.payload_json) : undefined,
+    createdAt: row.created_at
+  };
+}
+function isProcessAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
 function rowToLease(row) {
   return {
     id: row.id,
@@ -303,6 +470,11 @@ class Store {
   }
   migrate() {
     this.db.exec(`
+      CREATE TABLE IF NOT EXISTS schema_migrations (
+        id TEXT PRIMARY KEY,
+        applied_at TEXT NOT NULL
+      );
+
       CREATE TABLE IF NOT EXISTS loops (
         id TEXT PRIMARY KEY,
         name TEXT NOT NULL,
@@ -359,7 +531,82 @@ class Store {
         created_at TEXT NOT NULL,
         updated_at TEXT NOT NULL
       );
+
+      CREATE TABLE IF NOT EXISTS workflow_specs (
+        id TEXT PRIMARY KEY,
+        name TEXT NOT NULL,
+        description TEXT,
+        version INTEGER NOT NULL,
+        status TEXT NOT NULL,
+        steps_json TEXT NOT NULL,
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_workflows_status_name ON workflow_specs(status, name);
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_workflows_name_active ON workflow_specs(name) WHERE status = 'active';
+
+      CREATE TABLE IF NOT EXISTS workflow_runs (
+        id TEXT PRIMARY KEY,
+        workflow_id TEXT NOT NULL REFERENCES workflow_specs(id) ON DELETE CASCADE,
+        workflow_name TEXT NOT NULL,
+        loop_id TEXT REFERENCES loops(id) ON DELETE SET NULL,
+        loop_run_id TEXT REFERENCES loop_runs(id) ON DELETE SET NULL,
+        scheduled_for TEXT,
+        idempotency_key TEXT,
+        status TEXT NOT NULL,
+        started_at TEXT,
+        finished_at TEXT,
+        duration_ms INTEGER,
+        error TEXT,
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL
+      );
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_workflow_runs_idempotency
+        ON workflow_runs(workflow_id, idempotency_key)
+        WHERE idempotency_key IS NOT NULL;
+      CREATE INDEX IF NOT EXISTS idx_workflow_runs_workflow_created ON workflow_runs(workflow_id, created_at DESC);
+      CREATE INDEX IF NOT EXISTS idx_workflow_runs_loop_run ON workflow_runs(loop_run_id);
+      CREATE INDEX IF NOT EXISTS idx_workflow_runs_status ON workflow_runs(status);
+
+      CREATE TABLE IF NOT EXISTS workflow_step_runs (
+        id TEXT PRIMARY KEY,
+        workflow_run_id TEXT NOT NULL REFERENCES workflow_runs(id) ON DELETE CASCADE,
+        step_id TEXT NOT NULL,
+        sequence INTEGER NOT NULL,
+        status TEXT NOT NULL,
+        started_at TEXT,
+        finished_at TEXT,
+        exit_code INTEGER,
+        pid INTEGER,
+        duration_ms INTEGER,
+        stdout TEXT,
+        stderr TEXT,
+        error TEXT,
+        account_profile TEXT,
+        account_tool TEXT,
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL,
+        UNIQUE(workflow_run_id, step_id)
+      );
+      CREATE INDEX IF NOT EXISTS idx_workflow_step_runs_run_sequence ON workflow_step_runs(workflow_run_id, sequence);
+      CREATE INDEX IF NOT EXISTS idx_workflow_step_runs_status ON workflow_step_runs(status);
+
+      CREATE TABLE IF NOT EXISTS workflow_events (
+        id TEXT PRIMARY KEY,
+        workflow_run_id TEXT NOT NULL REFERENCES workflow_runs(id) ON DELETE CASCADE,
+        sequence INTEGER NOT NULL,
+        event_type TEXT NOT NULL,
+        step_id TEXT,
+        payload_json TEXT,
+        created_at TEXT NOT NULL,
+        UNIQUE(workflow_run_id, sequence)
+      );
+      CREATE INDEX IF NOT EXISTS idx_workflow_events_run_sequence ON workflow_events(workflow_run_id, sequence);
     `);
+    try {
+      this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
+    } catch {}
+    this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
   }
   createLoop(input, from = new Date) {
     const now = nowIso();
@@ -451,10 +698,343 @@ class Store {
     const res = this.db.query("DELETE FROM loops WHERE id = ?").run(loop.id);
     return res.changes > 0;
   }
+  createWorkflow(input) {
+    const normalized = normalizeCreateWorkflowInput(input);
+    const now = nowIso();
+    const workflow = {
+      id: genId(),
+      name: normalized.name,
+      description: normalized.description,
+      version: normalized.version ?? 1,
+      status: "active",
+      steps: normalized.steps,
+      createdAt: now,
+      updatedAt: now
+    };
+    this.db.query(`INSERT INTO workflow_specs (id, name, description, version, status, steps_json, created_at, updated_at)
+         VALUES ($id, $name, $description, $version, $status, $steps, $created, $updated)`).run({
+      $id: workflow.id,
+      $name: workflow.name,
+      $description: workflow.description ?? null,
+      $version: workflow.version,
+      $status: workflow.status,
+      $steps: JSON.stringify(workflow.steps),
+      $created: workflow.createdAt,
+      $updated: workflow.updatedAt
+    });
+    return workflow;
+  }
+  getWorkflow(id) {
+    const row = this.db.query("SELECT * FROM workflow_specs WHERE id = ?").get(id);
+    return row ? rowToWorkflow(row) : undefined;
+  }
+  findWorkflowByName(name) {
+    const row = this.db.query("SELECT * FROM workflow_specs WHERE name = ? AND status = 'active' ORDER BY updated_at DESC LIMIT 1").get(name);
+    return row ? rowToWorkflow(row) : undefined;
+  }
+  requireWorkflow(idOrName) {
+    return this.getWorkflow(idOrName) ?? this.findWorkflowByName(idOrName) ?? (() => {
+      throw new Error(`workflow not found: ${idOrName}`);
+    })();
+  }
+  listWorkflows(opts = {}) {
+    const limit = opts.limit ?? 200;
+    const rows = opts.status ? this.db.query("SELECT * FROM workflow_specs WHERE status = ? ORDER BY updated_at DESC LIMIT ?").all(opts.status, limit) : this.db.query("SELECT * FROM workflow_specs ORDER BY status ASC, updated_at DESC LIMIT ?").all(limit);
+    return rows.map(rowToWorkflow);
+  }
+  archiveWorkflow(idOrName) {
+    const workflow = this.requireWorkflow(idOrName);
+    const updated = nowIso();
+    this.db.query("UPDATE workflow_specs SET status='archived', updated_at=? WHERE id=?").run(updated, workflow.id);
+    const archived = this.getWorkflow(workflow.id);
+    if (!archived)
+      throw new Error(`workflow not found after archive: ${workflow.id}`);
+    return archived;
+  }
+  createWorkflowRun(input) {
+    const now = nowIso();
+    if (input.idempotencyKey) {
+      const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
+      if (existing)
+        return rowToWorkflowRun(existing);
+    }
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      if (input.idempotencyKey) {
+        const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
+        if (existing) {
+          this.db.exec("COMMIT");
+          return rowToWorkflowRun(existing);
+        }
+      }
+      const runId = genId();
+      this.db.query(`INSERT INTO workflow_runs (id, workflow_id, workflow_name, loop_id, loop_run_id, scheduled_for, idempotency_key,
+            status, started_at, finished_at, duration_ms, error, created_at, updated_at)
+           VALUES ($id, $workflowId, $workflowName, $loopId, $loopRunId, $scheduledFor, $idempotencyKey,
+            'running', $started, NULL, NULL, NULL, $created, $updated)`).run({
+        $id: runId,
+        $workflowId: input.workflow.id,
+        $workflowName: input.workflow.name,
+        $loopId: input.loop?.id ?? null,
+        $loopRunId: input.loopRun?.id ?? null,
+        $scheduledFor: input.scheduledFor ?? input.loopRun?.scheduledFor ?? null,
+        $idempotencyKey: input.idempotencyKey ?? null,
+        $started: now,
+        $created: now,
+        $updated: now
+      });
+      input.workflow.steps.forEach((step, sequence) => {
+        const account = step.account ?? step.target.account;
+        this.db.query(`INSERT INTO workflow_step_runs (id, workflow_run_id, step_id, sequence, status, started_at, finished_at,
+              exit_code, pid, duration_ms, stdout, stderr, error, account_profile, account_tool, created_at, updated_at)
+             VALUES ($id, $workflowRunId, $stepId, $sequence, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+              $accountProfile, $accountTool, $created, $updated)`).run({
+          $id: genId(),
+          $workflowRunId: runId,
+          $stepId: step.id,
+          $sequence: sequence,
+          $accountProfile: account?.profile ?? null,
+          $accountTool: account?.tool ?? null,
+          $created: now,
+          $updated: now
+        });
+      });
+      this.db.query(`INSERT INTO workflow_events (id, workflow_run_id, sequence, event_type, step_id, payload_json, created_at)
+           VALUES ($id, $workflowRunId, 1, 'created', NULL, $payload, $created)`).run({
+        $id: genId(),
+        $workflowRunId: runId,
+        $payload: JSON.stringify({
+          workflowId: input.workflow.id,
+          workflowName: input.workflow.name,
+          stepCount: input.workflow.steps.length,
+          loopId: input.loop?.id,
+          loopRunId: input.loopRun?.id
+        }),
+        $created: now
+      });
+      this.db.exec("COMMIT");
+      const run = this.getWorkflowRun(runId);
+      if (!run)
+        throw new Error(`workflow run not found after create: ${runId}`);
+      return run;
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
+  }
+  getWorkflowRun(id) {
+    const row = this.db.query("SELECT * FROM workflow_runs WHERE id = ?").get(id);
+    return row ? rowToWorkflowRun(row) : undefined;
+  }
+  requireWorkflowRun(id) {
+    const run = this.getWorkflowRun(id);
+    if (!run)
+      throw new Error(`workflow run not found: ${id}`);
+    return run;
+  }
+  listWorkflowRuns(opts = {}) {
+    const limit = opts.limit ?? 100;
+    let rows;
+    if (opts.workflowId) {
+      rows = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? ORDER BY created_at DESC LIMIT ?").all(opts.workflowId, limit);
+    } else if (opts.loopRunId) {
+      rows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? ORDER BY created_at DESC LIMIT ?").all(opts.loopRunId, limit);
+    } else {
+      rows = this.db.query("SELECT * FROM workflow_runs ORDER BY created_at DESC LIMIT ?").all(limit);
+    }
+    return rows.map(rowToWorkflowRun);
+  }
+  listWorkflowStepRuns(workflowRunId) {
+    const rows = this.db.query("SELECT * FROM workflow_step_runs WHERE workflow_run_id = ? ORDER BY sequence ASC").all(workflowRunId);
+    return rows.map(rowToWorkflowStepRun);
+  }
+  getWorkflowStepRun(workflowRunId, stepId) {
+    const row = this.db.query("SELECT * FROM workflow_step_runs WHERE workflow_run_id = ? AND step_id = ?").get(workflowRunId, stepId);
+    return row ? rowToWorkflowStepRun(row) : undefined;
+  }
+  isWorkflowRunTerminal(workflowRunId) {
+    const run = this.getWorkflowRun(workflowRunId);
+    return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
+  }
+  startWorkflowStepRun(workflowRunId, stepId) {
+    const now = nowIso();
+    const res = this.db.query(`UPDATE workflow_step_runs
+         SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
+          pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId
+           AND step_id=$stepId
+           AND status IN ('pending', 'failed', 'timed_out')
+           AND EXISTS (
+             SELECT 1 FROM workflow_runs
+             WHERE id=$workflowRunId AND status='running'
+           )`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
+    const run = this.getWorkflowStepRun(workflowRunId, stepId);
+    if (!run)
+      throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
+    if (res.changes !== 1) {
+      throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
+    }
+    this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
+    return run;
+  }
+  markWorkflowStepPid(workflowRunId, stepId, pid) {
+    const now = nowIso();
+    this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
+    const run = this.getWorkflowStepRun(workflowRunId, stepId);
+    if (!run)
+      throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
+    return run;
+  }
+  recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
+    const now = nowIso();
+    const before = this.listWorkflowStepRuns(workflowRunId).filter((step) => step.status === "running");
+    const live = before.filter((step) => step.pid !== undefined && isProcessAlive(step.pid));
+    if (live.length > 0) {
+      throw new Error(`cannot recover workflow run while step processes are still alive: ${live.map((step) => `${step.stepId} pid=${step.pid}`).join(", ")}`);
+    }
+    this.db.query(`UPDATE workflow_step_runs
+         SET status='pending', started_at=NULL, finished_at=NULL, exit_code=NULL, pid=NULL, duration_ms=NULL,
+          stdout=NULL, stderr=NULL, error=$reason, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND status='running'`).run({ $workflowRunId: workflowRunId, $reason: reason, $updated: now });
+    if (before.length > 0) {
+      this.appendWorkflowEvent(workflowRunId, "recovered", undefined, {
+        reason,
+        recoveredSteps: before.map((step) => step.stepId)
+      });
+    }
+    return {
+      run: this.requireWorkflowRun(workflowRunId),
+      recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
+    };
+  }
+  finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
+    const finishedAt = patch.finishedAt ?? nowIso();
+    const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
+         pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
+      $workflowRunId: workflowRunId,
+      $stepId: stepId,
+      $status: patch.status,
+      $finished: finishedAt,
+      $exitCode: patch.exitCode ?? null,
+      $durationMs: patch.durationMs ?? null,
+      $stdout: patch.stdout ?? null,
+      $stderr: patch.stderr ?? null,
+      $error: patch.error ?? null,
+      $updated: finishedAt
+    });
+    if (res.changes === 1) {
+      this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
+        exitCode: patch.exitCode,
+        error: patch.error
+      });
+    }
+    const run = this.getWorkflowStepRun(workflowRunId, stepId);
+    if (!run)
+      throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
+    return run;
+  }
+  skipWorkflowStepRun(workflowRunId, stepId, reason) {
+    const now = nowIso();
+    const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
+    if (res.changes === 1)
+      this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
+    const run = this.getWorkflowStepRun(workflowRunId, stepId);
+    if (!run)
+      throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
+    return run;
+  }
+  finalizeWorkflowRun(workflowRunId, status, patch = {}) {
+    const finishedAt = patch.finishedAt ?? nowIso();
+    const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
+         WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
+      $id: workflowRunId,
+      $status: status,
+      $finished: finishedAt,
+      $durationMs: patch.durationMs ?? null,
+      $error: patch.error ?? null,
+      $updated: finishedAt
+    });
+    const run = this.getWorkflowRun(workflowRunId);
+    if (!run)
+      throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
+    if (res.changes === 1)
+      this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
+    return run;
+  }
+  cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
+    const now = nowIso();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const run = this.requireWorkflowRun(workflowRunId);
+      if (!["succeeded", "failed", "timed_out", "cancelled"].includes(run.status)) {
+        this.db.query(`UPDATE workflow_runs
+             SET status='cancelled', finished_at=$finished, error=$reason, updated_at=$updated
+             WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRunId, $finished: now, $reason: reason, $updated: now });
+        this.db.query(`UPDATE workflow_step_runs
+             SET status='cancelled', finished_at=$finished, pid=NULL, error=$reason, updated_at=$updated
+             WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $finished: now, $reason: reason, $updated: now });
+        this.appendWorkflowEvent(workflowRunId, "cancelled", undefined, { reason });
+      }
+      this.db.exec("COMMIT");
+      return this.requireWorkflowRun(workflowRunId);
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
+  }
+  appendWorkflowEvent(workflowRunId, eventType, stepId, payload) {
+    const now = nowIso();
+    const current = this.db.query("SELECT MAX(sequence) AS sequence FROM workflow_events WHERE workflow_run_id = ?").get(workflowRunId);
+    const sequence = (current?.sequence ?? 0) + 1;
+    const id = genId();
+    this.db.query(`INSERT INTO workflow_events (id, workflow_run_id, sequence, event_type, step_id, payload_json, created_at)
+         VALUES ($id, $workflowRunId, $sequence, $eventType, $stepId, $payload, $created)`).run({
+      $id: id,
+      $workflowRunId: workflowRunId,
+      $sequence: sequence,
+      $eventType: eventType,
+      $stepId: stepId ?? null,
+      $payload: payload ? JSON.stringify(payload) : null,
+      $created: now
+    });
+    const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
+    if (!event)
+      throw new Error(`workflow event not found after append: ${id}`);
+    return rowToWorkflowEvent(event);
+  }
+  listWorkflowEvents(workflowRunId, limit = 200) {
+    const rows = this.db.query("SELECT * FROM workflow_events WHERE workflow_run_id = ? ORDER BY sequence ASC LIMIT ?").all(workflowRunId, limit);
+    return rows.map(rowToWorkflowEvent);
+  }
   hasRunningRun(loopId) {
     const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
     return (row?.count ?? 0) > 0;
   }
+  markRunPid(id, pid, claimedBy) {
+    const now = nowIso();
+    const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
+    if (res.changes !== 1)
+      return;
+    return this.getRun(id);
+  }
+  hasLiveWorkflowStepProcesses(loopRunId) {
+    const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid
+         FROM workflow_runs wr
+         JOIN workflow_step_runs wsr ON wsr.workflow_run_id = wr.id
+         WHERE wr.loop_run_id = ?
+           AND wr.status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
+           AND wsr.status = 'running'
+           AND wsr.pid IS NOT NULL`).all(loopRunId);
+    return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
+  }
   createSkippedRun(loop, scheduledFor, reason) {
     const now = nowIso();
     const run = {
@@ -502,6 +1082,14 @@ class Store {
       const existing = this.getRunBySlot(loop.id, scheduledFor);
       if (existing) {
         if (existing.status === "running") {
+          if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && existing.pid && isProcessAlive(existing.pid)) {
+            this.db.exec("COMMIT");
+            return;
+          }
+          if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && this.hasLiveWorkflowStepProcesses(existing.id)) {
+            this.db.exec("COMMIT");
+            return;
+          }
           const res3 = this.db.query(`UPDATE loop_runs SET status='running', started_at=$started, finished_at=NULL,
                claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
                duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
@@ -571,10 +1159,9 @@ class Store {
       throw error;
     }
   }
-  finalizeRun(id, patch) {
+  finalizeRun(id, patch, opts = {}) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
-         duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run({
+    const params = {
       $id: id,
       $status: patch.status,
       $finished: finishedAt,
@@ -584,13 +1171,29 @@ class Store {
       $stdout: patch.stdout ?? null,
       $stderr: patch.stderr ?? null,
       $error: patch.error ?? null,
-      $updated: finishedAt
-    });
+      $updated: finishedAt,
+      $claimedBy: opts.claimedBy ?? null,
+      $now: (opts.now ?? new Date).toISOString()
+    };
+    const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
+             duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
+             duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
     const run = this.getRun(id);
     if (!run)
       throw new Error(`run not found after finalize: ${id}`);
+    if (opts.claimedBy && res.changes !== 1)
+      return run;
     return run;
   }
+  heartbeatRunLease(id, claimedBy, leaseMs, now = new Date) {
+    const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
+    const res = this.db.query(`UPDATE loop_runs SET lease_expires_at=$expires, updated_at=$updated
+         WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $claimedBy: claimedBy, $expires: expiresAt, $updated: now.toISOString() });
+    if (res.changes !== 1)
+      return;
+    return this.getRun(id);
+  }
   listRuns(opts = {}) {
     const limit = opts.limit ?? 100;
     let rows;
@@ -609,8 +1212,26 @@ class Store {
     const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
     const recovered = [];
     for (const row of rows) {
+      if (row.pid && isProcessAlive(row.pid))
+        continue;
+      if (this.hasLiveWorkflowStepProcesses(row.id))
+        continue;
+      const finished = now.toISOString();
       this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
-           error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: now.toISOString(), $updated: now.toISOString() });
+           error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
+      const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
+      for (const workflowRow of workflowRows) {
+        this.db.query(`UPDATE workflow_runs
+             SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
+             WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
+        this.db.query(`UPDATE workflow_step_runs
+             SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
+             WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
+        this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
+          error: "parent loop run lease expired before completion",
+          loopRunId: row.id
+        });
+      }
       const run = this.getRun(row.id);
       if (run)
         recovered.push(run);