@hasna/loops 0.2.0 → 0.3.0

package/dist/lib/doctor.d.ts

@@ -0,0 +1,13 @@
+import type { Store } from "./store.js";
+export type DoctorSeverity = "ok" | "warn" | "fail";
+export interface DoctorCheck {
+    id: string;
+    status: DoctorSeverity;
+    message: string;
+    detail?: string;
+}
+export interface DoctorReport {
+    ok: boolean;
+    checks: DoctorCheck[];
+}
+export declare function runDoctor(store: Store): DoctorReport;

package/dist/lib/executor.d.ts

@@ -3,6 +3,8 @@ export interface ExecuteOptions {
     maxOutputBytes?: number;
     env?: NodeJS.ProcessEnv;
     log?: (message: string) => void;
+    signal?: AbortSignal;
+    onSpawn?: (pid: number) => void;
 }
 export interface ExecutionMetadata {
     loopId?: string;
@@ -14,5 +16,11 @@ export interface ExecutionMetadata {
     workflowRunId?: string;
     workflowStepId?: string;
 }
+export interface PreflightResult {
+    command: string;
+    accountProfile?: string;
+    accountTool?: string;
+}
+export declare function preflightTarget(target: ExecutableTarget, metadata?: ExecutionMetadata, opts?: ExecuteOptions): PreflightResult;
 export declare function executeTarget(target: ExecutableTarget, metadata?: ExecutionMetadata, opts?: ExecuteOptions): Promise<ExecutorResult>;
 export declare function executeLoop(loop: Loop, run: LoopRun, opts?: ExecuteOptions): Promise<ExecutorResult>;

package/dist/lib/scheduler.d.ts

@@ -15,4 +15,16 @@ export interface TickResult {
     recovered: LoopRun[];
     expired: Loop[];
 }
+export declare function manualRunScheduledFor(loop: Loop, now?: Date): string;
+export declare function shouldAdvanceManualRun(loop: Loop, scheduledFor: string, now?: Date): boolean;
+export declare function advanceLoop(store: Store, loop: Loop, run: LoopRun, finishedAt: Date, succeeded: boolean): void;
+export declare function executeClaimedRun(deps: {
+    store: Store;
+    runnerId: string;
+    loop: Loop;
+    run: LoopRun;
+    now?: () => Date;
+    execute?: (loop: Loop, run: LoopRun) => Promise<ExecutorResult>;
+    onError?: (loop: Loop, error: unknown) => void;
+}): Promise<LoopRun>;
 export declare function tick(deps: SchedulerDeps): Promise<TickResult>;

package/dist/lib/store.d.ts

@@ -45,6 +45,7 @@ export declare class Store {
     archiveWorkflow(idOrName: string): WorkflowSpec;
     createWorkflowRun(input: CreateWorkflowRunInput): WorkflowRun;
     getWorkflowRun(id: string): WorkflowRun | undefined;
+    requireWorkflowRun(id: string): WorkflowRun;
     listWorkflowRuns(opts?: {
         workflowId?: string;
         loopRunId?: string;
@@ -52,13 +53,22 @@ export declare class Store {
     }): WorkflowRun[];
     listWorkflowStepRuns(workflowRunId: string): WorkflowStepRun[];
     getWorkflowStepRun(workflowRunId: string, stepId: string): WorkflowStepRun | undefined;
+    isWorkflowRunTerminal(workflowRunId: string): boolean;
     startWorkflowStepRun(workflowRunId: string, stepId: string): WorkflowStepRun;
+    markWorkflowStepPid(workflowRunId: string, stepId: string, pid: number): WorkflowStepRun;
+    recoverWorkflowRun(workflowRunId: string, reason?: string): {
+        run: WorkflowRun;
+        recoveredSteps: WorkflowStepRun[];
+    };
     finalizeWorkflowStepRun(workflowRunId: string, stepId: string, patch: Pick<WorkflowStepRun, "status" | "finishedAt" | "durationMs" | "stdout" | "stderr"> & Partial<Pick<WorkflowStepRun, "exitCode" | "error">>): WorkflowStepRun;
     skipWorkflowStepRun(workflowRunId: string, stepId: string, reason: string): WorkflowStepRun;
     finalizeWorkflowRun(workflowRunId: string, status: WorkflowRunStatus, patch?: Partial<Pick<WorkflowRun, "finishedAt" | "durationMs" | "error">>): WorkflowRun;
+    cancelWorkflowRun(workflowRunId: string, reason?: string): WorkflowRun;
     appendWorkflowEvent(workflowRunId: string, eventType: string, stepId?: string, payload?: Record<string, unknown>): WorkflowEvent;
     listWorkflowEvents(workflowRunId: string, limit?: number): WorkflowEvent[];
     hasRunningRun(loopId: string): boolean;
+    markRunPid(id: string, pid: number, claimedBy?: string): LoopRun | undefined;
+    private hasLiveWorkflowStepProcesses;
     createSkippedRun(loop: Loop, scheduledFor: string, reason: string): LoopRun;
     getRun(id: string): LoopRun | undefined;
     getRunBySlot(loopId: string, scheduledFor: string): LoopRun | undefined;

package/dist/lib/store.js

@@ -247,6 +247,9 @@ function validateTarget(value, label) {
   assertObject(value, label);
   if (value.type === "command") {
     assertString(value.command, `${label}.command`);
+    if (value.shell !== true && /\s/.test(value.command.trim())) {
+      throw new Error(`${label}.command must be an executable without spaces when shell is false; put flags in args or set shell true`);
+    }
     return value;
   }
   if (value.type === "agent") {
@@ -412,6 +415,7 @@ function rowToWorkflowStepRun(row) {
     startedAt: row.started_at ?? undefined,
     finishedAt: row.finished_at ?? undefined,
     exitCode: row.exit_code ?? undefined,
+    pid: row.pid ?? undefined,
     durationMs: row.duration_ms ?? undefined,
     stdout: row.stdout ?? undefined,
     stderr: row.stderr ?? undefined,
@@ -433,6 +437,14 @@ function rowToWorkflowEvent(row) {
     createdAt: row.created_at
   };
 }
+function isProcessAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
 function rowToLease(row) {
   return {
     id: row.id,
@@ -565,6 +577,7 @@ class Store {
         started_at TEXT,
         finished_at TEXT,
         exit_code INTEGER,
+        pid INTEGER,
         duration_ms INTEGER,
         stdout TEXT,
         stderr TEXT,
@@ -590,6 +603,9 @@ class Store {
       );
       CREATE INDEX IF NOT EXISTS idx_workflow_events_run_sequence ON workflow_events(workflow_run_id, sequence);
     `);
+    try {
+      this.db.query("ALTER TABLE workflow_step_runs ADD COLUMN pid INTEGER").run();
+    } catch {}
     this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
   }
   createLoop(input, from = new Date) {
@@ -770,8 +786,8 @@ class Store {
       input.workflow.steps.forEach((step, sequence) => {
         const account = step.account ?? step.target.account;
         this.db.query(`INSERT INTO workflow_step_runs (id, workflow_run_id, step_id, sequence, status, started_at, finished_at,
-              exit_code, duration_ms, stdout, stderr, error, account_profile, account_tool, created_at, updated_at)
-             VALUES ($id, $workflowRunId, $stepId, $sequence, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+              exit_code, pid, duration_ms, stdout, stderr, error, account_profile, account_tool, created_at, updated_at)
+             VALUES ($id, $workflowRunId, $stepId, $sequence, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
               $accountProfile, $accountTool, $created, $updated)`).run({
           $id: genId(),
           $workflowRunId: runId,
@@ -812,6 +828,12 @@ class Store {
     const row = this.db.query("SELECT * FROM workflow_runs WHERE id = ?").get(id);
     return row ? rowToWorkflowRun(row) : undefined;
   }
+  requireWorkflowRun(id) {
+    const run = this.getWorkflowRun(id);
+    if (!run)
+      throw new Error(`workflow run not found: ${id}`);
+    return run;
+  }
   listWorkflowRuns(opts = {}) {
     const limit = opts.limit ?? 100;
     let rows;
@@ -832,23 +854,67 @@ class Store {
     const row = this.db.query("SELECT * FROM workflow_step_runs WHERE workflow_run_id = ? AND step_id = ?").get(workflowRunId, stepId);
     return row ? rowToWorkflowStepRun(row) : undefined;
   }
+  isWorkflowRunTerminal(workflowRunId) {
+    const run = this.getWorkflowRun(workflowRunId);
+    return Boolean(run && ["succeeded", "failed", "timed_out", "cancelled"].includes(run.status));
+  }
   startWorkflowStepRun(workflowRunId, stepId) {
     const now = nowIso();
-    this.db.query(`UPDATE workflow_step_runs
+    const res = this.db.query(`UPDATE workflow_step_runs
          SET status='running', started_at=$started, finished_at=NULL, exit_code=NULL, duration_ms=NULL,
-          stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running', 'failed', 'timed_out')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
-    this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
+          pid=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId
+           AND step_id=$stepId
+           AND status IN ('pending', 'failed', 'timed_out')
+           AND EXISTS (
+             SELECT 1 FROM workflow_runs
+             WHERE id=$workflowRunId AND status='running'
+           )`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $started: now, $updated: now });
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found: ${workflowRunId}/${stepId}`);
+    if (res.changes !== 1) {
+      throw new Error(`workflow step is not claimable: ${workflowRunId}/${stepId} status=${run.status}`);
+    }
+    this.appendWorkflowEvent(workflowRunId, "step_started", stepId);
+    return run;
+  }
+  markWorkflowStepPid(workflowRunId, stepId, pid) {
+    const now = nowIso();
+    this.db.query(`UPDATE workflow_step_runs SET pid=$pid, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $pid: pid, $updated: now });
+    const run = this.getWorkflowStepRun(workflowRunId, stepId);
+    if (!run)
+      throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
     return run;
   }
+  recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
+    const now = nowIso();
+    const before = this.listWorkflowStepRuns(workflowRunId).filter((step) => step.status === "running");
+    const live = before.filter((step) => step.pid !== undefined && isProcessAlive(step.pid));
+    if (live.length > 0) {
+      throw new Error(`cannot recover workflow run while step processes are still alive: ${live.map((step) => `${step.stepId} pid=${step.pid}`).join(", ")}`);
+    }
+    this.db.query(`UPDATE workflow_step_runs
+         SET status='pending', started_at=NULL, finished_at=NULL, exit_code=NULL, pid=NULL, duration_ms=NULL,
+          stdout=NULL, stderr=NULL, error=$reason, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND status='running'`).run({ $workflowRunId: workflowRunId, $reason: reason, $updated: now });
+    if (before.length > 0) {
+      this.appendWorkflowEvent(workflowRunId, "recovered", undefined, {
+        reason,
+        recoveredSteps: before.map((step) => step.stepId)
+      });
+    }
+    return {
+      run: this.requireWorkflowRun(workflowRunId),
+      recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
+    };
+  }
   finalizeWorkflowStepRun(workflowRunId, stepId, patch) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
-         stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId`).run({
+    const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
+         pid=NULL, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'`).run({
       $workflowRunId: workflowRunId,
       $stepId: stepId,
       $status: patch.status,
@@ -860,10 +926,12 @@ class Store {
       $error: patch.error ?? null,
       $updated: finishedAt
     });
-    this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
-      exitCode: patch.exitCode,
-      error: patch.error
-    });
+    if (res.changes === 1) {
+      this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
+        exitCode: patch.exitCode,
+        error: patch.error
+      });
+    }
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after finalize: ${workflowRunId}/${stepId}`);
@@ -871,9 +939,10 @@ class Store {
   }
   skipWorkflowStepRun(workflowRunId, stepId, reason) {
     const now = nowIso();
-    this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, error=$error, updated_at=$updated
-         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
-    this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
+    const res = this.db.query(`UPDATE workflow_step_runs SET status='skipped', finished_at=$finished, pid=NULL, error=$error, updated_at=$updated
+         WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $stepId: stepId, $finished: now, $error: reason, $updated: now });
+    if (res.changes === 1)
+      this.appendWorkflowEvent(workflowRunId, "step_skipped", stepId, { reason });
     const run = this.getWorkflowStepRun(workflowRunId, stepId);
     if (!run)
       throw new Error(`workflow step run not found after skip: ${workflowRunId}/${stepId}`);
@@ -881,8 +950,8 @@ class Store {
   }
   finalizeWorkflowRun(workflowRunId, status, patch = {}) {
     const finishedAt = patch.finishedAt ?? nowIso();
-    this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
-         WHERE id=$id`).run({
+    const res = this.db.query(`UPDATE workflow_runs SET status=$status, finished_at=$finished, duration_ms=$durationMs, error=$error, updated_at=$updated
+         WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({
       $id: workflowRunId,
       $status: status,
       $finished: finishedAt,
@@ -890,12 +959,36 @@ class Store {
       $error: patch.error ?? null,
       $updated: finishedAt
     });
-    this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
     const run = this.getWorkflowRun(workflowRunId);
     if (!run)
       throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
+    if (res.changes === 1)
+      this.appendWorkflowEvent(workflowRunId, status, undefined, { error: patch.error });
     return run;
   }
+  cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
+    const now = nowIso();
+    this.db.exec("BEGIN IMMEDIATE");
+    try {
+      const run = this.requireWorkflowRun(workflowRunId);
+      if (!["succeeded", "failed", "timed_out", "cancelled"].includes(run.status)) {
+        this.db.query(`UPDATE workflow_runs
+             SET status='cancelled', finished_at=$finished, error=$reason, updated_at=$updated
+             WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRunId, $finished: now, $reason: reason, $updated: now });
+        this.db.query(`UPDATE workflow_step_runs
+             SET status='cancelled', finished_at=$finished, pid=NULL, error=$reason, updated_at=$updated
+             WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRunId, $finished: now, $reason: reason, $updated: now });
+        this.appendWorkflowEvent(workflowRunId, "cancelled", undefined, { reason });
+      }
+      this.db.exec("COMMIT");
+      return this.requireWorkflowRun(workflowRunId);
+    } catch (error) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw error;
+    }
+  }
   appendWorkflowEvent(workflowRunId, eventType, stepId, payload) {
     const now = nowIso();
     const current = this.db.query("SELECT MAX(sequence) AS sequence FROM workflow_events WHERE workflow_run_id = ?").get(workflowRunId);
@@ -924,6 +1017,24 @@ class Store {
     const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND status = 'running'").get(loopId);
     return (row?.count ?? 0) > 0;
   }
+  markRunPid(id, pid, claimedBy) {
+    const now = nowIso();
+    const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
+             WHERE id=$id AND status='running' AND claimed_by=$claimedBy`).run({ $id: id, $pid: pid, $updated: now, $claimedBy: claimedBy }) : this.db.query("UPDATE loop_runs SET pid=$pid, updated_at=$updated WHERE id=$id AND status='running'").run({ $id: id, $pid: pid, $updated: now });
+    if (res.changes !== 1)
+      return;
+    return this.getRun(id);
+  }
+  hasLiveWorkflowStepProcesses(loopRunId) {
+    const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid
+         FROM workflow_runs wr
+         JOIN workflow_step_runs wsr ON wsr.workflow_run_id = wr.id
+         WHERE wr.loop_run_id = ?
+           AND wr.status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
+           AND wsr.status = 'running'
+           AND wsr.pid IS NOT NULL`).all(loopRunId);
+    return liveWorkflowSteps.some((step) => isProcessAlive(step.pid));
+  }
   createSkippedRun(loop, scheduledFor, reason) {
     const now = nowIso();
     const run = {
@@ -971,6 +1082,14 @@ class Store {
       const existing = this.getRunBySlot(loop.id, scheduledFor);
       if (existing) {
         if (existing.status === "running") {
+          if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && existing.pid && isProcessAlive(existing.pid)) {
+            this.db.exec("COMMIT");
+            return;
+          }
+          if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && this.hasLiveWorkflowStepProcesses(existing.id)) {
+            this.db.exec("COMMIT");
+            return;
+          }
           const res3 = this.db.query(`UPDATE loop_runs SET status='running', started_at=$started, finished_at=NULL,
                claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
                duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
@@ -1093,8 +1212,26 @@ class Store {
     const rows = this.db.query("SELECT * FROM loop_runs WHERE status = 'running' AND lease_expires_at <= ?").all(now.toISOString());
     const recovered = [];
     for (const row of rows) {
+      if (row.pid && isProcessAlive(row.pid))
+        continue;
+      if (this.hasLiveWorkflowStepProcesses(row.id))
+        continue;
+      const finished = now.toISOString();
       this.db.query(`UPDATE loop_runs SET status='abandoned', finished_at=$finished, lease_expires_at=NULL,
-           error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: now.toISOString(), $updated: now.toISOString() });
+           error='run lease expired before completion', updated_at=$updated WHERE id=$id`).run({ $id: row.id, $finished: finished, $updated: finished });
+      const workflowRows = this.db.query("SELECT * FROM workflow_runs WHERE loop_run_id = ? AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')").all(row.id);
+      for (const workflowRow of workflowRows) {
+        this.db.query(`UPDATE workflow_runs
+             SET status='failed', finished_at=$finished, error='parent loop run lease expired before completion', updated_at=$updated
+             WHERE id=$id AND status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')`).run({ $id: workflowRow.id, $finished: finished, $updated: finished });
+        this.db.query(`UPDATE workflow_step_runs
+             SET status='skipped', finished_at=$finished, pid=NULL, error='parent loop run lease expired before completion', updated_at=$updated
+             WHERE workflow_run_id=$workflowRunId AND status IN ('pending', 'running')`).run({ $workflowRunId: workflowRow.id, $finished: finished, $updated: finished });
+        this.appendWorkflowEvent(workflowRow.id, "failed", undefined, {
+          error: "parent loop run lease expired before completion",
+          loopRunId: row.id
+        });
+      }
       const run = this.getRun(row.id);
       if (run)
         recovered.push(run);

package/dist/lib/workflow-runner.d.ts

@@ -1,11 +1,14 @@
 import type { ExecutorResult, Loop, LoopRun, WorkflowSpec } from "../types.js";
-import { type ExecuteOptions } from "./executor.js";
+import { preflightTarget, type ExecuteOptions } from "./executor.js";
 import type { Store } from "./store.js";
 export interface ExecuteWorkflowOptions extends ExecuteOptions {
     loop?: Loop;
     loopRun?: LoopRun;
     scheduledFor?: string;
     idempotencyKey?: string;
+    cancelPollMs?: number;
+    signalTimeoutMessage?: () => string | undefined;
 }
 export declare function executeWorkflow(store: Store, workflow: WorkflowSpec, opts?: ExecuteWorkflowOptions): Promise<ExecutorResult>;
+export declare function preflightWorkflow(workflow: WorkflowSpec, opts?: ExecuteOptions): ReturnType<typeof preflightTarget>[];
 export declare function executeLoopTarget(store: Store, loop: Loop, run: LoopRun, opts?: ExecuteOptions): Promise<ExecutorResult>;