@osolmaz/pi-workflows 0.2.0 → 0.3.0

package/src/workflows/engine.ts

@@ -1,10 +1,20 @@
 import { randomUUID } from "node:crypto";
 import { isDeepStrictEqual } from "node:util";
-import { CancelledError, errorMessage, isAbortLikeError, TimeoutError } from "./errors.js";
+import { resolveArtifacts } from "./artifacts.js";
+import {
+  CancelledError,
+  errorMessage,
+  isAbortLikeError,
+  isClaimLostError,
+  isRunParkedError,
+  RunParkedError,
+  TimeoutError,
+  WorkflowSourceChangedError,
+} from "./errors.js";
 import { resolveNext, resolveNextForOutcome, validateWorkflowDefinition } from "./graph.js";
 import { extractJsonValue } from "./json.js";
 import { runShellAction, shellResultFromError } from "./shell.js";
-import { RUN_STATE_SCHEMA, WorkflowRunStore, createRunId } from "./store.js";
+import { RUN_STATE_SCHEMA, WorkflowRunStore, createRunId, readRunBundle } from "./store.js";
 import type {
   AgentNodeDefinition,
   AgentStepExecutor,
@@ -70,6 +80,7 @@ export class WorkflowEngine {
   private readonly onRunFinishing?: WorkflowEngineOptions["onRunFinishing"];
   private activeAbort: AbortController | null = null;
   private cancelled = false;
+  private parked = false;
   private paused = false;
   private wakePause: (() => void) | null = null;
 
@@ -96,6 +107,17 @@ export class WorkflowEngine {
     this.wakePause?.();
   }
 
+  /**
+   * Stop without a terminal event so another runner can claim and resume
+   * the run. The active node aborts; its partial attempt is never recorded,
+   * so resume reruns that node from its last persisted boundary.
+   */
+  park(): void {
+    this.parked = true;
+    this.activeAbort?.abort(new CancelledError());
+    this.wakePause?.();
+  }
+
   /**
    * Request a pause. The current step finishes normally; the engine then
    * holds before dispatching the next node until `resume` (or `cancel`).
@@ -118,17 +140,27 @@ export class WorkflowEngine {
   async run(
     workflow: WorkflowDefinition,
     input: unknown,
-    options: { workflowPath?: string } = {},
+    options: { workflowPath?: string; workflowHash?: string; runId?: string } = {},
   ): Promise<WorkflowRunResult> {
     validateWorkflowDefinition(workflow);
     // Fail before any bundle exists so bad input cannot leave a partial run
     // on disk or silently change shape when state.json round-trips.
     const normalizedInput = input === undefined ? null : input;
     assertJsonSerializable(normalizedInput, "Workflow run input");
+    if (options.runId !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(options.runId)) {
+      throw new Error(`Invalid workflow run id: ${JSON.stringify(options.runId)}`);
+    }
     this.cancelled = false;
     this.paused = false;
+    this.parked = false;
 
-    const state = await this.createRunState(workflow, normalizedInput, options.workflowPath);
+    const state = await this.createRunState(
+      workflow,
+      normalizedInput,
+      options.workflowPath,
+      options.workflowHash,
+      options.runId,
+    );
     const runDir = await this.store.initializeRunBundle(workflow, state);
     await this.persist(runDir, state, {
       scope: "run",
@@ -147,13 +179,273 @@ export class WorkflowEngine {
     try {
       await this.executeGraph(workflow, state, runDir);
     } catch (error) {
-      const cancelled = this.cancelled || isAbortLikeError(error);
+      if (isRunParkedError(error) || this.parked) {
+        return { runDir, state };
+      }
+      await this.finishAfterError(runDir, state, error);
+      return { runDir, state };
+    }
+    return { runDir, state };
+  }
+
+  /**
+   * Resume an interrupted run at the node it stopped on. The caller must
+   * hold the run's queue claim. Completed nodes replay from the recorded
+   * state; only the interrupted node and everything downstream rerun.
+   */
+  async resumeRun(
+    workflow: WorkflowDefinition,
+    runId: string,
+    options: { workflowHash?: string; force?: boolean } = {},
+  ): Promise<WorkflowRunResult> {
+    validateWorkflowDefinition(workflow);
+    // Reset before any await: a park or cancel landing during preparation
+    // must survive, or a host drain would hang while the run executes.
+    this.cancelled = false;
+    this.paused = false;
+    this.parked = false;
+    const bundle = await this.store.prepareRunResume(runId);
+    const { runDir } = bundle;
+    const state = bundle.state;
+    const hashMismatch =
+      state.workflowHash !== undefined &&
+      options.workflowHash !== undefined &&
+      state.workflowHash !== options.workflowHash;
+    if (hashMismatch && options.force !== true) {
+      throw new WorkflowSourceChangedError(runId);
+    }
+
+    const point = this.resumePointFor(workflow, state, "wait");
+    // A resumed run starts unpaused; the operator can pause again. The
+    // interrupted node's stale in-flight markers go away before the resume
+    // event so the projection matches what the engine is about to do.
+    delete state.paused;
+    delete state.currentNode;
+    delete state.currentAttemptId;
+    delete state.currentNodeStartedAt;
+    delete state.statusDetail;
+    await this.persist(runDir, state, {
+      scope: "run",
+      type: "run_resumed",
+      payload: {
+        ...(point.nodeId !== null ? { resumeAt: point.nodeId } : {}),
+        replayedSteps: state.steps.length,
+        ...(hashMismatch ? { workflowHashMismatch: true, forced: true } : {}),
+      },
+    });
+    await this.onRunStarted?.(runDir, state);
+
+    if (point.nodeId === null) {
+      // The last recorded transition already finished the graph; the crash
+      // happened before the terminal event was written. A finished
+      // checkpoint restores its waiting gate rather than completing.
+      if (point.waitingOn !== undefined) {
+        await this.finishRun(runDir, state, "waiting", {
+          waitingOn: point.waitingOn,
+          finalOutput: point.lastOutput,
+        });
+      } else if (point.failedResult === undefined) {
+        await this.finishRun(runDir, state, "completed", { finalOutput: point.lastOutput });
+      } else {
+        const timedOut = point.failedResult.outcome === "timed_out";
+        await this.finishRun(runDir, state, timedOut ? "timed_out" : "failed", {
+          error: point.failedResult.error ?? `Workflow node failed: ${point.failedResult.nodeId}`,
+        });
+      }
+      return { runDir, state };
+    }
+
+    try {
+      await this.executeGraph(
+        workflow,
+        state,
+        runDir,
+        point.nodeId,
+        state.steps.length,
+        point.lastOutput,
+      );
+    } catch (error) {
+      if (isRunParkedError(error) || this.parked) {
+        return { runDir, state };
+      }
+      await this.finishAfterError(runDir, state, error);
+      return { runDir, state };
+    }
+    return { runDir, state };
+  }
+
+  /**
+   * Start a continuation run from a checkpointed parent. The new run gets a
+   * fresh bundle and trace, carries forward the parent's outputs, results,
+   * and step accounting, and continues routing after the checkpoint.
+   */
+  async continueRun(
+    workflow: WorkflowDefinition,
+    parentRunId: string,
+    input: unknown,
+    options: { workflowPath?: string; workflowHash?: string; runId?: string; force?: boolean } = {},
+  ): Promise<WorkflowRunResult> {
+    validateWorkflowDefinition(workflow);
+    this.cancelled = false;
+    this.paused = false;
+    this.parked = false;
+    const parent = await readRunBundle(this.store.runDirFor(parentRunId));
+    if (parent === null) {
+      throw new Error(`Cannot continue from unreadable workflow run: ${parentRunId}`);
+    }
+    if (parent.state.status !== "waiting" || parent.state.waitingOn === undefined) {
+      throw new Error(
+        `Cannot continue workflow run ${parentRunId} with status ${parent.state.status}`,
+      );
+    }
+    const hashMismatch =
+      parent.state.workflowHash !== undefined &&
+      options.workflowHash !== undefined &&
+      parent.state.workflowHash !== options.workflowHash;
+    if (hashMismatch && options.force !== true) {
+      throw new WorkflowSourceChangedError(parentRunId);
+    }
+
+    const normalizedInput = input === undefined ? null : input;
+    assertJsonSerializable(normalizedInput, "Workflow run input");
+    if (options.runId !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(options.runId)) {
+      throw new Error(`Invalid workflow run id: ${JSON.stringify(options.runId)}`);
+    }
+
+    const state = await this.createRunState(
+      workflow,
+      normalizedInput,
+      options.workflowPath,
+      options.workflowHash,
+      options.runId,
+    );
+    state.parentRunId = parentRunId;
+    // Artifact references point into the parent's bundle, so carried values
+    // are fully resolved here and re-externalized into the new bundle.
+    state.outputs = (await resolveArtifacts(
+      parent.state.outputs,
+      parent.runDir,
+    )) as WorkflowRunState["outputs"];
+    state.results = (await resolveArtifacts(
+      parent.state.results,
+      parent.runDir,
+    )) as WorkflowRunState["results"];
+    state.steps = (await resolveArtifacts(
+      parent.state.steps,
+      parent.runDir,
+    )) as WorkflowRunState["steps"];
+    state.carriedStepCount = state.steps.length;
+
+    const runDir = await this.store.initializeRunBundle(workflow, state);
+    await this.persist(runDir, state, {
+      scope: "run",
+      type: "run_started",
+      payload: {
+        workflowName: workflow.name,
+        ...(state.runTitle ? { runTitle: state.runTitle } : {}),
+        input: state.input,
+        continuedFrom: parentRunId,
+        checkpoint: parent.state.waitingOn,
+        carriedSteps: state.steps.length,
+      },
+    });
+    await this.onRunStarted?.(runDir, state);
+
+    const point = this.resumePointFor(workflow, state, "continue");
+    if (point.nodeId === null) {
+      // The checkpoint was the final node; the answer completes the chain.
+      await this.finishRun(runDir, state, "completed", { finalOutput: point.lastOutput });
+      return { runDir, state };
+    }
+    try {
+      await this.executeGraph(
+        workflow,
+        state,
+        runDir,
+        point.nodeId,
+        state.steps.length,
+        point.lastOutput,
+      );
+    } catch (error) {
+      if (isRunParkedError(error) || this.parked) {
+        return { runDir, state };
+      }
+      await this.finishAfterError(runDir, state, error);
+      return { runDir, state };
+    }
+    return { runDir, state };
+  }
+
+  /**
+   * Find where a resumed run continues. An in-flight node reruns; otherwise
+   * routing continues from the last recorded step. A null nodeId means the
+   * graph was already done when the crash hit.
+   */
+  private resumePointFor(
+    workflow: WorkflowDefinition,
+    state: WorkflowRunState,
+    checkpointBehavior: "wait" | "continue",
+  ): {
+    nodeId: string | null;
+    lastOutput?: unknown;
+    failedResult?: WorkflowNodeResult;
+    waitingOn?: string;
+  } {
+    if (state.currentNode !== undefined) {
+      if (workflow.nodes[state.currentNode] === undefined) {
+        throw new Error(`Resume node is missing from the workflow: ${state.currentNode}`);
+      }
+      return { nodeId: state.currentNode };
+    }
+    const lastStep = state.steps.at(-1);
+    if (lastStep === undefined) {
+      return { nodeId: workflow.startAt };
+    }
+    const result = state.results[lastStep.nodeId];
+    if (result === undefined) {
+      return { nodeId: lastStep.nodeId };
+    }
+    if (result.outcome === "ok") {
+      // A recorded checkpoint means the run should be waiting; a crash
+      // before the run_waiting persist restores the gate instead of
+      // routing past it. The gate applies to this run's own checkpoint
+      // only: a continuation's carried steps end with the parent's
+      // already-answered checkpoint, and routing must continue from it.
+      const isCarriedStep = state.steps.length <= (state.carriedStepCount ?? 0);
+      if (
+        checkpointBehavior === "wait" &&
+        !isCarriedStep &&
+        workflow.nodes[lastStep.nodeId]?.nodeType === "checkpoint"
+      ) {
+        return { nodeId: null, waitingOn: lastStep.nodeId, lastOutput: result.output };
+      }
+      const next = resolveNext(workflow.edges, lastStep.nodeId, result.output, result);
+      return next === null
+        ? { nodeId: null, lastOutput: result.output }
+        : { nodeId: next, lastOutput: result.output };
+    }
+    const next = resolveNextForOutcome(workflow.edges, lastStep.nodeId, result);
+    return next === null ? { nodeId: null, failedResult: result } : { nodeId: next };
+  }
+
+  private async finishAfterError(
+    runDir: string,
+    state: WorkflowRunState,
+    error: unknown,
+  ): Promise<void> {
+    const cancelled = this.cancelled || isAbortLikeError(error);
+    try {
       await this.finishRun(runDir, state, cancelled ? "cancelled" : "failed", {
         error: errorMessage(error),
       });
-      return { runDir, state };
+    } catch (finishError) {
+      // A fenced-out runner must not touch the bundle, including terminal
+      // projections. Propagate the claim loss instead of the node error.
+      if (isClaimLostError(finishError)) {
+        throw finishError;
+      }
+      throw error;
     }
-    return { runDir, state };
   }
 
   /**
@@ -186,15 +478,18 @@ export class WorkflowEngine {
     workflow: WorkflowDefinition,
     input: unknown,
     workflowPath: string | undefined,
+    workflowHash: string | undefined,
+    runId: string | undefined,
   ): Promise<WorkflowRunState> {
     const now = new Date().toISOString();
     return {
       schema: RUN_STATE_SCHEMA,
       traceSeq: 0,
-      runId: createRunId(workflow.name),
+      runId: runId ?? createRunId(workflow.name),
       workflowName: workflow.name,
       ...(await this.resolveTitleBounded(workflow, input)),
       ...(workflowPath !== undefined ? { workflowPath } : {}),
+      ...(workflowHash !== undefined ? { workflowHash } : {}),
       startedAt: now,
       updatedAt: now,
       status: "running",
@@ -209,11 +504,14 @@ export class WorkflowEngine {
     workflow: WorkflowDefinition,
     state: WorkflowRunState,
     runDir: string,
+    startNodeId: string | null = workflow.startAt,
+    executedStepsBase = 0,
+    initialLastOutput?: unknown,
   ): Promise<void> {
     const maxSteps = workflow.maxSteps ?? this.maxSteps;
-    let currentNodeId: string | null = workflow.startAt;
-    let executedSteps = 0;
-    let lastOutput: unknown;
+    let currentNodeId: string | null = startNodeId;
+    let executedSteps = executedStepsBase;
+    let lastOutput: unknown = initialLastOutput;
 
     while (currentNodeId !== null) {
       await this.holdWhilePaused(state, runDir);
@@ -230,6 +528,11 @@ export class WorkflowEngine {
       }
 
       const attempt = await this.executeNode(workflow, state, runDir, currentNodeId, node);
+      if (this.parked) {
+        // Do not record the aborted attempt: the projection keeps the node
+        // as in-flight, and resume reruns it with a fresh attempt.
+        throw new RunParkedError();
+      }
       this.recordAttempt(state, attempt);
       // The terminal node event carries the output, receipt, and conversation
       // linkage so the trace alone is sufficient to reconstruct the run.
@@ -279,6 +582,9 @@ export class WorkflowEngine {
    * never interrupts a node mid-flight; it only delays the next dispatch.
    */
   private async holdWhilePaused(state: WorkflowRunState, runDir: string): Promise<void> {
+    if (this.parked) {
+      throw new RunParkedError();
+    }
     if (this.cancelled) {
       throw new CancelledError();
     }
@@ -287,12 +593,15 @@ export class WorkflowEngine {
     }
     state.paused = true;
     await this.persist(runDir, state, { scope: "run", type: "run_paused", payload: {} });
-    while (this.paused && !this.cancelled) {
+    while (this.paused && !this.cancelled && !this.parked) {
       await new Promise<void>((resolve) => {
         this.wakePause = resolve;
       });
     }
     this.wakePause = null;
+    if (this.parked) {
+      throw new RunParkedError();
+    }
     delete state.paused;
     if (this.cancelled) {
       throw new CancelledError();
@@ -452,6 +761,11 @@ export class WorkflowEngine {
     const timeoutMs = node.timeoutMs ?? this.defaultNodeTimeoutMs;
     const abort = new AbortController();
     this.activeAbort = abort;
+    if (this.parked) {
+      // A park that landed during the node_started persist must not let the
+      // node dispatch: its discarded side effects would rerun on resume.
+      throw new RunParkedError();
+    }
     if (this.cancelled) {
       throw new CancelledError();
     }
@@ -803,7 +1117,7 @@ export function appendStepContract(
     `Workflow step contract (workflow: ${workflowName}, step: ${nodeId}, attempt: ${attemptId})`,
     "",
     "Complete this step by calling the `workflow` tool exactly once with:",
-    `{"step": ${JSON.stringify(nodeId)}, "attempt": ${JSON.stringify(attemptId)}, "output": <your result>}`,
+    `{"action": "submit", "step": ${JSON.stringify(nodeId)}, "attempt": ${JSON.stringify(attemptId)}, "output": <your result>}`,
     `Expected output: ${expectedOutput ?? "a JSON object with your result"}`,
     "The step is complete only after the workflow tool accepts the output.",
     "If the tool reports a validation error, correct the output and call it again.",

package/src/workflows/errors.ts

@@ -15,6 +15,51 @@ export class CancelledError extends Error {
   }
 }
 
+/**
+ * Thrown when a runner writes to a run bundle after losing its queue claim.
+ * The current claim holder owns the run from that point on, so the fenced
+ * writer must stop touching the bundle immediately.
+ */
+export class ClaimLostError extends Error {
+  readonly runId: string;
+
+  constructor(runId: string) {
+    super(`Workflow run claim lost: ${runId}`);
+    this.name = "ClaimLostError";
+    this.runId = runId;
+  }
+}
+
+/**
+ * Internal stop signal for close-to-park: the engine halts without writing
+ * a terminal event, leaving a resumable bundle for the next claim holder.
+ */
+export class RunParkedError extends Error {
+  constructor() {
+    super("Workflow run was parked");
+    this.name = "RunParkedError";
+  }
+}
+
+export function isRunParkedError(error: unknown): error is RunParkedError {
+  return error instanceof RunParkedError;
+}
+
+/** The workflow source changed after the run started; resume needs force. */
+export class WorkflowSourceChangedError extends Error {
+  readonly runId: string;
+
+  constructor(runId: string) {
+    super(`Workflow source changed since run ${runId} started; pass force to resume anyway`);
+    this.name = "WorkflowSourceChangedError";
+    this.runId = runId;
+  }
+}
+
+export function isClaimLostError(error: unknown): error is ClaimLostError {
+  return error instanceof ClaimLostError;
+}
+
 export function isAbortLikeError(error: unknown): boolean {
   return error instanceof CancelledError || (error instanceof Error && error.name === "AbortError");
 }

package/src/workflows/graph.ts

@@ -58,11 +58,6 @@ function validateWorkflowEdge(
   outgoingEdges: Set<string>,
 ): void {
   assertKnownNode(workflow, edge.from, "Workflow edge references unknown from-node");
-  if (workflow.nodes[edge.from]?.nodeType === "checkpoint") {
-    // A checkpoint terminates the run as `waiting` and nothing resumes it,
-    // so an outgoing edge would make its targets silently unreachable.
-    throw new Error(`Workflow checkpoint node must not declare an outgoing edge: ${edge.from}`);
-  }
   if (outgoingEdges.has(edge.from)) {
     throw new Error(`Workflow node must not declare multiple outgoing edges: ${edge.from}`);
   }

package/src/workflows/loader.ts

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
@@ -11,7 +12,7 @@ const WORKFLOW_FILE_SUFFIXES = [".workflow.ts", ".workflow.js", ".workflow.mts",
 export type DiscoveredWorkflow = {
   name: string;
   path: string;
-  source: "project" | "global" | "path";
+  source: "project" | "global" | "builtin" | "path";
 };
 
 export type WorkflowSearchPaths = {
@@ -19,17 +20,26 @@ export type WorkflowSearchPaths = {
   homeDir?: string;
 };
 
-/** Directories scanned for `*.workflow.ts` files, in precedence order. */
+/** Directories scanned for workflow files, in precedence order. */
 export function workflowSearchDirs(
   options: WorkflowSearchPaths,
-): { dir: string; source: "project" | "global" }[] {
+): { dir: string; source: "project" | "global" | "builtin" }[] {
   const homeDir = options.homeDir ?? os.homedir();
+  const builtinDir = fileURLToPath(new URL("../builtins/", import.meta.url));
   return [
     { dir: path.join(options.cwd, ".pi", "workflows"), source: "project" },
     { dir: path.join(homeDir, ".pi", "agent", "workflows"), source: "global" },
+    { dir: builtinDir, source: "builtin" },
   ];
 }
 
+/** SHA-256 of a workflow source file, recorded in run state for resume pinning. */
+export async function hashWorkflowSource(filePath: string): Promise<string> {
+  return createHash("sha256")
+    .update(await fs.readFile(filePath))
+    .digest("hex");
+}
+
 function isWorkflowFile(fileName: string): boolean {
   return WORKFLOW_FILE_SUFFIXES.some((suffix) => fileName.endsWith(suffix));
 }

package/src/workflows/schema.ts

@@ -160,7 +160,7 @@ function assertValidEdgeShape(edge: WorkflowEdge, index: number): void {
  * Names claimed by `/workflow` subcommands; a workflow with one of these
  * names could never be started because the keyword wins the argument slot.
  */
-const RESERVED_WORKFLOW_NAMES = new Set(["cancel", "list", "pause", "resume"]);
+const RESERVED_WORKFLOW_NAMES = new Set(["answer", "cancel", "list", "pause", "resume", "status"]);
 
 export function assertValidWorkflowDefinitionShape(definition: WorkflowDefinition): void {
   assertRecord(definition, "workflow");