@osolmaz/pi-workflows 0.3.0 → 0.4.0

package/src/host/runner.ts

@@ -1,6 +1,7 @@
 import { randomUUID } from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
+import { builtinWorkflowCatalog } from "../builtins/catalog.js";
 import {
   ControllerManager,
   loadDiscoveredControllers,
@@ -17,8 +18,9 @@ import {
   isClaimLostError,
   WorkflowSourceChangedError,
 } from "../workflows/errors.js";
-import { hashWorkflowSource, loadWorkflowFile, resolveWorkflowRef } from "../workflows/loader.js";
-import { WorkflowRunStore } from "../workflows/store.js";
+import { resolveWorkflowRef, resolveWorkflowSource } from "../workflows/loader.js";
+import { migrateLegacyWorkflowSources } from "../workflows/migrate-sources.js";
+import { WorkflowRunStore, readRunBundle } from "../workflows/store.js";
 import type { WorkflowDefinition } from "../workflows/types.js";
 import { HostProcessRegistry } from "./processes.js";
 import { RpcStepExecutor } from "./rpc-executor.js";
@@ -59,6 +61,7 @@ export class WorkflowHost {
   private manager: ControllerManager | null = null;
   private pollTimer: ReturnType<typeof setInterval> | null = null;
   private readonly activeRuns = new Map<string, Promise<void>>();
+  private readonly migrationBlockedRuns = new Set<string>();
   private readonly schedulerExecutors = new Map<WorkflowEngine, RpcStepExecutor>();
   /** Runs whose resume refused (edited source); skipped until a host restart. */
   private readonly skippedRuns = new Set<string>();
@@ -94,14 +97,27 @@ export class WorkflowHost {
       this.log(`reaped ${reaped.length} orphaned headless session(s): ${reaped.join(", ")}`);
     }
 
+    const migration = await migrateLegacyWorkflowSources({
+      catalog: builtinWorkflowCatalog,
+      store: this.childRunStore,
+      queue: this.store,
+    });
+    for (const blocked of migration.blocked) {
+      this.migrationBlockedRuns.add(blocked.runId);
+      this.log(`run ${blocked.runId} parked: ${blocked.reason}`);
+    }
+
     const definitions = await loadDiscoveredControllers({ cwd: this.options.cwd });
     if (definitions.length > 0) {
       const scheduler = new WorkflowEngineScheduler({
         store: this.childRunStore,
         resolveWorkflow: async (name) => {
-          const resolved = await resolveWorkflowRef(name, { cwd: this.options.cwd });
-          const workflow = await loadWorkflowFile(resolved.path);
-          return { workflow };
+          const resolved = await resolveWorkflowRef(
+            name,
+            { cwd: this.options.cwd },
+            builtinWorkflowCatalog,
+          );
+          return { workflow: resolved.definition, workflowSource: resolved.source };
         },
         createEngine: () => {
           const executor = new RpcStepExecutor({
@@ -176,7 +192,7 @@ export class WorkflowHost {
         runnerId: this.runnerId,
         claimToken: randomUUID(),
         leaseMs: RUN_CLAIM_LEASE_MS,
-        excludeRunIds: [...this.skippedRuns],
+        excludeRunIds: [...this.skippedRuns, ...this.migrationBlockedRuns],
       });
     } catch (error) {
       // Store contention or corruption must not kill the host's loop.
@@ -195,13 +211,36 @@ export class WorkflowHost {
   private async runClaimed(record: WorkflowRunQueueRecord): Promise<void> {
     const claimToken = record.claimToken as string;
     const runId = record.runId;
-    this.log(`resuming ${record.workflowRef} run ${runId}`);
+    this.log(`resuming ${record.workflowName} run ${runId}`);
     let workflow: WorkflowDefinition;
-    let workflowHash: string;
+    let workflowSource: import("../workflows/types.js").WorkflowSource;
     try {
-      workflow = await loadWorkflowFile(record.workflowPath);
-      workflowHash = await hashWorkflowSource(record.workflowPath);
+      const bundle = await readRunBundle(this.childRunStore.runDirFor(runId));
+      if (bundle?.state.workflowSource === undefined) {
+        throw new Error(`Workflow run ${runId} has no canonical workflow source`);
+      }
+      workflow = await resolveWorkflowSource(
+        bundle.state.workflowSource,
+        builtinWorkflowCatalog,
+        runId,
+      );
+      workflowSource = bundle.state.workflowSource;
     } catch (error) {
+      if (error instanceof WorkflowSourceChangedError) {
+        try {
+          this.store.parkWorkflowRun({ runId, claimToken });
+        } catch {
+          // Best-effort.
+        }
+        this.skippedRuns.add(runId);
+        this.recordEvent(runId, record.workflowName, "parked", {
+          reason: "workflow source changed",
+        });
+        this.log(
+          `run ${runId} skipped: workflow source changed; install the matching package revision, then restart the host`,
+        );
+        return;
+      }
       await this.failUnresumable(record, claimToken, errorMessage(error));
       return;
     }
@@ -245,23 +284,23 @@ export class WorkflowHost {
     }, RUN_CLAIM_RENEW_MS);
     renewTimer.unref?.();
 
-    this.recordEvent(runId, record.workflowRef, "resumed", { runnerId: this.runnerId });
+    this.recordEvent(runId, record.workflowName, "resumed", { runnerId: this.runnerId });
     try {
-      const result = await engine.resumeRun(workflow, runId, { workflowHash });
+      const result = await engine.resumeRun(workflow, runId, { workflowSource });
       clearInterval(renewTimer);
       if (result.state.status === "running") {
         // Parked again mid-drain: leave it claimable for the next runner.
         this.store.parkWorkflowRun({ runId, claimToken });
-        this.recordEvent(runId, record.workflowRef, "parked", {});
-        this.log(`parked ${record.workflowRef} run ${runId}`);
+        this.recordEvent(runId, record.workflowName, "parked", {});
+        this.log(`parked ${record.workflowName} run ${runId}`);
         return;
       }
       this.store.completeWorkflowRun({ runId, claimToken });
-      this.recordEvent(runId, record.workflowRef, result.state.status, {
+      this.recordEvent(runId, record.workflowName, result.state.status, {
         ...(result.state.error !== undefined ? { error: result.state.error } : {}),
         ...(result.state.waitingOn !== undefined ? { waitingOn: result.state.waitingOn } : {}),
       });
-      this.log(`${record.workflowRef} run ${runId} ${result.state.status}`);
+      this.log(`${record.workflowName} run ${runId} ${result.state.status}`);
     } catch (error) {
       clearInterval(renewTimer);
       if (isClaimLostError(error)) {
@@ -277,7 +316,7 @@ export class WorkflowHost {
           // Best-effort.
         }
         this.skippedRuns.add(runId);
-        this.recordEvent(runId, record.workflowRef, "parked", {
+        this.recordEvent(runId, record.workflowName, "parked", {
           reason: "workflow source changed",
         });
         this.log(
@@ -327,9 +366,9 @@ export class WorkflowHost {
     // no-op (the bundle was already waiting or completed), so the feed
     // stays truthful for sessions syncing from it.
     if (actualStatus !== undefined && actualStatus !== "failed") {
-      this.recordEvent(record.runId, record.workflowRef, actualStatus, {});
+      this.recordEvent(record.runId, record.workflowName, actualStatus, {});
     } else {
-      this.recordEvent(record.runId, record.workflowRef, "failed", { error: message });
+      this.recordEvent(record.runId, record.workflowName, "failed", { error: message });
     }
     this.log(`run ${record.runId} cannot resume: ${message}`);
   }

package/src/workflows/catalog.ts

@@ -0,0 +1,135 @@
+import { isWorkflowDefinition } from "./definition.js";
+import { WorkflowSourceChangedError } from "./errors.js";
+import type { WorkflowDefinition, WorkflowSource } from "./types.js";
+
+const BUILTIN_ID_PATTERN = /^[a-z][a-z0-9-]{0,63}$/;
+const REVISION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+
+export type LegacyBuiltinSource = {
+  workflowHash: string;
+  revision: string;
+  pathSuffixes: readonly string[];
+};
+
+export type BuiltinWorkflowRegistration = {
+  id: string;
+  revision: string;
+  definition: WorkflowDefinition;
+  legacySources?: LegacyBuiltinSource[];
+};
+
+export type BuiltinWorkflowEntry = Readonly<{
+  id: string;
+  ref: string;
+  revision: string;
+  definition: WorkflowDefinition;
+  legacySources: readonly LegacyBuiltinSource[];
+}>;
+
+export type LegacyBuiltinMatch = {
+  entry: BuiltinWorkflowEntry;
+  revision: string;
+};
+
+/** Process-local catalog of package-provided workflow definitions. */
+export class BuiltinWorkflowCatalog {
+  private readonly byId = new Map<string, BuiltinWorkflowEntry>();
+  private readonly byName = new Map<string, BuiltinWorkflowEntry>();
+
+  constructor(registrations: BuiltinWorkflowRegistration[]) {
+    for (const registration of registrations) {
+      if (!BUILTIN_ID_PATTERN.test(registration.id)) {
+        throw new Error(`Invalid built-in workflow id: ${JSON.stringify(registration.id)}`);
+      }
+      if (!REVISION_PATTERN.test(registration.revision)) {
+        throw new Error(
+          `Invalid built-in workflow revision: ${JSON.stringify(registration.revision)}`,
+        );
+      }
+      if (!isWorkflowDefinition(registration.definition)) {
+        throw new Error(`Built-in workflow ${registration.id} is not defined with defineWorkflow`);
+      }
+      if (this.byId.has(registration.id)) {
+        throw new Error(`Duplicate built-in workflow id: ${registration.id}`);
+      }
+      if (this.byName.has(registration.definition.name)) {
+        throw new Error(`Duplicate built-in workflow name: ${registration.definition.name}`);
+      }
+      const entry: BuiltinWorkflowEntry = Object.freeze({
+        id: registration.id,
+        ref: `builtin:${registration.id}`,
+        revision: registration.revision,
+        definition: registration.definition,
+        legacySources: Object.freeze(
+          (registration.legacySources ?? []).map((legacy) =>
+            Object.freeze({ ...legacy, pathSuffixes: Object.freeze([...legacy.pathSuffixes]) }),
+          ),
+        ),
+      });
+      this.byId.set(entry.id, entry);
+      this.byName.set(entry.definition.name, entry);
+    }
+  }
+
+  list(): BuiltinWorkflowEntry[] {
+    return [...this.byId.values()];
+  }
+
+  get(id: string): BuiltinWorkflowEntry | undefined {
+    return this.byId.get(id);
+  }
+
+  getByName(name: string): BuiltinWorkflowEntry | undefined {
+    return this.byName.get(name);
+  }
+
+  resolve(
+    source: WorkflowSource,
+    runId = `builtin:${source.kind === "builtin" ? source.id : "unknown"}`,
+  ): WorkflowDefinition {
+    if (source.kind !== "builtin") {
+      throw new Error("A file workflow source cannot be resolved by the built-in catalog");
+    }
+    const entry = this.byId.get(source.id);
+    if (entry === undefined) {
+      throw new Error(`Unknown built-in workflow: ${source.id}`);
+    }
+    if (entry.revision !== source.revision) {
+      throw new WorkflowSourceChangedError(runId);
+    }
+    return entry.definition;
+  }
+
+  matchLegacy(options: {
+    workflowName: string;
+    workflowPath: string;
+    workflowHash: string;
+  }): LegacyBuiltinMatch | undefined {
+    const entry = this.legacyPathEntry(options);
+    if (entry === undefined) return undefined;
+    const workflowPath = options.workflowPath.replaceAll("\\", "/");
+    const legacy = entry.legacySources.find(
+      (candidate) =>
+        candidate.workflowHash === options.workflowHash &&
+        candidate.pathSuffixes.some((suffix) =>
+          workflowPath.endsWith(suffix.replaceAll("\\", "/")),
+        ),
+    );
+    return legacy === undefined ? undefined : { entry, revision: legacy.revision };
+  }
+
+  /** Identify a registered old built-in path without accepting its revision. */
+  legacyPathEntry(options: {
+    workflowName: string;
+    workflowPath: string;
+  }): BuiltinWorkflowEntry | undefined {
+    const entry = this.byName.get(options.workflowName);
+    if (entry === undefined) return undefined;
+    const workflowPath = options.workflowPath.replaceAll("\\", "/");
+    return entry.legacySources.some((legacy) =>
+      legacy.pathSuffixes.some((suffix) => workflowPath.endsWith(suffix.replaceAll("\\", "/"))),
+    )
+      ? entry
+      : undefined;
+  }
+}

package/src/workflows/engine.ts

@@ -32,6 +32,7 @@ import type {
   WorkflowNodeResult,
   WorkflowRunResult,
   WorkflowRunState,
+  WorkflowSource,
   WorkflowStepRecord,
   WorkflowTraceEventDraft,
 } from "./types.js";
@@ -39,6 +40,7 @@ import type {
 const DEFAULT_NODE_TIMEOUT_MS = 15 * 60_000;
 const DEFAULT_MAX_STEPS = 100;
 const TITLE_TIMEOUT_MS = 30_000;
+const TIMEOUT_RESOLUTION_TIMEOUT_MS = 30_000;
 // Covers the shell SIGTERM → SIGKILL escalation (1s) plus stdio close.
 const ABORT_CLEANUP_GRACE_MS = 2_000;
 
@@ -140,7 +142,7 @@ export class WorkflowEngine {
   async run(
     workflow: WorkflowDefinition,
     input: unknown,
-    options: { workflowPath?: string; workflowHash?: string; runId?: string } = {},
+    options: { workflowSource?: WorkflowSource; runId?: string } = {},
   ): Promise<WorkflowRunResult> {
     validateWorkflowDefinition(workflow);
     // Fail before any bundle exists so bad input cannot leave a partial run
@@ -157,8 +159,7 @@ export class WorkflowEngine {
     const state = await this.createRunState(
       workflow,
       normalizedInput,
-      options.workflowPath,
-      options.workflowHash,
+      options.workflowSource,
       options.runId,
     );
     const runDir = await this.store.initializeRunBundle(workflow, state);
@@ -196,7 +197,7 @@ export class WorkflowEngine {
   async resumeRun(
     workflow: WorkflowDefinition,
     runId: string,
-    options: { workflowHash?: string; force?: boolean } = {},
+    options: { workflowSource?: WorkflowSource; force?: boolean } = {},
   ): Promise<WorkflowRunResult> {
     validateWorkflowDefinition(workflow);
     // Reset before any await: a park or cancel landing during preparation
@@ -207,11 +208,8 @@ export class WorkflowEngine {
     const bundle = await this.store.prepareRunResume(runId);
     const { runDir } = bundle;
     const state = bundle.state;
-    const hashMismatch =
-      state.workflowHash !== undefined &&
-      options.workflowHash !== undefined &&
-      state.workflowHash !== options.workflowHash;
-    if (hashMismatch && options.force !== true) {
+    const sourceMismatch = workflowSourceMismatch(state, options.workflowSource);
+    if (sourceMismatch && options.force !== true) {
       throw new WorkflowSourceChangedError(runId);
     }
 
@@ -230,7 +228,7 @@ export class WorkflowEngine {
       payload: {
         ...(point.nodeId !== null ? { resumeAt: point.nodeId } : {}),
         replayedSteps: state.steps.length,
-        ...(hashMismatch ? { workflowHashMismatch: true, forced: true } : {}),
+        ...(sourceMismatch ? { workflowSourceMismatch: true, forced: true } : {}),
       },
     });
     await this.onRunStarted?.(runDir, state);
@@ -283,7 +281,7 @@ export class WorkflowEngine {
     workflow: WorkflowDefinition,
     parentRunId: string,
     input: unknown,
-    options: { workflowPath?: string; workflowHash?: string; runId?: string; force?: boolean } = {},
+    options: { workflowSource?: WorkflowSource; runId?: string; force?: boolean } = {},
   ): Promise<WorkflowRunResult> {
     validateWorkflowDefinition(workflow);
     this.cancelled = false;
@@ -298,11 +296,8 @@ export class WorkflowEngine {
         `Cannot continue workflow run ${parentRunId} with status ${parent.state.status}`,
       );
     }
-    const hashMismatch =
-      parent.state.workflowHash !== undefined &&
-      options.workflowHash !== undefined &&
-      parent.state.workflowHash !== options.workflowHash;
-    if (hashMismatch && options.force !== true) {
+    const sourceMismatch = workflowSourceMismatch(parent.state, options.workflowSource);
+    if (sourceMismatch && options.force !== true) {
       throw new WorkflowSourceChangedError(parentRunId);
     }
 
@@ -315,8 +310,7 @@ export class WorkflowEngine {
     const state = await this.createRunState(
       workflow,
       normalizedInput,
-      options.workflowPath,
-      options.workflowHash,
+      options.workflowSource,
       options.runId,
     );
     state.parentRunId = parentRunId;
@@ -477,8 +471,7 @@ export class WorkflowEngine {
   private async createRunState(
     workflow: WorkflowDefinition,
     input: unknown,
-    workflowPath: string | undefined,
-    workflowHash: string | undefined,
+    workflowSource: WorkflowSource | undefined,
     runId: string | undefined,
   ): Promise<WorkflowRunState> {
     const now = new Date().toISOString();
@@ -488,8 +481,7 @@ export class WorkflowEngine {
       runId: runId ?? createRunId(workflow.name),
       workflowName: workflow.name,
       ...(await this.resolveTitleBounded(workflow, input)),
-      ...(workflowPath !== undefined ? { workflowPath } : {}),
-      ...(workflowHash !== undefined ? { workflowHash } : {}),
+      ...(workflowSource !== undefined ? { workflowSource } : {}),
       startedAt: now,
       updatedAt: now,
       status: "running",
@@ -758,36 +750,43 @@ export class WorkflowEngine {
     node: WorkflowNodeDefinition,
     meta: NodeExecutionMeta,
   ): Promise<NodeExecution> {
-    const timeoutMs = node.timeoutMs ?? this.defaultNodeTimeoutMs;
     const abort = new AbortController();
+    const context = this.createNodeContext(state, abort.signal);
+    let timer: NodeJS.Timeout | undefined;
+    let dispatchSettled: Promise<void> | undefined;
     this.activeAbort = abort;
-    if (this.parked) {
-      // A park that landed during the node_started persist must not let the
-      // node dispatch: its discarded side effects would rerun on resume.
-      throw new RunParkedError();
-    }
-    if (this.cancelled) {
-      throw new CancelledError();
-    }
-
-    const timer = setTimeout(() => {
-      abort.abort(new TimeoutError(timeoutMs));
-    }, timeoutMs);
-    const dispatched = this.dispatchNode(
-      workflow,
-      state,
-      runDir,
-      nodeId,
-      attemptId,
-      node,
-      abort.signal,
-      meta,
-    );
-    const dispatchSettled = dispatched.then(
-      () => undefined,
-      () => undefined,
-    );
     try {
+      if (this.parked) {
+        // A park that landed during the node_started persist must not let the
+        // node dispatch: its discarded side effects would rerun on resume.
+        throw new RunParkedError();
+      }
+      if (this.cancelled) {
+        throw new CancelledError();
+      }
+
+      const timeoutMs = await this.resolveNodeTimeout(node, context, abort);
+      if (abort.signal.aborted) {
+        throw abortError(abort.signal);
+      }
+      timer = setTimeout(() => {
+        abort.abort(new TimeoutError(timeoutMs));
+      }, timeoutMs);
+      const dispatched = this.dispatchNode(
+        workflow,
+        state,
+        runDir,
+        nodeId,
+        attemptId,
+        node,
+        context,
+        abort.signal,
+        meta,
+      );
+      dispatchSettled = dispatched.then(
+        () => undefined,
+        () => undefined,
+      );
       // Race the dispatch against the abort signal so timeouts and cancel
       // take effect even for node callbacks that never observe the signal.
       const execution = await Promise.race([dispatched, abortRejection(abort.signal)]);
@@ -799,7 +798,7 @@ export class WorkflowEngine {
       assertJsonSerializable(execution.output, `Node ${nodeId} output`);
       return execution;
     } catch (error) {
-      if (node.nodeType === "action" && "exec" in node) {
+      if (node.nodeType === "action" && "exec" in node && dispatchSettled !== undefined) {
         // Give the killed shell command a short grace period to close so its
         // action receipt lands in `meta` before the failed attempt persists.
         await Promise.race([
@@ -809,9 +808,37 @@ export class WorkflowEngine {
       }
       const reason: unknown = abort.signal.aborted ? abort.signal.reason : undefined;
       throw reason instanceof TimeoutError || reason instanceof CancelledError ? reason : error;
+    } finally {
+      if (timer !== undefined) {
+        clearTimeout(timer);
+      }
+      if (this.activeAbort === abort) {
+        this.activeAbort = null;
+      }
+    }
+  }
+
+  private async resolveNodeTimeout(
+    node: WorkflowNodeDefinition,
+    context: WorkflowNodeContext,
+    abort: AbortController,
+  ): Promise<number> {
+    const configured = node.timeoutMs;
+    if (typeof configured !== "function") {
+      return assertValidTimeout(configured ?? this.defaultNodeTimeoutMs);
+    }
+    const timer = setTimeout(
+      () => abort.abort(new TimeoutError(TIMEOUT_RESOLUTION_TIMEOUT_MS)),
+      TIMEOUT_RESOLUTION_TIMEOUT_MS,
+    );
+    try {
+      const resolved = await Promise.race([
+        Promise.resolve(configured(context)),
+        abortRejection(abort.signal),
+      ]);
+      return assertValidTimeout(resolved);
     } finally {
       clearTimeout(timer);
-      this.activeAbort = null;
     }
   }
 
@@ -822,10 +849,10 @@ export class WorkflowEngine {
     nodeId: string,
     attemptId: string,
     node: WorkflowNodeDefinition,
+    context: WorkflowNodeContext,
     signal: AbortSignal,
     meta: NodeExecutionMeta,
   ): Promise<NodeExecution> {
-    const context = this.createNodeContext(state, signal);
     switch (node.nodeType) {
       case "agent":
         return await this.runAgentNode(
@@ -1044,7 +1071,13 @@ async function runShellActionNode(
   return { output, promptText: null, action: shellReceipt(result) };
 }
 
-/** Rejects with the abort reason once the signal fires; never resolves. */
+function assertValidTimeout(value: number): number {
+  if (!Number.isFinite(value) || value <= 0) {
+    throw new Error("Node timeoutMs must resolve to a finite positive number");
+  }
+  return value;
+}
+
 /** The error carried by an aborted signal, normalized to an Error. */
 function abortError(signal: AbortSignal): Error {
   const reason: unknown = signal.reason ?? new CancelledError();
@@ -1069,6 +1102,22 @@ function abortRejection(signal: AbortSignal): Promise<never> {
  * Failing here turns a bad callback return value into a normal node failure
  * instead of corrupting the run state.
  */
+function workflowSourceMismatch(
+  state: WorkflowRunState,
+  source: WorkflowSource | undefined,
+): boolean {
+  if (source === undefined) return false;
+  if (state.workflowSource !== undefined) {
+    return !isDeepStrictEqual(state.workflowSource, source);
+  }
+  // Bounded compatibility check for pre-catalog file runs. Startup normally
+  // converts these records with migrateLegacyWorkflowSources first.
+  return (
+    state.workflowHash !== undefined &&
+    (source.kind !== "file" || state.workflowHash !== source.hash)
+  );
+}
+
 function assertJsonSerializable(value: unknown, what: string): void {
   let encoded: string | undefined;
   try {

package/src/workflows/index.ts

@@ -21,6 +21,7 @@ export {
   discoverWorkflows,
   loadWorkflowFile,
   resolveWorkflowRef,
+  resolveWorkflowSource,
   workflowFileStem,
   workflowSearchDirs,
   type DiscoveredWorkflow,
@@ -83,6 +84,7 @@ export type {
   WorkflowRunResult,
   WorkflowRunState,
   WorkflowRunStatus,
+  WorkflowSource,
   WorkflowSessionBinding,
   WorkflowSessionEntryRecord,
   WorkflowStepRecord,