@osolmaz/pi-workflows 0.3.0 → 0.4.0

package/src/workflows/loader.ts

@@ -4,14 +4,16 @@ import os from "node:os";
 import path from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { createJiti } from "jiti";
+import type { BuiltinWorkflowCatalog } from "./catalog.js";
 import { isWorkflowDefinition } from "./definition.js";
-import type { WorkflowDefinition } from "./types.js";
+import { WorkflowSourceChangedError } from "./errors.js";
+import type { WorkflowDefinition, WorkflowSource } from "./types.js";
 
 const WORKFLOW_FILE_SUFFIXES = [".workflow.ts", ".workflow.js", ".workflow.mts", ".workflow.mjs"];
 
 export type DiscoveredWorkflow = {
   name: string;
-  path: string;
+  ref: string;
   source: "project" | "global" | "builtin" | "path";
 };
 
@@ -20,23 +22,27 @@ export type WorkflowSearchPaths = {
   homeDir?: string;
 };
 
-/** Directories scanned for workflow files, in precedence order. */
+export type ResolvedWorkflow = {
+  definition: WorkflowDefinition;
+  source: WorkflowSource;
+  sourceKind: DiscoveredWorkflow["source"];
+};
+
+/** Directories scanned for user workflow files, in precedence order. */
 export function workflowSearchDirs(
   options: WorkflowSearchPaths,
-): { dir: string; source: "project" | "global" | "builtin" }[] {
+): { dir: string; source: "project" | "global" }[] {
   const homeDir = options.homeDir ?? os.homedir();
-  const builtinDir = fileURLToPath(new URL("../builtins/", import.meta.url));
   return [
     { dir: path.join(options.cwd, ".pi", "workflows"), source: "project" },
     { dir: path.join(homeDir, ".pi", "agent", "workflows"), source: "global" },
-    { dir: builtinDir, source: "builtin" },
   ];
 }
 
-/** SHA-256 of a workflow source file, recorded in run state for resume pinning. */
+/** SHA-256 of a user workflow source file. */
 export async function hashWorkflowSource(filePath: string): Promise<string> {
   return createHash("sha256")
-    .update(await fs.readFile(filePath))
+    .update(await fs.readFile(path.resolve(filePath)))
     .digest("hex");
 }
 
@@ -50,12 +56,11 @@ export function workflowFileStem(filePath: string): string {
   return suffix ? base.slice(0, -suffix.length) : base;
 }
 
-// Alias the package name to this module's own entry so workflow files can
-// `import { agent } from "@osolmaz/pi-workflows"` whether the engine runs from src
-// (tests, tsx) or from the built dist inside the installed package.
+// Alias package imports to this process's workflow API. User files can reload,
+// but their node constructors and validators remain from one engine version.
 const SELF_ENTRY = path.join(path.dirname(fileURLToPath(import.meta.url)), "index");
 
-/** Load a workflow module from disk. The default export must be `defineWorkflow(...)`. */
+/** Load a user workflow module from disk. */
 export async function loadWorkflowFile(filePath: string): Promise<WorkflowDefinition> {
   const absolutePath = path.resolve(filePath);
   const jiti = createJiti(pathToFileURL(absolutePath).href, {
@@ -70,22 +75,26 @@ export async function loadWorkflowFile(filePath: string): Promise<WorkflowDefini
   return loaded;
 }
 
-/** Discover named workflows in the project and global workflow directories. */
+/** Discover user workflows first, then unshadowed catalog built-ins. */
 export async function discoverWorkflows(
   options: WorkflowSearchPaths,
+  catalog?: BuiltinWorkflowCatalog,
 ): Promise<DiscoveredWorkflow[]> {
   const discovered: DiscoveredWorkflow[] = [];
   const seenNames = new Set<string>();
   for (const { dir, source } of workflowSearchDirs(options)) {
     for (const filePath of await listWorkflowFiles(dir)) {
       const name = workflowFileStem(filePath);
-      if (seenNames.has(name)) {
-        continue;
-      }
+      if (seenNames.has(name)) continue;
       seenNames.add(name);
-      discovered.push({ name, path: filePath, source });
+      discovered.push({ name, ref: filePath, source });
     }
   }
+  for (const builtin of catalog?.list() ?? []) {
+    if (seenNames.has(builtin.definition.name)) continue;
+    seenNames.add(builtin.definition.name);
+    discovered.push({ name: builtin.definition.name, ref: builtin.ref, source: "builtin" });
+  }
   return discovered;
 }
 
@@ -102,26 +111,61 @@ async function listWorkflowFiles(dir: string): Promise<string[]> {
     .sort();
 }
 
-/**
- * Resolve a `/workflow` argument to a workflow file. Accepts a discovered
- * workflow name or a direct path to a `*.workflow.ts` file.
- */
+/** Resolve a workflow name, stable built-in ref, or direct user file path. */
 export async function resolveWorkflowRef(
   ref: string,
   options: WorkflowSearchPaths,
-): Promise<{ path: string; source: DiscoveredWorkflow["source"] }> {
+  catalog?: BuiltinWorkflowCatalog,
+): Promise<ResolvedWorkflow> {
+  if (ref.startsWith("builtin:")) {
+    const id = ref.slice("builtin:".length);
+    const builtin = catalog?.get(id);
+    if (builtin === undefined) throw new Error(`Unknown built-in workflow ${JSON.stringify(ref)}`);
+    return {
+      definition: builtin.definition,
+      source: { kind: "builtin", id: builtin.id, revision: builtin.revision },
+      sourceKind: "builtin",
+    };
+  }
   if (looksLikePath(ref)) {
     const absolutePath = path.resolve(options.cwd, ref);
     await fs.access(absolutePath);
-    return { path: absolutePath, source: "path" };
+    return {
+      definition: await loadWorkflowFile(absolutePath),
+      source: { kind: "file", path: absolutePath, hash: await hashWorkflowSource(absolutePath) },
+      sourceKind: "path",
+    };
   }
-  const discovered = await discoverWorkflows(options);
+  const discovered = await discoverWorkflows(options, catalog);
   const match = discovered.find((workflow) => workflow.name === ref);
-  if (!match) {
+  if (match === undefined) {
     const available = discovered.map((workflow) => workflow.name).join(", ") || "(none)";
     throw new Error(`Unknown workflow ${JSON.stringify(ref)}. Available workflows: ${available}`);
   }
-  return { path: match.path, source: match.source };
+  if (match.source === "builtin") return await resolveWorkflowRef(match.ref, options, catalog);
+  const absolutePath = path.resolve(match.ref);
+  return {
+    definition: await loadWorkflowFile(absolutePath),
+    source: { kind: "file", path: absolutePath, hash: await hashWorkflowSource(absolutePath) },
+    sourceKind: match.source,
+  };
+}
+
+/** Resolve an already persisted canonical source. */
+export async function resolveWorkflowSource(
+  source: WorkflowSource,
+  catalog?: BuiltinWorkflowCatalog,
+  runId = source.kind === "builtin" ? `builtin:${source.id}` : source.path,
+): Promise<WorkflowDefinition> {
+  if (source.kind === "builtin") {
+    if (catalog === undefined) throw new Error(`No built-in workflow catalog for ${source.id}`);
+    return catalog.resolve(source, runId);
+  }
+  const actualHash = await hashWorkflowSource(source.path);
+  if (actualHash !== source.hash) {
+    throw new WorkflowSourceChangedError(runId);
+  }
+  return await loadWorkflowFile(source.path);
 }
 
 function looksLikePath(ref: string): boolean {

package/src/workflows/migrate-sources.ts

@@ -0,0 +1,167 @@
+import { randomUUID } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { BuiltinWorkflowCatalog } from "./catalog.js";
+import { WorkflowRunStore, listRunBundles } from "./store.js";
+import type { WorkflowRunState, WorkflowSource } from "./types.js";
+
+export type LegacySourceMigrationQueue = {
+  repairCanonicalWorkflowSourceRun(options: {
+    runId: string;
+    workflowName: string;
+    workflowSourceRef: string;
+    runnerId: string;
+    claimToken: string;
+    leaseMs: number;
+  }): "unchanged" | "claimed" | false;
+  claimLegacyWorkflowSourceRun(options: {
+    runId: string;
+    workflowName: string;
+    oldWorkflowPath: string;
+    workflowSourceRef: string;
+    runnerId: string;
+    claimToken: string;
+    leaseMs: number;
+  }): boolean;
+  parkWorkflowRun(options: { runId: string; claimToken: string }): boolean;
+  getWorkflowRun(runId: string): { status: "claimed" | "parked" | "done" } | undefined;
+};
+
+const MIGRATION_LEASE_MS = 30_000;
+
+export type LegacySourceMigrationResult = {
+  migratedRunIds: string[];
+  blocked: { runId: string; reason: string }[];
+};
+
+/** One bounded migration for nonterminal runs created before canonical sources. */
+export async function migrateLegacyWorkflowSources(options: {
+  catalog: BuiltinWorkflowCatalog;
+  store?: WorkflowRunStore;
+  queue?: LegacySourceMigrationQueue;
+}): Promise<LegacySourceMigrationResult> {
+  const store = options.store ?? new WorkflowRunStore();
+  const result: LegacySourceMigrationResult = { migratedRunIds: [], blocked: [] };
+  for (const bundle of await listRunBundles(store.outputRoot)) {
+    const state = bundle.state;
+    if (state.status !== "running" && state.status !== "waiting") continue;
+    if (state.workflowSource !== undefined) {
+      if (options.queue === undefined) continue;
+      const claimToken = randomUUID();
+      const repaired = options.queue.repairCanonicalWorkflowSourceRun({
+        runId: state.runId,
+        workflowName: state.workflowName,
+        workflowSourceRef: sourceRef(state.workflowSource),
+        runnerId: `source-migration-${process.pid}`,
+        claimToken,
+        leaseMs: MIGRATION_LEASE_MS,
+      });
+      if (repaired === false) {
+        // Another runner can own a canonical run during startup. Its source is
+        // already safe; the normal queue lease will make it claimable later.
+      } else if (
+        repaired === "claimed" &&
+        !options.queue.parkWorkflowRun({ runId: state.runId, claimToken })
+      ) {
+        result.blocked.push({
+          runId: state.runId,
+          reason: "canonical queue repair could not release its claim",
+        });
+      }
+      continue;
+    }
+    if (state.workflowPath === undefined || state.workflowHash === undefined) {
+      result.blocked.push({ runId: state.runId, reason: "legacy workflow identity is incomplete" });
+      continue;
+    }
+    const legacyPath = {
+      workflowName: state.workflowName,
+      workflowPath: state.workflowPath,
+    };
+    const pathEntry = options.catalog.legacyPathEntry(legacyPath);
+    let workflowSource: WorkflowSource;
+    let workflowSourceRef: string;
+    if (pathEntry === undefined) {
+      workflowSource = {
+        kind: "file",
+        path: path.resolve(state.workflowPath),
+        hash: state.workflowHash,
+      };
+      workflowSourceRef = workflowSource.path;
+    } else {
+      const legacy = options.catalog.matchLegacy({
+        ...legacyPath,
+        workflowHash: state.workflowHash,
+      });
+      if (legacy === undefined) {
+        const reason = `legacy built-in ${pathEntry.id} has an unknown source revision`;
+        result.blocked.push({ runId: state.runId, reason });
+        continue;
+      }
+      workflowSource = {
+        kind: "builtin",
+        id: legacy.entry.id,
+        revision: legacy.revision,
+      };
+      workflowSourceRef = legacy.entry.ref;
+    }
+    const claimToken = randomUUID();
+    const queueIsDone = options.queue?.getWorkflowRun(state.runId)?.status === "done";
+    if (
+      options.queue !== undefined &&
+      !queueIsDone &&
+      !options.queue.claimLegacyWorkflowSourceRun({
+        runId: state.runId,
+        workflowName: state.workflowName,
+        oldWorkflowPath: state.workflowPath,
+        workflowSourceRef,
+        runnerId: `source-migration-${process.pid}`,
+        claimToken,
+        leaseMs: MIGRATION_LEASE_MS,
+      })
+    ) {
+      result.blocked.push({
+        runId: state.runId,
+        reason: "matching queue row is active or unavailable",
+      });
+      continue;
+    }
+    const migrated: WorkflowRunState = {
+      ...state,
+      workflowSource,
+    };
+    delete migrated.workflowPath;
+    delete migrated.workflowHash;
+    const migratedManifest = {
+      ...bundle.manifest,
+      workflowSource: migrated.workflowSource,
+    };
+    delete (migratedManifest as typeof migratedManifest & { workflowPath?: string }).workflowPath;
+    // State is the migration commit point. Queue and manifest updates are
+    // idempotent, so a crash before this write can safely retry.
+    await writeJsonAtomic(path.join(bundle.runDir, "manifest.json"), migratedManifest);
+    await writeJsonAtomic(path.join(bundle.runDir, "state.json"), migrated);
+    if (
+      options.queue !== undefined &&
+      !queueIsDone &&
+      !options.queue.parkWorkflowRun({ runId: state.runId, claimToken })
+    ) {
+      result.blocked.push({
+        runId: state.runId,
+        reason: "migration completed but its queue claim could not be released",
+      });
+    }
+    result.migratedRunIds.push(state.runId);
+  }
+  return result;
+}
+
+function sourceRef(source: WorkflowSource): string {
+  return source.kind === "builtin" ? `builtin:${source.id}` : source.path;
+}
+
+async function writeJsonAtomic(filePath: string, value: unknown): Promise<void> {
+  const tempPath = `${filePath}.${process.pid}.${randomUUID()}.source-migration.tmp`;
+  await fs.writeFile(tempPath, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
+  await fs.rename(tempPath, filePath);
+}

package/src/workflows/schema.ts

@@ -34,9 +34,10 @@ function assertOptionalFunction(value: unknown, description: string): void {
 function assertCommonNodeFields(node: WorkflowNodeDefinition, nodeId: string): void {
   if (
     node.timeoutMs !== undefined &&
+    typeof node.timeoutMs !== "function" &&
     (typeof node.timeoutMs !== "number" || !Number.isFinite(node.timeoutMs) || node.timeoutMs <= 0)
   ) {
-    fail(`node ${nodeId} timeoutMs must be a finite positive number`);
+    fail(`node ${nodeId} timeoutMs must be a finite positive number or function`);
   }
   if (node.statusDetail !== undefined && typeof node.statusDetail !== "string") {
     fail(`node ${nodeId} statusDetail must be a string`);

package/src/workflows/store.ts

@@ -1284,7 +1284,7 @@ function createManifest(
     runId: state.runId,
     workflowName: state.workflowName,
     ...(state.runTitle !== undefined ? { runTitle: state.runTitle } : {}),
-    ...(state.workflowPath !== undefined ? { workflowPath: state.workflowPath } : {}),
+    ...(state.workflowSource !== undefined ? { workflowSource: state.workflowSource } : {}),
     startedAt: state.startedAt,
     ...(state.finishedAt !== undefined ? { finishedAt: state.finishedAt } : {}),
     status: state.status,
@@ -1317,7 +1317,7 @@ export function createDefinitionSnapshot(workflow: WorkflowDefinition): Workflow
 function snapshotNode(node: WorkflowNodeDefinition): WorkflowNodeSnapshot {
   const common: WorkflowNodeSnapshot = {
     nodeType: node.nodeType,
-    ...(node.timeoutMs !== undefined ? { timeoutMs: node.timeoutMs } : {}),
+    ...(typeof node.timeoutMs === "number" ? { timeoutMs: node.timeoutMs } : {}),
     ...(node.statusDetail !== undefined ? { statusDetail: node.statusDetail } : {}),
   };
   if (node.nodeType === "agent" && node.expectedOutput !== undefined) {

package/src/workflows/types.ts

@@ -20,8 +20,11 @@ export type WorkflowNodeContext<TInput = unknown> = {
 };
 
 export type WorkflowNodeCommon = {
-  /** Per-node timeout. Falls back to the engine default (15 minutes). */
-  timeoutMs?: number;
+  /**
+   * Per-node timeout or a callback that derives it from the run context.
+   * Falls back to the engine default (15 minutes).
+   */
+  timeoutMs?: number | ((context: WorkflowNodeContext) => MaybePromise<number>);
   /** Short human-readable label shown in the viewer while the node runs. */
   statusDetail?: string;
 };
@@ -230,6 +233,10 @@ export type WorkflowRunStatus =
   | "timed_out"
   | "cancelled";
 
+export type WorkflowSource =
+  | { kind: "builtin"; id: string; revision: string }
+  | { kind: "file"; path: string; hash: string };
+
 export type WorkflowRunState = {
   schema: "pi-workflows.run-state.v1";
   /**
@@ -249,8 +256,10 @@ export type WorkflowRunState = {
    */
   carriedStepCount?: number;
   runTitle?: string;
+  /** Stable built-in identity or immutable file source used by this run. */
+  workflowSource?: WorkflowSource;
+  /** Legacy fields accepted only by the bounded built-in migration. */
   workflowPath?: string;
-  /** SHA-256 of the workflow source at run start; resume refuses mismatches. */
   workflowHash?: string;
   startedAt: string;
   finishedAt?: string;
@@ -377,7 +386,7 @@ export type WorkflowRunManifest = {
   runId: string;
   workflowName: string;
   runTitle?: string;
-  workflowPath?: string;
+  workflowSource?: WorkflowSource;
   startedAt: string;
   finishedAt?: string;
   status: WorkflowRunStatus;