@cascade-flow/runner 0.1.0 → 0.2.0

package/README.md

@@ -76,8 +76,13 @@ workflows/my-workflow/
 ├── input-schema.ts     # Optional: Zod schema
 ├── defineStep.ts       # Optional: Typed helper
 └── steps/
-    ├── step-1/step.ts
-    └── step-2/step.ts
+    ├── step-1/step.ts                    # Flat step
+    ├── step-2/step.ts                    # Flat step
+    └── data-processing/                  # Optional: Group directory
+        ├── extract/
+        │   └── fetch-data/step.ts       # Nested step
+        └── transform/
+            └── normalize/step.ts         # Nested step
 ```
 
 ## Key Features
@@ -87,6 +92,7 @@ workflows/my-workflow/
 - **Cycle detection** - DFS validation prevents circular dependencies
 - **Resume capability** - Continue from previous runs
 - **Pluggable backend** - Abstract persistence layer
+- **Nested step groups** - Organize steps in arbitrary directory hierarchies (purely organizational)
 
 ## Exports
 

package/dist/discovery.d.ts

@@ -17,6 +17,7 @@ export declare function loadWorkflowConfig(workflowDir: string): Promise<Workflo
 export declare function discoverWorkflows(workflowsRoot?: string): Promise<LocalWorkflowMetadata[]>;
 /**
  * Discovers and loads all steps from a directory
+ * Supports arbitrary deep nesting of steps in group subdirectories
  *
  * @param root - Root directory containing step subdirectories (defaults to "./steps")
  * @returns Array of loaded steps with wired dependencies

package/dist/discovery.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../src/discovery.ts"],"names":[],"mappings":"AAMA,OAAO,EAA0C,KAAK,UAAU,EAAyB,KAAK,cAAc,EAAE,KAAK,qBAAqB,EAAE,MAAM,SAAS,CAAC;AAG1J;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,cAAc,CAAC,CA0BzB;AAED;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CACrC,aAAa,SAA4B,GACxC,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAyElC;AAED;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,IAAI,SAAwB,GAC3B,OAAO,CAAC,UAAU,EAAE,CAAC,CA2FvB"}
+{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../src/discovery.ts"],"names":[],"mappings":"AAMA,OAAO,EAA0C,KAAK,UAAU,EAAyB,KAAK,cAAc,EAAE,KAAK,qBAAqB,EAAE,MAAM,SAAS,CAAC;AAG1J;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,cAAc,CAAC,CA0BzB;AAED;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CACrC,aAAa,SAA4B,GACxC,OAAO,CAAC,qBAAqB,EAAE,CAAC,CA+ElC;AA4CD;;;;;;GAMG;AACH,wBAAsB,aAAa,CACjC,IAAI,SAAwB,GAC3B,OAAO,CAAC,UAAU,EAAE,CAAC,CAkFvB"}

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 import type { Backend, LogEntry } from "@cascade-flow/backend-interface";
-import type { StepOutput, LoadedStep, RunnerContext } from "./types";
+import type { StepOutput, RunnerContext } from "@cascade-flow/workflow";
+import type { LoadedStep } from "./types";
 export type { LoadedStep };
 /**
  * Execute a step in an isolated child process
@@ -22,4 +23,5 @@ export declare function runAll(options: {
     input?: unknown;
 }): Promise<Record<string, StepOutput>>;
 export { discoverWorkflows, discoverSteps } from "./discovery";
+export { calculateWorkflowHash, getGitInfo } from "./versioning";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAa,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAMpF,OAAO,KAAK,EACV,UAAU,EACV,UAAU,EACV,aAAa,EACd,MAAM,SAAS,CAAC;AAGjB,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACrC,GAAG,EAAE,aAAa,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,OAAO,EAChB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,QAAQ,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAC/C,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,GACjC,OAAO,CAAC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,QAAQ,EAAE,CAAA;CAAE,CAAC,CAoBnD;AAED,wBAAsB,MAAM,CAAC,OAAO,EAAE;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,GAAG,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC7B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CA4ZtC;AAGD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAa,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAMpF,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAI1C,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACrC,GAAG,EAAE,aAAa,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,OAAO,EAChB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,QAAQ,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAC/C,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,GACjC,OAAO,CAAC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,QAAQ,EAAE,CAAA;CAAE,CAAC,CAoBnD;AAED,wBAAsB,MAAM,CAAC,OAAO,EAAE;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,GAAG,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC7B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAkctC;AAGD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG/D,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC"}

package/dist/index.js

@@ -10,7 +10,7 @@ var __export = (target, all) => {
 };
 
 // src/index.ts
-import { join } from "node:path";
+import { join as join2 } from "node:path";
 
 // src/discovery.ts
 import fs from "node:fs/promises";
@@ -12532,7 +12532,11 @@ async function discoverWorkflows(workflowsRoot = path.resolve("workflows")) {
         if (schemaMod?.inputSchema) {
           inputSchema = schemaMod.inputSchema;
         }
-      } catch {}
+      } catch (error46) {
+        if (error46?.code === "ENOENT" || error46?.code === "ERR_MODULE_NOT_FOUND") {} else {
+          throw new Error(`Failed to load input schema at ${inputSchemaPath}: ${error46?.message || error46}`);
+        }
+      }
       workflows.push({
         slug: entry.name,
         name: config2.name,
@@ -12547,18 +12551,32 @@ async function discoverWorkflows(workflowsRoot = path.resolve("workflows")) {
   }
   return workflows;
 }
-async function discoverSteps(root = path.resolve("steps")) {
+async function discoverStepDirectories(root, relativePath = "") {
   const entries = await fs.readdir(root, { withFileTypes: true });
-  const stepDirs = entries.filter((e) => e.isDirectory()).map((e) => path.join(root, e.name));
-  const loaded = [];
-  const byModuleStepObject = new Map;
-  for (const dir of stepDirs) {
-    const stepFile = path.join(dir, "step.ts");
+  const results = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory()) {
+      continue;
+    }
+    const fullPath = path.join(root, entry.name);
+    const stepFile = path.join(fullPath, "step.ts");
+    const currentPath = relativePath ? `${relativePath}/${entry.name}` : entry.name;
     try {
       await fs.access(stepFile);
+      results.push({ dir: fullPath, stepId: currentPath });
     } catch {
-      continue;
+      const nestedSteps = await discoverStepDirectories(fullPath, currentPath);
+      results.push(...nestedSteps);
     }
+  }
+  return results;
+}
+async function discoverSteps(root = path.resolve("steps")) {
+  const stepDirs = await discoverStepDirectories(root);
+  const loaded = [];
+  const byModuleStepObject = new Map;
+  for (const { dir, stepId } of stepDirs) {
+    const stepFile = path.join(dir, "step.ts");
     const modUrl = pathToFileURL(stepFile).toString();
     const mod = await import(modUrl);
     if (!mod?.step || typeof mod.step.fn !== "function") {
@@ -12569,10 +12587,9 @@ async function discoverSteps(root = path.resolve("steps")) {
       const errors3 = validationResult.error.issues.map((e) => `${e.path.join(".")}: ${e.message}`).join(", ");
       throw new Error(`Invalid step configuration at ${stepFile}: ${errors3}`);
     }
-    const dirName = path.basename(dir);
     const placeholder = {
-      id: dirName,
-      name: mod.step.name ?? dirName,
+      id: stepId,
+      name: mod.step.name ?? stepId.split("/").pop(),
       dir,
       fn: mod.step.fn,
       exportOutput: mod.step.exportOutput,
@@ -12635,11 +12652,10 @@ import { spawn } from "node:child_process";
 import { resolve, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { mkdir, readFile, unlink } from "node:fs/promises";
-import { getMicrosecondTimestamp } from "@cascade-flow/backend-interface";
+import { getMicrosecondTimestamp, ensureErrorMessage } from "@cascade-flow/backend-interface";
 function createStreamHandler(streamType, attemptNumber, emitLog) {
   let buffer = "";
   const handler = (chunk) => {
-    const timestamp = getMicrosecondTimestamp();
     buffer += chunk.toString();
     const lines = buffer.split(`
 `);
@@ -12647,6 +12663,7 @@ function createStreamHandler(streamType, attemptNumber, emitLog) {
     for (const line of lines) {
       if (!line.trim())
         continue;
+      const timestamp = getMicrosecondTimestamp();
       emitLog({
         timestamp,
         stream: streamType,
@@ -12670,7 +12687,7 @@ function createStreamHandler(streamType, attemptNumber, emitLog) {
   return { handler, getBuffer, flushBuffer };
 }
 async function executeStepInSubprocess(stepFile, stepId, dependencies, ctx, attemptNumber, outputPath, onLog, options) {
-  const executorPath = resolve(dirname(fileURLToPath(import.meta.url)), "step-executor.ts");
+  const executorPath = resolve(dirname(fileURLToPath(import.meta.url)), "step-executor");
   await mkdir(dirname(outputPath), { recursive: true });
   return new Promise((resolve2, reject) => {
     const child = spawn("bun", [executorPath], {
@@ -12762,7 +12779,8 @@ async function executeStepInSubprocess(stepFile, stepId, dependencies, ctx, atte
         if (lastStderrLog) {
           try {
             const errorObj = JSON.parse(lastStderrLog.message);
-            const error46 = new Error(errorObj.message);
+            const errorMessage = ensureErrorMessage(errorObj.message);
+            const error46 = new Error(errorMessage);
             error46.stack = errorObj.stack;
             error46.name = errorObj.name || "Error";
             reject(error46);
@@ -12785,6 +12803,87 @@ async function executeStepInSubprocess(stepFile, stepId, dependencies, ctx, atte
 // src/index.ts
 import { getMicrosecondTimestamp as getMicrosecondTimestamp2 } from "@cascade-flow/backend-interface";
 import { Skip, isOptional as isOptional2 } from "@cascade-flow/workflow";
+
+// src/versioning.ts
+import { createHash } from "node:crypto";
+import { readFile as readFile2, readdir } from "node:fs/promises";
+import { join } from "node:path";
+import { exec } from "node:child_process";
+import { promisify } from "node:util";
+var execAsync = promisify(exec);
+async function calculateWorkflowHash(workflow) {
+  const hash2 = createHash("sha256");
+  try {
+    const workflowJsonPath = join(workflow.dir, "workflow.json");
+    const workflowJsonContent = await readFile2(workflowJsonPath, "utf-8");
+    hash2.update(`workflow.json:${workflowJsonContent}`);
+  } catch (error46) {
+    throw new Error(`Failed to read workflow.json for ${workflow.slug}: ${error46 instanceof Error ? error46.message : "Unknown error"}`);
+  }
+  try {
+    const inputSchemaPath = join(workflow.dir, "input-schema.ts");
+    const inputSchemaContent = await readFile2(inputSchemaPath, "utf-8");
+    hash2.update(`input-schema.ts:${inputSchemaContent}`);
+  } catch (error46) {
+    if (error46 && typeof error46 === "object" && "code" in error46 && error46.code !== "ENOENT") {
+      throw error46;
+    }
+  }
+  const stepFiles = await collectStepFiles(workflow.stepsDir);
+  stepFiles.sort();
+  for (const stepFilePath of stepFiles) {
+    try {
+      const content = await readFile2(stepFilePath, "utf-8");
+      const relativePath = stepFilePath.replace(workflow.stepsDir + "/", "");
+      hash2.update(`step:${relativePath}:${content}`);
+    } catch (error46) {
+      throw new Error(`Failed to read step file ${stepFilePath}: ${error46 instanceof Error ? error46.message : "Unknown error"}`);
+    }
+  }
+  const digest = hash2.digest("hex");
+  return `sha256:${digest}`;
+}
+async function collectStepFiles(stepsDir) {
+  const files = [];
+  async function scan(dir) {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        await scan(fullPath);
+      } else if (entry.isFile() && entry.name === "step.ts") {
+        files.push(fullPath);
+      }
+    }
+  }
+  try {
+    await scan(stepsDir);
+  } catch (error46) {
+    throw new Error(`Failed to scan steps directory ${stepsDir}: ${error46 instanceof Error ? error46.message : "Unknown error"}`);
+  }
+  return files;
+}
+async function getGitInfo(workflowDir) {
+  try {
+    const { stdout: commit } = await execAsync("git rev-parse HEAD", {
+      cwd: workflowDir
+    });
+    const { stdout: branch } = await execAsync("git rev-parse --abbrev-ref HEAD", { cwd: workflowDir });
+    const { stdout: status } = await execAsync("git status --porcelain", {
+      cwd: workflowDir
+    });
+    const dirty = status.trim().length > 0;
+    return {
+      commit: commit.trim(),
+      branch: branch.trim(),
+      dirty
+    };
+  } catch (error46) {
+    return;
+  }
+}
+
+// src/index.ts
 async function executeStepInProcess(stepFile, stepId, dependencies, ctx, attemptNumber, backend, onLog, options) {
   const outputPath = backend.getStepOutputPath(ctx.workflow.slug, ctx.runId, stepId, attemptNumber);
   return executeStepInSubprocess(stepFile, stepId, dependencies, ctx, attemptNumber, outputPath, onLog, options);
@@ -12825,9 +12924,21 @@ async function runAll(options) {
   };
   const workflowStartTime = getMicrosecondTimestamp2();
   await backend.initializeRun(workflowSlug, runId);
+  const versionId = await calculateWorkflowHash(workflow);
+  const git = await getGitInfo(workflow.dir);
+  const stepManifest = steps.map((s) => s.id);
+  await backend.createWorkflowVersion({
+    workflowSlug,
+    versionId,
+    createdAt: getMicrosecondTimestamp2(),
+    stepManifest,
+    totalSteps: steps.length,
+    git
+  });
   const hasInputSchema = workflow.inputSchema !== undefined;
   const hasInput = options.input !== undefined;
   await backend.saveWorkflowStart(workflowSlug, runId, {
+    versionId,
     workflowAttemptNumber: 1,
     hasInputSchema,
     hasInput
@@ -12876,10 +12987,25 @@ ${errorMessage}`
   }
   if (options?.resume) {
     defaultCtx.log(`Resuming run ${defaultCtx.runId}...`);
+    const workflowEvents = await backend.loadEvents(workflowSlug, defaultCtx.runId, { category: "workflow" });
+    const workflowStartedEvent = workflowEvents.find((e) => e.type === "WorkflowStarted");
+    if (workflowStartedEvent && workflowStartedEvent.type === "WorkflowStarted") {
+      const previousVersionId = workflowStartedEvent.versionId;
+      if (previousVersionId !== versionId) {
+        const previousVersion = await backend.getWorkflowVersion(workflowSlug, previousVersionId);
+        const currentVersion = await backend.getWorkflowVersion(workflowSlug, versionId);
+        defaultCtx.log(`⚠️  Workflow definition changed since original run`);
+        defaultCtx.log(`   Original: ${previousVersionId}`);
+        defaultCtx.log(`   Current:  ${versionId}`);
+        if (previousVersion?.git && currentVersion?.git) {
+          defaultCtx.log(`   Git: ${previousVersion.git.commit} → ${currentVersion.git.commit}`);
+        }
+      }
+    }
     const existingRecords = await backend.loadRun(workflowSlug, defaultCtx.runId);
     let resumedSteps = 0;
     for (const record2 of existingRecords) {
-      if (record2.status === "completed" && record2.output) {
+      if (record2.status === "completed" && record2.output !== undefined && record2.output !== null) {
         try {
           const output = JSON.parse(record2.output);
           cache.set(record2.stepId, Promise.resolve(output));
@@ -12941,7 +13067,7 @@ ${errorMessage}`
           defaultCtx.log(`⊘ ${step.name} (skipped: dependency '${cascadedFromStep.name}' was skipped)`);
           return {};
         }
-        const stepFile = join(step.dir, "step.ts");
+        const stepFile = join2(step.dir, "step.ts");
         const maxRetries = step.maxRetries ?? 0;
         let lastError = null;
         for (let attemptNumber = 1;attemptNumber <= maxRetries + 1; attemptNumber++) {
@@ -13077,9 +13203,11 @@ ${errorMessage}`
 }
 export {
   runAll,
+  getGitInfo,
   executeStepInProcess,
   discoverWorkflows,
-  discoverSteps
+  discoverSteps,
+  calculateWorkflowHash
 };
 
-//# debugId=A5F45B2C15F619F364756E2164756E21
+//# debugId=0A6F5DFCD4D7954F64756E2164756E21