@cascade-flow/runner 0.2.4 → 0.2.6

package/README.md

@@ -1,6 +1,6 @@
 # Runner
 
-Core workflow execution engine for direct (non-queue) execution.
+Workflow discovery and step execution utilities used by the worker package.
 
 ## Installation
 
@@ -8,65 +8,35 @@ Core workflow execution engine for direct (non-queue) execution.
 bun install
 ```
 
-## Usage
+## Exports
 
 ```typescript
-import { runAll, defineStep, baseDefineStep } from "runner";
-
-// Execute workflow
-const results = await runAll({
-  workflow: "my-workflow",
-  input: { userId: "123" },
-  only: ["step1", "step2"],  // Optional: run subset
-  resume: true,              // Optional: resume from previous run
-  runId: "previous-run-id",  // Optional: specific run to resume
-});
-```
+// Step execution
+executeStepInProcess(stepFile, stepId, dependencies, ctx, attemptNumber, backend, onLog?, options?): Promise<{ result, logs }>;
 
-## Defining Steps
+// Discovery
+discoverWorkflows(dir?: string): Promise<WorkflowMetadata[]>;
+discoverSteps(workflowDir: string): Promise<LoadedStep[]>;
 
-### Basic Step
+// Versioning
+calculateWorkflowHash(workflow): Promise<string>;
+getGitInfo(dir): Promise<GitInfo | undefined>;
 
-```typescript
-// workflows/my-workflow/steps/my-step/step.ts
-import { defineStep } from "runner";
-import { step as dependency } from "../other-step/step.ts";
+// Validation
+getAllDependents(stepId, steps): string[];
+validateWorkflowVersion(workflowSlug, runId, currentVersionId, backend, log): Promise<string>;
 
-export const step = defineStep({
-  dependencies: { dependency },
-  exportOutput: true,
-  fn: async ({ dependencies, ctx }) => {
-    return { result: dependencies.dependency.value * 2 };
-  }
-});
+// Types
+LoadedStep  // Step with resolved dependencies
 ```
 
-### Typed Workflow Input
-
-```typescript
-// workflows/my-workflow/input-schema.ts
-import { z } from "zod";
-export const inputSchema = z.object({
-  userId: z.string(),
-  count: z.number()
-});
-export type WorkflowInput = z.infer<typeof inputSchema>;
-
-// workflows/my-workflow/defineStep.ts
-import { baseDefineStep } from "runner";
-import type { WorkflowInput } from "./input-schema";
-export const defineStep = baseDefineStep<WorkflowInput>();
-
-// workflows/my-workflow/steps/my-step/step.ts
-import { defineStep } from "../../defineStep";
+## Key Features
 
-export const step = defineStep({
-  fn: async ({ ctx }) => {
-    const userId = ctx.input.userId;  // Fully typed!
-    return { processed: true };
-  }
-});
-```
+- **Workflow discovery** - Auto-discovers workflows from `./workflows/` directory
+- **Step discovery** - Loads and wires step dependencies with full TypeScript inference
+- **Subprocess execution** - Executes steps in isolated child processes
+- **Cycle detection** - DFS validation prevents circular dependencies
+- **Version tracking** - SHA-256 hashing for workflow version comparison
 
 ## Workflow Structure
 
@@ -74,37 +44,14 @@ export const step = defineStep({
 workflows/my-workflow/
 ├── workflow.json       # {"name": "My Workflow"}
 ├── input-schema.ts     # Optional: Zod schema
-├── defineStep.ts       # Optional: Typed helper
 └── steps/
     ├── step-1/step.ts                    # Flat step
     ├── step-2/step.ts                    # Flat step
     └── data-processing/                  # Optional: Group directory
-        ├── extract/
-        │   └── fetch-data/step.ts       # Nested step
-        └── transform/
-            └── normalize/step.ts         # Nested step
+        └── extract/
+            └── fetch-data/step.ts        # Nested step
 ```
 
-## Key Features
-
-- **Promise caching** - Automatic parallelization via cached promises
-- **Type-safe dependencies** - Import step objects for full TypeScript inference
-- **Cycle detection** - DFS validation prevents circular dependencies
-- **Resume capability** - Continue from previous runs
-- **Pluggable backend** - Abstract persistence layer
-- **Nested step groups** - Organize steps in arbitrary directory hierarchies (purely organizational)
+## Note
 
-## Exports
-
-```typescript
-// Main execution
-runAll(options: RunAllOptions): Promise<WorkflowOutput>;
-
-// Step helpers
-defineStep(config: StepConfig): StepDefinition;
-baseDefineStep<TInput>(): typeof defineStep;
-
-// Discovery
-discoverWorkflows(dir?: string): Promise<WorkflowMetadata[]>;
-discoverSteps(workflowDir: string): Promise<LoadedStep[]>;
-```
+For workflow authoring APIs (`defineStep`, `baseDefineStep`, `Skip`, `optional`), see the `@cascade-flow/workflow` package.

package/dist/checkpoint-ipc.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Checkpoint IPC Types and Utilities
+ *
+ * Handles communication between the subprocess (step execution) and parent process
+ * (worker) for checkpoint operations. Uses file-based IPC for simplicity.
+ *
+ * Flow:
+ * 1. Subprocess writes request to req-{id}.json
+ * 2. Parent reads request, checks cache
+ * 3. Parent writes response to res-{id}.json (hit/miss)
+ * 4. If miss: subprocess executes fn, writes result to result-{id}.json
+ * 5. Parent persists checkpoint to backend
+ */
+/**
+ * Request from subprocess to parent for checkpoint lookup/save
+ */
+export type CheckpointRequest = {
+    requestId: string;
+    name: string;
+    sequenceNumber: number;
+};
+/**
+ * Response from parent to subprocess
+ */
+export type CheckpointResponse = {
+    requestId: string;
+    hit: boolean;
+    data?: string;
+};
+/**
+ * Result from subprocess after executing checkpoint function (on cache miss)
+ */
+export type CheckpointResult = {
+    requestId: string;
+    data: string;
+};
+/**
+ * Data passed to onCheckpoint callback
+ */
+export type CheckpointData = {
+    name: string;
+    sequenceNumber: number;
+    data: string;
+};
+/**
+ * Failure from subprocess when checkpoint function throws (on cache miss)
+ */
+export type CheckpointFailure = {
+    requestId: string;
+    name: string;
+    sequenceNumber: number;
+    error: string;
+};
+/**
+ * Data passed to onCheckpointFailed callback
+ */
+export type CheckpointFailedData = {
+    name: string;
+    sequenceNumber: number;
+    error: string;
+};
+/**
+ * Get path for checkpoint request file
+ */
+export declare function getRequestPath(dir: string, requestId: string): string;
+/**
+ * Get path for checkpoint response file
+ */
+export declare function getResponsePath(dir: string, requestId: string): string;
+/**
+ * Get path for checkpoint result file
+ */
+export declare function getResultPath(dir: string, requestId: string): string;
+/**
+ * Get path for checkpoint failure file
+ */
+export declare function getFailurePath(dir: string, requestId: string): string;
+/**
+ * Generate unique request ID for checkpoint IPC
+ */
+export declare function generateRequestId(): string;
+//# sourceMappingURL=checkpoint-ipc.d.ts.map

package/dist/checkpoint-ipc.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint-ipc.d.ts","sourceRoot":"","sources":["../src/checkpoint-ipc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,OAAO,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF;;GAEG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAErE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAEtE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAErE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C"}

package/dist/index.d.ts

@@ -1,27 +1,33 @@
 import type { Backend, LogEntry } from "@cascade-flow/backend-interface";
 import type { StepOutput, RunnerContext } from "@cascade-flow/workflow";
 import type { LoadedStep } from "./types";
+import type { CheckpointData, CheckpointFailedData } from "./checkpoint-ipc";
 export type { LoadedStep };
+export type { CheckpointData, CheckpointFailedData } from "./checkpoint-ipc";
 /**
  * Execute a step in an isolated child process
  *
  * Wrapper around executeStepInSubprocess that generates the output path.
+ *
+ * @param stepFile - Absolute path to the step.ts file
+ * @param stepId - Unique identifier of the step
+ * @param dependencies - Resolved dependency outputs
+ * @param ctx - Runner context passed to step
+ * @param attemptNumber - Current attempt number (for retries)
+ * @param backend - Backend for generating output path
+ * @param onLog - Optional callback for real-time log emission
+ * @param onCheckpoint - Optional callback for persisting checkpoints
+ * @param onCheckpointFailed - Optional callback for persisting checkpoint failures
+ * @param existingCheckpoints - Existing checkpoints to replay (name -> data[] by sequence)
+ * @param options - Additional options (signal for abort)
  */
-export declare function executeStepInProcess(stepFile: string, stepId: string, dependencies: Record<string, unknown>, ctx: RunnerContext, attemptNumber: number, backend: Backend, onLog?: (log: LogEntry) => void | Promise<void>, options?: {
+export declare function executeStepInProcess(stepFile: string, stepId: string, dependencies: Record<string, unknown>, ctx: RunnerContext, attemptNumber: number, backend: Backend, onLog?: (log: LogEntry) => void | Promise<void>, onCheckpoint?: (checkpoint: CheckpointData) => Promise<void>, onCheckpointFailed?: (checkpoint: CheckpointFailedData) => Promise<void>, existingCheckpoints?: Map<string, string[]>, options?: {
     signal?: AbortSignal;
 }): Promise<{
     result: StepOutput;
     logs: LogEntry[];
 }>;
-export declare function runAll(options: {
-    workflow: string;
-    only?: string[];
-    ctx?: Partial<RunnerContext>;
-    resume?: boolean;
-    runId?: string;
-    backend: Backend;
-    input?: unknown;
-}): Promise<Record<string, StepOutput>>;
 export { discoverWorkflows, discoverSteps } from "./discovery";
 export { calculateWorkflowHash, getGitInfo } from "./versioning";
+export { getAllDependents, validateWorkflowVersion } from "./validation";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAa,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAMpF,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAI1C,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACrC,GAAG,EAAE,aAAa,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,OAAO,EAChB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,QAAQ,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAC/C,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,GACjC,OAAO,CAAC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,QAAQ,EAAE,CAAA;CAAE,CAAC,CAoBnD;AAED,wBAAsB,MAAM,CAAC,OAAO,EAAE;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,GAAG,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC7B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAkctC;AAGD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG/D,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAEzE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,KAAK,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAG7E,YAAY,EAAE,UAAU,EAAE,CAAC;AAG3B,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE7E;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACrC,GAAG,EAAE,aAAa,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,OAAO,EAChB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,QAAQ,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAC/C,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,EAC5D,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,EACxE,mBAAmB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAC3C,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,GACjC,OAAO,CAAC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,QAAQ,EAAE,CAAA;CAAE,CAAC,CAuBnD;AAGD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG/D,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAGjE,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC"}