@elaraai/e3-core 0.0.2-beta.3 → 0.0.2-beta.31

package/dist/src/dataflow.d.ts

@@ -2,7 +2,19 @@
  * Copyright (c) 2025 Elara AI Pty Ltd
  * Licensed under BSL 1.1. See LICENSE for details.
  */
-import { type WorkspaceLockHandle } from './workspaceLock.js';
+import { type TreePath } from '@elaraai/e3-types';
+import type { TaskRunner } from './execution/interfaces.js';
+import type { StorageBackend, LockHandle } from './storage/interfaces.js';
+/**
+ * Parse a keypath string (from pathToString) back to TreePath.
+ *
+ * The keypath format is: .field1.field2 (dot-separated field names)
+ * Quoted identifiers use backticks: .field1.`complex/name`
+ *
+ * @param pathStr - The path string in keypath format
+ * @returns TreePath array of path segments
+ */
+export declare function parsePathString(pathStr: string): TreePath;
 /**
  * Result of executing a single task in the dataflow.
  */
@@ -11,6 +23,8 @@ export interface TaskExecutionResult {
     name: string;
     /** Whether the task was cached */
     cached: boolean;
+    /** Execution ID (UUIDv7) - present for executed or cached tasks */
+    executionId?: string;
     /** Final state */
     state: 'success' | 'failed' | 'error' | 'skipped';
     /** Error message if state is 'error' */
@@ -26,6 +40,8 @@ export interface TaskExecutionResult {
 export interface DataflowResult {
     /** Overall success - true if all tasks completed successfully */
     success: boolean;
+    /** Dataflow run ID (UUIDv7) */
+    runId: string;
     /** Number of tasks executed (not from cache) */
     executed: number;
     /** Number of tasks served from cache */
@@ -57,7 +73,7 @@ export interface DataflowOptions {
      * Use an external lock when you need to hold the lock across multiple
      * operations (e.g., API server that cancels and restarts dataflow on writes).
      */
-    lock?: WorkspaceLockHandle;
+    lock?: LockHandle;
     /**
      * AbortSignal for cancellation. When aborted:
      * - No new tasks will be started
@@ -65,6 +81,14 @@ export interface DataflowOptions {
      * - DataflowAbortedError will be thrown with partial results
      */
     signal?: AbortSignal;
+    /**
+     * Task runner to use for executing individual tasks.
+     * Defaults to using taskExecute() directly if not provided.
+     *
+     * Use MockTaskRunner for testing dataflow orchestration logic
+     * without spawning real processes.
+     */
+    runner?: TaskRunner;
     /** Callback when a task starts */
     onTaskStart?: (name: string) => void;
     /** Callback when a task completes */
@@ -85,7 +109,8 @@ export interface DataflowOptions {
  * to prevent concurrent modifications. If options.lock is provided, uses that
  * lock instead (caller is responsible for releasing it).
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier (for local storage, the path to e3 repository directory)
  * @param ws - Workspace name
  * @param options - Execution options
  * @returns Result of the dataflow execution
@@ -95,14 +120,15 @@ export interface DataflowOptions {
  * @throws {TaskNotFoundError} If filter specifies a task that doesn't exist
  * @throws {DataflowError} If execution fails for other reasons
  */
-export declare function dataflowExecute(repoPath: string, ws: string, options?: DataflowOptions): Promise<DataflowResult>;
+export declare function dataflowExecute(storage: StorageBackend, repo: string, ws: string, options?: DataflowOptions): Promise<DataflowResult>;
 /**
  * Start dataflow execution in the background (non-blocking).
  *
  * Returns a promise immediately without awaiting execution. The lock is
  * released automatically when execution completes.
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier (for local storage, the path to e3 repository directory)
  * @param ws - Workspace name
  * @param options - Execution options (lock must be provided)
  * @returns Promise that resolves when execution completes
@@ -111,20 +137,21 @@ export declare function dataflowExecute(repoPath: string, ws: string, options?:
  * @throws {TaskNotFoundError} If filter specifies a task that doesn't exist
  * @throws {DataflowError} If execution fails for other reasons
  */
-export declare function dataflowStart(repoPath: string, ws: string, options: DataflowOptions & {
-    lock: WorkspaceLockHandle;
+export declare function dataflowStart(storage: StorageBackend, repo: string, ws: string, options: DataflowOptions & {
+    lock: LockHandle;
 }): Promise<DataflowResult>;
 /**
  * Get the dependency graph for a workspace (for visualization/debugging).
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier (for local storage, the path to e3 repository directory)
  * @param ws - Workspace name
  * @returns Graph information
  * @throws {WorkspaceNotFoundError} If workspace doesn't exist
  * @throws {WorkspaceNotDeployedError} If workspace has no package deployed
  * @throws {DataflowError} If graph building fails for other reasons
  */
-export declare function dataflowGetGraph(repoPath: string, ws: string): Promise<{
+export declare function dataflowGetGraph(storage: StorageBackend, repo: string, ws: string): Promise<{
     tasks: Array<{
         name: string;
         hash: string;
@@ -133,4 +160,110 @@ export declare function dataflowGetGraph(repoPath: string, ws: string): Promise<
         dependsOn: string[];
     }>;
 }>;
+/**
+ * Graph structure returned by dataflowGetGraph.
+ */
+export interface DataflowGraph {
+    tasks: Array<{
+        name: string;
+        hash: string;
+        inputs: string[];
+        output: string;
+        dependsOn: string[];
+    }>;
+}
+/**
+ * Get tasks that are ready to execute given the set of completed tasks.
+ *
+ * A task is ready when all tasks it depends on have completed.
+ * This is useful for distributed execution (e.g., AWS Step Functions)
+ * where a coordinator needs to determine which tasks can run next.
+ *
+ * @param graph - The dependency graph from dataflowGetGraph
+ * @param completedTasks - Set of task names that have completed
+ * @returns Array of task names that are ready to execute
+ *
+ * @example
+ * ```typescript
+ * const graph = await dataflowGetGraph(storage, repo, 'production');
+ * const ready = dataflowGetReadyTasks(graph, new Set()); // Initial ready tasks
+ * // Execute ready[0]...
+ * const nextReady = dataflowGetReadyTasks(graph, new Set([ready[0]]));
+ * ```
+ */
+export declare function dataflowGetReadyTasks(graph: DataflowGraph, completedTasks: Set<string>): string[];
+/**
+ * Check if a task execution is cached for the given inputs.
+ *
+ * This is useful for distributed execution where a Lambda handler needs
+ * to check if a task can be skipped before spawning execution.
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository path
+ * @param taskHash - Hash of the TaskObject
+ * @param inputHashes - Array of input dataset hashes (in order)
+ * @returns Output hash if cached, null if execution needed
+ *
+ * @example
+ * ```typescript
+ * const outputHash = await dataflowCheckCache(storage, repo, taskHash, inputHashes);
+ * if (outputHash) {
+ *   // Task is cached, use outputHash directly
+ * } else {
+ *   // Need to execute task
+ * }
+ * ```
+ */
+export declare function dataflowCheckCache(storage: StorageBackend, repo: string, taskHash: string, inputHashes: string[]): Promise<string | null>;
+/**
+ * Find tasks that should be skipped when a task fails.
+ *
+ * Returns all tasks that transitively depend on the failed task
+ * (directly or through other tasks), excluding already completed
+ * or already skipped tasks.
+ *
+ * This is useful for distributed execution where the coordinator
+ * needs to mark downstream tasks as skipped after a failure.
+ *
+ * @param graph - The dependency graph from dataflowGetGraph
+ * @param failedTask - Name of the task that failed
+ * @param completedTasks - Set of task names already completed (won't be skipped)
+ * @param skippedTasks - Set of task names already skipped (won't be returned again)
+ * @returns Array of task names that should be skipped
+ *
+ * @example
+ * ```typescript
+ * const graph = await dataflowGetGraph(storage, repo, 'production');
+ * // Task 'etl' failed...
+ * const toSkip = dataflowGetDependentsToSkip(graph, 'etl', completed, skipped);
+ * // toSkip might be ['transform', 'aggregate', 'report'] - all downstream tasks
+ * ```
+ */
+export declare function dataflowGetDependentsToSkip(graph: DataflowGraph, failedTask: string, completedTasks: Set<string>, skippedTasks: Set<string>): string[];
+/**
+ * Resolve input hashes for a task from current workspace state.
+ *
+ * Returns an array of hashes in the same order as the task's inputs.
+ * If any input is unassigned, returns null for that position.
+ *
+ * This is useful for distributed execution where the input hashes
+ * need to be resolved before checking cache or executing.
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository path
+ * @param ws - Workspace name
+ * @param task - Task info from the graph (needs inputs array)
+ * @returns Array of hashes (null if input is unassigned)
+ *
+ * @example
+ * ```typescript
+ * const graph = await dataflowGetGraph(storage, repo, 'production');
+ * const task = graph.tasks.find(t => t.name === 'etl')!;
+ * const inputHashes = await dataflowResolveInputHashes(storage, repo, 'production', task);
+ * if (!inputHashes.includes(null)) {
+ *   const cached = await dataflowCheckCache(storage, repo, task.hash, inputHashes);
+ * }
+ * ```
+ */
+export declare function dataflowResolveInputHashes(storage: StorageBackend, repo: string, ws: string, task: DataflowGraph['tasks'][0]): Promise<Array<string | null>>;
 //# sourceMappingURL=dataflow.d.ts.map

package/dist/src/dataflow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dataflow.d.ts","sourceRoot":"","sources":["../../src/dataflow.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAmDH,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,oBAAoB,CAAC;AA2E5B;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,kCAAkC;IAClC,MAAM,EAAE,OAAO,CAAC;IAChB,kBAAkB;IAClB,KAAK,EAAE,SAAS,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;IAClD,wCAAwC;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,iEAAiE;IACjE,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,KAAK,EAAE,mBAAmB,EAAE,CAAC;IAC7B,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;OAOG;IACH,IAAI,CAAC,EAAE,mBAAmB,CAAC;IAC3B;;;;;OAKG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,kCAAkC;IAClC,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACrC,qCAAqC;IACrC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,mBAAmB,KAAK,IAAI,CAAC;IACvD,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACpD,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACrD;AAqHD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,EAAE,EAAE,MAAM,EACV,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,CAAC,CAazB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,MAAM,EAChB,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,eAAe,GAAG;IAAE,IAAI,EAAE,mBAAmB,CAAA;CAAE,GACvD,OAAO,CAAC,cAAc,CAAC,CAGzB;AAmVD;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,EAAE,EAAE,MAAM,GACT,OAAO,CAAC;IACT,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC,CAAC;CACJ,CAAC,CA0CD"}
+{"version":3,"file":"dataflow.d.ts","sourceRoot":"","sources":["../../src/dataflow.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuBH,OAAO,EAML,KAAK,QAAQ,EAGd,MAAM,mBAAmB,CAAC;AAM3B,OAAO,KAAK,EAAE,UAAU,EAAsB,MAAM,2BAA2B,CAAC;AAehF,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAM1E;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,CAsCzD;AAyED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,kCAAkC;IAClC,MAAM,EAAE,OAAO,CAAC;IAChB,mEAAmE;IACnE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB;IAClB,KAAK,EAAE,SAAS,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;IAClD,wCAAwC;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,iEAAiE;IACjE,OAAO,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,KAAK,EAAE,mBAAmB,EAAE,CAAC;IAC7B,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;OAOG;IACH,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,kCAAkC;IAClC,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACrC,qCAAqC;IACrC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,mBAAmB,KAAK,IAAI,CAAC;IACvD,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACpD,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACrD;AAgHD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,MAAM,EACV,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,CAAC,CAkBzB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,eAAe,GAAG;IAAE,IAAI,EAAE,UAAU,CAAA;CAAE,GAC9C,OAAO,CAAC,cAAc,CAAC,CAGzB;AA6fD;;;;;;;;;;GAUG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,MAAM,GACT,OAAO,CAAC;IACT,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC,CAAC;CACJ,CAAC,CA0CD;AAMD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC,CAAC;CACJ;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,aAAa,EACpB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,GAC1B,MAAM,EAAE,CAiBV;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,GACpB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGxB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,aAAa,EACpB,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,GACxB,MAAM,EAAE,CA8CV;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAC9B,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAgB/B"}