graphai 0.0.3 → 0.0.4

package/.github/workflows/node.js.yml

@@ -0,0 +1,27 @@
+# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
+
+name: Node.js CI
+
+on:
+  pull_request
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [18.x, 20.x]
+        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v4
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'npm'
+    - run: yarn install
+    - run: yarn test

package/lib/graphai.d.ts

@@ -5,54 +5,64 @@ export declare enum NodeState {
     TimedOut = 3,
     Completed = 4
 }
-type ResultData = Record<string, any>;
+type ResultData<ResultType = Record<string, any>> = ResultType | undefined;
+type ResultDataDictonary<ResultType = Record<string, any>> = Record<string, ResultData<ResultType>>;
 export type NodeDataParams = Record<string, any>;
 type NodeData = {
     inputs: undefined | Array<string>;
     params: NodeDataParams;
     retry: undefined | number;
     timeout: undefined | number;
+    functionName: undefined | string;
 };
 type GraphData = {
     nodes: Record<string, NodeData>;
+    concurrency: number;
 };
-export type NodeExecuteContext = {
+export type NodeExecuteContext<ResultType> = {
     nodeId: string;
     retry: number;
     params: NodeDataParams;
-    payload: ResultData;
+    payload: ResultDataDictonary<ResultType>;
 };
-type NodeExecute = (context: NodeExecuteContext) => Promise<ResultData>;
-declare class Node {
+type NodeExecute<ResultType> = (context: NodeExecuteContext<ResultType>) => Promise<ResultData<ResultType>>;
+declare class Node<ResultType = Record<string, any>> {
     nodeId: string;
     params: NodeDataParams;
     inputs: Array<string>;
     pendings: Set<string>;
     waitlist: Set<string>;
     state: NodeState;
-    result: ResultData;
+    functionName: string;
+    result: ResultData<ResultType>;
     retryLimit: number;
     retryCount: number;
     transactionId: undefined | number;
     timeout: number;
-    constructor(nodeId: string, data: NodeData);
+    private graph;
+    constructor(nodeId: string, data: NodeData, graph: GraphAI<ResultType>);
     asString(): string;
     private retry;
-    removePending(nodeId: string, graph: GraphAI): void;
-    payload(graph: GraphAI): ResultData;
-    executeIfReady(graph: GraphAI): void;
-    private execute;
+    removePending(nodeId: string): void;
+    payload(): ResultDataDictonary<ResultType>;
+    pushQueueIfReady(): void;
+    execute(): Promise<void>;
 }
-type GraphNodes = Record<string, Node>;
-export declare class GraphAI {
-    nodes: GraphNodes;
-    callback: NodeExecute;
+type GraphNodes<ResultType> = Record<string, Node<ResultType>>;
+type NodeExecuteDictonary<ResultType> = Record<string, NodeExecute<ResultType>>;
+export declare class GraphAI<ResultType = Record<string, any>> {
+    nodes: GraphNodes<ResultType>;
+    callbackDictonary: NodeExecuteDictonary<ResultType>;
     private runningNodes;
+    private nodeQueue;
     private onComplete;
-    constructor(data: GraphData, callback: NodeExecute);
+    private concurrency;
+    constructor(data: GraphData, callbackDictonary: NodeExecuteDictonary<ResultType> | NodeExecute<ResultType>);
+    getCallback(functionName: string): NodeExecute<ResultType>;
     asString(): string;
     run(): Promise<unknown>;
-    addRunning(node: Node): void;
-    removeRunning(node: Node): void;
+    private runNode;
+    pushQueue(node: Node<ResultType>): void;
+    removeRunning(node: Node<ResultType>): void;
 }
 export {};

package/lib/graphai.js

@@ -10,90 +10,102 @@ var NodeState;
     NodeState[NodeState["Completed"] = 4] = "Completed";
 })(NodeState || (exports.NodeState = NodeState = {}));
 class Node {
-    constructor(nodeId, data) {
+    constructor(nodeId, data, graph) {
         this.nodeId = nodeId;
         this.inputs = data.inputs ?? [];
         this.pendings = new Set(this.inputs);
         this.params = data.params;
         this.waitlist = new Set();
         this.state = NodeState.Waiting;
-        this.result = {};
+        this.functionName = data.functionName ?? "default";
+        this.result = undefined;
         this.retryLimit = data.retry ?? 0;
         this.retryCount = 0;
         this.timeout = data.timeout ?? 0;
+        this.graph = graph;
     }
     asString() {
         return `${this.nodeId}: ${this.state} ${[...this.waitlist]}`;
     }
-    retry(graph, state, result) {
+    retry(state, result) {
         if (this.retryCount < this.retryLimit) {
             this.retryCount++;
-            this.execute(graph);
+            this.execute();
         }
         else {
             this.state = state;
             this.result = result;
-            graph.removeRunning(this);
+            this.graph.removeRunning(this);
         }
     }
-    removePending(nodeId, graph) {
+    removePending(nodeId) {
         this.pendings.delete(nodeId);
-        this.executeIfReady(graph);
+        this.pushQueueIfReady();
     }
-    payload(graph) {
+    payload() {
         return this.inputs.reduce((results, nodeId) => {
-            results[nodeId] = graph.nodes[nodeId].result;
+            results[nodeId] = this.graph.nodes[nodeId].result;
             return results;
         }, {});
     }
-    executeIfReady(graph) {
-        if (this.pendings.size == 0) {
-            graph.addRunning(this);
-            this.execute(graph);
+    pushQueueIfReady() {
+        if (this.pendings.size === 0) {
+            this.graph.pushQueue(this);
         }
     }
-    async execute(graph) {
+    async execute() {
         this.state = NodeState.Executing;
         const transactionId = Date.now();
         this.transactionId = transactionId;
         if (this.timeout > 0) {
             setTimeout(() => {
-                if (this.state == NodeState.Executing && this.transactionId == transactionId) {
+                if (this.state === NodeState.Executing && this.transactionId === transactionId) {
                     console.log("*** timeout", this.timeout);
-                    this.retry(graph, NodeState.TimedOut, {});
+                    this.retry(NodeState.TimedOut, undefined);
                 }
             }, this.timeout);
         }
         try {
-            const result = await graph.callback({ nodeId: this.nodeId, retry: this.retryCount, params: this.params, payload: this.payload(graph) });
+            const callback = this.graph.getCallback(this.functionName);
+            const result = await callback({
+                nodeId: this.nodeId,
+                retry: this.retryCount,
+                params: this.params,
+                payload: this.payload(),
+            });
             if (this.transactionId !== transactionId) {
-                console.log("****** tid mismatch (success)");
+                console.log("****** transactionId mismatch (success)");
                 return;
             }
             this.state = NodeState.Completed;
             this.result = result;
             this.waitlist.forEach((nodeId) => {
-                const node = graph.nodes[nodeId];
-                node.removePending(this.nodeId, graph);
+                const node = this.graph.nodes[nodeId];
+                node.removePending(this.nodeId);
             });
-            graph.removeRunning(this);
+            this.graph.removeRunning(this);
         }
         catch (e) {
             if (this.transactionId !== transactionId) {
-                console.log("****** tid mismatch (failed)");
+                console.log("****** transactionId mismatch (failed)");
                 return;
             }
-            this.retry(graph, NodeState.Failed, {});
+            this.retry(NodeState.Failed, undefined);
         }
     }
 }
 class GraphAI {
-    constructor(data, callback) {
-        this.callback = callback;
+    constructor(data, callbackDictonary) {
+        this.callbackDictonary = typeof callbackDictonary === "function" ? { default: callbackDictonary } : callbackDictonary;
+        if (this.callbackDictonary["default"] === undefined) {
+            throw new Error("No default function");
+        }
+        this.concurrency = data.concurrency ?? 2;
         this.runningNodes = new Set();
+        this.nodeQueue = [];
         this.onComplete = () => { };
         this.nodes = Object.keys(data.nodes).reduce((nodes, nodeId) => {
-            nodes[nodeId] = new Node(nodeId, data.nodes[nodeId]);
+            nodes[nodeId] = new Node(nodeId, data.nodes[nodeId], this);
             return nodes;
         }, {});
         // Generate the waitlist for each node
@@ -105,6 +117,12 @@ class GraphAI {
             });
         });
     }
+    getCallback(functionName) {
+        if (functionName && this.callbackDictonary[functionName]) {
+            return this.callbackDictonary[functionName];
+        }
+        return this.callbackDictonary["default"];
+    }
     asString() {
         return Object.keys(this.nodes)
             .map((nodeId) => {
@@ -116,7 +134,7 @@ class GraphAI {
         // Nodes without pending data should run immediately.
         Object.keys(this.nodes).forEach((nodeId) => {
             const node = this.nodes[nodeId];
-            node.executeIfReady(this);
+            node.pushQueueIfReady();
         });
         return new Promise((resolve, reject) => {
             this.onComplete = () => {
@@ -128,12 +146,27 @@ class GraphAI {
             };
         });
     }
-    addRunning(node) {
+    runNode(node) {
         this.runningNodes.add(node.nodeId);
+        node.execute();
+    }
+    pushQueue(node) {
+        if (this.runningNodes.size < this.concurrency) {
+            this.runNode(node);
+        }
+        else {
+            this.nodeQueue.push(node);
+        }
     }
     removeRunning(node) {
         this.runningNodes.delete(node.nodeId);
-        if (this.runningNodes.size == 0) {
+        if (this.nodeQueue.length > 0) {
+            const n = this.nodeQueue.shift();
+            if (n) {
+                this.runNode(n);
+            }
+        }
+        if (this.runningNodes.size === 0) {
             this.onComplete();
         }
     }

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "graphai",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "description": "Asynchronous data flow execution engine to make it simple to build LLM apps.",
   "main": "lib/index.js",
   "scripts": {
     "build": "tsc",
     "eslint": "eslint --fix --ext src/**/*.{ts} tests/**/*.ts",
     "format": "prettier --write '{src,tests}/**/*.ts'",
-    "test": "node --test --require ts-node/register ./tests/sample_flow.ts"
+    "test": "node --test --require ts-node/register ./tests/test_*.ts"
   },
   "repository": {
     "type": "git",
@@ -25,13 +25,13 @@
     "@typescript-eslint/parser": "^6.8.0",
     "eslint": "^7.32.0 || ^8.2.0",
     "eslint-plugin-import": "^2.25.2",
+    "openai": "^4.12.4",
     "prettier": "^3.0.3",
     "slashgpt": "^0.0.6",
     "ts-node": "^10.9.1",
     "typescript": "^5.2.2"
   },
   "dependencies": {
-    "openai": "^4.12.4",
     "yaml": "^2.3.3"
   },
   "types": "./lib/index.d.ts",

package/src/graphai.ts

@@ -7,7 +7,9 @@ export enum NodeState {
   TimedOut,
   Completed,
 }
-type ResultData = Record<string, any>;
+type ResultData<ResultType = Record<string, any>> = ResultType | undefined;
+type ResultDataDictonary<ResultType = Record<string, any>> = Record<string, ResultData<ResultType>>;
+
 export type NodeDataParams = Record<string, any>; // App-specific parameters
 
 type NodeData = {
@@ -15,132 +17,154 @@ type NodeData = {
   params: NodeDataParams;
   retry: undefined | number;
   timeout: undefined | number; // msec
+  functionName: undefined | string;
 };
 
 type GraphData = {
   nodes: Record<string, NodeData>;
+  concurrency: number;
 };
 
-export type NodeExecuteContext = {
+export type NodeExecuteContext<ResultType> = {
   nodeId: string;
   retry: number;
   params: NodeDataParams;
-  payload: ResultData;
+  payload: ResultDataDictonary<ResultType>;
 };
 
-type NodeExecute = (context: NodeExecuteContext) => Promise<ResultData>;
+type NodeExecute<ResultType> = (context: NodeExecuteContext<ResultType>) => Promise<ResultData<ResultType>>;
 
-class Node {
+class Node<ResultType = Record<string, any>> {
   public nodeId: string;
   public params: NodeDataParams; // App-specific parameters
   public inputs: Array<string>; // List of nodes this node needs data from.
   public pendings: Set<string>; // List of nodes this node is waiting data from.
   public waitlist: Set<string>; // List of nodes which need data from this node.
   public state: NodeState;
-  public result: ResultData;
+  public functionName: string;
+  public result: ResultData<ResultType>;
   public retryLimit: number;
   public retryCount: number;
   public transactionId: undefined | number; // To reject callbacks from timed-out transactions
   public timeout: number; // msec
 
-  constructor(nodeId: string, data: NodeData) {
+  private graph: GraphAI<ResultType>;
+
+  constructor(nodeId: string, data: NodeData, graph: GraphAI<ResultType>) {
     this.nodeId = nodeId;
     this.inputs = data.inputs ?? [];
     this.pendings = new Set(this.inputs);
     this.params = data.params;
     this.waitlist = new Set<string>();
     this.state = NodeState.Waiting;
-    this.result = {};
+    this.functionName = data.functionName ?? "default";
+    this.result = undefined;
     this.retryLimit = data.retry ?? 0;
     this.retryCount = 0;
     this.timeout = data.timeout ?? 0;
+
+    this.graph = graph;
   }
 
   public asString() {
     return `${this.nodeId}: ${this.state} ${[...this.waitlist]}`;
   }
 
-  private retry(graph: GraphAI, state: NodeState, result: ResultData) {
+  private retry(state: NodeState, result: ResultData<ResultType>) {
     if (this.retryCount < this.retryLimit) {
       this.retryCount++;
-      this.execute(graph);
+      this.execute();
     } else {
       this.state = state;
       this.result = result;
-      graph.removeRunning(this);
+      this.graph.removeRunning(this);
     }
   }
 
-  public removePending(nodeId: string, graph: GraphAI) {
+  public removePending(nodeId: string) {
     this.pendings.delete(nodeId);
-    this.executeIfReady(graph);
+    this.pushQueueIfReady();
   }
 
-  public payload(graph: GraphAI) {
-    return this.inputs.reduce((results: ResultData, nodeId) => {
-      results[nodeId] = graph.nodes[nodeId].result;
+  public payload() {
+    return this.inputs.reduce((results: ResultDataDictonary<ResultType>, nodeId) => {
+      results[nodeId] = this.graph.nodes[nodeId].result;
       return results;
     }, {});
   }
 
-  public executeIfReady(graph: GraphAI) {
-    if (this.pendings.size == 0) {
-      graph.addRunning(this);
-      this.execute(graph);
+  public pushQueueIfReady() {
+    if (this.pendings.size === 0) {
+      this.graph.pushQueue(this);
     }
   }
 
-  private async execute(graph: GraphAI) {
+  public async execute() {
     this.state = NodeState.Executing;
     const transactionId = Date.now();
     this.transactionId = transactionId;
 
     if (this.timeout > 0) {
       setTimeout(() => {
-        if (this.state == NodeState.Executing && this.transactionId == transactionId) {
+        if (this.state === NodeState.Executing && this.transactionId === transactionId) {
           console.log("*** timeout", this.timeout);
-          this.retry(graph, NodeState.TimedOut, {});
+          this.retry(NodeState.TimedOut, undefined);
         }
       }, this.timeout);
     }
 
     try {
-      const result = await graph.callback({ nodeId: this.nodeId, retry: this.retryCount, params: this.params, payload: this.payload(graph) });
+      const callback = this.graph.getCallback(this.functionName);
+      const result = await callback({
+        nodeId: this.nodeId,
+        retry: this.retryCount,
+        params: this.params,
+        payload: this.payload(),
+      });
       if (this.transactionId !== transactionId) {
-        console.log("****** tid mismatch (success)");
+        console.log("****** transactionId mismatch (success)");
         return;
       }
       this.state = NodeState.Completed;
       this.result = result;
       this.waitlist.forEach((nodeId) => {
-        const node = graph.nodes[nodeId];
-        node.removePending(this.nodeId, graph);
+        const node = this.graph.nodes[nodeId];
+        node.removePending(this.nodeId);
       });
-      graph.removeRunning(this);
+      this.graph.removeRunning(this);
     } catch (e) {
       if (this.transactionId !== transactionId) {
-        console.log("****** tid mismatch (failed)");
+        console.log("****** transactionId mismatch (failed)");
         return;
       }
-      this.retry(graph, NodeState.Failed, {});
+      this.retry(NodeState.Failed, undefined);
     }
   }
 }
 
-type GraphNodes = Record<string, Node>;
+type GraphNodes<ResultType> = Record<string, Node<ResultType>>;
 
-export class GraphAI {
-  public nodes: GraphNodes;
-  public callback: NodeExecute;
+type NodeExecuteDictonary<ResultType> = Record<string, NodeExecute<ResultType>>;
+
+export class GraphAI<ResultType = Record<string, any>> {
+  public nodes: GraphNodes<ResultType>;
+  public callbackDictonary: NodeExecuteDictonary<ResultType>;
   private runningNodes: Set<string>;
+  private nodeQueue: Array<Node<ResultType>>;
   private onComplete: () => void;
+  private concurrency: number;
 
-  constructor(data: GraphData, callback: NodeExecute) {
-    this.callback = callback;
+  constructor(data: GraphData, callbackDictonary: NodeExecuteDictonary<ResultType> | NodeExecute<ResultType>) {
+    this.callbackDictonary = typeof callbackDictonary === "function" ? { default: callbackDictonary } : callbackDictonary;
+    if (this.callbackDictonary["default"] === undefined) {
+      throw new Error("No default function");
+    }
+    this.concurrency = data.concurrency ?? 2;
     this.runningNodes = new Set<string>();
+    this.nodeQueue = [];
     this.onComplete = () => {};
-    this.nodes = Object.keys(data.nodes).reduce((nodes: GraphNodes, nodeId: string) => {
-      nodes[nodeId] = new Node(nodeId, data.nodes[nodeId]);
+    this.nodes = Object.keys(data.nodes).reduce((nodes: GraphNodes<ResultType>, nodeId: string) => {
+      nodes[nodeId] = new Node<ResultType>(nodeId, data.nodes[nodeId], this);
       return nodes;
     }, {});
 
@@ -154,6 +178,13 @@ export class GraphAI {
     });
   }
 
+  public getCallback(functionName: string) {
+    if (functionName && this.callbackDictonary[functionName]) {
+      return this.callbackDictonary[functionName];
+    }
+    return this.callbackDictonary["default"];
+  }
+
   public asString() {
     return Object.keys(this.nodes)
       .map((nodeId) => {
@@ -166,12 +197,12 @@ export class GraphAI {
     // Nodes without pending data should run immediately.
     Object.keys(this.nodes).forEach((nodeId) => {
       const node = this.nodes[nodeId];
-      node.executeIfReady(this);
+      node.pushQueueIfReady();
     });
 
     return new Promise((resolve, reject) => {
       this.onComplete = () => {
-        const results = Object.keys(this.nodes).reduce((results: ResultData, nodeId) => {
+        const results = Object.keys(this.nodes).reduce((results: ResultDataDictonary<ResultType>, nodeId) => {
           results[nodeId] = this.nodes[nodeId].result;
           return results;
         }, {});
@@ -180,13 +211,28 @@ export class GraphAI {
     });
   }
 
-  public addRunning(node: Node) {
+  private runNode(node: Node<ResultType>) {
     this.runningNodes.add(node.nodeId);
+    node.execute();
   }
 
-  public removeRunning(node: Node) {
+  public pushQueue(node: Node<ResultType>) {
+    if (this.runningNodes.size < this.concurrency) {
+      this.runNode(node);
+    } else {
+      this.nodeQueue.push(node);
+    }
+  }
+
+  public removeRunning(node: Node<ResultType>) {
     this.runningNodes.delete(node.nodeId);
-    if (this.runningNodes.size == 0) {
+    if (this.nodeQueue.length > 0) {
+      const n = this.nodeQueue.shift();
+      if (n) {
+        this.runNode(n);
+      }
+    }
+    if (this.runningNodes.size === 0) {
       this.onComplete();
     }
   }

package/tests/graphs/test_multiple_functions_1.yml

@@ -0,0 +1,21 @@
+nodes:
+  node1:
+    params:
+      delay: 500
+  node2:
+    params:
+      delay: 100
+  node3:
+    params:
+      delay: 500
+    inputs: [node1, node2]
+    functionName: test2
+  node4:
+    params:
+      delay: 100
+    inputs: [node3]
+  node5:
+    params:
+      delay: 500
+    inputs: [node2, node4]
+    functionName: test2

package/tests/sample_gpt.ts

@@ -1,15 +1,15 @@
 import path from "path";
 import { GraphAI, NodeExecuteContext } from "../src/graphai";
 import { ChatSession, ChatConfig } from "slashgpt";
-import { readManifestData } from "../src/file_utils";
+import { readManifestData } from "./file_utils";
 
 const config = new ChatConfig(path.resolve(__dirname));
 
-const testFunction = async (context: NodeExecuteContext) => {
+const testFunction = async (context: NodeExecuteContext<Record<string, string>>) => {
   console.log("executing", context.nodeId, context.params, context.payload);
   const session = new ChatSession(config, context.params.manifest ?? {});
   const prompt = Object.keys(context.payload).reduce((prompt, key) => {
-    return prompt.replace("${" + key + "}", context.payload[key]["answer"]);
+    return prompt.replace("${" + key + "}", context.payload[key]!["answer"]);
   }, context.params.prompt);
   session.append_user_question(prompt);
 

package/tests/test_multiple_functions.ts

@@ -0,0 +1,47 @@
+import path from "path";
+import { GraphAI, NodeExecuteContext } from "../src/graphai";
+import { readManifestData } from "../src/file_utils";
+import { sleep } from "./utils";
+
+import test from "node:test";
+import assert from "node:assert";
+
+const testFunction1 = async (context: NodeExecuteContext<Record<string, string>>) => {
+  const { nodeId, retry, params } = context;
+  console.log("executing", nodeId, params);
+
+  const result = { [nodeId]: "output 1" };
+  console.log("completing", nodeId, result);
+  return result;
+};
+
+const testFunction2 = async (context: NodeExecuteContext<Record<string, string>>) => {
+  const { nodeId, retry, params } = context;
+  console.log("executing", nodeId, params);
+
+  const result = { [nodeId]: "output 2" };
+  console.log("completing", nodeId, result);
+  return result;
+};
+
+const runTest = async (file: string) => {
+  const file_path = path.resolve(__dirname) + file;
+  const graph_data = readManifestData(file_path);
+
+  const graph = new GraphAI(graph_data, { default: testFunction1, test2: testFunction2 });
+
+  const results = await graph.run();
+  console.log(results);
+  return results;
+};
+
+test("test sample1", async () => {
+  const result = await runTest("/graphs/test_multiple_functions_1.yml");
+  assert.deepStrictEqual(result, {
+    node1: { node1: "output 1" },
+    node2: { node2: "output 1" },
+    node3: { node3: "output 2" },
+    node4: { node4: "output 1" },
+    node5: { node5: "output 2" },
+  });
+});

package/tests/{sample_flow.ts → test_sample_flow.ts}

@@ -1,13 +1,13 @@
 import path from "path";
 import { GraphAI, NodeExecuteContext } from "../src/graphai";
-import { readManifestData } from "../src/file_utils";
+import { readManifestData } from "./file_utils";
 import { sleep } from "./utils";
 
 import test from "node:test";
 import assert from "node:assert";
 
-const testFunction = async (context: NodeExecuteContext) => {
-  const { nodeId, retry, params } = context;
+const testFunction = async (context: NodeExecuteContext<Record<string, string>>) => {
+  const { nodeId, retry, params, payload } = context;
   console.log("executing", nodeId, params);
   await sleep(params.delay / (retry + 1));
 
@@ -16,7 +16,13 @@ const testFunction = async (context: NodeExecuteContext) => {
     console.log("failed", nodeId, result, retry);
     throw new Error("Intentional Failure");
   } else {
-    const result = { [nodeId]: "output" };
+    const result = Object.keys(payload).reduce(
+      (result, key) => {
+        result = { ...result, ...payload[key] };
+        return result;
+      },
+      { [nodeId]: "output" },
+    );
     console.log("completing", nodeId, result);
     return result;
   }
@@ -38,9 +44,9 @@ test("test sample1", async () => {
   assert.deepStrictEqual(result, {
     node1: { node1: "output" },
     node2: { node2: "output" },
-    node3: { node3: "output" },
-    node4: { node4: "output" },
-    node5: { node5: "output" },
+    node3: { node3: "output", node1: "output", node2: "output" },
+    node4: { node4: "output", node3: "output", node1: "output", node2: "output" },
+    node5: { node5: "output", node4: "output", node3: "output", node1: "output", node2: "output" },
   });
 });
 
@@ -49,9 +55,9 @@ test("test sample1", async () => {
   assert.deepStrictEqual(result, {
     node1: { node1: "output" },
     node2: { node2: "output" },
-    node3: { node3: "output" },
-    node4: { node4: "output" },
-    node5: { node5: "output" },
+    node3: { node3: "output", node1: "output", node2: "output" },
+    node4: { node4: "output", node3: "output", node1: "output", node2: "output" },
+    node5: { node5: "output", node4: "output", node3: "output", node1: "output", node2: "output" },
   });
   console.log("COMPLETE 2");
 });

package/lib/file_utils.d.ts

@@ -1,3 +0,0 @@
-export declare const readManifestData: (file: string) => any;
-export declare const readJsonManifest: (fileName: string) => any;
-export declare const readYamlManifest: (fileName: string) => any;

package/lib/file_utils.js

@@ -1,30 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.readYamlManifest = exports.readJsonManifest = exports.readManifestData = void 0;
-const fs_1 = __importDefault(require("fs"));
-const yaml_1 = __importDefault(require("yaml"));
-const readManifestData = (file) => {
-    if (file.endsWith(".yaml") || file.endsWith(".yml")) {
-        return (0, exports.readYamlManifest)(file);
-    }
-    if (file.endsWith(".json")) {
-        return (0, exports.readJsonManifest)(file);
-    }
-    throw new Error("No file exists " + file);
-};
-exports.readManifestData = readManifestData;
-const readJsonManifest = (fileName) => {
-    const manifest_file = fs_1.default.readFileSync(fileName, "utf8");
-    const manifest = JSON.parse(manifest_file);
-    return manifest;
-};
-exports.readJsonManifest = readJsonManifest;
-const readYamlManifest = (fileName) => {
-    const manifest_file = fs_1.default.readFileSync(fileName, "utf8");
-    const manifest = yaml_1.default.parse(manifest_file);
-    return manifest;
-};
-exports.readYamlManifest = readYamlManifest;

package/lib/flow.d.ts

@@ -1,51 +0,0 @@
-export declare enum NodeState {
-    Waiting = 0,
-    Executing = 1,
-    Failed = 2,
-    Completed = 3
-}
-type NodeData = {
-    inputs: undefined | Array<string>;
-    params: any;
-    retry: undefined | number;
-};
-type FlowData = {
-    nodes: Record<string, NodeData>;
-};
-export declare enum FlowCommand {
-    Log = 0,
-    Execute = 1,
-    OnComplete = 2
-}
-type FlowCallback = (params: Record<string, any>) => void;
-declare class Node {
-    key: string;
-    inputs: Array<string>;
-    pendings: Set<string>;
-    params: any;
-    waitlist: Set<string>;
-    state: NodeState;
-    result: Record<string, any>;
-    retryLimit: number;
-    retryCount: number;
-    constructor(key: string, data: NodeData);
-    asString(): string;
-    complete(result: Record<string, any>, nodes: Record<string, Node>, graph: GraphAI): void;
-    reportError(result: Record<string, any>, nodes: Record<string, Node>, graph: GraphAI): void;
-    removePending(key: string, graph: GraphAI): void;
-    payload(graph: GraphAI): Record<string, any>;
-    executeIfReady(graph: GraphAI): void;
-}
-export declare class GraphAI {
-    nodes: Record<string, Node>;
-    callback: FlowCallback;
-    private runningNodes;
-    constructor(data: FlowData, callback: FlowCallback);
-    asString(): string;
-    run(): void;
-    feed(key: string, result: Record<string, any>): void;
-    reportError(key: string, result: Record<string, any>): void;
-    add(node: Node): void;
-    remove(node: Node): void;
-}
-export {};

package/lib/flow.js

@@ -1,118 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.GraphAI = exports.FlowCommand = exports.NodeState = void 0;
-var NodeState;
-(function (NodeState) {
-    NodeState[NodeState["Waiting"] = 0] = "Waiting";
-    NodeState[NodeState["Executing"] = 1] = "Executing";
-    NodeState[NodeState["Failed"] = 2] = "Failed";
-    NodeState[NodeState["Completed"] = 3] = "Completed";
-})(NodeState || (exports.NodeState = NodeState = {}));
-var FlowCommand;
-(function (FlowCommand) {
-    FlowCommand[FlowCommand["Log"] = 0] = "Log";
-    FlowCommand[FlowCommand["Execute"] = 1] = "Execute";
-    FlowCommand[FlowCommand["OnComplete"] = 2] = "OnComplete";
-})(FlowCommand || (exports.FlowCommand = FlowCommand = {}));
-class Node {
-    constructor(key, data) {
-        this.key = key;
-        this.inputs = data.inputs ?? [];
-        this.pendings = new Set(this.inputs);
-        this.params = data.params;
-        this.waitlist = new Set();
-        this.state = NodeState.Waiting;
-        this.result = {};
-        this.retryLimit = data.retry ?? 0;
-        this.retryCount = 0;
-    }
-    asString() {
-        return `${this.key}: ${this.state} ${[...this.waitlist]}`;
-    }
-    complete(result, nodes, graph) {
-        this.state = NodeState.Completed;
-        this.result = result;
-        this.waitlist.forEach(key => {
-            const node = nodes[key];
-            node.removePending(this.key, graph);
-        });
-        graph.remove(this);
-    }
-    reportError(result, nodes, graph) {
-        this.state = NodeState.Failed;
-        this.result = result;
-        if (this.retryCount < this.retryLimit) {
-            this.retryCount++;
-            this.state = NodeState.Executing;
-            graph.callback({ cmd: FlowCommand.Execute, node: this.key, params: this.params, retry: this.retryCount, payload: this.payload(graph) });
-        }
-        else {
-            graph.remove(this);
-        }
-    }
-    removePending(key, graph) {
-        this.pendings.delete(key);
-        this.executeIfReady(graph);
-    }
-    payload(graph) {
-        const foo = {};
-        return this.inputs.reduce((payload, key) => {
-            payload[key] = graph.nodes[key].result;
-            return payload;
-        }, foo);
-    }
-    executeIfReady(graph) {
-        if (this.pendings.size == 0) {
-            this.state = NodeState.Executing;
-            graph.add(this);
-            graph.callback({ cmd: FlowCommand.Execute, node: this.key, params: this.params, retry: 0, payload: this.payload(graph) });
-        }
-    }
-}
-class GraphAI {
-    constructor(data, callback) {
-        this.callback = callback;
-        this.runningNodes = new Set();
-        const foo = {}; // HACK: Work around
-        this.nodes = Object.keys(data.nodes).reduce((nodes, key) => {
-            nodes[key] = new Node(key, data.nodes[key]);
-            return nodes;
-        }, foo);
-        // Generate the waitlist for each node
-        Object.keys(this.nodes).forEach(key => {
-            const node = this.nodes[key];
-            node.pendings.forEach(pending => {
-                const node2 = this.nodes[pending];
-                node2.waitlist.add(key);
-            });
-        });
-    }
-    asString() {
-        return Object.keys(this.nodes).map((key) => { return this.nodes[key].asString(); }).join('\n');
-    }
-    run() {
-        // Nodes without pending data should run immediately.
-        Object.keys(this.nodes).forEach(key => {
-            const node = this.nodes[key];
-            node.executeIfReady(this);
-        });
-    }
-    feed(key, result) {
-        const node = this.nodes[key];
-        node.complete(result, this.nodes, this);
-    }
-    reportError(key, result) {
-        const node = this.nodes[key];
-        node.reportError(result, this.nodes, this);
-    }
-    add(node) {
-        this.runningNodes.add(node.key);
-    }
-    remove(node) {
-        this.runningNodes.delete(node.key);
-        if (this.runningNodes.size == 0) {
-            this.callback({ cmd: FlowCommand.OnComplete });
-        }
-    }
-}
-exports.GraphAI = GraphAI;