@uploadista/core 0.0.2

package/src/flow/README.md

@@ -0,0 +1,352 @@
+# Flow Engine
+
+A flexible and extensible flow engine for executing directed acyclic graphs (DAGs) of processing nodes.
+
+## Overview
+
+The Flow Engine allows you to create complex processing pipelines by connecting nodes together. Each node represents a processing step, and edges define the data flow between nodes.
+
+## Architecture
+
+### Core Components
+
+- **Flow**: The main container that holds nodes and edges
+- **Node**: Individual processing units with input/output capabilities
+- **Edge**: Connections between nodes defining data flow
+- **Result**: Typed results for success, error, and cancellation states
+
+### Node Types
+
+- **Input**: Entry point nodes that receive initial data
+- **Process**: Processing nodes that transform data
+- **Output**: Final nodes that produce the end result
+- **Conditional**: Nodes that route flow based on conditions
+- **Multiplex**: Nodes that split input into multiple parallel outputs
+- **Merge**: Nodes that combine multiple inputs into a single output
+
+## New Advanced Features
+
+### 1. Zip Node
+
+The zip node allows you to combine multiple files into a single archive:
+
+```typescript
+import { createZipNode } from "./nodes/zip-node";
+
+const zipNode = createZipNode("zip-files", {
+  zipName: "archive.zip",
+  includeMetadata: true,
+});
+```
+
+**Features:**
+
+- Combines multiple input files into a single zip archive
+- Supports both single files and file batches
+- Optional metadata inclusion
+- Customizable archive name
+
+### 2. Conditional Node
+
+The conditional node routes flow based on file properties:
+
+```typescript
+import { createConditionalNode } from "./nodes/conditional-node";
+
+const conditionalNode = createConditionalNode("size-check", {
+  field: "size",
+  operator: "greaterThan",
+  value: 1024 * 1024, // 1MB
+  trueBranch: "large-file-processor",
+  falseBranch: "small-file-processor",
+});
+```
+
+**Supported Conditions:**
+
+- **Fields**: `mimeType`, `size`, `width`, `height`, `extension`
+- **Operators**: `equals`, `notEquals`, `greaterThan`, `lessThan`, `contains`, `startsWith`
+
+### 3. Multiplex Node
+
+The multiplex node splits a single input into multiple parallel outputs:
+
+```typescript
+import { createMultiplexNode } from "./nodes/multiplex-node";
+
+const multiplexNode = createMultiplexNode("multiplex", {
+  outputCount: 3,
+  strategy: "copy", // or "split"
+});
+```
+
+**Strategies:**
+
+- **Copy**: Creates multiple copies of the same file
+- **Split**: Splits the file into chunks
+
+### 4. Merge Node
+
+The merge node combines multiple inputs into a single output:
+
+```typescript
+import { createMergeNode } from "./nodes/merge-node";
+
+const mergeNode = createMergeNode("merge-files", {
+  strategy: "batch", // or "concat"
+});
+```
+
+**Strategies:**
+
+- **Batch**: Returns files as a batch
+- **Concat**: Concatenates all files into one
+
+## Usage
+
+### Basic Example
+
+```typescript
+import type { FlowNode } from "./node";
+
+import { createFlow } from "./flow";
+import { NodeType } from "./node";
+import { successResult } from "./result";
+
+// Define your data type
+type MyData = {
+  value: number;
+  processed: boolean;
+};
+
+// Create nodes
+const inputNode: FlowNode<MyData, MyData> = {
+  id: "input",
+  name: "Input",
+  description: "Input node",
+  type: NodeType.input,
+  run: async ({ data }) => {
+    return successResult(data);
+  },
+};
+
+const processNode: FlowNode<MyData, MyData> = {
+  id: "process",
+  name: "Process",
+  description: "Process node",
+  type: NodeType.process,
+  run: async ({ data }) => {
+    return successResult({
+      ...data,
+      value: data.value * 2,
+      processed: true,
+    });
+  },
+};
+
+const outputNode: FlowNode<MyData, MyData> = {
+  id: "output",
+  name: "Output",
+  description: "Output node",
+  type: NodeType.output,
+  run: async ({ data }) => {
+    return successResult(data);
+  },
+};
+
+// Create the flow
+const flow = createFlow({
+  flowId: "simple-flow",
+  nodes: [inputNode, processNode, outputNode],
+  edges: [
+    { source: "input", target: "process" },
+    { source: "process", target: "output" },
+  ],
+});
+
+// Run the flow
+const result = await flow.run({
+  value: 5,
+  processed: false,
+});
+
+if (result.type === "success") {
+  console.log("Result:", result.value);
+  // Output: { value: 10, processed: true }
+}
+```
+
+### Error Handling
+
+The flow engine provides comprehensive error handling:
+
+```typescript
+const errorNode: FlowNode<MyData, MyData> = {
+  id: "error",
+  name: "Error Node",
+  type: NodeType.process,
+  run: async () => {
+    throw new Error("Something went wrong");
+  },
+};
+
+const flow = createFlow({
+  nodes: [inputNode, errorNode],
+  edges: [{ source: "input", target: "error" }],
+});
+
+const result = await flow.run(inputData);
+
+if (result.type === "error") {
+  console.error("Flow failed:", result.error);
+}
+```
+
+### Cancellation
+
+Flows can be cancelled during execution:
+
+```typescript
+const longRunningNode: FlowNode<MyData, MyData> = {
+  id: "long-running",
+  name: "Long Running",
+  type: NodeType.process,
+  run: async ({ runId }) => {
+    // Simulate long-running task
+    await new Promise(resolve => setTimeout(resolve, 5000));
+    return successResult(data);
+  },
+  cancel: (runId) => {
+    // Handle cancellation
+    console.log(`Cancelling node ${runId}`);
+  },
+};
+
+const flow = createFlow({
+  nodes: [inputNode, longRunningNode],
+  edges: [{ source: "input", target: "long-running" }],
+});
+
+// Start the flow
+const promise = flow.run(inputData);
+
+// Cancel after 1 second
+setTimeout(() => {
+  flow.cancel?.(runId);
+}, 1000);
+
+const result = await promise;
+if (result.type === "cancelled") {
+  console.log("Flow was cancelled");
+}
+```
+
+## Features
+
+### Topological Sorting
+
+The engine automatically determines the correct execution order using topological sorting, ensuring nodes are executed in the right sequence based on their dependencies.
+
+### Cycle Detection
+
+The engine detects cycles in the flow graph and returns an error if any are found.
+
+### Type Safety
+
+Full TypeScript support with generic types for input and output data.
+
+### Async Support
+
+All nodes support asynchronous operations and proper error propagation.
+
+### Cancellation Support
+
+Nodes can implement cancellation logic to handle graceful shutdowns.
+
+## Advanced Usage
+
+### Multiple Inputs
+
+For nodes with multiple incoming edges, you can implement custom logic to merge inputs:
+
+```typescript
+const mergeNode: FlowNode<MyData[], MyData> = {
+  id: "merge",
+  name: "Merge",
+  type: NodeType.process,
+  run: async ({ data }) => {
+    // Merge multiple inputs
+    const merged = data.reduce((acc, item) => ({
+      value: acc.value + item.value,
+      processed: acc.processed && item.processed,
+    }));
+    return successResult(merged);
+  },
+};
+```
+
+### Conditional Execution
+
+You can implement conditional logic in nodes:
+
+```typescript
+const conditionalNode: FlowNode<MyData, MyData> = {
+  id: "conditional",
+  name: "Conditional",
+  type: NodeType.process,
+  run: async ({ data }) => {
+    if (data.value > 10) {
+      return successResult({ ...data, processed: true });
+    }
+    else {
+      return successResult({ ...data, processed: false });
+    }
+  },
+};
+```
+
+## Best Practices
+
+1. **Use descriptive node names**: Make your nodes easy to understand and debug
+2. **Handle errors gracefully**: Always return proper error results from nodes
+3. **Keep nodes focused**: Each node should have a single responsibility
+4. **Test your flows**: Create unit tests for individual nodes and integration tests for flows
+5. **Monitor performance**: Use the runId and flowId for logging and monitoring
+
+## API Reference
+
+### Flow
+
+```typescript
+type Flow<Input, Output> = {
+  nodes: FlowNode<Input, Output>[];
+  edges: FlowEdge[];
+  run: (input?: Input) => Promise<FlowResult<Output>>;
+  cancel?: (runId: string) => void;
+};
+```
+
+### FlowNode
+
+```typescript
+type FlowNode<Input, Output> = {
+  id: string;
+  name: string;
+  type: NodeType;
+  run: (args: {
+    data: Input;
+    runId: string;
+    flowId: string;
+  }) => Promise<FlowResult<Output>>;
+  cancel?: (runId: string) => void;
+};
+```
+
+### FlowResult
+
+```typescript
+type FlowResult<Value>
+  = | { type: "success"; value: Value }
+    | { type: "error"; error: string }
+    | { type: "cancelled" };
+```

package/src/flow/edge.test.ts

@@ -0,0 +1,146 @@
+import { describe, expect, it } from "vitest";
+import { createFlowEdge, type FlowEdge } from "./edge";
+
+describe("FlowEdge", () => {
+  describe("createFlowEdge", () => {
+    it("should create a basic edge with source and target", () => {
+      const edge = createFlowEdge({
+        source: "node1",
+        target: "node2",
+      });
+
+      expect(edge).toEqual({
+        source: "node1",
+        target: "node2",
+        sourcePort: undefined,
+        targetPort: undefined,
+      });
+    });
+
+    it("should create edge with source and target ports", () => {
+      const edge = createFlowEdge({
+        source: "node1",
+        target: "node2",
+        sourcePort: "output1",
+        targetPort: "input1",
+      });
+
+      expect(edge).toEqual({
+        source: "node1",
+        target: "node2",
+        sourcePort: "output1",
+        targetPort: "input1",
+      });
+    });
+
+    it("should create edge with only source port", () => {
+      const edge = createFlowEdge({
+        source: "node1",
+        target: "node2",
+        sourcePort: "output1",
+      });
+
+      expect(edge).toEqual({
+        source: "node1",
+        target: "node2",
+        sourcePort: "output1",
+        targetPort: undefined,
+      });
+    });
+
+    it("should create edge with only target port", () => {
+      const edge = createFlowEdge({
+        source: "node1",
+        target: "node2",
+        targetPort: "input1",
+      });
+
+      expect(edge).toEqual({
+        source: "node1",
+        target: "node2",
+        sourcePort: undefined,
+        targetPort: "input1",
+      });
+    });
+
+    it("should handle empty string ports", () => {
+      const edge = createFlowEdge({
+        source: "node1",
+        target: "node2",
+        sourcePort: "",
+        targetPort: "",
+      });
+
+      expect(edge).toEqual({
+        source: "node1",
+        target: "node2",
+        sourcePort: "",
+        targetPort: "",
+      });
+    });
+
+    it("should preserve all provided properties", () => {
+      const edgeData = {
+        source: "input-node",
+        target: "transform-node",
+        sourcePort: "file-output",
+        targetPort: "file-input",
+      };
+
+      const edge = createFlowEdge(edgeData);
+
+      expect(edge.source).toBe(edgeData.source);
+      expect(edge.target).toBe(edgeData.target);
+      expect(edge.sourcePort).toBe(edgeData.sourcePort);
+      expect(edge.targetPort).toBe(edgeData.targetPort);
+    });
+
+    it("should create valid FlowEdge type", () => {
+      const edge: FlowEdge = createFlowEdge({
+        source: "node1",
+        target: "node2",
+      });
+
+      // Type assertion test - if this compiles, the types are compatible
+      expect(edge).toHaveProperty("source");
+      expect(edge).toHaveProperty("target");
+      expect(edge).toHaveProperty("sourcePort");
+      expect(edge).toHaveProperty("targetPort");
+    });
+
+    it("should handle special characters in node names", () => {
+      const edge = createFlowEdge({
+        source: "node-with-dashes_and_underscores.123",
+        target: "another.node@domain.com",
+        sourcePort: "port/with/slashes",
+        targetPort: "port:with:colons",
+      });
+
+      expect(edge.source).toBe("node-with-dashes_and_underscores.123");
+      expect(edge.target).toBe("another.node@domain.com");
+      expect(edge.sourcePort).toBe("port/with/slashes");
+      expect(edge.targetPort).toBe("port:with:colons");
+    });
+
+    it("should create multiple edges with different configurations", () => {
+      const edges = [
+        createFlowEdge({ source: "a", target: "b" }),
+        createFlowEdge({ source: "b", target: "c", sourcePort: "out" }),
+        createFlowEdge({ source: "c", target: "d", targetPort: "in" }),
+        createFlowEdge({
+          source: "d",
+          target: "e",
+          sourcePort: "out",
+          targetPort: "in",
+        }),
+      ];
+
+      expect(edges).toHaveLength(4);
+      expect(edges[0]?.sourcePort).toBeUndefined();
+      expect(edges[1]?.sourcePort).toBe("out");
+      expect(edges[2]?.targetPort).toBe("in");
+      expect(edges[3]?.sourcePort).toBe("out");
+      expect(edges[3]?.targetPort).toBe("in");
+    });
+  });
+});

package/src/flow/edge.ts

@@ -0,0 +1,60 @@
+import type { FlowEdge as EnhancedFlowEdge } from "./types/flow-types";
+
+/**
+ * Represents a connection between two nodes in a flow, defining the data flow direction.
+ *
+ * Edges connect the output of a source node to the input of a target node,
+ * enabling data to flow through the processing pipeline in a directed acyclic graph (DAG).
+ */
+export type FlowEdge = EnhancedFlowEdge;
+
+/**
+ * Creates a flow edge connecting two nodes in a processing pipeline.
+ *
+ * Edges define how data flows between nodes. The data output from the source node
+ * becomes the input for the target node. For nodes with multiple inputs/outputs,
+ * ports can be specified to route data to specific connections.
+ *
+ * @param config - Edge configuration
+ * @param config.source - ID of the source node (data originates here)
+ * @param config.target - ID of the target node (data flows to here)
+ * @param config.sourcePort - Optional port name on the source node for multi-output nodes
+ * @param config.targetPort - Optional port name on the target node for multi-input nodes
+ *
+ * @returns A FlowEdge object representing the connection
+ *
+ * @example
+ * ```typescript
+ * // Simple edge connecting two nodes
+ * const edge = createFlowEdge({
+ *   source: "input-1",
+ *   target: "process-1"
+ * });
+ *
+ * // Edge with ports for multi-input/output nodes
+ * const portEdge = createFlowEdge({
+ *   source: "multiplex-1",
+ *   target: "merge-1",
+ *   sourcePort: "out-a",
+ *   targetPort: "in-1"
+ * });
+ * ```
+ */
+export function createFlowEdge({
+  source,
+  target,
+  sourcePort,
+  targetPort,
+}: {
+  source: string;
+  target: string;
+  sourcePort?: string;
+  targetPort?: string;
+}): FlowEdge {
+  return {
+    source,
+    target,
+    sourcePort,
+    targetPort,
+  };
+}