@uploadista/core 0.0.2

package/src/flow/index.ts

@@ -0,0 +1,28 @@
+// Edge types
+export type { FlowEdge } from "./edge";
+export * from "./edge";
+
+export * from "./event";
+export type { Flow, FlowData } from "./flow";
+export * from "./flow";
+// Core flow engine
+export { createFlowWithSchema } from "./flow";
+export * from "./flow-server";
+export * from "./node";
+// Node types and interfaces
+export { createFlowNode, NodeType } from "./node";
+export * from "./nodes";
+// Parallel execution
+export * from "./parallel-scheduler";
+export * from "./plugins/credential-provider";
+export * from "./plugins/image-ai-plugin";
+export * from "./plugins/image-plugin";
+export * from "./plugins/types";
+export * from "./plugins/zip-plugin";
+export * from "./typed-flow";
+export { createFlow } from "./typed-flow";
+export * from "./types/flow-file";
+export * from "./types/flow-job";
+export * from "./types/flow-types";
+export * from "./types/run-args";
+export * from "./utils/resolve-upload-metadata";

package/src/flow/node.ts

@@ -0,0 +1,249 @@
+import { Effect } from "effect";
+import type { z } from "zod";
+import { UploadistaError } from "../errors";
+import type {
+  FlowNode,
+  FlowNodeData,
+  NodeExecutionResult,
+} from "./types/flow-types";
+
+/**
+ * Defines the type of node in a flow, determining its role in the processing pipeline.
+ */
+export enum NodeType {
+  /** Entry point for data into the flow */
+  input = "input",
+  /** Transforms data during flow execution */
+  process = "process",
+  /** Saves data to storage backends */
+  output = "output",
+  /** Routes data based on conditions */
+  conditional = "conditional",
+  /** Splits data to multiple outputs */
+  multiplex = "multiplex",
+  /** Combines multiple inputs into one output */
+  merge = "merge",
+}
+
+/**
+ * Fields that can be evaluated in conditional node conditions.
+ * These fields are typically found in file metadata.
+ */
+export type ConditionField =
+  | "mimeType"
+  | "size"
+  | "width"
+  | "height"
+  | "extension";
+
+/**
+ * Operators available for comparing values in conditional node conditions.
+ */
+export type ConditionOperator =
+  | "equals"
+  | "notEquals"
+  | "greaterThan"
+  | "lessThan"
+  | "contains"
+  | "startsWith";
+
+/**
+ * Value used in conditional node comparisons.
+ * Can be either a string or number depending on the field being evaluated.
+ */
+export type ConditionValue = string | number;
+
+/**
+ * Creates a flow node with automatic input/output validation and retry logic.
+ *
+ * Flow nodes are the building blocks of processing pipelines. Each node:
+ * - Validates its input against a Zod schema
+ * - Executes its processing logic
+ * - Validates its output against a Zod schema
+ * - Can optionally retry on failure with exponential backoff
+ *
+ * @template Input - The expected input type for this node
+ * @template Output - The output type produced by this node
+ *
+ * @param config - Node configuration
+ * @param config.id - Unique identifier for this node in the flow
+ * @param config.name - Human-readable name for the node
+ * @param config.description - Description of what this node does
+ * @param config.type - The type of node (input, process, output, conditional, multiplex, merge)
+ * @param config.inputSchema - Zod schema for validating input data
+ * @param config.outputSchema - Zod schema for validating output data
+ * @param config.run - The processing function to execute for this node
+ * @param config.condition - Optional condition for conditional nodes to determine if they should execute
+ * @param config.multiInput - If true, node receives all inputs as a record instead of a single input
+ * @param config.multiOutput - If true, node can output to multiple targets
+ * @param config.pausable - If true, node can pause execution and wait for additional data
+ * @param config.retry - Optional retry configuration for handling transient failures
+ * @param config.retry.maxRetries - Maximum number of retry attempts (default: 0)
+ * @param config.retry.retryDelay - Base delay in milliseconds between retries (default: 1000)
+ * @param config.retry.exponentialBackoff - Whether to use exponential backoff for retries (default: true)
+ *
+ * @returns An Effect that succeeds with the created FlowNode
+ *
+ * @example
+ * ```typescript
+ * const resizeNode = createFlowNode({
+ *   id: "resize-1",
+ *   name: "Resize Image",
+ *   description: "Resizes images to 800x600",
+ *   type: NodeType.process,
+ *   inputSchema: z.object({
+ *     stream: z.instanceof(Uint8Array),
+ *     metadata: z.object({ width: z.number(), height: z.number() })
+ *   }),
+ *   outputSchema: z.object({
+ *     stream: z.instanceof(Uint8Array),
+ *     metadata: z.object({ width: z.literal(800), height: z.literal(600) })
+ *   }),
+ *   run: ({ data }) => Effect.gen(function* () {
+ *     const resized = yield* resizeImage(data.stream, 800, 600);
+ *     return {
+ *       type: "complete",
+ *       data: { stream: resized, metadata: { width: 800, height: 600 } }
+ *     };
+ *   }),
+ *   retry: {
+ *     maxRetries: 3,
+ *     retryDelay: 1000,
+ *     exponentialBackoff: true
+ *   }
+ * });
+ * ```
+ */
+export function createFlowNode<Input, Output>({
+  id,
+  name,
+  description,
+  type,
+  inputSchema,
+  outputSchema,
+  run,
+  condition,
+  multiInput = false,
+  multiOutput = false,
+  pausable = false,
+  retry,
+}: {
+  id: string;
+  name: string;
+  description: string;
+  type: NodeType;
+  inputSchema: z.ZodSchema<Input>;
+  outputSchema: z.ZodSchema<Output>;
+  run: (args: {
+    data: Input;
+    jobId: string;
+    storageId: string;
+    flowId: string;
+    clientId: string | null;
+  }) => Effect.Effect<NodeExecutionResult<Output>, UploadistaError>;
+  condition?: {
+    field: ConditionField;
+    operator: ConditionOperator;
+    value: ConditionValue;
+  };
+  multiInput?: boolean;
+  multiOutput?: boolean;
+  pausable?: boolean;
+  retry?: {
+    maxRetries?: number;
+    retryDelay?: number;
+    exponentialBackoff?: boolean;
+  };
+}): Effect.Effect<FlowNode<Input, Output, UploadistaError>> {
+  return Effect.succeed({
+    id,
+    name,
+    description,
+    type,
+    inputSchema,
+    outputSchema,
+    pausable,
+    run: ({
+      data,
+      jobId,
+      flowId,
+      storageId,
+      clientId,
+    }: {
+      data: Input;
+      jobId: string;
+      flowId: string;
+      storageId: string;
+      clientId: string | null;
+    }) =>
+      Effect.gen(function* () {
+        // Validate input data against schema
+        const validatedData = yield* Effect.try({
+          try: () => inputSchema.parse(data),
+          catch: (error) => {
+            const errorMessage =
+              error instanceof Error ? error.message : String(error);
+            return UploadistaError.fromCode("FLOW_INPUT_VALIDATION_ERROR", {
+              body: `Node '${name}' (${id}) input validation failed: ${errorMessage}`,
+              cause: error,
+            });
+          },
+        });
+
+        // Run the node logic
+        const result = yield* run({
+          data: validatedData,
+          jobId,
+          storageId,
+          flowId,
+          clientId,
+        });
+
+        // If the node returned waiting state, pass it through
+        if (result.type === "waiting") {
+          return result;
+        }
+
+        // Validate output data against schema for completed results
+        const validatedResult = yield* Effect.try({
+          try: () => outputSchema.parse(result.data),
+          catch: (error) => {
+            const errorMessage =
+              error instanceof Error ? error.message : String(error);
+            return UploadistaError.fromCode("FLOW_OUTPUT_VALIDATION_ERROR", {
+              body: `Node '${name}' (${id}) output validation failed: ${errorMessage}`,
+              cause: error,
+            });
+          },
+        });
+
+        return { type: "complete" as const, data: validatedResult };
+      }),
+    condition,
+    multiInput,
+    multiOutput,
+    retry,
+  });
+}
+
+/**
+ * Extracts serializable node metadata from a FlowNode instance.
+ *
+ * This function is useful for serializing flow configurations or
+ * transmitting node information over the network without including
+ * the executable run function or schemas.
+ *
+ * @param node - The flow node to extract data from
+ * @returns A plain object containing the node's metadata (id, name, description, type)
+ */
+export const getNodeData = (
+  // biome-ignore lint/suspicious/noExplicitAny: maybe type later
+  node: FlowNode<any, any, UploadistaError>,
+): FlowNodeData => {
+  return {
+    id: node.id,
+    name: node.name,
+    description: node.description,
+    type: node.type,
+  };
+};

package/src/flow/nodes/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Re-exports all flow node implementations.
+ * This module provides a centralized way to import all available node types.
+ */
+
+export * from "./input-node";
+export * from "./storage-node";
+export * from "./transform-node";

package/src/flow/nodes/input-node.ts

@@ -0,0 +1,296 @@
+import { Effect } from "effect";
+import { z } from "zod";
+import { UploadistaError } from "../../errors";
+import type { InputFile } from "../../types";
+import { uploadFileSchema } from "../../types";
+import { UploadServer } from "../../upload";
+import { arrayBuffer, fetchFile } from "../../upload/upload-url";
+import { createFlowNode, NodeType } from "../node";
+import { completeNodeExecution, waitingNodeExecution } from "../types";
+import { resolveUploadMetadata } from "../utils/resolve-upload-metadata";
+
+/**
+ * Schema for initializing a streaming upload operation.
+ * Creates a new upload session for chunked file uploads.
+ */
+const initStreamingInputSchema = z.object({
+  /** Operation type identifier */
+  operation: z.literal("init"),
+  /** Storage ID where the file will be stored */
+  storageId: z.string(),
+  /** Optional metadata for the file */
+  metadata: z.record(z.string(), z.any()).optional(),
+});
+
+/**
+ * Schema for finalizing a streaming upload operation.
+ * Completes the upload process after all chunks have been uploaded.
+ */
+const finalizeStreamingInputSchema = z.object({
+  /** Operation type identifier */
+  operation: z.literal("finalize"),
+  /** Upload ID from the init operation */
+  uploadId: z.string(),
+});
+
+/**
+ * Schema for fetching a file from a URL.
+ * Downloads and processes a file from a remote URL.
+ */
+const urlInputSchema = z.object({
+  /** Operation type identifier */
+  operation: z.literal("url"),
+  /** URL to fetch the file from */
+  url: z.string(),
+  /** Optional storage ID where the file will be stored */
+  storageId: z.string().optional(),
+  /** Optional metadata for the file */
+  metadata: z.record(z.string(), z.any()).optional(),
+});
+
+/**
+ * Union schema for all input operations.
+ * Defines the possible input data structures for the input node.
+ */
+export const inputDataSchema = z.union([
+  initStreamingInputSchema,
+  finalizeStreamingInputSchema,
+  urlInputSchema,
+]);
+
+/**
+ * Type representing the input data for the input node.
+ * Can be one of three operation types: init, finalize, or url.
+ */
+export type InputData = z.infer<typeof inputDataSchema>;
+
+/**
+ * Schema for input node filtering parameters.
+ * Defines validation rules for incoming files.
+ */
+export const inputNodeParamsSchema = z.object({
+  /** Array of allowed MIME types (supports wildcards like "image/*") */
+  allowedMimeTypes: z.array(z.string()).optional(),
+  /** Minimum file size in bytes */
+  minSize: z.number().positive().optional(),
+  /** Maximum file size in bytes */
+  maxSize: z.number().positive().optional(),
+});
+
+/**
+ * Parameters for configuring input node validation.
+ * Controls which files are accepted based on type and size constraints.
+ */
+export type InputNodeParams = z.infer<typeof inputNodeParamsSchema>;
+
+/**
+ * Helper function to validate file against input parameters.
+ * Performs MIME type and size validation based on the provided parameters.
+ *
+ * @param file - File information to validate
+ * @param params - Validation parameters
+ * @returns An Effect that succeeds if validation passes or fails with validation error
+ */
+function validateFile(
+  file: { type: string; size: number },
+  params?: InputNodeParams
+): Effect.Effect<void, UploadistaError> {
+  return Effect.gen(function* () {
+    if (!params) return;
+
+    // Check MIME type
+    if (params.allowedMimeTypes && params.allowedMimeTypes.length > 0) {
+      const isAllowed = params.allowedMimeTypes.some((allowed) => {
+        // Support wildcard patterns like "image/*"
+        if (allowed.endsWith("/*")) {
+          const prefix = allowed.slice(0, -2);
+          return file.type.startsWith(prefix);
+        }
+        return file.type === allowed;
+      });
+
+      if (!isAllowed) {
+        throw yield* UploadistaError.fromCode("VALIDATION_ERROR", {
+          cause: new Error(
+            `File type "${
+              file.type
+            }" is not allowed. Allowed types: ${params.allowedMimeTypes.join(
+              ", "
+            )}`
+          ),
+        }).toEffect();
+      }
+    }
+
+    // Check minimum size
+    if (params.minSize !== undefined && file.size < params.minSize) {
+      throw yield* UploadistaError.fromCode("VALIDATION_ERROR", {
+        cause: new Error(
+          `File size (${file.size} bytes) is below minimum (${params.minSize} bytes)`
+        ),
+      }).toEffect();
+    }
+
+    // Check maximum size
+    if (params.maxSize !== undefined && file.size > params.maxSize) {
+      throw yield* UploadistaError.fromCode("VALIDATION_ERROR", {
+        cause: new Error(
+          `File size (${file.size} bytes) exceeds maximum (${params.maxSize} bytes)`
+        ),
+      }).toEffect();
+    }
+  });
+}
+
+/**
+ * Creates an input node for handling file input through multiple methods.
+ *
+ * The input node supports three operation types:
+ * - `init`: Initialize a streaming upload session
+ * - `finalize`: Complete a streaming upload after all chunks are uploaded
+ * - `url`: Fetch a file directly from a URL
+ *
+ * @param id - Unique identifier for the node
+ * @param params - Optional validation parameters for filtering incoming files
+ * @returns An Effect that creates a flow node configured for file input
+ *
+ * @example
+ * ```typescript
+ * // Create input node with validation
+ * const inputNode = yield* createInputNode("file-input", {
+ *   allowedMimeTypes: ["image/*", "application/pdf"],
+ *   maxSize: 10 * 1024 * 1024, // 10MB
+ * });
+ *
+ * // Create input node without validation
+ * const openInputNode = yield* createInputNode("open-input");
+ * ```
+ */
+export function createInputNode(id: string, params?: InputNodeParams) {
+  return Effect.gen(function* () {
+    const uploadServer = yield* UploadServer;
+    return yield* createFlowNode({
+      id,
+      name: "Input",
+      description:
+        "Handles file input through multiple methods - streaming upload (init/finalize) or direct URL fetch",
+      type: NodeType.input,
+      inputSchema: inputDataSchema,
+      outputSchema: uploadFileSchema,
+      run: ({ data, flowId, jobId, clientId }) => {
+        return Effect.gen(function* () {
+          switch (data.operation) {
+            case "init": {
+              // Create upload using upload server - it handles all state management
+              const inputFile: InputFile = {
+                storageId: data.storageId,
+                size: data.metadata?.size || 0,
+                type: data.metadata?.mimeType || "application/octet-stream",
+                fileName: data.metadata?.originalName,
+                lastModified: data.metadata?.size ? Date.now() : undefined,
+                metadata: data.metadata
+                  ? JSON.stringify(data.metadata)
+                  : undefined,
+                flow: {
+                  flowId,
+                  nodeId: id,
+                  jobId,
+                },
+              };
+
+              const uploadFile = yield* uploadServer.createUpload(
+                inputFile,
+                clientId
+              );
+
+              // Return waiting state with the upload file
+              // Client will upload chunks directly to the upload API
+              return waitingNodeExecution(uploadFile);
+            }
+
+            case "finalize": {
+              // Get final upload file from upload server's KV store
+              const finalUploadFile = yield* uploadServer.getUpload(
+                data.uploadId
+              );
+
+              // Extract type and size from metadata for validation
+              const { type } = resolveUploadMetadata(finalUploadFile.metadata);
+              const size = finalUploadFile.size || 0;
+
+              // Validate file against params
+              yield* validateFile({ type, size }, params);
+
+              // Complete the node execution with the final upload file
+              // Flow can now continue to next nodes (e.g., save to storage, optimize)
+              return completeNodeExecution(finalUploadFile);
+            }
+
+            case "url": {
+              // Fetch file from URL directly
+              const response = yield* fetchFile(data.url);
+              const buffer = yield* arrayBuffer(response);
+
+              // Extract metadata from response or use provided metadata
+              const mimeType =
+                data.metadata?.mimeType ||
+                response.headers.get("content-type") ||
+                "application/octet-stream";
+              const size =
+                data.metadata?.size ||
+                Number(response.headers.get("content-length") || 0);
+              const fileName =
+                data.metadata?.originalName ||
+                data.url.split("/").pop() ||
+                "file";
+
+              // Validate file against params
+              yield* validateFile({ type: mimeType, size }, params);
+
+              // Create a readable stream from the buffer
+              const stream = new ReadableStream({
+                start(controller) {
+                  controller.enqueue(new Uint8Array(buffer));
+                  controller.close();
+                },
+              });
+
+              // Use upload server to create and store the file
+              const inputFile: InputFile = {
+                storageId: data.storageId || "buffer",
+                size,
+                type: mimeType,
+                fileName,
+                lastModified: Date.now(),
+                metadata: data.metadata
+                  ? JSON.stringify(data.metadata)
+                  : undefined,
+              };
+
+              const uploadFile = yield* uploadServer.upload(
+                inputFile,
+                clientId,
+                stream
+              );
+
+              // Complete the node execution with the upload file
+              return completeNodeExecution({
+                ...uploadFile,
+                flow: {
+                  flowId,
+                  nodeId: id,
+                  jobId,
+                },
+              });
+            }
+
+            default:
+              throw yield* UploadistaError.fromCode("VALIDATION_ERROR", {
+                cause: new Error("Invalid operation"),
+              }).toEffect();
+          }
+        });
+      },
+    });
+  });
+}