@uploadista/core 0.0.2

package/src/flow/nodes/storage-node.ts

@@ -0,0 +1,128 @@
+import { Effect } from "effect";
+import { z } from "zod";
+import { UploadistaError } from "../../errors";
+import { type UploadFile, uploadFileSchema } from "../../types";
+import { UploadServer } from "../../upload";
+import { createFlowNode, NodeType } from "../node";
+import { completeNodeExecution } from "../types";
+import { resolveUploadMetadata } from "../utils/resolve-upload-metadata";
+
+/**
+ * Schema for storage node parameters.
+ * Currently empty but can be extended for storage-specific configuration.
+ */
+export const storageParamsSchema = z.object({});
+
+/**
+ * Parameters for the storage node.
+ * Currently no parameters are required, but the schema is available for future extensions.
+ */
+export type StorageParams = z.infer<typeof storageParamsSchema>;
+
+/**
+ * Creates a storage node for storing files in the specified storage.
+ *
+ * The storage node handles the process of:
+ * 1. Reading the input file from the upload server
+ * 2. Checking if the file is already in the target storage
+ * 3. If not, transferring the file to the target storage
+ * 4. Applying optional post-processing
+ * 5. Returning the final stored file
+ *
+ * @param id - Unique identifier for the node
+ * @param postProcessFile - Optional function to process the file after storage
+ * @returns An Effect that creates a flow node configured for file storage
+ *
+ * @example
+ * ```typescript
+ * // Create basic storage node
+ * const storageNode = yield* createStorageNode("store-file");
+ *
+ * // Create storage node with post-processing
+ * const storageWithProcessing = yield* createStorageNode("store-and-process", (file) => {
+ *   return Effect.succeed({
+ *     ...file,
+ *     metadata: { ...file.metadata, processed: true }
+ *   });
+ * });
+ * ```
+ */
+export function createStorageNode(
+  id: string,
+  postProcessFile: (file: UploadFile) => Effect.Effect<UploadFile> = (file) =>
+    Effect.succeed(file)
+) {
+  return Effect.gen(function* () {
+    const uploadServer = yield* UploadServer;
+    return yield* createFlowNode({
+      id,
+      name: "Storage",
+      description: "Stores a file in the storage",
+      type: NodeType.output,
+      inputSchema: uploadFileSchema,
+      outputSchema: uploadFileSchema,
+      run: ({ data: file, storageId, flowId, jobId, clientId }) => {
+        return Effect.gen(function* () {
+          const { type, fileName, metadata, metadataJson } =
+            resolveUploadMetadata(file.metadata);
+          const flow = {
+            flowId,
+            nodeId: id,
+            jobId,
+          };
+          const normalizedFile = metadata ? { ...file, metadata } : file;
+
+          const upload = yield* uploadServer.getUpload(file.id);
+          if (!upload.id) {
+            return yield* Effect.fail(
+              UploadistaError.fromCode(
+                "FILE_READ_ERROR",
+                new Error("Upload Key is undefined")
+              )
+            );
+          }
+          // If the upload is already in the correct storage, return the file, just update the flow
+          if (upload.storage.id === storageId) {
+            return completeNodeExecution(
+              yield* postProcessFile({ ...normalizedFile, flow })
+            );
+          }
+
+          const inputBytes = yield* uploadServer.read(file.id, clientId);
+          const stream = new ReadableStream({
+            start(controller) {
+              controller.enqueue(inputBytes);
+              controller.close();
+            },
+          });
+
+          const uploadResult = yield* uploadServer.upload(
+            {
+              storageId,
+              size: inputBytes.byteLength,
+              type,
+              fileName,
+              lastModified: 0,
+              metadata: metadataJson,
+              flow,
+            },
+            clientId,
+            stream
+          );
+
+          const resolvedUploadResult = resolveUploadMetadata(
+            uploadResult.metadata
+          );
+
+          const postProcessed = yield* postProcessFile(
+            resolvedUploadResult.metadata
+              ? { ...uploadResult, metadata: resolvedUploadResult.metadata }
+              : uploadResult
+          );
+
+          return completeNodeExecution(postProcessed);
+        });
+      },
+    });
+  });
+}

package/src/flow/nodes/transform-node.ts

@@ -0,0 +1,154 @@
+import { Effect } from "effect";
+import type { UploadistaError } from "../../errors";
+import type { UploadFile } from "../../types";
+import { uploadFileSchema } from "../../types";
+import { UploadServer } from "../../upload";
+import { createFlowNode, NodeType } from "../node";
+import { completeNodeExecution } from "../types";
+import { resolveUploadMetadata } from "../utils/resolve-upload-metadata";
+
+/**
+ * Configuration object for creating a transform node.
+ */
+export interface TransformNodeConfig {
+  /** Unique identifier for the node */
+  id: string;
+  /** Human-readable name for the node */
+  name: string;
+  /** Description of what the node does */
+  description: string;
+  /** Function that transforms file bytes */
+  transform: (
+    bytes: Uint8Array,
+    file: UploadFile
+  ) => Effect.Effect<
+    Uint8Array | { bytes: Uint8Array; type?: string; fileName?: string },
+    UploadistaError
+  >;
+}
+
+/**
+ * Creates a transform node that handles the common pattern of:
+ * 1. Reading bytes from an UploadFile
+ * 2. Transforming the bytes
+ * 3. Uploading the result as a new UploadFile
+ *
+ * This simplifies nodes that just need to transform file bytes without
+ * worrying about upload server interactions.
+ *
+ * @param config - Configuration object for the transform node
+ * @returns An Effect that creates a flow node configured for file transformation
+ *
+ * @example
+ * ```typescript
+ * // Create an image resize transform node
+ * const resizeNode = yield* createTransformNode({
+ *   id: "resize-image",
+ *   name: "Resize Image",
+ *   description: "Resizes images to specified dimensions",
+ *   transform: (bytes, file) => {
+ *     // Your transformation logic here
+ *     return Effect.succeed(transformedBytes);
+ *   }
+ * });
+ *
+ * // Create a transform node that changes file metadata
+ * const metadataTransformNode = yield* createTransformNode({
+ *   id: "add-metadata",
+ *   name: "Add Metadata",
+ *   description: "Adds custom metadata to files",
+ *   transform: (bytes, file) => {
+ *     return Effect.succeed({
+ *       bytes,
+ *       type: "application/custom",
+ *       fileName: `processed-${file.fileName}`
+ *     });
+ *   }
+ * });
+ * ```
+ */
+export function createTransformNode({
+  id,
+  name,
+  description,
+  transform,
+}: TransformNodeConfig) {
+  return Effect.gen(function* () {
+    const uploadServer = yield* UploadServer;
+
+    return yield* createFlowNode({
+      id,
+      name,
+      description,
+      type: NodeType.process,
+      inputSchema: uploadFileSchema,
+      outputSchema: uploadFileSchema,
+      run: ({ data: file, storageId, flowId, jobId, clientId }) => {
+        return Effect.gen(function* () {
+          const flow = {
+            flowId,
+            nodeId: id,
+            jobId,
+          };
+          // Read input bytes from upload server
+          const inputBytes = yield* uploadServer.read(file.id, clientId);
+
+          // Transform the bytes using the provided function
+          const transformResult = yield* transform(inputBytes, file);
+
+          // Handle both simple Uint8Array and object with metadata
+          const outputBytes =
+            transformResult instanceof Uint8Array
+              ? transformResult
+              : transformResult.bytes;
+
+          const outputType =
+            transformResult instanceof Uint8Array
+              ? undefined
+              : transformResult.type;
+
+          const outputFileName =
+            transformResult instanceof Uint8Array
+              ? undefined
+              : transformResult.fileName;
+
+          // Create a stream from the output bytes
+          const stream = new ReadableStream({
+            start(controller) {
+              controller.enqueue(outputBytes);
+              controller.close();
+            },
+          });
+
+          const { type, fileName, metadata, metadataJson } =
+            resolveUploadMetadata(file.metadata);
+
+          // Upload the transformed bytes back to the upload server
+          // Use output metadata if provided, otherwise fall back to original
+          const result = yield* uploadServer.upload(
+            {
+              storageId,
+              size: outputBytes.byteLength,
+              type: outputType ?? type,
+              fileName: outputFileName ?? fileName,
+              lastModified: 0,
+              metadata: metadataJson,
+              flow,
+            },
+            clientId,
+            stream
+          );
+
+          return completeNodeExecution(
+            metadata
+              ? {
+                  ...result,
+                  metadata,
+                }
+              : result
+          );
+        });
+      },
+    });
+  });
+}

package/src/flow/parallel-scheduler.ts

@@ -0,0 +1,259 @@
+/**
+ * Parallel execution scheduler for flow nodes.
+ *
+ * The ParallelScheduler analyzes flow dependencies and groups nodes into execution
+ * levels where nodes at the same level can run in parallel. It manages concurrency
+ * using Effect's built-in concurrency control to prevent resource exhaustion.
+ *
+ * @module flow/parallel-scheduler
+ * @see {@link ParallelScheduler} for the main scheduler class
+ *
+ * @remarks
+ * This scheduler groups nodes by execution level (respecting dependencies) and executes
+ * each level in parallel with controlled concurrency. Levels are executed sequentially
+ * to ensure dependencies are satisfied before dependent nodes execute.
+ *
+ * @example
+ * ```typescript
+ * const scheduler = new ParallelScheduler({ maxConcurrency: 4 });
+ *
+ * // Group nodes by execution level
+ * const levels = scheduler.groupNodesByExecutionLevel(nodes, edges);
+ *
+ * // Execute nodes in a level with Effect
+ * const results = yield* scheduler.executeNodesInParallel([
+ *   () => executeNode("node1"),
+ *   () => executeNode("node2"),
+ *   () => executeNode("node3")
+ * ]);
+ * ```
+ */
+
+import { Effect } from "effect";
+import type { FlowNode } from "./types/flow-types";
+
+/**
+ * Represents a level in the execution hierarchy where all nodes can run in parallel.
+ *
+ * @property level - The execution level (0 = first to execute, higher = later)
+ * @property nodes - Array of node IDs that can execute in parallel at this level
+ *
+ * @example
+ * ```
+ * Level 0: [input_node]           (no dependencies)
+ * Level 1: [resize, optimize]     (all depend on level 0)
+ * Level 2: [storage]              (depends on level 1)
+ * ```
+ */
+export interface ExecutionLevel {
+  level: number;
+  nodes: string[];
+}
+
+/**
+ * Configuration options for the ParallelScheduler.
+ *
+ * @property maxConcurrency - Maximum number of nodes to execute in parallel (default: 4)
+ *                           Controls how many nodes run simultaneously within a level
+ *
+ * @example
+ * ```typescript
+ * const scheduler = new ParallelScheduler({ maxConcurrency: 8 });
+ * ```
+ */
+export interface ParallelSchedulerConfig {
+  maxConcurrency?: number;
+}
+
+/**
+ * Scheduler for executing flow nodes in parallel while respecting dependencies.
+ *
+ * The scheduler performs topological sorting to identify nodes that can run
+ * concurrently, groups them into execution levels, and provides methods to
+ * execute them with controlled concurrency using Effect.
+ *
+ * Key responsibilities:
+ * - Analyze flow dependencies and detect cycles
+ * - Group nodes into parallel execution levels
+ * - Execute levels in parallel with concurrency limits
+ * - Provide utilities to check parallel execution feasibility
+ */
+export class ParallelScheduler {
+  private maxConcurrency: number;
+
+  /**
+   * Creates a new ParallelScheduler instance.
+   *
+   * @param config - Configuration for the scheduler
+   * @example
+   * ```typescript
+   * const scheduler = new ParallelScheduler({ maxConcurrency: 4 });
+   * ```
+   */
+  constructor(config: ParallelSchedulerConfig = {}) {
+    this.maxConcurrency = config.maxConcurrency ?? 4;
+  }
+
+  /**
+   * Groups nodes into execution levels where nodes in the same level can run in parallel.
+   *
+   * Uses Kahn's algorithm to perform topological sorting with level identification.
+   * Nodes are grouped by their distance from source nodes (input nodes with no dependencies).
+   *
+   * @param nodes - Array of flow nodes to analyze
+   * @param edges - Array of edges defining dependencies between nodes
+   * @returns Array of execution levels, ordered from 0 (no dependencies) onwards
+   * @throws Error if a cycle is detected in the flow graph
+   *
+   * @example
+   * ```typescript
+   * const levels = scheduler.groupNodesByExecutionLevel(nodes, edges);
+   * // levels = [
+   * //   { level: 0, nodes: ['input_1'] },
+   * //   { level: 1, nodes: ['resize_1', 'optimize_1'] },
+   * //   { level: 2, nodes: ['output_1'] }
+   * // ]
+   * ```
+   */
+  groupNodesByExecutionLevel(
+    nodes: FlowNode<unknown, unknown>[],
+    edges: Array<{ source: string; target: string }>,
+  ): ExecutionLevel[] {
+    // Build dependency graph
+    const graph: Record<string, string[]> = {};
+    const inDegree: Record<string, number> = {};
+
+    // Initialize graph structure
+    nodes.forEach((node) => {
+      graph[node.id] = [];
+      inDegree[node.id] = 0;
+    });
+
+    // Build edges and calculate in-degrees
+    edges.forEach((edge) => {
+      graph[edge.source]?.push(edge.target);
+      inDegree[edge.target] = (inDegree[edge.target] || 0) + 1;
+    });
+
+    const levels: ExecutionLevel[] = [];
+    const processedNodes = new Set<string>();
+    let levelIndex = 0;
+
+    // Use Kahn's algorithm to group nodes by level
+    while (processedNodes.size < nodes.length) {
+      // Find all nodes with zero in-degree that haven't been processed
+      const currentLevelNodes = Object.keys(inDegree).filter(
+        (nodeId) => inDegree[nodeId] === 0 && !processedNodes.has(nodeId),
+      );
+
+      if (currentLevelNodes.length === 0) {
+        throw new Error(
+          "Cycle detected in flow graph - cannot execute in parallel",
+        );
+      }
+
+      levels.push({
+        level: levelIndex++,
+        nodes: currentLevelNodes,
+      });
+
+      // Remove current level nodes and update in-degrees for dependent nodes
+      currentLevelNodes.forEach((nodeId) => {
+        processedNodes.add(nodeId);
+        delete inDegree[nodeId];
+
+        // Decrease in-degree for all nodes that depend on this node
+        graph[nodeId]?.forEach((dependentId) => {
+          if (inDegree[dependentId] !== undefined) {
+            inDegree[dependentId]--;
+          }
+        });
+      });
+    }
+
+    return levels;
+  }
+
+  /**
+   * Executes a batch of Effect-based node executors in parallel with concurrency control.
+   *
+   * All executors are run in parallel, but the number of concurrent executions is limited
+   * by maxConcurrency. This prevents resource exhaustion while maximizing parallelism.
+   *
+   * @template T - The return type of each executor
+   * @template E - The error type of the Effects
+   * @template R - The requirements type of the Effects
+   *
+   * @param nodeExecutors - Array of Effect-returning functions to execute in parallel
+   * @returns Effect that resolves to array of results in the same order as input
+   *
+   * @example
+   * ```typescript
+   * const results = yield* scheduler.executeNodesInParallel([
+   *   () => executeNode("node1"),
+   *   () => executeNode("node2"),
+   *   () => executeNode("node3")
+   * ]);
+   * // results will be in order: [result1, result2, result3]
+   * ```
+   */
+  executeNodesInParallel<T, E, R>(
+    nodeExecutors: Array<() => Effect.Effect<T, E, R>>,
+  ): Effect.Effect<T[], E, R> {
+    return Effect.all(
+      nodeExecutors.map((executor) => executor()),
+      {
+        concurrency: this.maxConcurrency,
+      },
+    );
+  }
+
+  /**
+   * Determines if a set of nodes can be safely executed in parallel.
+   *
+   * Nodes can execute in parallel if all their dependencies have been completed.
+   * This is typically called to verify that nodes in an execution level are ready
+   * to run given the current node results.
+   *
+   * @param nodeIds - Array of node IDs to check
+   * @param nodeResults - Map of completed node IDs to their results
+   * @param reverseGraph - Dependency graph mapping node IDs to their incoming dependencies
+   * @returns true if all dependencies for all nodes are in nodeResults, false otherwise
+   *
+   * @example
+   * ```typescript
+   * const canRun = scheduler.canExecuteInParallel(
+   *   ['resize_1', 'optimize_1'],
+   *   nodeResults,
+   *   reverseGraph
+   * );
+   * ```
+   */
+  canExecuteInParallel(
+    nodeIds: string[],
+    nodeResults: Map<string, unknown>,
+    reverseGraph: Record<string, string[]>,
+  ): boolean {
+    return nodeIds.every((nodeId) => {
+      const dependencies = reverseGraph[nodeId] || [];
+      return dependencies.every((depId) => nodeResults.has(depId));
+    });
+  }
+
+  /**
+   * Gets execution statistics for monitoring and debugging.
+   *
+   * @returns Object containing current scheduler configuration
+   *
+   * @example
+   * ```typescript
+   * const stats = scheduler.getStats();
+   * console.log(`Max concurrency: ${stats.maxConcurrency}`);
+   * ```
+   */
+  getStats() {
+    return {
+      maxConcurrency: this.maxConcurrency,
+    };
+  }
+}

package/src/flow/plugins/credential-provider.ts

@@ -0,0 +1,48 @@
+import { Context, type Effect } from "effect";
+import type { UploadistaError } from "@/errors";
+
+/**
+ * Shape definition for the Credential Provider interface.
+ * Defines the contract for retrieving credentials for various services.
+ */
+export interface CredentialProviderShape {
+  /**
+   * Retrieves credentials for a specific service and client.
+   *
+   * @param params - Parameters for credential retrieval
+   * @param params.clientId - Unique identifier for the client, or null if not available
+   * @param params.serviceType - Optional service type to get specific credentials for
+   * @returns An Effect that resolves to a record of credential key-value pairs
+   * @throws {UploadistaError} When credential retrieval fails
+   */
+  getCredential: (params: {
+    clientId: string | null;
+    serviceType?: string;
+  }) => Effect.Effect<Record<string, unknown>, UploadistaError>;
+}
+
+/**
+ * Context tag for the Credential Provider.
+ *
+ * This tag provides a type-safe way to access credential functionality
+ * throughout the application using Effect's dependency injection system.
+ *
+ * @example
+ * ```typescript
+ * import { CredentialProvider } from "@uploadista/core/flow/plugins";
+ *
+ * // In your flow node
+ * const program = Effect.gen(function* () {
+ *   const credentialProvider = yield* CredentialProvider;
+ *   const credentials = yield* credentialProvider.getCredential({
+ *     clientId: "user123",
+ *     serviceType: "replicate"
+ *   });
+ *   return credentials;
+ * });
+ * ```
+ */
+export class CredentialProvider extends Context.Tag("CredentialProvider")<
+  CredentialProvider,
+  CredentialProviderShape
+>() {}

package/src/flow/plugins/image-ai-plugin.ts

@@ -0,0 +1,66 @@
+import type { UploadistaError } from "@uploadista/core/errors";
+import { Context, type Effect } from "effect";
+
+/**
+ * Context information for AI image processing operations.
+ * Contains client identification for tracking and billing purposes.
+ */
+export type ImageAiContext = {
+  /** Unique identifier for the client making the request, or null if not available */
+  clientId: string | null;
+};
+
+/**
+ * Shape definition for the Image AI Plugin interface.
+ * Defines the contract that all image AI implementations must follow.
+ */
+export type ImageAiPluginShape = {
+  /**
+   * Removes the background from an image using AI processing.
+   *
+   * @param inputUrl - The URL of the input image to process
+   * @param context - Context information including client ID for tracking
+   * @returns An Effect that resolves to an object containing the output image URL
+   * @throws {UploadistaError} When the background removal fails
+   */
+  removeBackground: (
+    inputUrl: string,
+    context: ImageAiContext
+  ) => Effect.Effect<{ outputUrl: string }, UploadistaError>;
+
+  /**
+   * Generates a textual description of an image using AI analysis.
+   *
+   * @param inputUrl - The URL of the input image to analyze
+   * @param context - Context information including client ID for tracking
+   * @returns An Effect that resolves to an object containing the image description
+   * @throws {UploadistaError} When the image analysis fails
+   */
+  describeImage: (
+    inputUrl: string,
+    context: ImageAiContext
+  ) => Effect.Effect<{ description: string }, UploadistaError>;
+};
+
+/**
+ * Context tag for the Image AI Plugin.
+ *
+ * This tag provides a type-safe way to access image AI functionality
+ * throughout the application using Effect's dependency injection system.
+ *
+ * @example
+ * ```typescript
+ * import { ImageAiPlugin } from "@uploadista/core/flow/plugins";
+ *
+ * // In your flow node
+ * const program = Effect.gen(function* () {
+ *   const imageAi = yield* ImageAiPlugin;
+ *   const result = yield* imageAi.removeBackground(imageUrl, { clientId: "user123" });
+ *   return result.outputUrl;
+ * });
+ * ```
+ */
+export class ImageAiPlugin extends Context.Tag("ImageAiPlugin")<
+  ImageAiPlugin,
+  ImageAiPluginShape
+>() {}