@uploadista/core 0.0.18 → 0.0.20-beta.1

package/src/flow/nodes/transform-node.ts

@@ -1,7 +1,7 @@
-import { Effect } from "effect";
+import { Effect, Stream } from "effect";
 import type { UploadistaError } from "../../errors";
-import type { UploadFile } from "../../types";
-import { uploadFileSchema } from "../../types";
+import type { StreamingConfig, UploadFile } from "../../types";
+import { DEFAULT_STREAMING_CONFIG, uploadFileSchema } from "../../types";
 import { UploadServer } from "../../upload";
 import { createFlowNode, NodeType } from "../node";
 import { completeNodeExecution, type FileNamingConfig } from "../types";
@@ -12,6 +12,38 @@ import {
 } from "../utils/file-naming";
 import { resolveUploadMetadata } from "../utils/resolve-upload-metadata";
 
+/**
+ * Transform mode for controlling how file data is processed.
+ *
+ * - `buffered`: Always load entire file into memory before transforming (default, backward compatible)
+ * - `streaming`: Process file as a stream of chunks for memory efficiency
+ * - `auto`: Automatically select mode based on file size and DataStore capabilities
+ */
+export type TransformMode = "buffered" | "streaming" | "auto";
+
+/**
+ * Result type for streaming transforms.
+ * Can return just the transformed stream, or include metadata changes.
+ */
+export type StreamingTransformResult =
+  | Stream.Stream<Uint8Array, UploadistaError>
+  | {
+      stream: Stream.Stream<Uint8Array, UploadistaError>;
+      type?: string;
+      fileName?: string;
+      /** Estimated output size in bytes (for progress tracking) */
+      estimatedSize?: number;
+    };
+
+/**
+ * Function type for streaming transforms.
+ * Receives an input stream and file metadata, returns a transformed stream.
+ */
+export type StreamingTransformFn = (
+  stream: Stream.Stream<Uint8Array, UploadistaError>,
+  file: UploadFile,
+) => Effect.Effect<StreamingTransformResult, UploadistaError>;
+
 /**
  * Configuration object for creating a transform node.
  */
@@ -60,14 +92,47 @@ export interface TransformNodeConfig {
    * Overrides flow-level circuit breaker defaults for this node.
    */
   circuitBreaker?: FlowCircuitBreakerConfig;
-  /** Function that transforms file bytes */
-  transform: (
+  /**
+   * Transform mode controlling how file data is processed.
+   * - `buffered`: Always load entire file into memory
+   * - `streaming`: Process file as a stream of chunks
+   * - `auto`: Select mode based on file size and DataStore capabilities (default)
+   *
+   * @default "auto"
+   */
+  mode?: TransformMode;
+  /**
+   * Configuration for streaming mode (file size threshold, chunk size).
+   * Only used when mode is "streaming" or "auto".
+   */
+  streamingConfig?: StreamingConfig;
+  /**
+   * Function that transforms file bytes (buffered mode).
+   * Required unless streamingTransform is provided and mode is "streaming".
+   */
+  transform?: (
     bytes: Uint8Array,
     file: UploadFile,
   ) => Effect.Effect<
     Uint8Array | { bytes: Uint8Array; type?: string; fileName?: string },
     UploadistaError
   >;
+  /**
+   * Function that transforms file as a stream (streaming mode).
+   * For memory-efficient processing of large files.
+   * Used when mode is "streaming" or when "auto" selects streaming.
+   */
+  streamingTransform?: StreamingTransformFn;
+}
+
+/**
+ * Helper to check if a StreamingTransformResult is a stream or an object with metadata.
+ */
+function isStreamResult(
+  result: StreamingTransformResult,
+): result is Stream.Stream<Uint8Array, UploadistaError> {
+  // Check if it has the 'stream' property (object form) vs is a Stream directly
+  return !("stream" in result);
 }
 
 /**
@@ -79,12 +144,17 @@ export interface TransformNodeConfig {
  * This simplifies nodes that just need to transform file bytes without
  * worrying about upload server interactions.
  *
+ * Supports both buffered and streaming modes:
+ * - **Buffered mode**: Loads entire file into memory, transforms, uploads
+ * - **Streaming mode**: Processes file as chunks for memory efficiency with large files
+ * - **Auto mode** (default): Selects mode based on file size and DataStore capabilities
+ *
  * @param config - Configuration object for the transform node
  * @returns An Effect that creates a flow node configured for file transformation
  *
  * @example
  * ```typescript
- * // Create an image resize transform node
+ * // Create a transform node with auto mode (default) - uses streaming for large files
  * const resizeNode = yield* createTransformNode({
  *   id: "resize-image",
  *   name: "Resize Image",
@@ -92,31 +162,31 @@ export interface TransformNodeConfig {
  *   transform: (bytes, file) => {
  *     // Your transformation logic here
  *     return Effect.succeed(transformedBytes);
+ *   },
+ *   streamingTransform: (stream, file) => {
+ *     const transformed = Stream.map(stream, (chunk) => processChunk(chunk));
+ *     return Effect.succeed(transformed);
  *   }
  * });
  *
- * // Create a transform node with keepOutput enabled
- * const processedNode = yield* createTransformNode({
- *   id: "process-image",
- *   name: "Process Image",
- *   description: "Processes images and preserves output",
- *   keepOutput: true, // Output will be included in flow results
- *   transform: (bytes, file) => {
- *     return Effect.succeed(transformedBytes);
- *   }
+ * // Force buffered mode for specific use cases
+ * const bufferedNode = yield* createTransformNode({
+ *   id: "optimize-small",
+ *   name: "Optimize Small Files",
+ *   description: "Optimizes small files with buffered mode",
+ *   mode: "buffered",
+ *   transform: (bytes, file) => Effect.succeed(transformBytes(bytes)),
  * });
  *
- * // Create a transform node that changes file metadata
- * const metadataTransformNode = yield* createTransformNode({
- *   id: "add-metadata",
- *   name: "Add Metadata",
- *   description: "Adds custom metadata to files",
- *   transform: (bytes, file) => {
- *     return Effect.succeed({
- *       bytes,
- *       type: "application/custom",
- *       fileName: `processed-${file.fileName}`
- *     });
+ * // Force streaming mode for memory efficiency
+ * const streamingNode = yield* createTransformNode({
+ *   id: "optimize-large",
+ *   name: "Optimize Large Files",
+ *   description: "Optimizes large files with streaming",
+ *   mode: "streaming",
+ *   streamingTransform: (stream, file) => {
+ *     const transformed = Stream.map(stream, (chunk) => processChunk(chunk));
+ *     return Effect.succeed(transformed);
  *   }
  * });
  * ```
@@ -132,8 +202,34 @@ export function createTransformNode({
   nodeTypeId,
   namingVars,
   circuitBreaker,
+  mode = "auto",
+  streamingConfig,
   transform,
+  streamingTransform,
 }: TransformNodeConfig) {
+  // Validate configuration
+  if (mode === "streaming" && !streamingTransform) {
+    throw new Error(
+      `Transform node "${id}": mode is "streaming" but no streamingTransform function provided`,
+    );
+  }
+  if (mode === "buffered" && !transform) {
+    throw new Error(
+      `Transform node "${id}": mode is "buffered" but no transform function provided`,
+    );
+  }
+  if (mode === "auto" && !transform && !streamingTransform) {
+    throw new Error(
+      `Transform node "${id}": mode is "auto" but neither transform nor streamingTransform provided`,
+    );
+  }
+
+  // Merge streaming config with defaults
+  const effectiveStreamingConfig = {
+    ...DEFAULT_STREAMING_CONFIG,
+    ...streamingConfig,
+  };
+
   return Effect.gen(function* () {
     const uploadServer = yield* UploadServer;
 
@@ -155,6 +251,205 @@ export function createTransformNode({
             nodeId: id,
             jobId,
           };
+
+          // Determine which mode to use
+          const shouldUseStreaming = yield* Effect.gen(function* () {
+            if (mode === "buffered") return false;
+            if (mode === "streaming") return true;
+
+            // Auto mode: check file size and capabilities
+            const fileSize = file.size ?? 0;
+            const threshold = effectiveStreamingConfig.fileSizeThreshold;
+
+            // If file is smaller than threshold, use buffered
+            if (fileSize > 0 && fileSize < threshold) {
+              yield* Effect.logDebug(
+                `File ${file.id} (${fileSize} bytes) below threshold (${threshold}), using buffered mode`,
+              );
+              return false;
+            }
+
+            // Check if we have the required functions
+            if (!streamingTransform) {
+              yield* Effect.logDebug(
+                `No streamingTransform function, using buffered mode`,
+              );
+              return false;
+            }
+
+            // Check DataStore capabilities via UploadServer
+            const capabilities = yield* uploadServer.getCapabilities(
+              storageId,
+              clientId,
+            );
+            if (!capabilities.supportsStreamingRead) {
+              yield* Effect.logDebug(
+                `DataStore doesn't support streaming read, using buffered mode`,
+              );
+              return false;
+            }
+
+            yield* Effect.logDebug(
+              `File ${file.id} qualifies for streaming mode`,
+            );
+            return true;
+          });
+
+          const { type, fileName, metadata, metadataJson } =
+            resolveUploadMetadata(file.metadata);
+
+          if (shouldUseStreaming && streamingTransform) {
+            // STREAMING PATH - True end-to-end streaming
+            yield* Effect.logDebug(`Using streaming transform for ${file.id}`);
+
+            // Get input stream
+            const inputStream = yield* uploadServer.readStream(
+              file.id,
+              clientId,
+              effectiveStreamingConfig,
+            );
+
+            // Transform the stream
+            const transformResult = yield* streamingTransform(
+              inputStream,
+              file,
+            );
+
+            // Extract stream and metadata from result
+            const outputStream = isStreamResult(transformResult)
+              ? transformResult
+              : transformResult.stream;
+            const outputType = isStreamResult(transformResult)
+              ? undefined
+              : transformResult.type;
+            const estimatedSize = isStreamResult(transformResult)
+              ? undefined
+              : transformResult.estimatedSize;
+
+            // Get fileName from transform result or apply naming config
+            let outputFileName = isStreamResult(transformResult)
+              ? undefined
+              : transformResult.fileName;
+
+            if (!outputFileName && naming) {
+              const namingContext = buildNamingContext(
+                file,
+                { flowId, jobId, nodeId: id, nodeType: namingNodeType },
+                namingVars,
+              );
+              outputFileName = applyFileNaming(file, namingContext, naming);
+            }
+
+            // Check if DataStore supports streaming writes
+            const capabilities = yield* uploadServer.getCapabilities(
+              storageId,
+              clientId,
+            );
+
+            let result: UploadFile;
+
+            if (capabilities.supportsStreamingWrite) {
+              // True end-to-end streaming: pipe transform output directly to storage
+              yield* Effect.logDebug(
+                `Using streaming write for ${file.id} - no intermediate buffering`,
+              );
+
+              result = yield* uploadServer.uploadStream(
+                {
+                  storageId,
+                  uploadLengthDeferred: true,
+                  sizeHint: estimatedSize,
+                  type: outputType ?? type,
+                  fileName: outputFileName ?? fileName,
+                  lastModified: 0,
+                  metadata: metadataJson,
+                  flow,
+                },
+                clientId,
+                outputStream,
+              );
+            } else {
+              // Fallback: buffer the output before uploading
+              // This path is for DataStores that don't support streaming writes
+              yield* Effect.logDebug(
+                `Falling back to buffered upload for ${file.id} (streaming write not supported)`,
+              );
+
+              const outputChunks: Uint8Array[] = [];
+              yield* Stream.runForEach(outputStream, (chunk) =>
+                Effect.sync(() => {
+                  outputChunks.push(chunk);
+                }),
+              );
+
+              // Concatenate chunks into a single Uint8Array
+              const totalLength = outputChunks.reduce(
+                (sum, chunk) => sum + chunk.byteLength,
+                0,
+              );
+              const outputBytes = new Uint8Array(totalLength);
+              let offset = 0;
+              for (const chunk of outputChunks) {
+                outputBytes.set(chunk, offset);
+                offset += chunk.byteLength;
+              }
+
+              // Create a ReadableStream for upload
+              const bufferedUploadStream = new ReadableStream({
+                start(controller) {
+                  controller.enqueue(outputBytes);
+                  controller.close();
+                },
+              });
+
+              result = yield* uploadServer.upload(
+                {
+                  storageId,
+                  size: outputBytes.byteLength,
+                  type: outputType ?? type,
+                  fileName: outputFileName ?? fileName,
+                  lastModified: 0,
+                  metadata: metadataJson,
+                  flow,
+                },
+                clientId,
+                bufferedUploadStream,
+              );
+            }
+
+            // Merge updated metadata
+            const updatedMetadata = metadata
+              ? {
+                  ...metadata,
+                  ...(outputType && {
+                    mimeType: outputType,
+                    type: outputType,
+                    "content-type": outputType,
+                  }),
+                  ...(outputFileName && {
+                    fileName: outputFileName,
+                    originalName: outputFileName,
+                    name: outputFileName,
+                    extension:
+                      outputFileName.split(".").pop() || metadata.extension,
+                  }),
+                }
+              : result.metadata;
+
+            return completeNodeExecution(
+              updatedMetadata
+                ? { ...result, metadata: updatedMetadata }
+                : result,
+            );
+          }
+
+          // BUFFERED PATH (default, backward compatible)
+          if (!transform) {
+            throw new Error(
+              `Transform node "${id}": buffered mode selected but no transform function provided`,
+            );
+          }
+
           // Read input bytes from upload server
           const inputBytes = yield* uploadServer.read(file.id, clientId);
 
@@ -201,9 +496,6 @@ export function createTransformNode({
             },
           });
 
-          const { type, fileName, metadata, metadataJson } =
-            resolveUploadMetadata(file.metadata);
-
           // Upload the transformed bytes back to the upload server
           // Use output metadata if provided, otherwise fall back to original
           const result = yield* uploadServer.upload(

package/src/flow/plugins/image-plugin.ts

@@ -1,4 +1,4 @@
-import { Context, type Effect, type Layer } from "effect";
+import { Context, type Effect, type Layer, type Stream } from "effect";
 import type { UploadistaError } from "../../errors";
 import type { OptimizeParams } from "./types/optimize-node";
 import type { ResizeParams } from "./types/resize-node";
@@ -73,6 +73,106 @@ export type ImagePluginShape = {
     input: Uint8Array,
     transformation: Transformation,
   ) => Effect.Effect<Uint8Array, UploadistaError>;
+
+  /**
+   * Optimizes an image using streaming for memory-efficient processing of large files.
+   *
+   * This method processes image data as a stream, which is beneficial for large images
+   * where loading the entire file into memory would be problematic.
+   *
+   * Note: Image processing inherently requires decoding the full image, so memory
+   * savings are primarily from avoiding double-buffering. The streaming interface
+   * allows better pipeline integration with DataStore streaming reads.
+   *
+   * @param input - The input image as an Effect Stream of Uint8Array chunks
+   * @param options - Optimization parameters including quality and format
+   * @returns An Effect that resolves to a Stream of the optimized image bytes
+   * @throws {UploadistaError} When image optimization fails
+   *
+   * @example
+   * ```typescript
+   * const program = Effect.gen(function* () {
+   *   const imagePlugin = yield* ImagePlugin;
+   *   const inputStream = yield* dataStore.readStream(fileId);
+   *   const outputStream = yield* imagePlugin.optimizeStream(inputStream, {
+   *     quality: 80,
+   *     format: "webp"
+   *   });
+   *   return outputStream;
+   * });
+   * ```
+   */
+  optimizeStream?: (
+    input: Stream.Stream<Uint8Array, UploadistaError>,
+    options: OptimizeParams,
+  ) => Effect.Effect<Stream.Stream<Uint8Array, UploadistaError>, UploadistaError>;
+
+  /**
+   * Resizes an image using streaming for memory-efficient processing of large files.
+   *
+   * This method processes image data as a stream. Like other image operations,
+   * the full image must be decoded before processing, but the streaming interface
+   * avoids double-buffering when combined with streaming DataStore reads and writes.
+   *
+   * @param input - The input image as an Effect Stream of Uint8Array chunks
+   * @param options - Resize parameters including width, height, and fit mode
+   * @returns An Effect that resolves to a Stream of the resized image bytes
+   * @throws {UploadistaError} When image resizing fails
+   *
+   * @example
+   * ```typescript
+   * const program = Effect.gen(function* () {
+   *   const imagePlugin = yield* ImagePlugin;
+   *   const inputStream = yield* dataStore.readStream(fileId);
+   *   const outputStream = yield* imagePlugin.resizeStream(inputStream, {
+   *     width: 800,
+   *     height: 600,
+   *     fit: "cover"
+   *   });
+   *   return outputStream;
+   * });
+   * ```
+   */
+  resizeStream?: (
+    input: Stream.Stream<Uint8Array, UploadistaError>,
+    options: ResizeParams,
+  ) => Effect.Effect<Stream.Stream<Uint8Array, UploadistaError>, UploadistaError>;
+
+  /**
+   * Applies a single transformation using streaming for memory-efficient processing.
+   *
+   * This method processes image data as a stream. The streaming interface
+   * allows better pipeline integration with DataStore streaming reads and writes,
+   * reducing peak memory usage for large files.
+   *
+   * @param input - The input image as an Effect Stream of Uint8Array chunks
+   * @param transformation - The transformation to apply
+   * @returns An Effect that resolves to a Stream of the transformed image bytes
+   * @throws {UploadistaError} When transformation fails
+   *
+   * @example
+   * ```typescript
+   * const program = Effect.gen(function* () {
+   *   const imagePlugin = yield* ImagePlugin;
+   *   const inputStream = yield* dataStore.readStream(fileId);
+   *   const outputStream = yield* imagePlugin.transformStream(inputStream, {
+   *     type: 'blur',
+   *     sigma: 5.0
+   *   });
+   *   return outputStream;
+   * });
+   * ```
+   */
+  transformStream?: (
+    input: Stream.Stream<Uint8Array, UploadistaError>,
+    transformation: Transformation,
+  ) => Effect.Effect<Stream.Stream<Uint8Array, UploadistaError>, UploadistaError>;
+
+  /**
+   * Indicates whether this plugin supports streaming operations.
+   * Returns true if streaming methods (optimizeStream, resizeStream, transformStream) are available.
+   */
+  supportsStreaming?: boolean;
 };
 
 /**

package/src/flow/plugins/video-plugin.ts

@@ -1,4 +1,4 @@
-import { Context, type Effect, type Layer } from "effect";
+import { Context, type Effect, type Layer, type Stream } from "effect";
 import type { UploadistaError } from "../../errors";
 import type { DescribeVideoMetadata } from "./types/describe-video-node";
 import type { ExtractFrameVideoParams } from "./types/extract-frame-video-node";
@@ -6,6 +6,26 @@ import type { ResizeVideoParams } from "./types/resize-video-node";
 import type { TranscodeVideoParams } from "./types/transcode-video-node";
 import type { TrimVideoParams } from "./types/trim-video-node";
 
+/**
+ * Input type for streaming video operations.
+ * Accepts either buffered input (Uint8Array) or streaming input (Effect Stream).
+ * Streaming input is only supported for specific formats like MPEG-TS.
+ */
+export type VideoStreamInput =
+  | Uint8Array
+  | Stream.Stream<Uint8Array, UploadistaError>;
+
+/**
+ * Options for streaming video operations.
+ */
+export type VideoStreamOptions = {
+  /**
+   * Hint for input format to help determine if streaming input is possible.
+   * MPEG-TS format supports true streaming input; other formats require buffering.
+   */
+  inputFormat?: string;
+};
+
 /**
  * Shape definition for the Video Plugin interface.
  * Defines the contract that all video processing implementations must follow.
@@ -73,6 +93,109 @@ export type VideoPluginShape = {
   describe: (
     input: Uint8Array,
   ) => Effect.Effect<DescribeVideoMetadata, UploadistaError>;
+
+  /**
+   * Transcodes a video using streaming for memory-efficient processing of large files.
+   *
+   * This method outputs the transcoded video as a stream, reducing peak memory usage.
+   * For input, it accepts either a buffered Uint8Array or a Stream. Streaming input
+   * is only supported for MPEG-TS format; other formats will be buffered internally.
+   *
+   * @param input - The input video as Uint8Array or Stream (MPEG-TS only for streaming)
+   * @param options - Transcode parameters including format, codec, and bitrates
+   * @param streamOptions - Optional streaming configuration including input format hint
+   * @returns An Effect that resolves to a Stream of the transcoded video bytes
+   * @throws {UploadistaError} When video transcoding fails
+   *
+   * @example
+   * ```typescript
+   * const program = Effect.gen(function* () {
+   *   const videoPlugin = yield* VideoPlugin;
+   *   const inputStream = yield* dataStore.readStream(fileId);
+   *   const outputStream = yield* videoPlugin.transcodeStream(inputStream, {
+   *     format: "mp4",
+   *     codec: "h264"
+   *   }, { inputFormat: "video/mp2t" });
+   *   return outputStream;
+   * });
+   * ```
+   */
+  transcodeStream?: (
+    input: VideoStreamInput,
+    options: TranscodeVideoParams,
+    streamOptions?: VideoStreamOptions,
+  ) => Effect.Effect<Stream.Stream<Uint8Array, UploadistaError>, UploadistaError>;
+
+  /**
+   * Resizes a video using streaming for memory-efficient processing of large files.
+   *
+   * This method outputs the resized video as a stream, reducing peak memory usage.
+   * For input, it accepts either a buffered Uint8Array or a Stream. Streaming input
+   * is only supported for MPEG-TS format; other formats will be buffered internally.
+   *
+   * @param input - The input video as Uint8Array or Stream (MPEG-TS only for streaming)
+   * @param options - Resize parameters including width, height, and aspect ratio
+   * @param streamOptions - Optional streaming configuration including input format hint
+   * @returns An Effect that resolves to a Stream of the resized video bytes
+   * @throws {UploadistaError} When video resizing fails
+   *
+   * @example
+   * ```typescript
+   * const program = Effect.gen(function* () {
+   *   const videoPlugin = yield* VideoPlugin;
+   *   const inputStream = yield* dataStore.readStream(fileId);
+   *   const outputStream = yield* videoPlugin.resizeStream(inputStream, {
+   *     width: 1280,
+   *     height: 720,
+   *     aspectRatio: "keep"
+   *   });
+   *   return outputStream;
+   * });
+   * ```
+   */
+  resizeStream?: (
+    input: VideoStreamInput,
+    options: ResizeVideoParams,
+    streamOptions?: VideoStreamOptions,
+  ) => Effect.Effect<Stream.Stream<Uint8Array, UploadistaError>, UploadistaError>;
+
+  /**
+   * Trims a video using streaming for memory-efficient processing of large files.
+   *
+   * This method outputs the trimmed video as a stream, reducing peak memory usage.
+   * For input, it accepts either a buffered Uint8Array or a Stream. Streaming input
+   * is only supported for MPEG-TS format; other formats will be buffered internally.
+   *
+   * @param input - The input video as Uint8Array or Stream (MPEG-TS only for streaming)
+   * @param options - Trim parameters including start time and end time/duration
+   * @param streamOptions - Optional streaming configuration including input format hint
+   * @returns An Effect that resolves to a Stream of the trimmed video bytes
+   * @throws {UploadistaError} When video trimming fails
+   *
+   * @example
+   * ```typescript
+   * const program = Effect.gen(function* () {
+   *   const videoPlugin = yield* VideoPlugin;
+   *   const inputStream = yield* dataStore.readStream(fileId);
+   *   const outputStream = yield* videoPlugin.trimStream(inputStream, {
+   *     startTime: 10,
+   *     endTime: 30
+   *   });
+   *   return outputStream;
+   * });
+   * ```
+   */
+  trimStream?: (
+    input: VideoStreamInput,
+    options: TrimVideoParams,
+    streamOptions?: VideoStreamOptions,
+  ) => Effect.Effect<Stream.Stream<Uint8Array, UploadistaError>, UploadistaError>;
+
+  /**
+   * Indicates whether this plugin supports streaming operations.
+   * Returns true if streaming methods are available and functional.
+   */
+  supportsStreaming?: boolean;
 };
 
 /**