@uploadista/flow-images-nodes 0.0.18 → 0.0.20-beta.1

package/src/optimize-node.ts

@@ -1,3 +1,4 @@
+import { UploadistaError } from "@uploadista/core";
 import {
   applyFileNaming,
   buildNamingContext,
@@ -7,6 +8,8 @@ import {
   ImagePlugin,
   type OptimizeParams,
   STORAGE_OUTPUT_TYPE_ID,
+  type StreamingConfig,
+  type TransformMode,
 } from "@uploadista/core/flow";
 import { Effect } from "effect";
 
@@ -29,28 +32,111 @@ const formatToExtension: Record<OptimizeParams["format"], string> = {
 /**
  * Creates an optimize node that optimizes images for web delivery.
  *
+ * Supports both buffered and streaming modes for memory-efficient processing
+ * of large images. In streaming mode, the image is read and processed
+ * incrementally, reducing peak memory usage.
+ *
  * @param id - Unique node identifier
  * @param params - Optimize parameters (quality, format)
  * @param options - Optional configuration
  * @param options.keepOutput - Whether to keep output in flow results
  * @param options.naming - File naming configuration (auto suffix: `${format}`)
+ * @param options.mode - Transform mode: "buffered" (default), "streaming", or "auto"
+ * @param options.streamingConfig - Streaming configuration (file size threshold, chunk size)
  *
  * @example
  * ```typescript
- * // With auto-naming: "photo.jpg" -> "photo-webp.webp"
+ * // Buffered mode (default) - "photo.jpg" -> "photo-webp.webp"
  * const optimize = yield* createOptimizeNode("opt-1", { quality: 80, format: "webp" }, {
  *   naming: { mode: "auto" }
  * });
+ *
+ * // Streaming mode for large files
+ * const optimizeStreaming = yield* createOptimizeNode("opt-2", { quality: 80, format: "webp" }, {
+ *   mode: "streaming",
+ *   naming: { mode: "auto" }
+ * });
+ *
+ * // Auto mode - uses streaming for files > 1MB
+ * const optimizeAuto = yield* createOptimizeNode("opt-3", { quality: 80, format: "webp" }, {
+ *   mode: "auto",
+ *   streamingConfig: { fileSizeThreshold: 1_048_576 },
+ *   naming: { mode: "auto" }
+ * });
  * ```
  */
 export function createOptimizeNode(
   id: string,
   { quality, format }: OptimizeParams,
-  options?: { keepOutput?: boolean; naming?: FileNamingConfig },
+  options?: {
+    keepOutput?: boolean;
+    naming?: FileNamingConfig;
+    mode?: TransformMode;
+    streamingConfig?: StreamingConfig;
+  },
 ) {
   return Effect.gen(function* () {
     const imageService = yield* ImagePlugin;
 
+    // Determine if streaming is available and requested
+    const supportsStreaming = imageService.supportsStreaming ?? false;
+    const requestedMode = options?.mode ?? "buffered";
+
+    // If streaming requested but not supported, fall back to buffered
+    const effectiveMode: TransformMode =
+      requestedMode === "buffered"
+        ? "buffered"
+        : supportsStreaming
+          ? requestedMode
+          : "buffered";
+
+    // Helper to build output metadata from optimized result
+    const buildOutputMetadata = (file: {
+      metadata?: Record<string, unknown>;
+      flow?: { flowId?: string; jobId?: string };
+    }) => {
+      const newType = formatToMimeType[format];
+      const newExtension = formatToExtension[format];
+
+      // Get original fileName
+      const fileName = file.metadata?.fileName;
+      let newFileName: string | undefined;
+
+      if (fileName && typeof fileName === "string") {
+        // Apply naming if configured
+        if (options?.naming) {
+          const namingConfig: FileNamingConfig = {
+            ...options.naming,
+            autoSuffix:
+              options.naming.autoSuffix ?? ((ctx) => ctx.format ?? format),
+          };
+          const namingContext = buildNamingContext(
+            file as Parameters<typeof buildNamingContext>[0],
+            {
+              flowId: file.flow?.flowId ?? "",
+              jobId: file.flow?.jobId ?? "",
+              nodeId: id,
+              nodeType: "optimize",
+            },
+            { format, quality },
+          );
+          // Apply naming to get base name with suffix
+          const namedFile = applyFileNaming(
+            file as Parameters<typeof applyFileNaming>[0],
+            namingContext,
+            namingConfig,
+          );
+          // Replace extension with new format extension
+          newFileName = `${getBaseName(namedFile)}.${newExtension}`;
+        } else {
+          // No naming config, just update extension
+          newFileName = fileName.replace(/\.[^.]+$/, `.${newExtension}`);
+        }
+      }
+
+      return { newType, newFileName };
+    };
+
     return yield* createTransformNode({
       id,
       name: "Optimize",
@@ -61,46 +147,14 @@ export function createOptimizeNode(
       // Note: naming is handled in transform since format changes extension
       nodeType: "optimize",
       namingVars: { format, quality },
+      mode: effectiveMode,
+      streamingConfig: options?.streamingConfig,
+      // Buffered transform (used when mode is "buffered" or "auto" selects buffered)
       transform: (inputBytes, file) =>
         Effect.map(
           imageService.optimize(inputBytes, { quality, format }),
           (optimizedBytes) => {
-            // Return bytes with updated metadata if format changes
-            const newType = formatToMimeType[format];
-            const newExtension = formatToExtension[format];
-
-            // Get original fileName
-            const fileName = file.metadata?.fileName;
-            let newFileName: string | undefined;
-
-            if (fileName && typeof fileName === "string") {
-              // Apply naming if configured
-              if (options?.naming) {
-                const namingConfig: FileNamingConfig = {
-                  ...options.naming,
-                  autoSuffix:
-                    options.naming.autoSuffix ?? ((ctx) => ctx.format ?? format),
-                };
-                const namingContext = buildNamingContext(
-                  file,
-                  {
-                    flowId: file.flow?.flowId ?? "",
-                    jobId: file.flow?.jobId ?? "",
-                    nodeId: id,
-                    nodeType: "optimize",
-                  },
-                  { format, quality },
-                );
-                // Apply naming to get base name with suffix
-                const namedFile = applyFileNaming(file, namingContext, namingConfig);
-                // Replace extension with new format extension
-                newFileName = `${getBaseName(namedFile)}.${newExtension}`;
-              } else {
-                // No naming config, just update extension
-                newFileName = fileName.replace(/\.[^.]+$/, `.${newExtension}`);
-              }
-            }
-
+            const { newType, newFileName } = buildOutputMetadata(file);
             return {
               bytes: optimizedBytes,
               type: newType,
@@ -110,6 +164,30 @@ export function createOptimizeNode(
               | { bytes: Uint8Array; type: string; fileName?: string };
           },
         ),
+      // Streaming transform (used when mode is "streaming" or "auto" selects streaming)
+      streamingTransform: imageService.optimizeStream
+        ? (inputStream, file) => {
+            const optimizeStreamFn = imageService.optimizeStream;
+            if (!optimizeStreamFn) {
+              throw UploadistaError.fromCode("UNKNOWN_ERROR");
+            }
+            return Effect.gen(function* () {
+              // Use the streaming optimization
+              const outputStream = yield* optimizeStreamFn(inputStream, {
+                quality,
+                format,
+              });
+
+              const { newType, newFileName } = buildOutputMetadata(file);
+
+              return {
+                stream: outputStream,
+                type: newType,
+                fileName: newFileName,
+              };
+            });
+          }
+        : undefined,
     });
   });
 }

package/src/resize-node.ts

@@ -4,34 +4,71 @@ import {
   ImagePlugin,
   type ResizeParams,
   STORAGE_OUTPUT_TYPE_ID,
+  type StreamingConfig,
+  type TransformMode,
 } from "@uploadista/core/flow";
 import { Effect } from "effect";
 
 /**
  * Creates a resize node that resizes images to specified dimensions.
  *
+ * Supports both buffered and streaming modes for memory-efficient processing
+ * of large images. In streaming mode, the image is read and processed
+ * incrementally, reducing peak memory usage.
+ *
  * @param id - Unique node identifier
  * @param params - Resize parameters (width, height, fit)
  * @param options - Optional configuration
  * @param options.keepOutput - Whether to keep output in flow results
  * @param options.naming - File naming configuration (auto suffix: `${width}x${height}`)
+ * @param options.mode - Transform mode: "buffered", "streaming", or "auto" (default)
+ * @param options.streamingConfig - Streaming configuration (file size threshold, chunk size)
  *
  * @example
  * ```typescript
- * // With auto-naming: "photo.jpg" -> "photo-800x600.jpg"
+ * // Auto mode (default) - uses streaming for files > 1MB, otherwise buffered
  * const resize = yield* createResizeNode("resize-1", { width: 800, height: 600 }, {
  *   naming: { mode: "auto" }
  * });
+ *
+ * // Force buffered mode for small files
+ * const resizeBuffered = yield* createResizeNode("resize-2", { width: 800, height: 600 }, {
+ *   mode: "buffered",
+ *   naming: { mode: "auto" }
+ * });
+ *
+ * // Force streaming mode for memory efficiency
+ * const resizeStreaming = yield* createResizeNode("resize-3", { width: 800, height: 600 }, {
+ *   mode: "streaming",
+ *   naming: { mode: "auto" }
+ * });
  * ```
  */
 export function createResizeNode(
   id: string,
   { width, height, fit }: ResizeParams,
-  options?: { keepOutput?: boolean; naming?: FileNamingConfig },
+  options?: {
+    keepOutput?: boolean;
+    naming?: FileNamingConfig;
+    mode?: TransformMode;
+    streamingConfig?: StreamingConfig;
+  },
 ) {
   return Effect.gen(function* () {
     const imageService = yield* ImagePlugin;
 
+    // Determine if streaming is available and requested
+    const supportsStreaming = imageService.supportsStreaming ?? false;
+    const requestedMode = options?.mode ?? "auto";
+
+    // If streaming requested but not supported, fall back to buffered
+    const effectiveMode: TransformMode =
+      requestedMode === "buffered"
+        ? "buffered"
+        : supportsStreaming
+          ? requestedMode
+          : "buffered";
+
     // Build naming config with auto suffix for resize
     const namingConfig: FileNamingConfig | undefined = options?.naming
       ? {
@@ -53,8 +90,27 @@ export function createResizeNode(
       naming: namingConfig,
       nodeType: "resize",
       namingVars: { width, height },
+      mode: effectiveMode,
+      streamingConfig: options?.streamingConfig,
+      // Buffered transform (used when mode is "buffered" or "auto" selects buffered)
       transform: (inputBytes) =>
         imageService.resize(inputBytes, { height, width, fit }),
+      // Streaming transform (used when mode is "streaming" or "auto" selects streaming)
+      streamingTransform: imageService.resizeStream
+        ? (inputStream) =>
+            Effect.gen(function* () {
+              const resizeStreamFn = imageService.resizeStream;
+              if (!resizeStreamFn) {
+                throw new Error("resizeStream not available");
+              }
+              const outputStream = yield* resizeStreamFn(inputStream, {
+                width,
+                height,
+                fit,
+              });
+              return { stream: outputStream };
+            })
+        : undefined,
     });
   });
 }

package/src/transform-image-node.ts

@@ -3,9 +3,12 @@ import {
   type FileNamingConfig,
   ImagePlugin,
   STORAGE_OUTPUT_TYPE_ID,
+  type StreamingConfig,
   type TransformImageParams,
+  type TransformMode,
 } from "@uploadista/core/flow";
-import { Effect } from "effect";
+import type { UploadistaError } from "@uploadista/core/errors";
+import { Effect, Stream } from "effect";
 
 /**
  * Apply a chain of transformations to an image by reducing over the transformations array.
@@ -26,6 +29,62 @@ function applyTransformationChain(
   );
 }
 
+/**
+ * Apply a chain of transformations using streaming where possible.
+ * Falls back to buffered processing for transformations that don't support streaming.
+ *
+ * @param imageService - The image plugin service to use for transformations
+ * @param inputStream - The input image as a stream
+ * @param transformations - Array of transformations to apply in sequence
+ * @returns Effect that resolves to the final transformed image stream
+ */
+function applyStreamingTransformationChain(
+  imageService: ReturnType<typeof ImagePlugin.of>,
+  inputStream: Stream.Stream<Uint8Array, UploadistaError>,
+  transformations: TransformImageParams["transformations"],
+): Effect.Effect<Stream.Stream<Uint8Array, UploadistaError>, UploadistaError> {
+  const transformStreamFn = imageService.transformStream;
+
+  if (!transformStreamFn) {
+    // If streaming not supported, collect to buffer and use buffered chain
+    return Effect.gen(function* () {
+      // Collect stream to buffer
+      const chunks: Uint8Array[] = [];
+      yield* Stream.runForEach(inputStream, (chunk) =>
+        Effect.sync(() => {
+          chunks.push(chunk);
+        }),
+      );
+      const totalLength = chunks.reduce((sum, c) => sum + c.byteLength, 0);
+      const inputBuffer = new Uint8Array(totalLength);
+      let offset = 0;
+      for (const chunk of chunks) {
+        inputBuffer.set(chunk, offset);
+        offset += chunk.byteLength;
+      }
+
+      // Apply transformations
+      const result = yield* applyTransformationChain(
+        imageService,
+        inputBuffer,
+        transformations,
+      );
+      return Stream.make(result);
+    });
+  }
+
+  // Apply each transformation in sequence using streaming
+  return Effect.reduce(
+    transformations,
+    inputStream,
+    (currentStream, transformation) =>
+      Effect.flatMap(
+        transformStreamFn(currentStream, transformation),
+        (outputStream) => Effect.succeed(outputStream),
+      ),
+  );
+}
+
 /**
  * Creates a transform image node that applies multiple transformations sequentially.
  *
@@ -33,24 +92,32 @@ function applyTransformationChain(
  * together. Each transformation is applied to the output of the previous transformation,
  * allowing for powerful image manipulation pipelines.
  *
+ * Supports both buffered and streaming modes for memory-efficient processing
+ * of large images. In streaming mode, each transformation is applied in sequence
+ * using streaming where supported.
+ *
  * Supported transformations include:
  * - Basic: resize, blur, rotate, flip
  * - Filters: grayscale, sepia, brightness, contrast
  * - Effects: sharpen
- * - Advanced: watermark, logo, text
+ * - Advanced: watermark, logo, text (streaming not supported for these)
  *
  * Note: Watermark and logo transformations require imagePath to be a valid URL.
  * Images will be fetched from the provided URL during transformation.
+ * Streaming mode is not supported for watermark, logo, and text transformations;
+ * these will cause fallback to buffered mode.
  *
  * @param id - Unique identifier for this node
  * @param params - Parameters including the transformations array
  * @param options - Optional configuration
  * @param options.keepOutput - Whether to keep output in flow results
  * @param options.naming - File naming configuration (auto suffix: `transformed`)
+ * @param options.mode - Transform mode: "buffered", "streaming", or "auto" (default)
+ * @param options.streamingConfig - Streaming configuration (file size threshold, chunk size)
  *
  * @example
  * ```typescript
- * // With auto-naming: "photo.jpg" -> "photo-transformed.jpg"
+ * // Auto mode (default) - uses streaming for files > 1MB, otherwise buffered
  * const node = yield* createTransformImageNode("transform-1", {
  *   transformations: [
  *     { type: 'resize', width: 800, height: 600, fit: 'cover' },
@@ -59,16 +126,57 @@ function applyTransformationChain(
  * }, {
  *   naming: { mode: "auto" }
  * });
+ *
+ * // Force buffered mode for small files
+ * const nodeBuffered = yield* createTransformImageNode("transform-2", {
+ *   transformations: [
+ *     { type: 'resize', width: 800, height: 600, fit: 'cover' },
+ *     { type: 'blur', sigma: 5 }
+ *   ]
+ * }, {
+ *   mode: "buffered",
+ *   naming: { mode: "auto" }
+ * });
+ *
+ * // Force streaming mode for memory efficiency
+ * const nodeStreaming = yield* createTransformImageNode("transform-3", {
+ *   transformations: [{ type: 'grayscale' }]
+ * }, {
+ *   mode: "streaming",
+ *   naming: { mode: "auto" }
+ * });
  * ```
  */
 export function createTransformImageNode(
   id: string,
   { transformations }: TransformImageParams,
-  options?: { keepOutput?: boolean; naming?: FileNamingConfig },
+  options?: {
+    keepOutput?: boolean;
+    naming?: FileNamingConfig;
+    mode?: TransformMode;
+    streamingConfig?: StreamingConfig;
+  },
 ) {
   return Effect.gen(function* () {
     const imageService = yield* ImagePlugin;
 
+    // Determine if streaming is available and requested
+    const supportsStreaming = imageService.supportsStreaming ?? false;
+    const requestedMode = options?.mode ?? "auto";
+
+    // Check if any transformations don't support streaming
+    const hasUnsupportedTransformations = transformations.some(
+      (t) => t.type === "watermark" || t.type === "logo" || t.type === "text",
+    );
+
+    // If streaming requested but not supported or unsupported transformations, fall back to buffered
+    const effectiveMode: TransformMode =
+      requestedMode === "buffered"
+        ? "buffered"
+        : supportsStreaming && !hasUnsupportedTransformations
+          ? requestedMode
+          : "buffered";
+
     // Build naming config with auto suffix for transform-image
     const namingConfig: FileNamingConfig | undefined = options?.naming
       ? {
@@ -86,8 +194,25 @@ export function createTransformImageNode(
       keepOutput: options?.keepOutput,
       naming: namingConfig,
       nodeType: "transform-image",
+      mode: effectiveMode,
+      streamingConfig: options?.streamingConfig,
+      // Buffered transform (used when mode is "buffered" or "auto" selects buffered)
       transform: (inputBytes) =>
         applyTransformationChain(imageService, inputBytes, transformations),
+      // Streaming transform (used when mode is "streaming" or "auto" selects streaming)
+      streamingTransform:
+        supportsStreaming && !hasUnsupportedTransformations
+          ? (inputStream) =>
+              Effect.gen(function* () {
+                const outputStream =
+                  yield* applyStreamingTransformationChain(
+                    imageService,
+                    inputStream,
+                    transformations,
+                  );
+                return { stream: outputStream };
+              })
+          : undefined,
     });
   });
 }