@uploadista/core 0.0.19 → 0.0.20-beta.1

package/src/testing/mock-upload-server.ts

@@ -1,6 +1,11 @@
-import { Effect, Layer } from "effect";
+import { Effect, Layer, Stream } from "effect";
+import type { UploadistaError } from "../errors";
 import type { InputFile, UploadFile, WebSocketConnection } from "../types";
-import type { DataStoreCapabilities } from "../types/data-store";
+import {
+  DEFAULT_STREAMING_CONFIG,
+  type DataStoreCapabilities,
+  type StreamingConfig,
+} from "../types/data-store";
 import { UploadServer } from "../upload";
 
 /**
@@ -29,6 +34,78 @@ export const TestUploadServer = Layer.succeed(
         const text = `Content of file ${fileId}`;
         return new TextEncoder().encode(text);
       }),
+    readStream: (
+      fileId: string,
+      _clientId: string | null,
+      config?: StreamingConfig,
+    ) =>
+      Effect.sync(() => {
+        const effectiveConfig = { ...DEFAULT_STREAMING_CONFIG, ...config };
+        // Generate mock file data based on fileId
+        const text = `Content of file ${fileId}`;
+        const fullData = new TextEncoder().encode(text);
+
+        // Split data into chunks based on chunkSize
+        const chunkSize = effectiveConfig.chunkSize;
+        const chunks: Uint8Array[] = [];
+        for (let i = 0; i < fullData.length; i += chunkSize) {
+          chunks.push(fullData.slice(i, i + chunkSize));
+        }
+
+        // Return as a stream of chunks
+        return Stream.fromIterable(chunks);
+      }),
+    uploadStream: (
+      file: Omit<InputFile, "size"> & { size?: number; sizeHint?: number },
+      _clientId: string | null,
+      stream: Stream.Stream<Uint8Array, UploadistaError>,
+    ) =>
+      Effect.gen(function* () {
+        // Collect stream to calculate total size
+        const chunks: Uint8Array[] = [];
+        yield* Stream.runForEach(stream, (chunk) =>
+          Effect.sync(() => {
+            chunks.push(chunk);
+          }),
+        );
+
+        const totalSize = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
+
+        // Parse existing metadata
+        const existingMetadata =
+          typeof file.metadata === "string"
+            ? JSON.parse(file.metadata)
+            : file.metadata || {};
+
+        // Extract extension from fileName
+        const extension = file.fileName
+          ? file.fileName.split(".").pop()
+          : existingMetadata.extension;
+
+        // Create new UploadFile with final size
+        const uploadId = `stream-uploaded-${Date.now()}-${Math.random().toString(36).substring(7)}`;
+        return {
+          id: uploadId,
+          offset: totalSize,
+          size: totalSize,
+          storage: {
+            id: file.storageId,
+            type: "memory",
+          },
+          metadata: {
+            ...existingMetadata,
+            mimeType: file.type,
+            type: file.type,
+            "content-type": file.type,
+            fileName: file.fileName,
+            originalName: file.fileName,
+            name: file.fileName,
+            extension,
+          },
+          url: `http://test-storage/${uploadId}`,
+          creationDate: new Date().toISOString(),
+        } satisfies UploadFile;
+      }),
     upload: (file, _clientId, stream) =>
       Effect.gen(function* () {
         // Read stream to completion
@@ -126,6 +203,8 @@ export const TestUploadServer = Layer.succeed(
         supportsDeferredLength: true,
         supportsResumableUploads: true,
         supportsTransactionalUploads: false,
+        supportsStreamingRead: true,
+        supportsStreamingWrite: true,
         maxConcurrentUploads: 10,
         minChunkSize: 5 * 1024 * 1024, // 5MB
         maxChunkSize: 100 * 1024 * 1024, // 100MB

package/src/types/data-store.ts

@@ -24,6 +24,98 @@ export type DataStoreWriteOptions = {
  */
 export type UploadStrategy = "single" | "parallel";
 
+/**
+ * Configuration options for streaming file reads.
+ *
+ * Used to control streaming behavior in transform nodes and data stores.
+ *
+ * @property fileSizeThreshold - Files below this size use buffered mode (default: 1MB)
+ * @property chunkSize - Chunk size for streaming reads in bytes (default: 64KB)
+ *
+ * @example
+ * ```typescript
+ * const config: StreamingConfig = {
+ *   fileSizeThreshold: 1_048_576, // 1MB - use buffered for smaller files
+ *   chunkSize: 65_536, // 64KB chunks
+ * };
+ * ```
+ */
+export type StreamingConfig = {
+  /** Files below this size use buffered mode (default: 1MB = 1_048_576 bytes) */
+  fileSizeThreshold?: number;
+  /** Chunk size for streaming reads in bytes (default: 64KB = 65_536 bytes) */
+  chunkSize?: number;
+};
+
+/**
+ * Default streaming configuration values.
+ */
+export const DEFAULT_STREAMING_CONFIG: Required<StreamingConfig> = {
+  fileSizeThreshold: 1_048_576, // 1MB
+  chunkSize: 65_536, // 64KB
+};
+
+/**
+ * Default multipart part size for S3/R2 streaming writes.
+ * S3 requires minimum 5MB parts (except for the last part).
+ */
+export const DEFAULT_MULTIPART_PART_SIZE = 5 * 1024 * 1024; // 5MB
+
+/**
+ * Options for streaming write operations.
+ *
+ * Used when writing file content from a stream with unknown final size.
+ * The store will finalize the upload when the stream completes.
+ *
+ * @property stream - Effect Stream of byte chunks to write
+ * @property contentType - Optional MIME type for the file
+ * @property metadata - Optional metadata to store with the file
+ * @property sizeHint - Optional estimated size for optimization (e.g., multipart part sizing)
+ *
+ * @example
+ * ```typescript
+ * const options: StreamWriteOptions = {
+ *   stream: transformedStream,
+ *   contentType: "image/webp",
+ *   metadata: { originalName: "photo.jpg" },
+ *   sizeHint: 5_000_000, // ~5MB expected
+ * };
+ * ```
+ */
+export type StreamWriteOptions = {
+  stream: Stream.Stream<Uint8Array, UploadistaError>;
+  contentType?: string;
+  metadata?: Record<string, string>;
+  /** Optional size hint for optimization (not required) */
+  sizeHint?: number;
+};
+
+/**
+ * Result of a streaming write operation.
+ *
+ * Contains the final size after the stream completes, along with
+ * storage location information.
+ *
+ * @property id - Unique identifier of the written file
+ * @property size - Final size in bytes after stream completed
+ * @property path - Storage path or key where file was written
+ * @property bucket - Optional bucket/container name (for cloud storage)
+ *
+ * @example
+ * ```typescript
+ * const result = yield* dataStore.writeStream(fileId, options);
+ * console.log(`Wrote ${result.size} bytes to ${result.path}`);
+ * ```
+ */
+export type StreamWriteResult = {
+  id: string;
+  size: number;
+  path: string;
+  bucket?: string;
+  /** Public URL for accessing the uploaded file (if available) */
+  url?: string;
+};
+
 /**
  * Capabilities and constraints of a DataStore implementation.
  *
@@ -36,6 +128,7 @@ export type UploadStrategy = "single" | "parallel";
  * @property supportsDeferredLength - Can start upload without knowing final size
  * @property supportsResumableUploads - Can resume interrupted uploads from last offset
  * @property supportsTransactionalUploads - Guarantees atomic upload success/failure
+ * @property supportsStreamingRead - Can read file content as a stream instead of buffering
  * @property maxConcurrentUploads - Maximum parallel upload parts (if parallel supported)
  * @property minChunkSize - Minimum size in bytes for each chunk (except last)
  * @property maxChunkSize - Maximum size in bytes for each chunk
@@ -57,6 +150,12 @@ export type UploadStrategy = "single" | "parallel";
  *   // Use single upload
  *   uploadAsSingleChunk(file);
  * }
+ *
+ * // Check for streaming support
+ * if (capabilities.supportsStreamingRead) {
+ *   // Use streaming for memory-efficient processing
+ *   const stream = yield* dataStore.readStream(fileId);
+ * }
  * ```
  */
 export type DataStoreCapabilities = {
@@ -65,6 +164,10 @@ export type DataStoreCapabilities = {
   supportsDeferredLength: boolean;
   supportsResumableUploads: boolean;
   supportsTransactionalUploads: boolean;
+  /** Whether the store supports streaming reads via readStream() */
+  supportsStreamingRead?: boolean;
+  /** Whether the store supports streaming writes via writeStream() with unknown final size */
+  supportsStreamingWrite?: boolean;
   maxConcurrentUploads?: number;
   minChunkSize?: number;
   maxChunkSize?: number;
@@ -149,15 +252,69 @@ export type DataStore<TData = unknown> = {
   readonly path?: string;
   readonly create: (file: TData) => Effect.Effect<TData, UploadistaError>;
   readonly remove: (file_id: string) => Effect.Effect<void, UploadistaError>;
+  /**
+   * Reads the complete file contents as bytes (buffered mode).
+   * For large files, consider using readStream() if available.
+   */
   readonly read: (
     file_id: string,
   ) => Effect.Effect<Uint8Array, UploadistaError>;
+  /**
+   * Reads file content as a stream of chunks for memory-efficient processing.
+   * Optional - check getCapabilities().supportsStreamingRead before using.
+   *
+   * @param file_id - The unique identifier of the file to read
+   * @param config - Optional streaming configuration (chunk size)
+   * @returns An Effect that resolves to a Stream of byte chunks
+   *
+   * @example
+   * ```typescript
+   * const capabilities = dataStore.getCapabilities();
+   * if (capabilities.supportsStreamingRead && dataStore.readStream) {
+   *   const stream = yield* dataStore.readStream(fileId, { chunkSize: 65536 });
+   *   // Process stream chunk by chunk
+   * }
+   * ```
+   */
+  readonly readStream?: (
+    file_id: string,
+    config?: StreamingConfig,
+  ) => Effect.Effect<Stream.Stream<Uint8Array, UploadistaError>, UploadistaError>;
   readonly write: (
     options: DataStoreWriteOptions,
     dependencies: {
       onProgress?: (chunkSize: number) => void;
     },
   ) => Effect.Effect<number, UploadistaError>;
+  /**
+   * Writes file content from a stream with unknown final size.
+   * Optional - check getCapabilities().supportsStreamingWrite before using.
+   *
+   * This method is optimized for end-to-end streaming where the output
+   * size isn't known until the stream completes. It uses store-specific
+   * mechanisms like multipart uploads (S3/R2), resumable uploads (GCS),
+   * or block staging (Azure) to efficiently handle streaming data.
+   *
+   * @param fileId - Unique identifier for the file being written
+   * @param options - Stream and optional metadata
+   * @returns StreamWriteResult containing final size after completion
+   *
+   * @example
+   * ```typescript
+   * const capabilities = dataStore.getCapabilities();
+   * if (capabilities.supportsStreamingWrite && dataStore.writeStream) {
+   *   const result = yield* dataStore.writeStream(fileId, {
+   *     stream: transformedStream,
+   *     contentType: "image/webp",
+   *   });
+   *   console.log(`Wrote ${result.size} bytes`);
+   * }
+   * ```
+   */
+  readonly writeStream?: (
+    fileId: string,
+    options: StreamWriteOptions,
+  ) => Effect.Effect<StreamWriteResult, UploadistaError>;
   readonly deleteExpired?: () => Effect.Effect<number, UploadistaError>;
   readonly getCapabilities: () => DataStoreCapabilities;
   readonly validateUploadStrategy: (

package/src/types/input-file.ts

@@ -8,24 +8,41 @@ import { z } from "zod";
  *
  * @see {@link InputFile} for the TypeScript type
  */
-export const inputFileSchema = z.object({
-  uploadLengthDeferred: z.boolean().optional(),
-  storageId: z.string(),
-  size: z.number(),
-  type: z.string(),
-  fileName: z.string().optional(),
-  lastModified: z.number().optional(),
-  metadata: z.string().optional(),
-  checksum: z.string().optional(),
-  checksumAlgorithm: z.string().optional(),
-  flow: z
-    .object({
-      flowId: z.string(),
-      nodeId: z.string(),
-      jobId: z.string(),
-    })
-    .optional(),
-});
+export const inputFileSchema = z
+  .object({
+    uploadLengthDeferred: z.boolean().optional(),
+    storageId: z.string(),
+    /** File size in bytes. Optional when uploadLengthDeferred is true. */
+    size: z.number().optional(),
+    /** Optional size hint for optimization when size is unknown */
+    sizeHint: z.number().optional(),
+    type: z.string(),
+    fileName: z.string().optional(),
+    lastModified: z.number().optional(),
+    metadata: z.string().optional(),
+    checksum: z.string().optional(),
+    checksumAlgorithm: z.string().optional(),
+    flow: z
+      .object({
+        flowId: z.string(),
+        nodeId: z.string(),
+        jobId: z.string(),
+      })
+      .optional(),
+  })
+  .refine(
+    (data) => {
+      // Size is required unless uploadLengthDeferred is true
+      if (data.uploadLengthDeferred === true) {
+        return true; // Size can be omitted
+      }
+      return data.size !== undefined && data.size >= 0;
+    },
+    {
+      message: "size is required when uploadLengthDeferred is not true",
+      path: ["size"],
+    },
+  );
 
 /**
  * Represents the input data for creating a new file upload.
@@ -34,7 +51,8 @@ export const inputFileSchema = z.object({
  * It's used by clients to provide upload metadata before sending file data.
  *
  * @property storageId - Target storage backend identifier (e.g., "s3-production", "azure-blob")
- * @property size - File size in bytes
+ * @property size - File size in bytes. Optional when uploadLengthDeferred is true.
+ * @property sizeHint - Optional size hint for optimization when exact size is unknown
  * @property type - MIME type of the file (e.g., "image/jpeg", "application/pdf")
  * @property uploadLengthDeferred - If true, file size is not known upfront (streaming upload)
  * @property fileName - Original filename from the client
@@ -94,14 +112,22 @@ export const inputFileSchema = z.object({
  *   }
  * };
  *
- * // Streaming upload (size unknown)
+ * // Streaming upload (size unknown) - size can be omitted
  * const streamingInput: InputFile = {
  *   storageId: "s3-production",
- *   size: 0, // Will be updated as data arrives
  *   type: "video/mp4",
  *   uploadLengthDeferred: true,
  *   fileName: "live-stream.mp4"
  * };
+ *
+ * // Streaming upload with size hint for optimization
+ * const streamingWithHint: InputFile = {
+ *   storageId: "s3-production",
+ *   type: "image/webp",
+ *   uploadLengthDeferred: true,
+ *   sizeHint: 5_000_000, // ~5MB expected
+ *   fileName: "optimized-image.webp"
+ * };
  * ```
  */
 export type InputFile = z.infer<typeof inputFileSchema>;

package/src/upload/upload-server.ts

@@ -1,4 +1,4 @@
-import { Context, Effect, Layer } from "effect";
+import { Context, Effect, Layer, Stream } from "effect";
 import type { UploadistaError } from "../errors";
 import type {
   DataStore,
@@ -7,12 +7,14 @@ import type {
   InputFile,
   KvStore,
   Middleware,
+  StreamingConfig,
   UploadEvent,
   UploadFile,
   WebSocketConnection,
 } from "../types";
 import {
   UploadEventEmitter,
+  UploadEventType,
   UploadFileDataStores,
   UploadFileKVStore,
 } from "../types";
@@ -151,10 +153,70 @@ export type UploadServerShape = {
     url: string,
   ) => Effect.Effect<UploadFile, UploadistaError>;
   getUpload: (uploadId: string) => Effect.Effect<UploadFile, UploadistaError>;
+  /**
+   * Reads the complete uploaded file data as bytes (buffered mode).
+   * For large files, consider using readStream() for memory efficiency.
+   */
   read: (
     uploadId: string,
     clientId: string | null,
   ) => Effect.Effect<Uint8Array, UploadistaError>;
+  /**
+   * Reads file content as a stream of chunks for memory-efficient processing.
+   * Falls back to buffered read if the underlying DataStore doesn't support streaming.
+   *
+   * @param uploadId - The unique identifier of the upload to read
+   * @param clientId - Client identifier for multi-tenant routing
+   * @param config - Optional streaming configuration (chunk size)
+   * @returns An Effect that resolves to a Stream of byte chunks
+   *
+   * @example
+   * ```typescript
+   * const server = yield* UploadServer;
+   * const stream = yield* server.readStream(uploadId, clientId, { chunkSize: 65536 });
+   * // Process stream chunk by chunk with bounded memory
+   * yield* Stream.runForEach(stream, (chunk) => processChunk(chunk));
+   * ```
+   */
+  readStream: (
+    uploadId: string,
+    clientId: string | null,
+    config?: StreamingConfig,
+  ) => Effect.Effect<Stream.Stream<Uint8Array, UploadistaError>, UploadistaError>;
+  /**
+   * Uploads file content from a stream with unknown final size.
+   * Creates upload with deferred length, streams content to storage,
+   * and updates the upload record with final size when complete.
+   *
+   * Falls back to buffered upload if the underlying DataStore
+   * doesn't support streaming writes.
+   *
+   * @param file - Input file configuration (size is optional)
+   * @param clientId - Client identifier for multi-tenant routing
+   * @param stream - Effect Stream of byte chunks to upload
+   * @returns The completed UploadFile with final size
+   *
+   * @example
+   * ```typescript
+   * const server = yield* UploadServer;
+   * const result = yield* server.uploadStream(
+   *   {
+   *     storageId: "s3-production",
+   *     type: "image/webp",
+   *     uploadLengthDeferred: true,
+   *     fileName: "optimized.webp",
+   *   },
+   *   clientId,
+   *   transformedStream,
+   * );
+   * console.log(`Uploaded ${result.size} bytes`);
+   * ```
+   */
+  uploadStream: (
+    file: Omit<InputFile, "size"> & { size?: number; sizeHint?: number },
+    clientId: string | null,
+    stream: Stream.Stream<Uint8Array, UploadistaError>,
+  ) => Effect.Effect<UploadFile, UploadistaError>;
   delete: (
     uploadId: string,
     clientId: string | null,
@@ -323,6 +385,177 @@ export function createUploadServer() {
           );
           return yield* dataStore.read(uploadId);
         }),
+      readStream: (
+        uploadId: string,
+        clientId: string | null,
+        config?: StreamingConfig,
+      ) =>
+        Effect.gen(function* () {
+          const upload = yield* kvStore.get(uploadId);
+          const dataStore = yield* dataStoreService.getDataStore(
+            upload.storage.id,
+            clientId,
+          );
+
+          // Check if the DataStore supports streaming reads
+          const capabilities = dataStore.getCapabilities();
+          if (capabilities.supportsStreamingRead && dataStore.readStream) {
+            // Use native streaming
+            yield* Effect.logDebug(
+              `Using streaming read for file ${uploadId}`,
+            );
+            return yield* dataStore.readStream(uploadId, config);
+          }
+
+          // Fallback: read entire file and convert to stream
+          yield* Effect.logDebug(
+            `Falling back to buffered read for file ${uploadId} (streaming not supported)`,
+          );
+          const bytes = yield* dataStore.read(uploadId);
+
+          // Convert buffered bytes to a single-chunk stream
+          return Stream.succeed(bytes);
+        }),
+      uploadStream: (
+        file: Omit<InputFile, "size"> & { size?: number; sizeHint?: number },
+        clientId: string | null,
+        stream: Stream.Stream<Uint8Array, UploadistaError>,
+      ) =>
+        Effect.gen(function* () {
+          // Get the data store for this storage
+          const dataStore = yield* dataStoreService.getDataStore(
+            file.storageId,
+            clientId,
+          );
+
+          // Check if the DataStore supports streaming writes
+          const capabilities = dataStore.getCapabilities();
+
+          // Generate upload ID
+          const uploadId = yield* generateId.generateId();
+
+          if (capabilities.supportsStreamingWrite && dataStore.writeStream) {
+            // Use native streaming write - DO NOT call createUpload as it would
+            // create an S3 multipart upload that we won't use (writeStream creates its own)
+            yield* Effect.logDebug(
+              `Using streaming write for file ${uploadId}`,
+            );
+
+            // Parse metadata
+            const metadata =
+              typeof file.metadata === "string"
+                ? JSON.parse(file.metadata)
+                : file.metadata || {};
+
+            // Convert metadata to Record<string, string> if present
+            const stringMetadata = Object.fromEntries(
+              Object.entries(metadata).map(([k, v]) => [k, String(v)]),
+            );
+
+            // Create initial upload record in KV store (without creating S3 multipart upload)
+            const initialUpload: UploadFile = {
+              id: uploadId,
+              offset: 0,
+              size: file.size ?? 0,
+              storage: {
+                id: file.storageId,
+                type: dataStore.getCapabilities().supportsStreamingWrite
+                  ? "streaming"
+                  : "default",
+              },
+              metadata,
+              creationDate: new Date().toISOString(),
+            };
+            yield* kvStore.set(uploadId, initialUpload);
+
+            // Emit started event
+            yield* eventEmitter.emit(uploadId, {
+              type: UploadEventType.UPLOAD_STARTED,
+              data: initialUpload,
+            });
+
+            const result = yield* dataStore.writeStream(uploadId, {
+              stream,
+              contentType: file.type,
+              sizeHint: file.sizeHint,
+              metadata: stringMetadata,
+            });
+
+            // Update the upload record with the final size and URL
+            const completedUpload: UploadFile = {
+              ...initialUpload,
+              size: result.size,
+              offset: result.size,
+              storage: {
+                ...initialUpload.storage,
+                path: result.path,
+              },
+              ...(result.url && { url: result.url }),
+            };
+
+            yield* kvStore.set(uploadId, completedUpload);
+
+            // Emit completion event
+            yield* eventEmitter.emit(uploadId, {
+              type: UploadEventType.UPLOAD_COMPLETE,
+              data: completedUpload,
+            });
+
+            return completedUpload;
+          }
+
+          // Fallback: buffer the stream and use regular upload (which calls createUpload + uploadChunk)
+          yield* Effect.logWarning(
+            `Falling back to buffered upload for file ${uploadId} (streaming write not supported)`,
+          );
+
+          // Collect stream into a buffer
+          const chunks: Uint8Array[] = [];
+          yield* Stream.runForEach(stream, (chunk) =>
+            Effect.sync(() => {
+              chunks.push(chunk);
+            }),
+          );
+
+          // Calculate total size
+          const totalSize = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
+
+          // Create a combined buffer
+          const buffer = new Uint8Array(totalSize);
+          let offset = 0;
+          for (const chunk of chunks) {
+            buffer.set(chunk, offset);
+            offset += chunk.length;
+          }
+
+          // Create a readable stream from the buffer
+          const readableStream = new ReadableStream({
+            start(controller) {
+              controller.enqueue(buffer);
+              controller.close();
+            },
+          });
+
+          // For fallback, use the regular flow with createUpload + uploadChunk
+          const inputFile: InputFile = {
+            ...file,
+            size: totalSize,
+          };
+
+          const uploadFile = yield* createUpload(inputFile, clientId, {
+            dataStoreService,
+            kvStore,
+            eventEmitter,
+            generateId: { generateId: () => Effect.succeed(uploadId) },
+          });
+
+          // Use regular uploadChunk
+          return yield* uploadChunk(uploadId, clientId, readableStream, {
+            dataStoreService,
+            kvStore,
+            eventEmitter,
+          });
+        }),
       delete: (uploadId: string, clientId: string | null) =>
         Effect.gen(function* () {
           const upload = yield* kvStore.get(uploadId);