@uploadista/core 0.0.2

package/src/upload/upload-server.ts

@@ -0,0 +1,390 @@
+import { Context, Effect, Layer } from "effect";
+import type { UploadistaError } from "../errors";
+import type {
+  DataStore,
+  DataStoreCapabilities,
+  EventEmitter,
+  InputFile,
+  KvStore,
+  Middleware,
+  UploadEvent,
+  UploadFile,
+  WebSocketConnection,
+} from "../types";
+import {
+  UploadEventEmitter,
+  UploadFileDataStores,
+  UploadFileKVStore,
+} from "../types";
+import { GenerateId, type GenerateIdShape } from "../utils/generate-id";
+import { createUpload } from "./create-upload";
+import { uploadChunk } from "./upload-chunk";
+import { arrayBuffer, fetchFile } from "./upload-url";
+
+/**
+ * Legacy configuration options for UploadServer.
+ *
+ * @deprecated Use Effect Layers instead of this configuration object.
+ * This type is kept for backward compatibility.
+ *
+ * @property dataStore - DataStore instance or factory function
+ * @property kvStore - KV store for upload metadata
+ * @property eventEmitter - Event emitter for upload progress
+ * @property generateId - Optional ID generator (defaults to UUID)
+ * @property middlewares - Optional request middlewares
+ * @property withTracing - Enable Effect tracing for debugging
+ */
+export type UploadServerOptions = {
+  dataStore:
+    | ((storageId: string) => Promise<DataStore<UploadFile>>)
+    | DataStore<UploadFile>;
+  kvStore: KvStore<UploadFile>;
+  eventEmitter: EventEmitter<UploadEvent>;
+  generateId?: GenerateIdShape;
+  middlewares?: Middleware[];
+  withTracing?: boolean;
+};
+
+/**
+ * UploadServer service interface.
+ *
+ * This is the core upload handling service that provides all file upload operations.
+ * It manages upload lifecycle, resumable uploads, progress tracking, and storage integration.
+ *
+ * All operations return Effect types for composable, type-safe error handling.
+ *
+ * @property createUpload - Initiates a new upload and returns metadata
+ * @property uploadChunk - Uploads a chunk of data for an existing upload
+ * @property getCapabilities - Returns storage backend capabilities
+ * @property upload - Complete upload in one operation (create + upload data)
+ * @property uploadFromUrl - Uploads a file from a remote URL
+ * @property getUpload - Retrieves upload metadata by ID
+ * @property read - Reads the complete uploaded file data
+ * @property delete - Deletes an upload and its data
+ * @property subscribeToUploadEvents - Subscribes WebSocket to upload progress events
+ * @property unsubscribeFromUploadEvents - Unsubscribes from upload events
+ *
+ * @example
+ * ```typescript
+ * // Basic upload flow
+ * const program = Effect.gen(function* () {
+ *   const server = yield* UploadServer;
+ *
+ *   // 1. Create upload
+ *   const inputFile: InputFile = {
+ *     storageId: "s3-production",
+ *     size: 1024000,
+ *     type: "image/jpeg",
+ *     fileName: "photo.jpg"
+ *   };
+ *   const upload = yield* server.createUpload(inputFile, "client123");
+ *
+ *   // 2. Upload chunks
+ *   const chunk = new ReadableStream(...);
+ *   const updated = yield* server.uploadChunk(upload.id, "client123", chunk);
+ *
+ *   // 3. Read the uploaded file
+ *   const data = yield* server.read(upload.id, "client123");
+ *
+ *   return upload;
+ * });
+ *
+ * // Upload with WebSocket progress tracking
+ * const uploadWithProgress = Effect.gen(function* () {
+ *   const server = yield* UploadServer;
+ *
+ *   // Subscribe to progress events
+ *   yield* server.subscribeToUploadEvents(uploadId, websocket);
+ *
+ *   // Upload (events will be emitted automatically)
+ *   const result = yield* server.upload(inputFile, clientId, stream);
+ *
+ *   // Unsubscribe when done
+ *   yield* server.unsubscribeFromUploadEvents(uploadId);
+ *
+ *   return result;
+ * });
+ *
+ * // Upload from URL
+ * const urlUpload = Effect.gen(function* () {
+ *   const server = yield* UploadServer;
+ *
+ *   const inputFile: InputFile = {
+ *     storageId: "s3-production",
+ *     size: 0, // Unknown initially
+ *     type: "image/png",
+ *     fileName: "remote-image.png"
+ *   };
+ *
+ *   const upload = yield* server.uploadFromUrl(
+ *     inputFile,
+ *     "client123",
+ *     "https://example.com/image.png"
+ *   );
+ *
+ *   return upload;
+ * });
+ * ```
+ */
+export type UploadServerShape = {
+  createUpload: (
+    inputFile: InputFile,
+    clientId: string | null,
+  ) => Effect.Effect<UploadFile, UploadistaError>;
+  uploadChunk: (
+    uploadId: string,
+    clientId: string | null,
+    chunk: ReadableStream,
+  ) => Effect.Effect<UploadFile, UploadistaError>;
+  getCapabilities: (
+    storageId: string,
+    clientId: string | null,
+  ) => Effect.Effect<DataStoreCapabilities, UploadistaError>;
+  upload: (
+    file: InputFile,
+    clientId: string | null,
+    stream: ReadableStream,
+  ) => Effect.Effect<UploadFile, UploadistaError>;
+  uploadFromUrl: (
+    inputFile: InputFile,
+    clientId: string | null,
+    url: string,
+  ) => Effect.Effect<UploadFile, UploadistaError>;
+  getUpload: (uploadId: string) => Effect.Effect<UploadFile, UploadistaError>;
+  read: (
+    uploadId: string,
+    clientId: string | null,
+  ) => Effect.Effect<Uint8Array, UploadistaError>;
+  delete: (
+    uploadId: string,
+    clientId: string | null,
+  ) => Effect.Effect<void, UploadistaError>;
+  subscribeToUploadEvents: (
+    uploadId: string,
+    connection: WebSocketConnection,
+  ) => Effect.Effect<void, UploadistaError>;
+  unsubscribeFromUploadEvents: (
+    uploadId: string,
+  ) => Effect.Effect<void, UploadistaError>;
+};
+
+/**
+ * Effect-TS context tag for the UploadServer service.
+ *
+ * Use this tag to access the UploadServer in an Effect context.
+ * The server must be provided via a Layer or dependency injection.
+ *
+ * @example
+ * ```typescript
+ * // Access UploadServer in an Effect
+ * const uploadEffect = Effect.gen(function* () {
+ *   const server = yield* UploadServer;
+ *   const upload = yield* server.createUpload(inputFile, clientId);
+ *   return upload;
+ * });
+ *
+ * // Provide UploadServer layer
+ * const program = uploadEffect.pipe(
+ *   Effect.provide(uploadServer),
+ *   Effect.provide(uploadFileKvStore),
+ *   Effect.provide(dataStoreLayer),
+ *   Effect.provide(eventEmitterLayer)
+ * );
+ * ```
+ */
+export class UploadServer extends Context.Tag("UploadServer")<
+  UploadServer,
+  UploadServerShape
+>() {}
+
+/**
+ * Creates the UploadServer implementation.
+ *
+ * This function constructs the UploadServer service by composing all required
+ * dependencies (KV store, data stores, event emitter, ID generator). It implements
+ * all upload operations defined in UploadServerShape.
+ *
+ * The server automatically handles:
+ * - Upload lifecycle management (create, resume, complete)
+ * - Progress tracking and event emission
+ * - Storage backend routing based on storageId
+ * - Error handling with proper UploadistaError types
+ *
+ * @returns An Effect that yields the UploadServerShape implementation
+ *
+ * @example
+ * ```typescript
+ * // Create a custom UploadServer layer
+ * const myUploadServer = Layer.effect(
+ *   UploadServer,
+ *   createUploadServer()
+ * );
+ *
+ * // Use in a program
+ * const program = Effect.gen(function* () {
+ *   const server = yield* UploadServer;
+ *   // Use server operations...
+ * }).pipe(Effect.provide(myUploadServer));
+ * ```
+ */
+export function createUploadServer() {
+  return Effect.gen(function* () {
+    const kvStore = yield* UploadFileKVStore;
+    const eventEmitter = yield* UploadEventEmitter;
+    const generateId = yield* GenerateId;
+    const dataStoreService = yield* UploadFileDataStores;
+
+    return {
+      upload: (
+        inputFile: InputFile,
+        clientId: string | null,
+        stream: ReadableStream,
+      ) =>
+        Effect.gen(function* () {
+          const fileCreated = yield* createUpload(inputFile, clientId, {
+            dataStoreService,
+            kvStore,
+            eventEmitter,
+            generateId,
+          });
+          return yield* uploadChunk(fileCreated.id, clientId, stream, {
+            dataStoreService,
+            kvStore,
+            eventEmitter,
+          });
+        }),
+      uploadFromUrl: (
+        inputFile: InputFile,
+        clientId: string | null,
+        url: string,
+      ) =>
+        Effect.gen(function* () {
+          const response = yield* fetchFile(url);
+          const buffer = yield* arrayBuffer(response);
+
+          // Create a readable stream from the buffer
+          const stream = new ReadableStream({
+            start(controller) {
+              controller.enqueue(new Uint8Array(buffer));
+              controller.close();
+            },
+          });
+
+          const fileCreated = yield* createUpload(
+            { ...inputFile, size: buffer.byteLength },
+            clientId,
+            {
+              dataStoreService,
+              kvStore,
+              eventEmitter,
+              generateId,
+            },
+          );
+          return yield* uploadChunk(fileCreated.id, clientId, stream, {
+            dataStoreService,
+            kvStore,
+            eventEmitter,
+          });
+        }),
+      createUpload: (inputFile: InputFile, clientId: string | null) =>
+        Effect.gen(function* () {
+          const fileCreated = yield* createUpload(inputFile, clientId, {
+            dataStoreService,
+            kvStore,
+            eventEmitter,
+            generateId,
+          });
+          return fileCreated;
+        }),
+      uploadChunk: (
+        uploadId: string,
+        clientId: string | null,
+        chunk: ReadableStream,
+      ) =>
+        Effect.gen(function* () {
+          const file = yield* uploadChunk(uploadId, clientId, chunk, {
+            dataStoreService,
+            kvStore,
+            eventEmitter,
+          });
+          return file;
+        }),
+      getUpload: (uploadId: string) =>
+        Effect.gen(function* () {
+          const file = yield* kvStore.get(uploadId);
+          return file;
+        }),
+      read: (uploadId: string, clientId: string | null) =>
+        Effect.gen(function* () {
+          const upload = yield* kvStore.get(uploadId);
+          const dataStore = yield* dataStoreService.getDataStore(
+            upload.storage.id,
+            clientId,
+          );
+          return yield* dataStore.read(uploadId);
+        }),
+      delete: (uploadId: string, clientId: string | null) =>
+        Effect.gen(function* () {
+          const upload = yield* kvStore.get(uploadId);
+          const dataStore = yield* dataStoreService.getDataStore(
+            upload.storage.id,
+            clientId,
+          );
+          yield* dataStore.remove(uploadId);
+          yield* kvStore.delete(uploadId);
+          return;
+        }),
+      getCapabilities: (storageId: string, clientId: string | null) =>
+        Effect.gen(function* () {
+          const dataStore = yield* dataStoreService.getDataStore(
+            storageId,
+            clientId,
+          );
+          return dataStore.getCapabilities();
+        }),
+      subscribeToUploadEvents: (
+        uploadId: string,
+        connection: WebSocketConnection,
+      ) =>
+        Effect.gen(function* () {
+          yield* eventEmitter.subscribe(uploadId, connection);
+        }),
+      unsubscribeFromUploadEvents: (uploadId: string) =>
+        Effect.gen(function* () {
+          yield* eventEmitter.unsubscribe(uploadId);
+        }),
+    } satisfies UploadServerShape;
+  });
+}
+
+/**
+ * Pre-built UploadServer Effect Layer.
+ *
+ * This layer provides a ready-to-use UploadServer implementation that can be
+ * composed with other layers to build a complete upload system.
+ *
+ * Required dependencies:
+ * - UploadFileKVStore: For storing upload metadata
+ * - UploadFileDataStores: For routing to storage backends
+ * - UploadEventEmitter: For progress events
+ * - GenerateId: For creating upload IDs
+ *
+ * @example
+ * ```typescript
+ * // Compose a complete upload system
+ * const fullUploadSystem = Layer.mergeAll(
+ *   uploadServer,
+ *   uploadFileKvStore,
+ *   dataStoreLayer,
+ *   uploadEventEmitter,
+ *   generateIdLayer
+ * );
+ *
+ * // Use in application
+ * const app = Effect.gen(function* () {
+ *   const server = yield* UploadServer;
+ *   // Perform uploads...
+ * }).pipe(Effect.provide(fullUploadSystem));
+ * ```
+ */
+export const uploadServer = Layer.effect(UploadServer, createUploadServer());

package/src/upload/upload-strategy-negotiator.ts

@@ -0,0 +1,316 @@
+import type { DataStoreCapabilities, UploadStrategy } from "../types";
+
+/**
+ * Configuration options for upload strategy negotiation.
+ *
+ * @property fileSize - Size of the file to be uploaded in bytes
+ * @property preferredStrategy - Preferred upload strategy (single, parallel, resumable)
+ * @property preferredChunkSize - Preferred chunk size in bytes
+ * @property parallelUploads - Number of parallel upload connections
+ * @property minChunkSizeForParallel - Minimum file size to consider parallel uploads
+ */
+export type UploadStrategyOptions = {
+  fileSize: number;
+  preferredStrategy?: UploadStrategy;
+  preferredChunkSize?: number;
+  parallelUploads?: number;
+  minChunkSizeForParallel?: number;
+};
+
+/**
+ * Result of upload strategy negotiation.
+ *
+ * @property strategy - The negotiated upload strategy
+ * @property chunkSize - The negotiated chunk size in bytes
+ * @property parallelUploads - The negotiated number of parallel uploads
+ * @property reasoning - Array of reasoning strings explaining the decisions
+ * @property warnings - Array of warning messages about adjustments made
+ */
+export type NegotiatedStrategy = {
+  strategy: UploadStrategy;
+  chunkSize: number;
+  parallelUploads: number;
+  reasoning: string[];
+  warnings: string[];
+};
+
+/**
+ * Negotiates the optimal upload strategy based on data store capabilities and file characteristics.
+ *
+ * This class analyzes data store capabilities, file size, and user preferences to determine
+ * the best upload strategy (single, parallel, resumable) and optimal parameters like chunk size
+ * and parallel connection count.
+ *
+ * The negotiator considers:
+ * - Data store capabilities (parallel uploads, resumable uploads, concatenation)
+ * - File size and chunk size constraints
+ * - User preferences and requirements
+ * - Performance optimization opportunities
+ *
+ * @example
+ * ```typescript
+ * // Create negotiator for S3 data store
+ * const negotiator = new UploadStrategyNegotiator(
+ *   s3Capabilities,
+ *   (strategy) => s3Capabilities.supportsStrategy(strategy)
+ * );
+ *
+ * // Negotiate strategy for large file
+ * const result = negotiator.negotiateStrategy({
+ *   fileSize: 100_000_000, // 100MB
+ *   preferredStrategy: "parallel",
+ *   preferredChunkSize: 5_000_000, // 5MB chunks
+ *   parallelUploads: 4
+ * });
+ *
+ * console.log(result.strategy); // "parallel"
+ * console.log(result.chunkSize); // 5_000_000
+ * console.log(result.reasoning); // ["Using preferred strategy: parallel", ...]
+ * ```
+ */
+export class UploadStrategyNegotiator {
+  /**
+   * Creates a new upload strategy negotiator.
+   *
+   * @param capabilities - Data store capabilities and constraints
+   * @param validateUploadStrategy - Function to validate if a strategy is supported
+   */
+  constructor(
+    private capabilities: DataStoreCapabilities,
+    private validateUploadStrategy: (strategy: UploadStrategy) => boolean
+  ) {}
+
+  /**
+   * Negotiates the optimal upload strategy based on options and data store capabilities.
+   *
+   * This method analyzes the provided options and data store capabilities to determine
+   * the best upload strategy, chunk size, and parallel upload settings. It considers
+   * user preferences, file size, and data store constraints to make optimal decisions.
+   *
+   * The negotiation process:
+   * 1. Validates preferred strategy against data store capabilities
+   * 2. Automatically selects strategy based on file size and capabilities
+   * 3. Adjusts chunk size to fit within data store constraints
+   * 4. Validates parallel upload settings
+   * 5. Ensures final strategy is supported by the data store
+   *
+   * @param options - Upload strategy options including file size and preferences
+   * @returns Negotiated strategy with reasoning and warnings
+   *
+   * @example
+   * ```typescript
+   * const result = negotiator.negotiateStrategy({
+   *   fileSize: 50_000_000, // 50MB
+   *   preferredStrategy: "parallel",
+   *   preferredChunkSize: 5_000_000, // 5MB
+   *   parallelUploads: 3
+   * });
+   *
+   * console.log(result.strategy); // "parallel"
+   * console.log(result.chunkSize); // 5_000_000
+   * console.log(result.parallelUploads); // 3
+   * console.log(result.reasoning); // ["Using preferred strategy: parallel", ...]
+   * console.log(result.warnings); // [] (no warnings)
+   * ```
+   */
+  negotiateStrategy(options: UploadStrategyOptions): NegotiatedStrategy {
+    const reasoning: string[] = [];
+    const warnings: string[] = [];
+
+    let strategy: UploadStrategy = "single";
+    let chunkSize =
+      options.preferredChunkSize ??
+      this.capabilities.optimalChunkSize ??
+      1024 * 1024;
+    let parallelUploads = options.parallelUploads ?? 1;
+
+    // Check if data store supports the preferred strategy
+    if (options.preferredStrategy) {
+      if (!this.validateUploadStrategy(options.preferredStrategy)) {
+        warnings.push(
+          `Preferred strategy '${options.preferredStrategy}' not supported by data store, falling back`
+        );
+      } else {
+        strategy = options.preferredStrategy;
+        reasoning.push(`Using preferred strategy: ${strategy}`);
+      }
+    }
+
+    // Automatic strategy selection based on capabilities and file size
+    if (
+      !options.preferredStrategy ||
+      !this.validateUploadStrategy(options.preferredStrategy)
+    ) {
+      if (
+        this.capabilities.supportsParallelUploads &&
+        options.fileSize > (options.minChunkSizeForParallel ?? 10 * 1024 * 1024)
+      ) {
+        strategy = "parallel";
+        reasoning.push(
+          `Selected parallel upload for large file (${options.fileSize} bytes)`
+        );
+      } else {
+        strategy = "single";
+        reasoning.push(
+          this.capabilities.supportsParallelUploads
+            ? `Selected single upload for small file (${options.fileSize} bytes)`
+            : "Selected single upload (parallel not supported by data store)"
+        );
+      }
+    }
+
+    // Validate and adjust chunk size based on data store constraints
+    if (
+      this.capabilities.minChunkSize &&
+      chunkSize < this.capabilities.minChunkSize
+    ) {
+      warnings.push(
+        `Chunk size ${chunkSize} below minimum ${this.capabilities.minChunkSize}, adjusting`
+      );
+      chunkSize = this.capabilities.minChunkSize;
+    }
+
+    if (
+      this.capabilities.maxChunkSize &&
+      chunkSize > this.capabilities.maxChunkSize
+    ) {
+      warnings.push(
+        `Chunk size ${chunkSize} above maximum ${this.capabilities.maxChunkSize}, adjusting`
+      );
+      chunkSize = this.capabilities.maxChunkSize;
+    }
+
+    // Validate parallel upload settings
+    if (strategy === "parallel") {
+      if (
+        this.capabilities.maxConcurrentUploads &&
+        parallelUploads > this.capabilities.maxConcurrentUploads
+      ) {
+        warnings.push(
+          `Parallel uploads ${parallelUploads} exceeds maximum ${this.capabilities.maxConcurrentUploads}, adjusting`
+        );
+        parallelUploads = this.capabilities.maxConcurrentUploads;
+      }
+
+      // Check if file would exceed max parts limit
+      if (this.capabilities.maxParts) {
+        const estimatedParts = Math.ceil(options.fileSize / chunkSize);
+        if (estimatedParts > this.capabilities.maxParts) {
+          const minChunkForParts = Math.ceil(
+            options.fileSize / this.capabilities.maxParts
+          );
+          warnings.push(
+            `Estimated parts ${estimatedParts} exceeds maximum ${this.capabilities.maxParts}, increasing chunk size`
+          );
+          chunkSize = Math.max(chunkSize, minChunkForParts);
+        }
+      }
+    }
+
+    // Final validation - ensure strategy is still valid after adjustments
+    if (!this.validateUploadStrategy(strategy)) {
+      warnings.push(
+        `Final strategy validation failed, falling back to single upload`
+      );
+      strategy = "single";
+      parallelUploads = 1;
+    }
+
+    // Add capability information to reasoning
+    reasoning.push(
+      `Data store capabilities: parallel=${this.capabilities.supportsParallelUploads}, concatenation=${this.capabilities.supportsConcatenation}, resumable=${this.capabilities.supportsResumableUploads}`
+    );
+
+    return {
+      strategy,
+      chunkSize,
+      parallelUploads: strategy === "parallel" ? parallelUploads : 1,
+      reasoning,
+      warnings,
+    };
+  }
+
+  /**
+   * Gets the data store capabilities used by this negotiator.
+   *
+   * @returns The data store capabilities and constraints
+   */
+  getDataStoreCapabilities(): DataStoreCapabilities {
+    return this.capabilities;
+  }
+
+  /**
+   * Validates upload strategy configuration against data store capabilities.
+   *
+   * This method checks if the provided configuration is valid for the current
+   * data store capabilities without performing the actual negotiation. It's
+   * useful for pre-validation before attempting to negotiate a strategy.
+   *
+   * @param options - Upload strategy options to validate
+   * @returns Validation result with validity flag and error messages
+   *
+   * @example
+   * ```typescript
+   * const validation = negotiator.validateConfiguration({
+   *   fileSize: 10_000_000,
+   *   preferredStrategy: "parallel",
+   *   preferredChunkSize: 1_000_000,
+   *   parallelUploads: 5
+   * });
+   *
+   * if (!validation.valid) {
+   *   console.log("Configuration errors:", validation.errors);
+   *   // Handle validation errors
+   * }
+   * ```
+   */
+  validateConfiguration(options: UploadStrategyOptions): {
+    valid: boolean;
+    errors: string[];
+  } {
+    const errors: string[] = [];
+
+    if (
+      options.preferredStrategy &&
+      !this.validateUploadStrategy(options.preferredStrategy)
+    ) {
+      errors.push(
+        `Preferred strategy '${options.preferredStrategy}' not supported by data store`
+      );
+    }
+
+    if (options.preferredChunkSize) {
+      if (
+        this.capabilities.minChunkSize &&
+        options.preferredChunkSize < this.capabilities.minChunkSize
+      ) {
+        errors.push(
+          `Chunk size ${options.preferredChunkSize} below data store minimum ${this.capabilities.minChunkSize}`
+        );
+      }
+      if (
+        this.capabilities.maxChunkSize &&
+        options.preferredChunkSize > this.capabilities.maxChunkSize
+      ) {
+        errors.push(
+          `Chunk size ${options.preferredChunkSize} above data store maximum ${this.capabilities.maxChunkSize}`
+        );
+      }
+    }
+
+    if (
+      options.parallelUploads &&
+      this.capabilities.maxConcurrentUploads &&
+      options.parallelUploads > this.capabilities.maxConcurrentUploads
+    ) {
+      errors.push(
+        `Parallel uploads ${options.parallelUploads} exceeds data store maximum ${this.capabilities.maxConcurrentUploads}`
+      );
+    }
+
+    return {
+      valid: errors.length === 0,
+      errors,
+    };
+  }
+}