@uploadista/core 0.0.2

package/src/streams/stream-limiter.test.ts

@@ -0,0 +1,150 @@
+import { describe, expect, it, vi } from "vitest";
+import { UploadistaError } from "../errors/uploadista-error";
+import { streamLimiter } from "./stream-limiter";
+
+// Helper function to convert stream to array
+async function streamToArray(
+  stream: ReadableStream<Uint8Array>,
+): Promise<Uint8Array[]> {
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  return chunks;
+}
+
+// Helper function to write data to stream
+async function writeToStream(
+  stream: WritableStream<Uint8Array>,
+  data: Uint8Array[],
+): Promise<void> {
+  const writer = stream.getWriter();
+
+  try {
+    for (const chunk of data) {
+      await writer.write(chunk);
+    }
+  } finally {
+    await writer.close();
+  }
+}
+
+describe("streamLimiter", () => {
+  it("should create a TransformStream", () => {
+    const limiter = streamLimiter({ maxSize: 1024 });
+    expect(limiter).toBeInstanceOf(TransformStream);
+  });
+
+  it("should pass through data within size limit", async () => {
+    const limiter = streamLimiter({ maxSize: 1024 });
+    const testData = [new Uint8Array([1, 2, 3, 4, 5])];
+
+    const writePromise = writeToStream(limiter.writable, testData);
+    const readPromise = streamToArray(limiter.readable);
+
+    const [, result] = await Promise.all([writePromise, readPromise]);
+
+    expect(result).toHaveLength(1);
+    expect(result[0]).toEqual(testData[0]);
+  });
+
+  it("should call onData callback for each chunk", async () => {
+    const onDataCallback = vi.fn();
+    const limiter = streamLimiter({ maxSize: 1024, onData: onDataCallback });
+
+    const testData = [new Uint8Array([1, 2, 3])];
+
+    const writePromise = writeToStream(limiter.writable, testData);
+    const readPromise = streamToArray(limiter.readable);
+
+    await Promise.all([writePromise, readPromise]);
+
+    expect(onDataCallback).toHaveBeenCalledOnce();
+    expect(onDataCallback).toHaveBeenCalledWith(testData[0].byteLength);
+  });
+
+  it("should work without onData callback", async () => {
+    const limiter = streamLimiter({ maxSize: 1024 });
+    const testData = [new Uint8Array([1, 2, 3])];
+
+    const writePromise = writeToStream(limiter.writable, testData);
+    const readPromise = streamToArray(limiter.readable);
+
+    const [, result] = await Promise.all([writePromise, readPromise]);
+
+    expect(result).toHaveLength(1);
+    expect(result[0]).toEqual(testData[0]);
+  });
+
+  it("should handle zero-sized chunks", async () => {
+    const onDataCallback = vi.fn();
+    const limiter = streamLimiter({ maxSize: 1024, onData: onDataCallback });
+
+    const testData = [new Uint8Array(0)];
+
+    const writePromise = writeToStream(limiter.writable, testData);
+    const readPromise = streamToArray(limiter.readable);
+
+    const [, result] = await Promise.all([writePromise, readPromise]);
+
+    expect(onDataCallback).toHaveBeenCalledWith(testData[0].byteLength);
+    expect(result).toHaveLength(1);
+    expect(result[0]).toEqual(testData[0]);
+  });
+
+  it("should error when size limit is exceeded", async () => {
+    const limiter = streamLimiter({ maxSize: 10 });
+    const testData = [new Uint8Array(15).fill(1)]; // Exceeds limit of 10
+
+    const writePromise = writeToStream(limiter.writable, testData);
+    const readPromise = streamToArray(limiter.readable);
+
+    await expect(
+      Promise.all([writePromise, readPromise]),
+    ).rejects.toBeInstanceOf(UploadistaError);
+  });
+
+  it("should track cumulative size across multiple chunks", async () => {
+    const limiter = streamLimiter({ maxSize: 10 });
+
+    // First chunk (5 bytes) + second chunk (3 bytes) = 8 bytes (within limit)
+    // Third chunk (3 bytes) would make total 11 bytes (exceeds limit)
+    const testData = [
+      new Uint8Array(5).fill(1),
+      new Uint8Array(3).fill(2),
+      new Uint8Array(3).fill(3),
+    ];
+
+    const writePromise = writeToStream(limiter.writable, testData);
+    const readPromise = streamToArray(limiter.readable);
+
+    await expect(
+      Promise.all([writePromise, readPromise]),
+    ).rejects.toBeInstanceOf(UploadistaError);
+  });
+
+  it("should error with correct UploadistaError code", async () => {
+    const limiter = streamLimiter({ maxSize: 5 });
+    const testData = [new Uint8Array(10)];
+
+    try {
+      await Promise.all([
+        writeToStream(limiter.writable, testData),
+        streamToArray(limiter.readable),
+      ]);
+      expect.fail("Should have thrown an error");
+    } catch (error) {
+      expect(error).toBeInstanceOf(UploadistaError);
+      expect((error as UploadistaError).code).toBe("ERR_MAX_SIZE_EXCEEDED");
+    }
+  });
+});

package/src/streams/stream-limiter.ts

@@ -0,0 +1,75 @@
+import { Effect, Ref, Stream } from "effect";
+import { UploadistaError } from "../errors/uploadista-error";
+
+type StreamLimiterOptions = {
+  maxSize: number;
+  onData?: (chunkSize: number) => void;
+};
+
+export function streamLimiter({
+  maxSize,
+  onData,
+}: StreamLimiterOptions): TransformStream {
+  let currentSize = 0;
+
+  return new TransformStream({
+    transform(chunk, controller) {
+      currentSize += chunk.length;
+
+      onData?.(chunk.byteLength);
+
+      if (currentSize > maxSize) {
+        controller.error(UploadistaError.fromCode("ERR_MAX_SIZE_EXCEEDED"));
+      } else {
+        controller.enqueue(chunk);
+      }
+    },
+  });
+}
+
+/**
+ * Effect-based stream limiter that restricts stream size
+ */
+export const StreamLimiterEffect = {
+  /**
+   * Creates an Effect-based stream limiter
+   * @param maxSize - Maximum allowed stream size in bytes
+   * @param onData - Optional callback for data progress tracking
+   * @returns Effect stream transformation that enforces size limits
+   */
+  limit:
+    ({ maxSize, onData }: StreamLimiterOptions) =>
+    <A>(stream: Stream.Stream<A, UploadistaError>) => {
+      return Effect.gen(function* () {
+        const currentSize = yield* Ref.make(0);
+
+        return stream.pipe(
+          Stream.mapEffect((chunk) =>
+            Effect.gen(function* () {
+              const chunkSize =
+                chunk instanceof Uint8Array ? chunk.byteLength : 0;
+              yield* Ref.update(currentSize, (size) => size + chunkSize);
+
+              onData?.(chunkSize);
+              const size = yield* Ref.get(currentSize);
+              if (size > maxSize) {
+                yield* UploadistaError.fromCode(
+                  "ERR_MAX_SIZE_EXCEEDED",
+                ).toEffect();
+              }
+
+              return chunk;
+            }),
+          ),
+        );
+      }).pipe(Stream.unwrap);
+    },
+
+  /**
+   * Creates a legacy TransformStream-based limiter
+   * @param options - Stream limiter options
+   * @returns TransformStream that enforces size limits
+   */
+  createTransformStream: (options: StreamLimiterOptions): TransformStream =>
+    streamLimiter(options),
+};

package/src/types/data-store.ts

@@ -0,0 +1,427 @@
+import { Context, Effect, Layer, type Stream } from "effect";
+import { UploadistaError } from "../errors/uploadista-error";
+import type { UploadFileKVStore } from "./kv-store";
+import type { UploadFile } from "./upload-file";
+
+/**
+ * Options for writing data to a DataStore.
+ *
+ * @property file_id - Unique identifier for the file being written
+ * @property stream - Stream of byte chunks to write to storage
+ * @property offset - Byte offset where writing should begin (for resumable uploads)
+ */
+export type DataStoreWriteOptions = {
+  file_id: string;
+  stream: Stream.Stream<Uint8Array, UploadistaError>;
+  offset: number;
+};
+
+/**
+ * Upload strategy type indicating how chunks are uploaded.
+ *
+ * - `single`: Upload file in a single request (traditional upload)
+ * - `parallel`: Upload file chunks in parallel (for large files)
+ */
+export type UploadStrategy = "single" | "parallel";
+
+/**
+ * Capabilities and constraints of a DataStore implementation.
+ *
+ * This type describes what features a storage backend supports and what
+ * limitations it has. Use this to determine the optimal upload strategy
+ * and validate client requests.
+ *
+ * @property supportsParallelUploads - Can upload chunks in parallel (e.g., S3 multipart)
+ * @property supportsConcatenation - Can concatenate multiple uploads into one file
+ * @property supportsDeferredLength - Can start upload without knowing final size
+ * @property supportsResumableUploads - Can resume interrupted uploads from last offset
+ * @property supportsTransactionalUploads - Guarantees atomic upload success/failure
+ * @property maxConcurrentUploads - Maximum parallel upload parts (if parallel supported)
+ * @property minChunkSize - Minimum size in bytes for each chunk (except last)
+ * @property maxChunkSize - Maximum size in bytes for each chunk
+ * @property maxParts - Maximum number of parts in a multipart upload
+ * @property optimalChunkSize - Recommended chunk size for best performance
+ * @property requiresOrderedChunks - Must receive chunks in sequential order
+ * @property requiresMimeTypeValidation - Validates file MIME type matches declaration
+ * @property maxValidationSize - Maximum file size for MIME type validation
+ *
+ * @example
+ * ```typescript
+ * const capabilities = dataStore.getCapabilities();
+ *
+ * if (capabilities.supportsParallelUploads && fileSize > 10_000_000) {
+ *   // Use parallel upload for large files
+ *   const chunkSize = capabilities.optimalChunkSize || 5_242_880; // 5MB default
+ *   uploadInParallel(file, chunkSize);
+ * } else {
+ *   // Use single upload
+ *   uploadAsSingleChunk(file);
+ * }
+ * ```
+ */
+export type DataStoreCapabilities = {
+  supportsParallelUploads: boolean;
+  supportsConcatenation: boolean;
+  supportsDeferredLength: boolean;
+  supportsResumableUploads: boolean;
+  supportsTransactionalUploads: boolean;
+  maxConcurrentUploads?: number;
+  minChunkSize?: number;
+  maxChunkSize?: number;
+  maxParts?: number;
+  optimalChunkSize?: number;
+  requiresOrderedChunks: boolean;
+  requiresMimeTypeValidation?: boolean;
+  maxValidationSize?: number;
+};
+
+/**
+ * Core interface for all storage backend implementations.
+ *
+ * DataStore abstracts file storage operations across different backends
+ * (S3, Azure Blob, GCS, local filesystem, etc.). All storage adapters
+ * must implement this interface.
+ *
+ * @template TData - The data type stored (typically UploadFile)
+ *
+ * @property bucket - Optional storage bucket or container name
+ * @property path - Optional base path prefix for all stored files
+ * @property create - Creates a new file record in storage
+ * @property remove - Deletes a file from storage
+ * @property read - Reads complete file contents as bytes
+ * @property write - Writes data stream to storage at specified offset
+ * @property deleteExpired - Optional cleanup of expired files
+ * @property getCapabilities - Returns storage backend capabilities
+ * @property validateUploadStrategy - Validates if strategy is supported
+ *
+ * @example
+ * ```typescript
+ * // Implement a custom DataStore
+ * const myDataStore: DataStore<UploadFile> = {
+ *   bucket: "my-uploads",
+ *   path: "files/",
+ *
+ *   create: (file) => Effect.gen(function* () {
+ *     // Store file metadata
+ *     yield* saveMetadata(file);
+ *     return file;
+ *   }),
+ *
+ *   write: ({ file_id, stream, offset }, { onProgress }) => Effect.gen(function* () {
+ *     // Write chunks to storage
+ *     let bytesWritten = offset;
+ *     yield* Stream.runForEach(stream, (chunk) => Effect.sync(() => {
+ *       writeChunk(file_id, chunk, bytesWritten);
+ *       bytesWritten += chunk.byteLength;
+ *       onProgress?.(chunk.byteLength);
+ *     }));
+ *     return bytesWritten;
+ *   }),
+ *
+ *   read: (file_id) => Effect.gen(function* () {
+ *     // Read complete file
+ *     const data = yield* readFromStorage(file_id);
+ *     return data;
+ *   }),
+ *
+ *   remove: (file_id) => Effect.gen(function* () {
+ *     yield* deleteFromStorage(file_id);
+ *   }),
+ *
+ *   getCapabilities: () => ({
+ *     supportsParallelUploads: true,
+ *     supportsConcatenation: false,
+ *     supportsDeferredLength: true,
+ *     supportsResumableUploads: true,
+ *     supportsTransactionalUploads: false,
+ *     maxConcurrentUploads: 10,
+ *     optimalChunkSize: 5_242_880, // 5MB
+ *     requiresOrderedChunks: false,
+ *   }),
+ *
+ *   validateUploadStrategy: (strategy) =>
+ *     Effect.succeed(strategy === "parallel" || strategy === "single"),
+ * };
+ * ```
+ */
+export type DataStore<TData = unknown> = {
+  readonly bucket?: string;
+  readonly path?: string;
+  readonly create: (file: TData) => Effect.Effect<TData, UploadistaError>;
+  readonly remove: (file_id: string) => Effect.Effect<void, UploadistaError>;
+  readonly read: (
+    file_id: string,
+  ) => Effect.Effect<Uint8Array, UploadistaError>;
+  readonly write: (
+    options: DataStoreWriteOptions,
+    dependencies: {
+      onProgress?: (chunkSize: number) => void;
+    },
+  ) => Effect.Effect<number, UploadistaError>;
+  readonly deleteExpired?: Effect.Effect<number, UploadistaError>;
+  readonly getCapabilities: () => DataStoreCapabilities;
+  readonly validateUploadStrategy: (
+    strategy: UploadStrategy,
+  ) => Effect.Effect<boolean, never>;
+};
+
+/**
+ * Effect-TS context tag for UploadFile DataStore.
+ *
+ * Use this tag to access the primary DataStore in an Effect context.
+ * This is the standard storage backend for uploaded files.
+ *
+ * @example
+ * ```typescript
+ * const uploadEffect = Effect.gen(function* () {
+ *   const dataStore = yield* UploadFileDataStore;
+ *   const file = yield* dataStore.create(uploadFile);
+ *   return file;
+ * });
+ * ```
+ */
+export class UploadFileDataStore extends Context.Tag("UploadFileDataStore")<
+  UploadFileDataStore,
+  DataStore<UploadFile>
+>() {}
+
+/**
+ * Effect-TS context tag for buffered/temporary DataStore.
+ *
+ * This is an optional storage backend used for temporary or intermediate files
+ * during flow processing. Not all implementations provide a buffered store.
+ *
+ * @example
+ * ```typescript
+ * const processEffect = Effect.gen(function* () {
+ *   const bufferedStore = yield* BufferedUploadFileDataStore;
+ *   // Store intermediate processing results
+ *   const tempFile = yield* bufferedStore.create(intermediateFile);
+ *   return tempFile;
+ * });
+ * ```
+ */
+export class BufferedUploadFileDataStore extends Context.Tag(
+  "BufferedUploadFileDataStore",
+)<BufferedUploadFileDataStore, DataStore<UploadFile>>() {}
+
+/**
+ * Service interface for managing multiple DataStore instances.
+ *
+ * This allows routing files to different storage backends based on
+ * storageId (e.g., different S3 buckets, Azure containers, or storage tiers).
+ *
+ * @property getDataStore - Retrieves the appropriate DataStore for a given storage ID
+ * @property bufferedDataStore - Optional temporary storage for intermediate files
+ */
+export type UploadFileDataStoresShape = {
+  getDataStore: (
+    storageId: string,
+    clientId: string | null,
+  ) => Effect.Effect<DataStore<UploadFile>, UploadistaError>;
+  bufferedDataStore: Effect.Effect<
+    DataStore<UploadFile> | undefined,
+    UploadistaError
+  >;
+};
+
+/**
+ * Effect-TS context tag for the DataStore routing service.
+ *
+ * Provides access to multiple DataStore instances with routing logic.
+ *
+ * @example
+ * ```typescript
+ * const uploadEffect = Effect.gen(function* () {
+ *   const dataStores = yield* UploadFileDataStores;
+ *   // Route to specific storage based on storageId
+ *   const dataStore = yield* dataStores.getDataStore("s3-production", clientId);
+ *   const file = yield* dataStore.create(uploadFile);
+ *   return file;
+ * });
+ * ```
+ */
+export class UploadFileDataStores extends Context.Tag("UploadFileDataStores")<
+  UploadFileDataStores,
+  UploadFileDataStoresShape
+>() {}
+
+/**
+ * Simplified DataStore configuration for easy setup.
+ *
+ * This type allows flexible configuration:
+ * - Single DataStore instance
+ * - Multiple named stores with routing
+ * - Effect that resolves to a DataStore
+ * - Pre-built Effect Layer
+ *
+ * @example
+ * ```typescript
+ * // Single store
+ * const config: DataStoreConfig = s3DataStore;
+ *
+ * // Multiple stores with routing
+ * const config: DataStoreConfig = {
+ *   stores: {
+ *     "s3-prod": s3ProdStore,
+ *     "s3-dev": s3DevStore,
+ *     "local": localFileStore,
+ *   },
+ *   default: "s3-prod"
+ * };
+ *
+ * // Effect that creates a store
+ * const config: DataStoreConfig = Effect.gen(function* () {
+ *   const kvStore = yield* UploadFileKVStore;
+ *   return createS3Store(kvStore);
+ * });
+ *
+ * // Pre-built Layer
+ * const config: DataStoreConfig = Layer.succeed(UploadFileDataStores, {...});
+ * ```
+ */
+export type DataStoreConfig =
+  | DataStore<UploadFile>
+  | Effect.Effect<DataStore<UploadFile>, never, UploadFileKVStore>
+  | {
+      stores: Record<
+        string,
+        | DataStore<UploadFile>
+        | Effect.Effect<DataStore<UploadFile>, never, UploadFileKVStore>
+      >;
+      default?: string;
+    }
+  | Layer.Layer<UploadFileDataStores, never, UploadFileKVStore>;
+
+/**
+ * Type guard to check if a value is a DataStore instance.
+ *
+ * @param config - The value to check
+ * @returns True if the value is a DataStore
+ *
+ * @example
+ * ```typescript
+ * if (isDataStore(config)) {
+ *   const capabilities = config.getCapabilities();
+ * }
+ * ```
+ */
+export const isDataStore = (
+  config: DataStoreConfig,
+): config is DataStore<UploadFile> => {
+  return "create" in config && "write" in config;
+};
+
+/**
+ * Creates an Effect Layer from simplified DataStoreConfig.
+ *
+ * This function converts any DataStoreConfig format into a proper Effect Layer
+ * that can be provided to the UploadFileDataStores context tag.
+ *
+ * It handles:
+ * - Single DataStore: Wraps in a Layer that always returns that store
+ * - Multiple stores: Creates routing logic with optional default
+ * - Effect<DataStore>: Executes the Effect and wraps the result
+ * - Layer: Returns as-is
+ *
+ * @param config - The DataStore configuration
+ * @returns A Layer that provides UploadFileDataStores service
+ *
+ * @example
+ * ```typescript
+ * // Create from single store
+ * const layer = await createDataStoreLayer(s3DataStore);
+ *
+ * // Create from multiple stores
+ * const layer = await createDataStoreLayer({
+ *   stores: {
+ *     "production": s3Store,
+ *     "development": localStore,
+ *   },
+ *   default: "development"
+ * });
+ *
+ * // Use the layer
+ * const program = Effect.gen(function* () {
+ *   const stores = yield* UploadFileDataStores;
+ *   const store = yield* stores.getDataStore("production", null);
+ *   return store;
+ * }).pipe(Effect.provide(layer));
+ * ```
+ */
+export const createDataStoreLayer = async (
+  config: DataStoreConfig,
+): Promise<Layer.Layer<UploadFileDataStores, never, UploadFileKVStore>> => {
+  // Already a Layer, return as-is
+  if (Layer.isLayer(config)) {
+    return config as Layer.Layer<
+      UploadFileDataStores,
+      never,
+      UploadFileKVStore
+    >;
+  }
+
+  // Check if it's an Effect
+  if (Effect.isEffect(config)) {
+    return Layer.effect(
+      UploadFileDataStores,
+      Effect.gen(function* () {
+        const dataStore = config as Effect.Effect<
+          DataStore<UploadFile>,
+          never,
+          UploadFileKVStore
+        >;
+        const resolvedStore = yield* dataStore;
+        return {
+          getDataStore: (_storageId: string) => Effect.succeed(resolvedStore),
+          bufferedDataStore: Effect.succeed(undefined),
+        };
+      }),
+    );
+  }
+
+  // Single store (most common case)
+  if (isDataStore(config)) {
+    const store = config as DataStore<UploadFile>;
+    return Layer.succeed(UploadFileDataStores, {
+      getDataStore: (_storageId: string) => Effect.succeed(store),
+      bufferedDataStore: Effect.succeed(undefined),
+    });
+  }
+
+  // Multiple stores with routing
+  const multiConfig = config as {
+    stores: Record<
+      string,
+      DataStore<UploadFile> | Effect.Effect<DataStore<UploadFile>>
+    >;
+    default?: string;
+  };
+
+  const defaultKey = multiConfig.default || Object.keys(multiConfig.stores)[0];
+
+  // Resolve any Effects in the stores
+  const resolvedStores: Record<string, DataStore<UploadFile>> = {};
+  for (const [key, storeOrEffect] of Object.entries(multiConfig.stores)) {
+    if ("pipe" in storeOrEffect && !("create" in storeOrEffect)) {
+      resolvedStores[key] = await Effect.runPromise(
+        storeOrEffect as Effect.Effect<DataStore<UploadFile>>,
+      );
+    } else {
+      resolvedStores[key] = storeOrEffect as DataStore<UploadFile>;
+    }
+  }
+
+  return Layer.succeed(UploadFileDataStores, {
+    getDataStore: (storageId: string) => {
+      const store =
+        resolvedStores[storageId] ||
+        (defaultKey ? resolvedStores[defaultKey] : undefined);
+      if (store) {
+        return Effect.succeed(store);
+      }
+      return Effect.fail(UploadistaError.fromCode("FILE_NOT_FOUND"));
+    },
+    bufferedDataStore: Effect.succeed(undefined),
+  });
+};

package/src/types/event-broadcaster.ts

@@ -0,0 +1,39 @@
+import { Context, type Effect } from "effect";
+import type { UploadistaError } from "../errors";
+
+/**
+ * Event broadcaster interface for pub/sub messaging across distributed instances.
+ * Used by WebSocketManager to broadcast upload events to all connected instances.
+ */
+export interface EventBroadcaster {
+  /**
+   * Publish a message to a channel
+   */
+  readonly publish: (
+    channel: string,
+    message: string,
+  ) => Effect.Effect<void, UploadistaError>;
+
+  /**
+   * Subscribe to messages on a channel
+   */
+  readonly subscribe: (
+    channel: string,
+    handler: (message: string) => void,
+  ) => Effect.Effect<void, UploadistaError>;
+
+  /**
+   * Unsubscribe from a channel (optional - not all implementations may support)
+   */
+  readonly unsubscribe?: (
+    channel: string,
+  ) => Effect.Effect<void, UploadistaError>;
+}
+
+/**
+ * Context tag for EventBroadcaster service
+ */
+export class EventBroadcasterService extends Context.Tag("EventBroadcaster")<
+  EventBroadcasterService,
+  EventBroadcaster
+>() {}