@uploadista/core 0.0.2

package/src/upload/upload-url.ts

@@ -0,0 +1,173 @@
+import { Effect, Metric } from "effect";
+import { UploadistaError } from "../errors";
+
+/**
+ * Fetches a file from a remote URL.
+ *
+ * This function handles HTTP requests to remote URLs for file uploads,
+ * including proper error handling, metrics tracking, and observability.
+ *
+ * Features:
+ * - HTTP request with proper error handling
+ * - Effect tracing for performance monitoring
+ * - Metrics tracking for URL-based uploads
+ * - Structured logging for debugging
+ * - Response validation and error reporting
+ *
+ * @param url - The remote URL to fetch the file from
+ * @returns Effect that yields the Response object
+ *
+ * @example
+ * ```typescript
+ * // Fetch a file from URL
+ * const fetchEffect = fetchFile("https://example.com/image.jpg");
+ *
+ * // Run with error handling
+ * const response = await Effect.runPromise(
+ *   fetchEffect.pipe(
+ *     Effect.catchAll((error) =>
+ *       Effect.logError("Failed to fetch file").pipe(
+ *         Effect.andThen(Effect.fail(error))
+ *       )
+ *     )
+ *   )
+ * );
+ * ```
+ */
+export const fetchFile = (url: string) => {
+  return Effect.tryPromise({
+    try: async () => {
+      return await fetch(url);
+    },
+    catch: (error) => {
+      return UploadistaError.fromCode("UNKNOWN_ERROR", {
+        cause: error,
+      });
+    },
+  }).pipe(
+    // Add tracing span for URL fetch
+    Effect.withSpan("upload-fetch-url", {
+      attributes: {
+        "upload.url": url,
+        "upload.operation": "fetch",
+      },
+    }),
+    // Track URL fetch metrics
+    Effect.tap((response) =>
+      Effect.gen(function* () {
+        // Increment URL upload counter
+        yield* Metric.increment(
+          Metric.counter("upload_from_url_total", {
+            description: "Total number of URL-based uploads",
+          })
+        );
+
+        // Track success/failure
+        if (response.ok) {
+          yield* Metric.increment(
+            Metric.counter("upload_from_url_success_total", {
+              description: "Total number of successful URL-based uploads",
+            })
+          );
+        }
+      })
+    ),
+    // Add structured logging
+    Effect.tap((response) =>
+      Effect.logInfo("URL fetch completed").pipe(
+        Effect.annotateLogs({
+          "upload.url": url,
+          "response.status": response.status.toString(),
+          "response.ok": response.ok.toString(),
+          "response.content_length":
+            response.headers.get("content-length") ?? "unknown",
+        })
+      )
+    ),
+    // Handle errors with logging and metrics
+    Effect.tapError((error) =>
+      Effect.gen(function* () {
+        // Track failed URL upload
+        yield* Metric.increment(
+          Metric.counter("upload_from_url_failed_total", {
+            description: "Total number of failed URL-based uploads",
+          })
+        );
+
+        // Log error
+        yield* Effect.logError("URL fetch failed").pipe(
+          Effect.annotateLogs({
+            "upload.url": url,
+            error: String(error),
+          })
+        );
+      })
+    )
+  );
+};
+
+/**
+ * Converts a Response object to an ArrayBuffer.
+ *
+ * This function safely converts HTTP response data to binary format
+ * for processing and storage, with proper error handling and observability.
+ *
+ * Features:
+ * - Safe conversion from Response to ArrayBuffer
+ * - Effect tracing for performance monitoring
+ * - Structured logging for debugging
+ * - Error handling with proper UploadistaError types
+ *
+ * @param response - The HTTP Response object to convert
+ * @returns Effect that yields the ArrayBuffer data
+ *
+ * @example
+ * ```typescript
+ * // Convert response to buffer
+ * const bufferEffect = arrayBuffer(response);
+ *
+ * // Use in upload pipeline
+ * const buffer = await Effect.runPromise(
+ *   bufferEffect.pipe(
+ *     Effect.tap((buffer) =>
+ *       Effect.logInfo(`Buffer size: ${buffer.byteLength} bytes`)
+ *     )
+ *   )
+ * );
+ * ```
+ */
+export const arrayBuffer = (response: Response) => {
+  return Effect.tryPromise({
+    try: async () => {
+      return await response.arrayBuffer();
+    },
+    catch: (error) => {
+      return UploadistaError.fromCode("UNKNOWN_ERROR", {
+        cause: error,
+      });
+    },
+  }).pipe(
+    // Add tracing span for buffer conversion
+    Effect.withSpan("upload-convert-to-buffer", {
+      attributes: {
+        "upload.operation": "arrayBuffer",
+      },
+    }),
+    // Add structured logging
+    Effect.tap((buffer) =>
+      Effect.logDebug("Response converted to array buffer").pipe(
+        Effect.annotateLogs({
+          "buffer.size": buffer.byteLength.toString(),
+        })
+      )
+    ),
+    // Handle errors with logging
+    Effect.tapError((error) =>
+      Effect.logError("Failed to convert response to array buffer").pipe(
+        Effect.annotateLogs({
+          error: String(error),
+        })
+      )
+    )
+  );
+};

package/src/upload/write-to-store.ts

@@ -0,0 +1,211 @@
+import { Effect, Ref } from "effect";
+import { UploadistaError } from "../errors";
+import { StreamLimiterEffect } from "../streams/stream-limiter";
+import type { DataStore, UploadEvent, UploadFile } from "../types";
+import { type EventEmitter, UploadEventType } from "../types";
+import { convertToStream } from "./convert-to-stream";
+
+/**
+ * Configuration options for writing data to a data store.
+ *
+ * @property data - The stream of data to write
+ * @property upload - Upload file metadata
+ * @property dataStore - Target data store for writing
+ * @property maxFileSize - Maximum allowed file size in bytes
+ * @property controller - AbortController for cancellation
+ * @property eventEmitter - Event emitter for progress tracking
+ * @property uploadProgressInterval - Progress emission interval in milliseconds (default: 200)
+ */
+type WriteToStoreOptions = {
+  data: ReadableStream<Uint8Array>;
+  upload: UploadFile;
+  dataStore: DataStore<UploadFile>;
+  maxFileSize: number;
+  controller: AbortController;
+  eventEmitter: EventEmitter<UploadEvent>;
+  uploadProgressInterval?: number;
+};
+
+/**
+ * Writes data stream to a data store with progress tracking and size limits.
+ *
+ * This function handles the core data writing logic including:
+ * - Stream conversion and processing
+ * - File size validation and limiting
+ * - Progress tracking with throttled events
+ * - Abort signal handling for cancellation
+ * - Error handling and cleanup
+ *
+ * The function includes comprehensive observability with:
+ * - Effect tracing spans for performance monitoring
+ * - Structured logging for debugging and monitoring
+ * - Progress event emission with throttling
+ * - Error handling with proper UploadistaError types
+ *
+ * @param data - The stream of data to write to storage
+ * @param upload - Upload file metadata containing ID, offset, etc.
+ * @param dataStore - Target data store for writing the data
+ * @param maxFileSize - Maximum allowed file size in bytes
+ * @param controller - AbortController for handling cancellation
+ * @param eventEmitter - Event emitter for progress tracking
+ * @param uploadProgressInterval - Progress emission interval in milliseconds (default: 200)
+ * @returns Effect that yields the number of bytes written
+ *
+ * @example
+ * ```typescript
+ * // Write data to store with progress tracking
+ * const writeEffect = writeToStore({
+ *   data: fileStream,
+ *   upload: uploadMetadata,
+ *   dataStore: s3DataStore,
+ *   maxFileSize: 100_000_000, // 100MB
+ *   controller: abortController,
+ *   eventEmitter: progressEmitter,
+ *   uploadProgressInterval: 500 // Emit progress every 500ms
+ * });
+ *
+ * // Run with error handling
+ * const bytesWritten = await Effect.runPromise(
+ *   writeEffect.pipe(
+ *     Effect.catchAll((error) =>
+ *       Effect.logError("Failed to write to store").pipe(
+ *         Effect.andThen(Effect.fail(error))
+ *       )
+ *     )
+ *   )
+ * );
+ * ```
+ */
+export function writeToStore({
+  data,
+  upload,
+  dataStore,
+  maxFileSize,
+  controller,
+  eventEmitter,
+  uploadProgressInterval = 200,
+}: WriteToStoreOptions) {
+  return Effect.gen(function* () {
+    const stream = convertToStream(data);
+    // Check if already aborted
+    if (controller.signal.aborted) {
+      return yield* Effect.fail(UploadistaError.fromCode("ABORTED"));
+    }
+
+    // Create an AbortController to manage the stream pipeline
+    const abortController = new AbortController();
+    const { signal } = abortController;
+
+    // Set up abort handling
+    const onAbort = () => {
+      // stream.cancel();
+      abortController.abort();
+    };
+
+    controller.signal.addEventListener("abort", onAbort, { once: true });
+
+    return yield* Effect.acquireUseRelease(
+      Effect.sync(() => ({ signal, onAbort })),
+      ({ signal: _signal }) =>
+        Effect.gen(function* () {
+          // Create a ref to track the last progress emission time for throttling
+          const lastEmitTime = yield* Ref.make(0);
+
+          // Create the stream limiter
+          const limiter = StreamLimiterEffect.limit({
+            maxSize: maxFileSize,
+          });
+
+          // Pipe the data through the limiter
+          const limitedStream = limiter(stream);
+
+          // Write to the data store with progress tracking
+          const offset = yield* dataStore.write(
+            {
+              stream: limitedStream,
+              file_id: upload.id,
+              offset: upload.offset,
+            },
+            {
+              onProgress: (newOffset: number) => {
+                // Simple throttling using timestamp check
+                const now = Date.now();
+                Ref.get(lastEmitTime)
+                  .pipe(
+                    Effect.flatMap((lastTime) => {
+                      if (now - lastTime >= uploadProgressInterval) {
+                        return Effect.gen(function* () {
+                          yield* Ref.set(lastEmitTime, now);
+                          yield* eventEmitter.emit(upload.id, {
+                            type: UploadEventType.UPLOAD_PROGRESS,
+                            data: {
+                              id: upload.id,
+                              progress: newOffset,
+                              total: upload.size ?? 0,
+                            },
+                            flow: upload.flow,
+                          });
+                        });
+                      }
+                      return Effect.void;
+                    }),
+                    Effect.runPromise
+                  )
+                  .catch(() => {
+                    // Ignore errors during progress emission
+                  });
+              },
+            }
+          );
+
+          return offset;
+        }).pipe(
+          Effect.catchAll((error) => {
+            if (error instanceof Error && error.name === "AbortError") {
+              return Effect.fail(UploadistaError.fromCode("ABORTED"));
+            }
+            if (error instanceof UploadistaError) {
+              return Effect.fail(error);
+            }
+            return Effect.fail(
+              UploadistaError.fromCode("FILE_WRITE_ERROR", { cause: error })
+            );
+          })
+        ),
+      ({ onAbort }) =>
+        Effect.sync(() => {
+          controller.signal.removeEventListener("abort", onAbort);
+        })
+    );
+  }).pipe(
+    // Add tracing span for write operation
+    Effect.withSpan("upload-write-to-store", {
+      attributes: {
+        "upload.id": upload.id,
+        "upload.offset": upload.offset.toString(),
+        "upload.max_file_size": maxFileSize.toString(),
+        "upload.file_size": upload.size?.toString() ?? "0",
+      },
+    }),
+    // Add structured logging for write operation
+    Effect.tap((offset) =>
+      Effect.logDebug("Data written to store").pipe(
+        Effect.annotateLogs({
+          "upload.id": upload.id,
+          "write.offset": offset.toString(),
+          "write.bytes_written": (offset - upload.offset).toString(),
+        })
+      )
+    ),
+    // Handle errors with logging
+    Effect.tapError((error) =>
+      Effect.logError("Failed to write to store").pipe(
+        Effect.annotateLogs({
+          "upload.id": upload.id,
+          "upload.offset": upload.offset.toString(),
+          error: error instanceof UploadistaError ? error.code : String(error),
+        })
+      )
+    )
+  );
+}

package/src/utils/checksum.ts

@@ -0,0 +1,61 @@
+import { Effect } from "effect";
+import { UploadistaError } from "../errors/uploadista-error";
+
+/**
+ * Supported checksum algorithms
+ */
+const SUPPORTED_ALGORITHMS = ["sha256"] as const;
+export type ChecksumAlgorithm = (typeof SUPPORTED_ALGORITHMS)[number];
+
+/**
+ * Check if a checksum algorithm is supported
+ */
+export function isSupportedAlgorithm(algorithm: string): algorithm is ChecksumAlgorithm {
+  return SUPPORTED_ALGORITHMS.includes(algorithm as ChecksumAlgorithm);
+}
+
+/**
+ * Compute checksum of file bytes using the Web Crypto API.
+ * This works across all modern platforms: browsers, Node.js 15+, Deno, Bun, and Cloudflare Workers.
+ *
+ * @param bytes - File content as Uint8Array
+ * @param algorithm - Hashing algorithm to use (currently only 'sha256' is supported)
+ * @returns Effect that resolves to hex-encoded checksum string
+ */
+export function computeChecksum(
+  bytes: Uint8Array,
+  algorithm: string,
+): Effect.Effect<string, UploadistaError> {
+  return Effect.gen(function* () {
+    // Validate algorithm is supported
+    if (!isSupportedAlgorithm(algorithm)) {
+      return yield* UploadistaError.fromCode("UNSUPPORTED_CHECKSUM_ALGORITHM", {
+        body: `Checksum algorithm '${algorithm}' is not supported. Supported algorithms: ${SUPPORTED_ALGORITHMS.join(", ")}`,
+        details: { algorithm, supportedAlgorithms: SUPPORTED_ALGORITHMS },
+      }).toEffect();
+    }
+
+    // Map algorithm name to Web Crypto API algorithm name
+    const webCryptoAlgorithm = algorithm.toUpperCase().replace(/\d+/, "-$&"); // "sha256" -> "SHA-256"
+
+    // Compute hash using Web Crypto API (available in browsers, Node.js 15+, Deno, Bun, Cloudflare Workers)
+    // Pass Uint8Array directly - it's a valid BufferSource
+    const hashBuffer = yield* Effect.tryPromise({
+      try: () => crypto.subtle.digest(webCryptoAlgorithm, bytes as BufferSource),
+      catch: (error) =>
+        UploadistaError.fromCode("UNKNOWN_ERROR", {
+          body: `Failed to compute checksum: ${error instanceof Error ? error.message : "Unknown error"}`,
+          cause: error,
+          details: { algorithm },
+        }),
+    });
+
+    // Convert ArrayBuffer to hex string
+    const hashArray = Array.from(new Uint8Array(hashBuffer));
+    const hashHex = hashArray
+      .map((byte) => byte.toString(16).padStart(2, "0"))
+      .join("");
+
+    return hashHex;
+  });
+}

package/src/utils/debounce.test.ts

@@ -0,0 +1,126 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { debounce } from "./debounce";
+
+describe("debounce", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("should delay function execution by the specified time", () => {
+    const mockFn = vi.fn();
+    const debouncedFn = debounce(mockFn, 100);
+
+    debouncedFn();
+    expect(mockFn).not.toHaveBeenCalled();
+
+    vi.advanceTimersByTime(99);
+    expect(mockFn).not.toHaveBeenCalled();
+
+    vi.advanceTimersByTime(1);
+    expect(mockFn).toHaveBeenCalledOnce();
+  });
+
+  it("should reset delay on subsequent calls", () => {
+    const mockFn = vi.fn();
+    const debouncedFn = debounce(mockFn, 100);
+
+    debouncedFn();
+    vi.advanceTimersByTime(50);
+    debouncedFn(); // This should reset the timer
+
+    vi.advanceTimersByTime(99);
+    expect(mockFn).not.toHaveBeenCalled();
+
+    vi.advanceTimersByTime(1);
+    expect(mockFn).toHaveBeenCalledOnce();
+  });
+
+  it("should call function immediately with leading: true", () => {
+    const mockFn = vi.fn();
+    const debouncedFn = debounce(mockFn, 100, { leading: true });
+
+    debouncedFn();
+    expect(mockFn).toHaveBeenCalledOnce();
+
+    vi.advanceTimersByTime(100);
+    expect(mockFn).toHaveBeenCalledOnce(); // Should not be called again
+  });
+
+  it("should not call function on trailing edge with trailing: false", () => {
+    const mockFn = vi.fn();
+    const debouncedFn = debounce(mockFn, 100, { trailing: false });
+
+    debouncedFn();
+    vi.advanceTimersByTime(100);
+    expect(mockFn).not.toHaveBeenCalled();
+  });
+
+  it("should call function on both leading and trailing edges when both options are true", () => {
+    const mockFn = vi.fn();
+    const debouncedFn = debounce(mockFn, 100, {
+      leading: true,
+      trailing: true,
+    });
+
+    debouncedFn();
+    expect(mockFn).toHaveBeenCalledOnce();
+
+    debouncedFn(); // Second call within delay
+    vi.advanceTimersByTime(100);
+    expect(mockFn).toHaveBeenCalledTimes(2);
+  });
+
+  it("should preserve function arguments", () => {
+    const mockFn = vi.fn();
+    const debouncedFn = debounce(mockFn, 100);
+
+    debouncedFn("arg1", "arg2", 123);
+    vi.advanceTimersByTime(100);
+
+    expect(mockFn).toHaveBeenCalledWith("arg1", "arg2", 123);
+  });
+
+  it("should preserve function context (this)", () => {
+    const obj = {
+      value: 42,
+      fn: vi.fn(function (this: { value: number }) {
+        return this.value;
+      }),
+    };
+
+    const debouncedFn = debounce(obj.fn, 100);
+    debouncedFn.call(obj);
+
+    vi.advanceTimersByTime(100);
+    expect(obj.fn).toHaveBeenCalledOnce();
+  });
+
+  it("should handle multiple rapid calls correctly", () => {
+    const mockFn = vi.fn();
+    const debouncedFn = debounce(mockFn, 100);
+
+    // Make 5 rapid calls
+    for (let i = 0; i < 5; i++) {
+      debouncedFn(i);
+    }
+
+    vi.advanceTimersByTime(100);
+    expect(mockFn).toHaveBeenCalledOnce();
+    expect(mockFn).toHaveBeenCalledWith(4); // Should be called with the last argument
+  });
+
+  it("should use default options when none provided", () => {
+    const mockFn = vi.fn();
+    const debouncedFn = debounce(mockFn, 100);
+
+    debouncedFn();
+    expect(mockFn).not.toHaveBeenCalled(); // leading: false by default
+
+    vi.advanceTimersByTime(100);
+    expect(mockFn).toHaveBeenCalledOnce(); // trailing: true by default
+  });
+});

package/src/utils/debounce.ts

@@ -0,0 +1,89 @@
+type DebounceOptions = {
+  // Whether the first call to the debounced function is run immediately.
+  leading?: boolean;
+
+  // Whether the last call to the debounced function is run after delay
+  // milliseconds have elapsed since the last call.
+  trailing?: boolean;
+};
+
+const defaultDebounceOptions: DebounceOptions = {
+  leading: false,
+  trailing: true,
+};
+
+/**
+ * Returns `fn` wrapped by a function that delays invoking `fn` for `delay`
+ * milliseconds since the last call. Set `options.leading` to invoke `fn` on
+ * the leading edge of the delay, and/or set `options.trailing` to invoke `fn`
+ * on the trailing edge of the delay (true by default).
+ *
+ * Example for `debounce(fn, 30, {leading: true, trailing: true})`,
+ * where `fn` is called twice, with the second call made 20 ms after the first:
+ *
+ * Time:     0                   20                            50 (ms)
+ * Timeline: |----------------------------------------------------------------|
+ *           ^                   ^                             ^
+ *           |                   |                             |
+ *           | First call.       | Second call 20ms after the  | End of delay.
+ *           | (instant leading  | first.                      | (trailing edge
+ *           |  edge call)       |                             |  call)
+ *                               |-----------------------------|
+ *                               |  30 ms delay for debounce.  |
+ *
+ *
+ * Note that if both `options.leading` and `options.trailing` are true, `fn`
+ * will only be invoked on the trailing edge if the debounced function is called
+ * more than once during the delay.
+ *
+ * @param fn - Function to debounce.
+ * @param delay - Milliseconds to delay calling `fn` since the last call.
+ * @param debounceOptions - See `DebounceOptions` and `defaultDebounceOptions`.
+ * @returns A debounced `fn`.
+ */
+export function debounce<T, A extends unknown[]>(
+  fn: (this: T, ...args: A) => void,
+  delay: number,
+  debounceOptions: DebounceOptions = {},
+): (this: T, ...args: A) => void {
+  const options = { ...defaultDebounceOptions, ...debounceOptions };
+
+  let timer: ReturnType<typeof setTimeout> | undefined;
+
+  return function (this: T, ...args: A): void {
+    if (options.leading && !timer) {
+      // Leading edge.
+      // Call fn on the leading edge, when debouncing hasn't started yet.
+      console.log("leading");
+      fn.apply(this, args);
+
+      // Debounce the next call.
+      timer = setTimeout(() => {
+        timer = undefined;
+      }, delay);
+    } else {
+      // Trailing edge.
+      // Postpone calling fn until the delay has elapsed since the last call.
+      // Each call clears any previously delayed call and resets the delay, so
+      // the postponed call will always be the last one.
+      clearTimeout(timer);
+      timer = setTimeout(() => {
+        if (options.trailing) {
+          // Call fn on the trailing edge.
+          fn.apply(this, args);
+
+          if (options.leading) {
+            // Debounce next leading call since a trailing call was just made.
+            setTimeout(() => {
+              timer = undefined;
+            }, delay);
+          }
+        } else {
+          // No trailing call. Since the delay has elapsed since the last call,
+          // immediately reset the debouncing delay.
+          timer = undefined;
+        }
+      }, delay);
+    }
+  };
+}

package/src/utils/generate-id.ts

@@ -0,0 +1,35 @@
+import { Context, Effect, Layer } from "effect";
+
+export type GenerateIdShape = {
+  generateId: () => Effect.Effect<string>;
+};
+
+// Declaring a tag for a service that generates random id
+export class GenerateId extends Context.Tag("UploadistaGenerateIdService")<
+  GenerateId,
+  { readonly generateId: () => Effect.Effect<string> }
+>() {}
+
+/**
+ * Effect-based ID generation services
+ */
+export const GenerateIdService = GenerateId.Service;
+
+/**
+ * Generates a random UUID using Effect
+ * @returns Effect that produces a random UUID string
+ */
+export const GenerateIdRandom = GenerateId.of({
+  generateId: () => Effect.succeed(crypto.randomUUID()),
+});
+
+export const GenerateIdLive = Layer.succeed(GenerateId, GenerateIdRandom);
+
+/**
+ * Generates a timestamp-based ID using Effect
+ * @returns Effect that produces a timestamp-based ID
+ */
+export const GenerateIdTimestamp = GenerateId.of({
+  generateId: () =>
+    Effect.succeed(`${Date.now()}-${Math.random().toString(36).slice(2, 11)}`),
+});