@uploadista/core 0.0.2

package/dist/upload/upload-strategy-negotiator.js

@@ -0,0 +1,217 @@
+/**
+ * Negotiates the optimal upload strategy based on data store capabilities and file characteristics.
+ *
+ * This class analyzes data store capabilities, file size, and user preferences to determine
+ * the best upload strategy (single, parallel, resumable) and optimal parameters like chunk size
+ * and parallel connection count.
+ *
+ * The negotiator considers:
+ * - Data store capabilities (parallel uploads, resumable uploads, concatenation)
+ * - File size and chunk size constraints
+ * - User preferences and requirements
+ * - Performance optimization opportunities
+ *
+ * @example
+ * ```typescript
+ * // Create negotiator for S3 data store
+ * const negotiator = new UploadStrategyNegotiator(
+ *   s3Capabilities,
+ *   (strategy) => s3Capabilities.supportsStrategy(strategy)
+ * );
+ *
+ * // Negotiate strategy for large file
+ * const result = negotiator.negotiateStrategy({
+ *   fileSize: 100_000_000, // 100MB
+ *   preferredStrategy: "parallel",
+ *   preferredChunkSize: 5_000_000, // 5MB chunks
+ *   parallelUploads: 4
+ * });
+ *
+ * console.log(result.strategy); // "parallel"
+ * console.log(result.chunkSize); // 5_000_000
+ * console.log(result.reasoning); // ["Using preferred strategy: parallel", ...]
+ * ```
+ */
+export class UploadStrategyNegotiator {
+    capabilities;
+    validateUploadStrategy;
+    /**
+     * Creates a new upload strategy negotiator.
+     *
+     * @param capabilities - Data store capabilities and constraints
+     * @param validateUploadStrategy - Function to validate if a strategy is supported
+     */
+    constructor(capabilities, validateUploadStrategy) {
+        this.capabilities = capabilities;
+        this.validateUploadStrategy = validateUploadStrategy;
+    }
+    /**
+     * Negotiates the optimal upload strategy based on options and data store capabilities.
+     *
+     * This method analyzes the provided options and data store capabilities to determine
+     * the best upload strategy, chunk size, and parallel upload settings. It considers
+     * user preferences, file size, and data store constraints to make optimal decisions.
+     *
+     * The negotiation process:
+     * 1. Validates preferred strategy against data store capabilities
+     * 2. Automatically selects strategy based on file size and capabilities
+     * 3. Adjusts chunk size to fit within data store constraints
+     * 4. Validates parallel upload settings
+     * 5. Ensures final strategy is supported by the data store
+     *
+     * @param options - Upload strategy options including file size and preferences
+     * @returns Negotiated strategy with reasoning and warnings
+     *
+     * @example
+     * ```typescript
+     * const result = negotiator.negotiateStrategy({
+     *   fileSize: 50_000_000, // 50MB
+     *   preferredStrategy: "parallel",
+     *   preferredChunkSize: 5_000_000, // 5MB
+     *   parallelUploads: 3
+     * });
+     *
+     * console.log(result.strategy); // "parallel"
+     * console.log(result.chunkSize); // 5_000_000
+     * console.log(result.parallelUploads); // 3
+     * console.log(result.reasoning); // ["Using preferred strategy: parallel", ...]
+     * console.log(result.warnings); // [] (no warnings)
+     * ```
+     */
+    negotiateStrategy(options) {
+        const reasoning = [];
+        const warnings = [];
+        let strategy = "single";
+        let chunkSize = options.preferredChunkSize ??
+            this.capabilities.optimalChunkSize ??
+            1024 * 1024;
+        let parallelUploads = options.parallelUploads ?? 1;
+        // Check if data store supports the preferred strategy
+        if (options.preferredStrategy) {
+            if (!this.validateUploadStrategy(options.preferredStrategy)) {
+                warnings.push(`Preferred strategy '${options.preferredStrategy}' not supported by data store, falling back`);
+            }
+            else {
+                strategy = options.preferredStrategy;
+                reasoning.push(`Using preferred strategy: ${strategy}`);
+            }
+        }
+        // Automatic strategy selection based on capabilities and file size
+        if (!options.preferredStrategy ||
+            !this.validateUploadStrategy(options.preferredStrategy)) {
+            if (this.capabilities.supportsParallelUploads &&
+                options.fileSize > (options.minChunkSizeForParallel ?? 10 * 1024 * 1024)) {
+                strategy = "parallel";
+                reasoning.push(`Selected parallel upload for large file (${options.fileSize} bytes)`);
+            }
+            else {
+                strategy = "single";
+                reasoning.push(this.capabilities.supportsParallelUploads
+                    ? `Selected single upload for small file (${options.fileSize} bytes)`
+                    : "Selected single upload (parallel not supported by data store)");
+            }
+        }
+        // Validate and adjust chunk size based on data store constraints
+        if (this.capabilities.minChunkSize &&
+            chunkSize < this.capabilities.minChunkSize) {
+            warnings.push(`Chunk size ${chunkSize} below minimum ${this.capabilities.minChunkSize}, adjusting`);
+            chunkSize = this.capabilities.minChunkSize;
+        }
+        if (this.capabilities.maxChunkSize &&
+            chunkSize > this.capabilities.maxChunkSize) {
+            warnings.push(`Chunk size ${chunkSize} above maximum ${this.capabilities.maxChunkSize}, adjusting`);
+            chunkSize = this.capabilities.maxChunkSize;
+        }
+        // Validate parallel upload settings
+        if (strategy === "parallel") {
+            if (this.capabilities.maxConcurrentUploads &&
+                parallelUploads > this.capabilities.maxConcurrentUploads) {
+                warnings.push(`Parallel uploads ${parallelUploads} exceeds maximum ${this.capabilities.maxConcurrentUploads}, adjusting`);
+                parallelUploads = this.capabilities.maxConcurrentUploads;
+            }
+            // Check if file would exceed max parts limit
+            if (this.capabilities.maxParts) {
+                const estimatedParts = Math.ceil(options.fileSize / chunkSize);
+                if (estimatedParts > this.capabilities.maxParts) {
+                    const minChunkForParts = Math.ceil(options.fileSize / this.capabilities.maxParts);
+                    warnings.push(`Estimated parts ${estimatedParts} exceeds maximum ${this.capabilities.maxParts}, increasing chunk size`);
+                    chunkSize = Math.max(chunkSize, minChunkForParts);
+                }
+            }
+        }
+        // Final validation - ensure strategy is still valid after adjustments
+        if (!this.validateUploadStrategy(strategy)) {
+            warnings.push(`Final strategy validation failed, falling back to single upload`);
+            strategy = "single";
+            parallelUploads = 1;
+        }
+        // Add capability information to reasoning
+        reasoning.push(`Data store capabilities: parallel=${this.capabilities.supportsParallelUploads}, concatenation=${this.capabilities.supportsConcatenation}, resumable=${this.capabilities.supportsResumableUploads}`);
+        return {
+            strategy,
+            chunkSize,
+            parallelUploads: strategy === "parallel" ? parallelUploads : 1,
+            reasoning,
+            warnings,
+        };
+    }
+    /**
+     * Gets the data store capabilities used by this negotiator.
+     *
+     * @returns The data store capabilities and constraints
+     */
+    getDataStoreCapabilities() {
+        return this.capabilities;
+    }
+    /**
+     * Validates upload strategy configuration against data store capabilities.
+     *
+     * This method checks if the provided configuration is valid for the current
+     * data store capabilities without performing the actual negotiation. It's
+     * useful for pre-validation before attempting to negotiate a strategy.
+     *
+     * @param options - Upload strategy options to validate
+     * @returns Validation result with validity flag and error messages
+     *
+     * @example
+     * ```typescript
+     * const validation = negotiator.validateConfiguration({
+     *   fileSize: 10_000_000,
+     *   preferredStrategy: "parallel",
+     *   preferredChunkSize: 1_000_000,
+     *   parallelUploads: 5
+     * });
+     *
+     * if (!validation.valid) {
+     *   console.log("Configuration errors:", validation.errors);
+     *   // Handle validation errors
+     * }
+     * ```
+     */
+    validateConfiguration(options) {
+        const errors = [];
+        if (options.preferredStrategy &&
+            !this.validateUploadStrategy(options.preferredStrategy)) {
+            errors.push(`Preferred strategy '${options.preferredStrategy}' not supported by data store`);
+        }
+        if (options.preferredChunkSize) {
+            if (this.capabilities.minChunkSize &&
+                options.preferredChunkSize < this.capabilities.minChunkSize) {
+                errors.push(`Chunk size ${options.preferredChunkSize} below data store minimum ${this.capabilities.minChunkSize}`);
+            }
+            if (this.capabilities.maxChunkSize &&
+                options.preferredChunkSize > this.capabilities.maxChunkSize) {
+                errors.push(`Chunk size ${options.preferredChunkSize} above data store maximum ${this.capabilities.maxChunkSize}`);
+            }
+        }
+        if (options.parallelUploads &&
+            this.capabilities.maxConcurrentUploads &&
+            options.parallelUploads > this.capabilities.maxConcurrentUploads) {
+            errors.push(`Parallel uploads ${options.parallelUploads} exceeds data store maximum ${this.capabilities.maxConcurrentUploads}`);
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+        };
+    }
+}

package/dist/upload/upload-url.d.ts

@@ -0,0 +1,68 @@
+import { Effect } from "effect";
+import { UploadistaError } from "../errors";
+/**
+ * Fetches a file from a remote URL.
+ *
+ * This function handles HTTP requests to remote URLs for file uploads,
+ * including proper error handling, metrics tracking, and observability.
+ *
+ * Features:
+ * - HTTP request with proper error handling
+ * - Effect tracing for performance monitoring
+ * - Metrics tracking for URL-based uploads
+ * - Structured logging for debugging
+ * - Response validation and error reporting
+ *
+ * @param url - The remote URL to fetch the file from
+ * @returns Effect that yields the Response object
+ *
+ * @example
+ * ```typescript
+ * // Fetch a file from URL
+ * const fetchEffect = fetchFile("https://example.com/image.jpg");
+ *
+ * // Run with error handling
+ * const response = await Effect.runPromise(
+ *   fetchEffect.pipe(
+ *     Effect.catchAll((error) =>
+ *       Effect.logError("Failed to fetch file").pipe(
+ *         Effect.andThen(Effect.fail(error))
+ *       )
+ *     )
+ *   )
+ * );
+ * ```
+ */
+export declare const fetchFile: (url: string) => Effect.Effect<Response, UploadistaError, never>;
+/**
+ * Converts a Response object to an ArrayBuffer.
+ *
+ * This function safely converts HTTP response data to binary format
+ * for processing and storage, with proper error handling and observability.
+ *
+ * Features:
+ * - Safe conversion from Response to ArrayBuffer
+ * - Effect tracing for performance monitoring
+ * - Structured logging for debugging
+ * - Error handling with proper UploadistaError types
+ *
+ * @param response - The HTTP Response object to convert
+ * @returns Effect that yields the ArrayBuffer data
+ *
+ * @example
+ * ```typescript
+ * // Convert response to buffer
+ * const bufferEffect = arrayBuffer(response);
+ *
+ * // Use in upload pipeline
+ * const buffer = await Effect.runPromise(
+ *   bufferEffect.pipe(
+ *     Effect.tap((buffer) =>
+ *       Effect.logInfo(`Buffer size: ${buffer.byteLength} bytes`)
+ *     )
+ *   )
+ * );
+ * ```
+ */
+export declare const arrayBuffer: (response: Response) => Effect.Effect<ArrayBuffer, UploadistaError, never>;
+//# sourceMappingURL=upload-url.d.ts.map

package/dist/upload/upload-url.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upload-url.d.ts","sourceRoot":"","sources":["../../src/upload/upload-url.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAU,MAAM,QAAQ,CAAC;AACxC,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,eAAO,MAAM,SAAS,GAAI,KAAK,MAAM,oDAsEpC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,eAAO,MAAM,WAAW,GAAI,UAAU,QAAQ,uDAkC7C,CAAC"}

package/dist/upload/upload-url.js

@@ -0,0 +1,142 @@
+import { Effect, Metric } from "effect";
+import { UploadistaError } from "../errors";
+/**
+ * Fetches a file from a remote URL.
+ *
+ * This function handles HTTP requests to remote URLs for file uploads,
+ * including proper error handling, metrics tracking, and observability.
+ *
+ * Features:
+ * - HTTP request with proper error handling
+ * - Effect tracing for performance monitoring
+ * - Metrics tracking for URL-based uploads
+ * - Structured logging for debugging
+ * - Response validation and error reporting
+ *
+ * @param url - The remote URL to fetch the file from
+ * @returns Effect that yields the Response object
+ *
+ * @example
+ * ```typescript
+ * // Fetch a file from URL
+ * const fetchEffect = fetchFile("https://example.com/image.jpg");
+ *
+ * // Run with error handling
+ * const response = await Effect.runPromise(
+ *   fetchEffect.pipe(
+ *     Effect.catchAll((error) =>
+ *       Effect.logError("Failed to fetch file").pipe(
+ *         Effect.andThen(Effect.fail(error))
+ *       )
+ *     )
+ *   )
+ * );
+ * ```
+ */
+export const fetchFile = (url) => {
+    return Effect.tryPromise({
+        try: async () => {
+            return await fetch(url);
+        },
+        catch: (error) => {
+            return UploadistaError.fromCode("UNKNOWN_ERROR", {
+                cause: error,
+            });
+        },
+    }).pipe(
+    // Add tracing span for URL fetch
+    Effect.withSpan("upload-fetch-url", {
+        attributes: {
+            "upload.url": url,
+            "upload.operation": "fetch",
+        },
+    }), 
+    // Track URL fetch metrics
+    Effect.tap((response) => Effect.gen(function* () {
+        // Increment URL upload counter
+        yield* Metric.increment(Metric.counter("upload_from_url_total", {
+            description: "Total number of URL-based uploads",
+        }));
+        // Track success/failure
+        if (response.ok) {
+            yield* Metric.increment(Metric.counter("upload_from_url_success_total", {
+                description: "Total number of successful URL-based uploads",
+            }));
+        }
+    })), 
+    // Add structured logging
+    Effect.tap((response) => Effect.logInfo("URL fetch completed").pipe(Effect.annotateLogs({
+        "upload.url": url,
+        "response.status": response.status.toString(),
+        "response.ok": response.ok.toString(),
+        "response.content_length": response.headers.get("content-length") ?? "unknown",
+    }))), 
+    // Handle errors with logging and metrics
+    Effect.tapError((error) => Effect.gen(function* () {
+        // Track failed URL upload
+        yield* Metric.increment(Metric.counter("upload_from_url_failed_total", {
+            description: "Total number of failed URL-based uploads",
+        }));
+        // Log error
+        yield* Effect.logError("URL fetch failed").pipe(Effect.annotateLogs({
+            "upload.url": url,
+            error: String(error),
+        }));
+    })));
+};
+/**
+ * Converts a Response object to an ArrayBuffer.
+ *
+ * This function safely converts HTTP response data to binary format
+ * for processing and storage, with proper error handling and observability.
+ *
+ * Features:
+ * - Safe conversion from Response to ArrayBuffer
+ * - Effect tracing for performance monitoring
+ * - Structured logging for debugging
+ * - Error handling with proper UploadistaError types
+ *
+ * @param response - The HTTP Response object to convert
+ * @returns Effect that yields the ArrayBuffer data
+ *
+ * @example
+ * ```typescript
+ * // Convert response to buffer
+ * const bufferEffect = arrayBuffer(response);
+ *
+ * // Use in upload pipeline
+ * const buffer = await Effect.runPromise(
+ *   bufferEffect.pipe(
+ *     Effect.tap((buffer) =>
+ *       Effect.logInfo(`Buffer size: ${buffer.byteLength} bytes`)
+ *     )
+ *   )
+ * );
+ * ```
+ */
+export const arrayBuffer = (response) => {
+    return Effect.tryPromise({
+        try: async () => {
+            return await response.arrayBuffer();
+        },
+        catch: (error) => {
+            return UploadistaError.fromCode("UNKNOWN_ERROR", {
+                cause: error,
+            });
+        },
+    }).pipe(
+    // Add tracing span for buffer conversion
+    Effect.withSpan("upload-convert-to-buffer", {
+        attributes: {
+            "upload.operation": "arrayBuffer",
+        },
+    }), 
+    // Add structured logging
+    Effect.tap((buffer) => Effect.logDebug("Response converted to array buffer").pipe(Effect.annotateLogs({
+        "buffer.size": buffer.byteLength.toString(),
+    }))), 
+    // Handle errors with logging
+    Effect.tapError((error) => Effect.logError("Failed to convert response to array buffer").pipe(Effect.annotateLogs({
+        error: String(error),
+    }))));
+};

package/dist/upload/write-to-store.d.ts

@@ -0,0 +1,77 @@
+import { Effect } from "effect";
+import { UploadistaError } from "../errors";
+import type { DataStore, UploadEvent, UploadFile } from "../types";
+import { type EventEmitter } from "../types";
+/**
+ * Configuration options for writing data to a data store.
+ *
+ * @property data - The stream of data to write
+ * @property upload - Upload file metadata
+ * @property dataStore - Target data store for writing
+ * @property maxFileSize - Maximum allowed file size in bytes
+ * @property controller - AbortController for cancellation
+ * @property eventEmitter - Event emitter for progress tracking
+ * @property uploadProgressInterval - Progress emission interval in milliseconds (default: 200)
+ */
+type WriteToStoreOptions = {
+    data: ReadableStream<Uint8Array>;
+    upload: UploadFile;
+    dataStore: DataStore<UploadFile>;
+    maxFileSize: number;
+    controller: AbortController;
+    eventEmitter: EventEmitter<UploadEvent>;
+    uploadProgressInterval?: number;
+};
+/**
+ * Writes data stream to a data store with progress tracking and size limits.
+ *
+ * This function handles the core data writing logic including:
+ * - Stream conversion and processing
+ * - File size validation and limiting
+ * - Progress tracking with throttled events
+ * - Abort signal handling for cancellation
+ * - Error handling and cleanup
+ *
+ * The function includes comprehensive observability with:
+ * - Effect tracing spans for performance monitoring
+ * - Structured logging for debugging and monitoring
+ * - Progress event emission with throttling
+ * - Error handling with proper UploadistaError types
+ *
+ * @param data - The stream of data to write to storage
+ * @param upload - Upload file metadata containing ID, offset, etc.
+ * @param dataStore - Target data store for writing the data
+ * @param maxFileSize - Maximum allowed file size in bytes
+ * @param controller - AbortController for handling cancellation
+ * @param eventEmitter - Event emitter for progress tracking
+ * @param uploadProgressInterval - Progress emission interval in milliseconds (default: 200)
+ * @returns Effect that yields the number of bytes written
+ *
+ * @example
+ * ```typescript
+ * // Write data to store with progress tracking
+ * const writeEffect = writeToStore({
+ *   data: fileStream,
+ *   upload: uploadMetadata,
+ *   dataStore: s3DataStore,
+ *   maxFileSize: 100_000_000, // 100MB
+ *   controller: abortController,
+ *   eventEmitter: progressEmitter,
+ *   uploadProgressInterval: 500 // Emit progress every 500ms
+ * });
+ *
+ * // Run with error handling
+ * const bytesWritten = await Effect.runPromise(
+ *   writeEffect.pipe(
+ *     Effect.catchAll((error) =>
+ *       Effect.logError("Failed to write to store").pipe(
+ *         Effect.andThen(Effect.fail(error))
+ *       )
+ *     )
+ *   )
+ * );
+ * ```
+ */
+export declare function writeToStore({ data, upload, dataStore, maxFileSize, controller, eventEmitter, uploadProgressInterval, }: WriteToStoreOptions): Effect.Effect<number, UploadistaError, never>;
+export {};
+//# sourceMappingURL=write-to-store.d.ts.map

package/dist/upload/write-to-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"write-to-store.d.ts","sourceRoot":"","sources":["../../src/upload/write-to-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAO,MAAM,QAAQ,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACnE,OAAO,EAAE,KAAK,YAAY,EAAmB,MAAM,UAAU,CAAC;AAG9D;;;;;;;;;;GAUG;AACH,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;IACjC,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,eAAe,CAAC;IAC5B,YAAY,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC;IACxC,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AACH,wBAAgB,YAAY,CAAC,EAC3B,IAAI,EACJ,MAAM,EACN,SAAS,EACT,WAAW,EACX,UAAU,EACV,YAAY,EACZ,sBAA4B,GAC7B,EAAE,mBAAmB,iDA4HrB"}

package/dist/upload/write-to-store.js

@@ -0,0 +1,147 @@
+import { Effect, Ref } from "effect";
+import { UploadistaError } from "../errors";
+import { StreamLimiterEffect } from "../streams/stream-limiter";
+import { UploadEventType } from "../types";
+import { convertToStream } from "./convert-to-stream";
+/**
+ * Writes data stream to a data store with progress tracking and size limits.
+ *
+ * This function handles the core data writing logic including:
+ * - Stream conversion and processing
+ * - File size validation and limiting
+ * - Progress tracking with throttled events
+ * - Abort signal handling for cancellation
+ * - Error handling and cleanup
+ *
+ * The function includes comprehensive observability with:
+ * - Effect tracing spans for performance monitoring
+ * - Structured logging for debugging and monitoring
+ * - Progress event emission with throttling
+ * - Error handling with proper UploadistaError types
+ *
+ * @param data - The stream of data to write to storage
+ * @param upload - Upload file metadata containing ID, offset, etc.
+ * @param dataStore - Target data store for writing the data
+ * @param maxFileSize - Maximum allowed file size in bytes
+ * @param controller - AbortController for handling cancellation
+ * @param eventEmitter - Event emitter for progress tracking
+ * @param uploadProgressInterval - Progress emission interval in milliseconds (default: 200)
+ * @returns Effect that yields the number of bytes written
+ *
+ * @example
+ * ```typescript
+ * // Write data to store with progress tracking
+ * const writeEffect = writeToStore({
+ *   data: fileStream,
+ *   upload: uploadMetadata,
+ *   dataStore: s3DataStore,
+ *   maxFileSize: 100_000_000, // 100MB
+ *   controller: abortController,
+ *   eventEmitter: progressEmitter,
+ *   uploadProgressInterval: 500 // Emit progress every 500ms
+ * });
+ *
+ * // Run with error handling
+ * const bytesWritten = await Effect.runPromise(
+ *   writeEffect.pipe(
+ *     Effect.catchAll((error) =>
+ *       Effect.logError("Failed to write to store").pipe(
+ *         Effect.andThen(Effect.fail(error))
+ *       )
+ *     )
+ *   )
+ * );
+ * ```
+ */
+export function writeToStore({ data, upload, dataStore, maxFileSize, controller, eventEmitter, uploadProgressInterval = 200, }) {
+    return Effect.gen(function* () {
+        const stream = convertToStream(data);
+        // Check if already aborted
+        if (controller.signal.aborted) {
+            return yield* Effect.fail(UploadistaError.fromCode("ABORTED"));
+        }
+        // Create an AbortController to manage the stream pipeline
+        const abortController = new AbortController();
+        const { signal } = abortController;
+        // Set up abort handling
+        const onAbort = () => {
+            // stream.cancel();
+            abortController.abort();
+        };
+        controller.signal.addEventListener("abort", onAbort, { once: true });
+        return yield* Effect.acquireUseRelease(Effect.sync(() => ({ signal, onAbort })), ({ signal: _signal }) => Effect.gen(function* () {
+            // Create a ref to track the last progress emission time for throttling
+            const lastEmitTime = yield* Ref.make(0);
+            // Create the stream limiter
+            const limiter = StreamLimiterEffect.limit({
+                maxSize: maxFileSize,
+            });
+            // Pipe the data through the limiter
+            const limitedStream = limiter(stream);
+            // Write to the data store with progress tracking
+            const offset = yield* dataStore.write({
+                stream: limitedStream,
+                file_id: upload.id,
+                offset: upload.offset,
+            }, {
+                onProgress: (newOffset) => {
+                    // Simple throttling using timestamp check
+                    const now = Date.now();
+                    Ref.get(lastEmitTime)
+                        .pipe(Effect.flatMap((lastTime) => {
+                        if (now - lastTime >= uploadProgressInterval) {
+                            return Effect.gen(function* () {
+                                yield* Ref.set(lastEmitTime, now);
+                                yield* eventEmitter.emit(upload.id, {
+                                    type: UploadEventType.UPLOAD_PROGRESS,
+                                    data: {
+                                        id: upload.id,
+                                        progress: newOffset,
+                                        total: upload.size ?? 0,
+                                    },
+                                    flow: upload.flow,
+                                });
+                            });
+                        }
+                        return Effect.void;
+                    }), Effect.runPromise)
+                        .catch(() => {
+                        // Ignore errors during progress emission
+                    });
+                },
+            });
+            return offset;
+        }).pipe(Effect.catchAll((error) => {
+            if (error instanceof Error && error.name === "AbortError") {
+                return Effect.fail(UploadistaError.fromCode("ABORTED"));
+            }
+            if (error instanceof UploadistaError) {
+                return Effect.fail(error);
+            }
+            return Effect.fail(UploadistaError.fromCode("FILE_WRITE_ERROR", { cause: error }));
+        })), ({ onAbort }) => Effect.sync(() => {
+            controller.signal.removeEventListener("abort", onAbort);
+        }));
+    }).pipe(
+    // Add tracing span for write operation
+    Effect.withSpan("upload-write-to-store", {
+        attributes: {
+            "upload.id": upload.id,
+            "upload.offset": upload.offset.toString(),
+            "upload.max_file_size": maxFileSize.toString(),
+            "upload.file_size": upload.size?.toString() ?? "0",
+        },
+    }), 
+    // Add structured logging for write operation
+    Effect.tap((offset) => Effect.logDebug("Data written to store").pipe(Effect.annotateLogs({
+        "upload.id": upload.id,
+        "write.offset": offset.toString(),
+        "write.bytes_written": (offset - upload.offset).toString(),
+    }))), 
+    // Handle errors with logging
+    Effect.tapError((error) => Effect.logError("Failed to write to store").pipe(Effect.annotateLogs({
+        "upload.id": upload.id,
+        "upload.offset": upload.offset.toString(),
+        error: error instanceof UploadistaError ? error.code : String(error),
+    }))));
+}

package/dist/upload-DLuICjpP.cjs

@@ -0,0 +1 @@
+const e=require(`./chunk-CUT6urMc.cjs`),t=require(`./uploadista-error-BB-Wdiz9.cjs`),n=require(`./types-BT-cvi7T.cjs`),r=require(`./generate-id-xHp_Z7Cl.cjs`),i=require(`./stream-limiter-JgOwmkMa.cjs`);let a=require(`effect`);a=e.__toESM(a);const o=(e,t)=>{if(e.length>=4){let t=e.slice(0,4);if(t[0]===137&&t[1]===80&&t[2]===78&&t[3]===71)return`image/png`;if(t[0]===255&&t[1]===216)return`image/jpeg`;if(t[0]===71&&t[1]===73&&t[2]===70)return`image/gif`;if(t[0]===82&&t[1]===73&&t[2]===70&&t[3]===70)return`image/webp`}if(t)switch(t.split(`.`).pop()?.toLowerCase()){case`jpg`:case`jpeg`:return`image/jpeg`;case`png`:return`image/png`;case`gif`:return`image/gif`;case`webp`:return`image/webp`;case`pdf`:return`application/pdf`;case`txt`:return`text/plain`;case`json`:return`application/json`;default:return`application/octet-stream`}return`application/octet-stream`},s=(e,{dataStoreService:t,kvStore:r,eventEmitter:i,generateId:o,isBuffer:s})=>a.Effect.gen(function*(){let a=yield*t.getDataStore(e.storageId,s),c=yield*o.generateId(),{size:l,type:u,fileName:d,lastModified:f}=e,p={type:u,fileName:d??``};f&&(p.lastModified=f.toString());let m={id:c,size:l,metadata:p,offset:0,creationDate:new Date().toISOString(),storage:{id:e.storageId,isBuffer:s,type:u,path:``,bucket:a.bucket}},h=yield*a.create(m);return yield*r.set(c,h),yield*i.emit(c,{type:n.UploadEventType.UPLOAD_STARTED,data:h}),h});function c(e){return a.Stream.fromReadableStream(()=>e,e=>new t.UploadistaError({code:`UNKNOWN_ERROR`,status:500,body:String(e)}))}function l({data:e,upload:r,dataStore:o,maxFileSize:s,controller:l,eventEmitter:u,uploadProgressInterval:d=200}){return a.Effect.gen(function*(){let f=c(e);if(l.signal.aborted)return yield*a.Effect.fail(t.UploadistaError.fromCode(`ABORTED`));let p=new AbortController,{signal:m}=p,h=()=>{p.abort()};return l.signal.addEventListener(`abort`,h,{once:!0}),yield*a.Effect.acquireUseRelease(a.Effect.sync(()=>({signal:m,onAbort:h})),({signal:e})=>a.Effect.gen(function*(){let e=yield*a.Ref.make(0),t=i.StreamLimiterEffect.limit({maxSize:s})(f);return yield*o.write({stream:t,file_id:r.id,offset:r.offset},{onProgress:t=>{let i=Date.now();a.Ref.get(e).pipe(a.Effect.flatMap(o=>i-o>=d?a.Effect.gen(function*(){yield*a.Ref.set(e,i),yield*u.emit(r.id,{type:n.UploadEventType.UPLOAD_PROGRESS,data:{id:r.id,progress:t,total:r.size??0}})}):a.Effect.void),a.Effect.runPromise).catch(()=>{})}})}).pipe(a.Effect.catchAll(e=>e instanceof Error&&e.name===`AbortError`?a.Effect.fail(t.UploadistaError.fromCode(`ABORTED`)):e instanceof t.UploadistaError?a.Effect.fail(e):a.Effect.fail(t.UploadistaError.fromCode(`FILE_WRITE_ERROR`,{cause:e})))),({onAbort:e})=>a.Effect.sync(()=>{l.signal.removeEventListener(`abort`,e)}))})}const u=(e,t,{dataStoreService:r,kvStore:i,eventEmitter:o})=>a.Effect.gen(function*(){let a=yield*i.get(e),s=yield*r.getDataStore(a.storage.id,a.storage.isBuffer),c=new AbortController;return a.offset=yield*l({dataStore:s,data:t,upload:a,maxFileSize:1e8,controller:c,uploadProgressInterval:200,eventEmitter:o}),yield*i.set(e,a),yield*o.emit(a.id,{type:n.UploadEventType.UPLOAD_PROGRESS,data:{id:a.id,progress:a.offset,total:a.size??0}}),a});var d=class extends a.Context.Tag(`UploadServer`)(){};function f(){return a.Effect.gen(function*(){let e=yield*n.UploadFileKVStore,t=yield*n.UploadEventEmitter,i=yield*r.GenerateId,o=yield*n.UploadFileDataStores;return{upload:(n,r)=>a.Effect.gen(function*(){let a=yield*s(n,{dataStoreService:o,kvStore:e,eventEmitter:t,generateId:i});return yield*u(a.id,r,{dataStoreService:o,kvStore:e,eventEmitter:t})}),createUpload:n=>a.Effect.gen(function*(){return yield*s(n,{dataStoreService:o,kvStore:e,eventEmitter:t,generateId:i})}),uploadChunk:(n,r)=>a.Effect.gen(function*(){return yield*u(n,r,{dataStoreService:o,kvStore:e,eventEmitter:t})}),getUpload:t=>a.Effect.gen(function*(){return yield*e.get(t)}),read:e=>a.Effect.gen(function*(){return yield*(yield*o.getDataStore(e)).read(e)}),getCapabilities:e=>a.Effect.gen(function*(){return(yield*o.getDataStore(e)).getCapabilities()}),subscribeToUploadEvents:(e,n)=>a.Effect.gen(function*(){yield*t.subscribe(e,n)}),unsubscribeFromUploadEvents:e=>a.Effect.gen(function*(){yield*t.unsubscribe(e)})}})}const p=a.Layer.effect(d,f());var m=class{constructor(e,t){this.capabilities=e,this.validateUploadStrategy=t}negotiateStrategy(e){let t=[],n=[],r=`single`,i=e.preferredChunkSize??this.capabilities.optimalChunkSize??1024*1024,a=e.parallelUploads??1;if(e.preferredStrategy&&(this.validateUploadStrategy(e.preferredStrategy)?(r=e.preferredStrategy,t.push(`Using preferred strategy: ${r}`)):n.push(`Preferred strategy '${e.preferredStrategy}' not supported by data store, falling back`)),(!e.preferredStrategy||!this.validateUploadStrategy(e.preferredStrategy))&&(this.capabilities.supportsParallelUploads&&e.fileSize>(e.minChunkSizeForParallel??10*1024*1024)?(r=`parallel`,t.push(`Selected parallel upload for large file (${e.fileSize} bytes)`)):(r=`single`,t.push(this.capabilities.supportsParallelUploads?`Selected single upload for small file (${e.fileSize} bytes)`:`Selected single upload (parallel not supported by data store)`))),this.capabilities.minChunkSize&&i<this.capabilities.minChunkSize&&(n.push(`Chunk size ${i} below minimum ${this.capabilities.minChunkSize}, adjusting`),i=this.capabilities.minChunkSize),this.capabilities.maxChunkSize&&i>this.capabilities.maxChunkSize&&(n.push(`Chunk size ${i} above maximum ${this.capabilities.maxChunkSize}, adjusting`),i=this.capabilities.maxChunkSize),r===`parallel`&&(this.capabilities.maxConcurrentUploads&&a>this.capabilities.maxConcurrentUploads&&(n.push(`Parallel uploads ${a} exceeds maximum ${this.capabilities.maxConcurrentUploads}, adjusting`),a=this.capabilities.maxConcurrentUploads),this.capabilities.maxParts)){let t=Math.ceil(e.fileSize/i);if(t>this.capabilities.maxParts){let r=Math.ceil(e.fileSize/this.capabilities.maxParts);n.push(`Estimated parts ${t} exceeds maximum ${this.capabilities.maxParts}, increasing chunk size`),i=Math.max(i,r)}}return this.validateUploadStrategy(r)||(n.push(`Final strategy validation failed, falling back to single upload`),r=`single`,a=1),t.push(`Data store capabilities: parallel=${this.capabilities.supportsParallelUploads}, concatenation=${this.capabilities.supportsConcatenation}, resumable=${this.capabilities.supportsResumableUploads}`),{strategy:r,chunkSize:i,parallelUploads:r===`parallel`?a:1,reasoning:t,warnings:n}}getDataStoreCapabilities(){return this.capabilities}validateConfiguration(e){let t=[];return e.preferredStrategy&&!this.validateUploadStrategy(e.preferredStrategy)&&t.push(`Preferred strategy '${e.preferredStrategy}' not supported by data store`),e.preferredChunkSize&&(this.capabilities.minChunkSize&&e.preferredChunkSize<this.capabilities.minChunkSize&&t.push(`Chunk size ${e.preferredChunkSize} below data store minimum ${this.capabilities.minChunkSize}`),this.capabilities.maxChunkSize&&e.preferredChunkSize>this.capabilities.maxChunkSize&&t.push(`Chunk size ${e.preferredChunkSize} above data store maximum ${this.capabilities.maxChunkSize}`)),e.parallelUploads&&this.capabilities.maxConcurrentUploads&&e.parallelUploads>this.capabilities.maxConcurrentUploads&&t.push(`Parallel uploads ${e.parallelUploads} exceeds data store maximum ${this.capabilities.maxConcurrentUploads}`),{valid:t.length===0,errors:t}}};Object.defineProperty(exports,`UploadServer`,{enumerable:!0,get:function(){return d}}),Object.defineProperty(exports,`UploadStrategyNegotiator`,{enumerable:!0,get:function(){return m}}),Object.defineProperty(exports,`createUploadServer`,{enumerable:!0,get:function(){return f}}),Object.defineProperty(exports,`detectMimeType`,{enumerable:!0,get:function(){return o}}),Object.defineProperty(exports,`uploadServer`,{enumerable:!0,get:function(){return p}});