@uploadista/data-store-s3 0.0.3

package/dist/s3-store.d.ts

@@ -0,0 +1,9 @@
+import type { DataStore, UploadFile } from "@uploadista/core/types";
+import { UploadFileKVStore } from "@uploadista/core/types";
+import { Effect } from "effect";
+import { S3ClientService } from "./services/s3-client.service";
+import type { S3StoreConfig } from "./types";
+export declare function createS3StoreImplementation(config: S3StoreConfig): Effect.Effect<DataStore<UploadFile>, never, S3ClientService>;
+export declare const createS3Store: (options: S3StoreConfig) => Effect.Effect<DataStore<UploadFile>, never, UploadFileKVStore>;
+export declare const s3Store: (config: S3StoreConfig) => Promise<DataStore<UploadFile>>;
+//# sourceMappingURL=s3-store.d.ts.map

package/dist/s3-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3-store.d.ts","sourceRoot":"","sources":["../src/s3-store.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,SAAS,EAGT,UAAU,EAEX,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAe3D,OAAO,EAAE,MAAM,EAAyB,MAAM,QAAQ,CAAC;AACvD,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,KAAK,EAAa,aAAa,EAAE,MAAM,SAAS,CAAC;AAgCxD,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,aAAa,gEA+8BhE;AAGD,eAAO,MAAM,aAAa,GAAI,SAAS,aAAa,mEAUhD,CAAC;AAGL,eAAO,MAAM,OAAO,GAAI,QAAQ,aAAa,mCAS5C,CAAC"}

package/dist/s3-store.js

@@ -0,0 +1,666 @@
+import { UploadistaError } from "@uploadista/core/errors";
+import { UploadFileKVStore } from "@uploadista/core/types";
+import { s3ActiveUploadsGauge as activeUploadsGauge, s3FileSizeHistogram as fileSizeHistogram, logS3UploadCompletion, s3PartSizeHistogram as partSizeHistogram, s3PartUploadDurationHistogram as partUploadDurationHistogram, s3UploadDurationHistogram as uploadDurationHistogram, s3UploadErrorsTotal as uploadErrorsTotal, s3UploadPartsTotal as uploadPartsTotal, s3UploadRequestsTotal as uploadRequestsTotal, s3UploadSuccessTotal as uploadSuccessTotal, withS3TimingMetrics as withTimingMetrics, withS3UploadMetrics as withUploadMetrics, } from "@uploadista/observability";
+import { Effect, Ref, Schedule, Stream } from "effect";
+import { S3ClientLayer, S3ClientService } from "./services/s3-client.service";
+import { calcOffsetFromParts, calcOptimalPartSize, getExpirationDate, isUploadNotFoundError, } from "./utils";
+/**
+ * Generates an S3 key from an upload file, preserving the file extension if available.
+ * Looks for filename in metadata under common keys: 'filename', 'fileName', or 'name'.
+ * Falls back to just the upload ID if no filename is found.
+ */
+const getS3Key = (uploadFile) => {
+    const { id, metadata } = uploadFile;
+    if (!metadata) {
+        return id;
+    }
+    // Try common metadata keys for filename
+    const filename = metadata.filename || metadata.fileName || metadata.name;
+    if (typeof filename === "string" && filename.includes(".")) {
+        const extension = filename.substring(filename.lastIndexOf("."));
+        return `${id}${extension}`;
+    }
+    return id;
+};
+// Clean implementation using composed services
+export function createS3StoreImplementation(config) {
+    const { deliveryUrl, partSize, minPartSize = 5_242_880, useTags = true, maxMultipartParts = 10_000, kvStore, maxConcurrentPartUploads = 60, expirationPeriodInMilliseconds = 1000 * 60 * 60 * 24 * 7, // 1 week
+    s3ClientConfig: { bucket }, } = config;
+    return Effect.gen(function* () {
+        const s3Client = yield* S3ClientService;
+        const preferredPartSize = partSize || 8 * 1024 * 1024;
+        const getUploadId = (uploadFile) => {
+            const uploadId = uploadFile.storage.uploadId;
+            if (!uploadId) {
+                return Effect.fail(UploadistaError.fromCode("FILE_WRITE_ERROR", new Error("Upload ID is undefined")));
+            }
+            return Effect.succeed(uploadId);
+        };
+        const uploadPart = (uploadFile, data, partNumber) => {
+            const s3Key = getS3Key(uploadFile);
+            return withTimingMetrics(partUploadDurationHistogram, Effect.gen(function* () {
+                const uploadId = yield* getUploadId(uploadFile);
+                const etag = yield* s3Client
+                    .uploadPart({
+                    bucket: s3Client.bucket,
+                    key: s3Key,
+                    uploadId,
+                    partNumber,
+                    data,
+                })
+                    .pipe(Effect.retry(Schedule.exponential("1 second", 2.0).pipe(Schedule.intersect(Schedule.recurs(3)))), Effect.tapError((error) => Effect.logWarning("Retrying part upload").pipe(Effect.annotateLogs({
+                    upload_id: uploadFile.id,
+                    part_number: partNumber,
+                    error_message: error.message,
+                    retry_attempt: "unknown", // Will be overridden by the retry schedule
+                    part_size: data.length,
+                    s3_bucket: s3Client.bucket,
+                }))));
+                yield* uploadPartsTotal(Effect.succeed(1));
+                yield* Effect.logInfo("Part uploaded successfully").pipe(Effect.annotateLogs({
+                    upload_id: uploadFile.id,
+                    part_number: partNumber,
+                    part_size: data.length,
+                    etag: etag,
+                }));
+                return etag;
+            })).pipe(Effect.withSpan(`s3-upload-part-${partNumber}`, {
+                attributes: {
+                    "upload.id": uploadFile.id,
+                    "upload.part_number": partNumber,
+                    "upload.part_size": data.length,
+                    "s3.bucket": s3Client.bucket,
+                    "s3.key": s3Key,
+                },
+            }));
+        };
+        const uploadIncompletePart = (id, data) => s3Client.putIncompletePart(id, data);
+        const downloadIncompletePart = (id) => Effect.gen(function* () {
+            const incompletePart = yield* s3Client.getIncompletePart(id);
+            if (!incompletePart) {
+                return undefined;
+            }
+            // Read the stream and collect all chunks to calculate size
+            const reader = incompletePart.getReader();
+            const chunks = [];
+            let incompletePartSize = 0;
+            try {
+                while (true) {
+                    const { done, value } = yield* Effect.promise(() => reader.read());
+                    if (done)
+                        break;
+                    chunks.push(value);
+                    incompletePartSize += value.length;
+                }
+            }
+            finally {
+                reader.releaseLock();
+            }
+            const stream = Stream.fromIterable(chunks);
+            return {
+                size: incompletePartSize,
+                stream,
+            };
+        });
+        const deleteIncompletePart = (id) => s3Client.deleteIncompletePart(id);
+        const getIncompletePartSize = (id) => s3Client.getIncompletePartSize(id);
+        const complete = (uploadFile, parts) => {
+            const s3Key = getS3Key(uploadFile);
+            return Effect.gen(function* () {
+                const uploadId = yield* getUploadId(uploadFile);
+                return yield* s3Client.completeMultipartUpload({
+                    bucket: s3Client.bucket,
+                    key: s3Key,
+                    uploadId,
+                }, parts);
+            }).pipe(Effect.tap(() => uploadSuccessTotal(Effect.succeed(1))), Effect.withSpan("s3-complete-multipart-upload", {
+                attributes: {
+                    "upload.id": uploadFile.id,
+                    "upload.parts_count": parts.length,
+                    "s3.bucket": s3Client.bucket,
+                    "s3.key": s3Key,
+                },
+            }));
+        };
+        const abort = (uploadFile) => {
+            const s3Key = getS3Key(uploadFile);
+            return Effect.gen(function* () {
+                const uploadId = yield* getUploadId(uploadFile);
+                yield* s3Client.abortMultipartUpload({
+                    bucket: s3Client.bucket,
+                    key: s3Key,
+                    uploadId,
+                });
+                yield* s3Client.deleteObjects([s3Key]);
+            });
+        };
+        const retrievePartsRecursive = (s3Key, uploadId, uploadFileId, partNumberMarker) => Effect.gen(function* () {
+            try {
+                const result = yield* s3Client.listParts({
+                    bucket: s3Client.bucket,
+                    key: s3Key,
+                    uploadId,
+                    partNumberMarker,
+                });
+                let parts = result.parts;
+                if (result.isTruncated) {
+                    const rest = yield* retrievePartsRecursive(s3Key, uploadId, uploadFileId, result.nextPartNumberMarker);
+                    parts = [...parts, ...rest.parts];
+                }
+                if (!partNumberMarker) {
+                    parts.sort((a, b) => (a.PartNumber ?? 0) - (b.PartNumber ?? 0));
+                }
+                return { uploadFound: true, parts };
+            }
+            catch (error) {
+                if (isUploadNotFoundError(error)) {
+                    yield* Effect.logWarning("S3 upload not found during listParts").pipe(Effect.annotateLogs({
+                        upload_id: uploadFileId,
+                        error_code: error.code,
+                    }));
+                    return { uploadFound: false, parts: [] };
+                }
+                throw error;
+            }
+        });
+        const retrieveParts = (id, partNumberMarker) => Effect.gen(function* () {
+            const metadata = yield* kvStore.get(id);
+            const uploadId = yield* getUploadId(metadata);
+            const s3Key = getS3Key(metadata);
+            return yield* retrievePartsRecursive(s3Key, uploadId, id, partNumberMarker);
+        });
+        const completeMetadata = (upload, useTags) => Effect.gen(function* () {
+            if (!useTags) {
+                return 0;
+            }
+            const uploadFile = yield* kvStore.get(upload.id);
+            const uploadId = uploadFile.storage.uploadId;
+            if (!uploadId) {
+                return 0;
+            }
+            yield* kvStore.set(upload.id, {
+                ...uploadFile,
+                storage: { ...uploadFile.storage, uploadId },
+            });
+            return 0;
+        });
+        const clearCache = (id) => Effect.gen(function* () {
+            yield* Effect.logInfo("Clearing cache").pipe(Effect.annotateLogs({ upload_id: id }));
+            yield* kvStore.delete(id);
+        });
+        const createMultipartUpload = (upload) => {
+            const s3Key = getS3Key(upload);
+            return Effect.gen(function* () {
+                yield* Effect.logInfo("Initializing multipart upload").pipe(Effect.annotateLogs({ upload_id: upload.id }));
+                const multipartInfo = yield* s3Client.createMultipartUpload({
+                    bucket: s3Client.bucket,
+                    key: s3Key,
+                    uploadId: "", // Not needed for create
+                    contentType: upload.metadata?.contentType?.toString(),
+                    cacheControl: upload.metadata?.cacheControl?.toString(),
+                });
+                const uploadCreated = {
+                    ...upload,
+                    storage: {
+                        ...upload.storage,
+                        path: multipartInfo.key,
+                        uploadId: multipartInfo.uploadId,
+                        bucket: multipartInfo.bucket,
+                    },
+                    url: `${deliveryUrl}/${s3Key}`,
+                };
+                yield* kvStore.set(upload.id, uploadCreated);
+                yield* Effect.logInfo("Multipart upload created").pipe(Effect.annotateLogs({
+                    upload_id: upload.id,
+                    s3_upload_id: uploadCreated.storage.uploadId,
+                    s3_key: s3Key,
+                }));
+                yield* uploadRequestsTotal(Effect.succeed(1));
+                yield* fileSizeHistogram(Effect.succeed(upload.size || 0));
+                return uploadCreated;
+            }).pipe(Effect.withSpan("s3-create-upload", {
+                attributes: {
+                    "upload.id": upload.id,
+                    "upload.size": upload.size || 0,
+                    "s3.bucket": s3Client.bucket,
+                    "s3.key": s3Key,
+                },
+            }));
+        };
+        /**
+         * Creates a multipart upload on S3 attaching any metadata to it.
+         * Also, a `${file_id}.info` file is created which holds some information
+         * about the upload itself like: `upload-id`, `upload-length`, etc.
+         */
+        const create = (upload) => {
+            return Effect.gen(function* () {
+                yield* Effect.logInfo("Initializing multipart upload").pipe(Effect.annotateLogs({ upload_id: upload.id }));
+                const uploadCreated = yield* createMultipartUpload(upload);
+                yield* kvStore.set(upload.id, uploadCreated);
+                yield* Effect.logInfo("Multipart upload created").pipe(Effect.annotateLogs({
+                    upload_id: upload.id,
+                    s3_upload_id: uploadCreated.storage.uploadId,
+                }));
+                yield* uploadRequestsTotal(Effect.succeed(1));
+                return uploadCreated;
+            }).pipe(Effect.withSpan("s3-create-upload", {
+                attributes: {
+                    "upload.id": upload.id,
+                    "upload.size": upload.size || 0,
+                    "s3.bucket": bucket,
+                },
+            }));
+        };
+        const remove = (id) => Effect.gen(function* () {
+            const uploadFile = yield* kvStore.get(id);
+            yield* abort(uploadFile);
+            yield* clearCache(id);
+        });
+        const write = (options, dependencies) => withUploadMetrics(options.file_id, withTimingMetrics(uploadDurationHistogram, Effect.gen(function* () {
+            const { stream: initialData, file_id, offset: initialOffset, } = options;
+            const { onProgress } = dependencies;
+            // Capture start time for upload completion metrics
+            const startTime = Date.now();
+            // Track active upload
+            yield* activeUploadsGauge(Effect.succeed(1));
+            const prepareResult = yield* prepareUpload(file_id, initialOffset, initialData);
+            const { uploadFile, nextPartNumber, offset, data, existingPartSize, } = prepareResult;
+            // Use existing part size if parts already exist, otherwise calculate optimal size
+            const uploadPartSize = existingPartSize ||
+                calcOptimalPartSize(uploadFile.size, preferredPartSize, minPartSize, maxMultipartParts);
+            // Log part size decision for debugging
+            yield* Effect.logInfo("Part size decision").pipe(Effect.annotateLogs({
+                upload_id: file_id,
+                existing_part_size: existingPartSize,
+                calculated_part_size: calcOptimalPartSize(uploadFile.size, preferredPartSize, minPartSize, maxMultipartParts),
+                final_part_size: uploadPartSize,
+                next_part_number: nextPartNumber,
+            }));
+            const bytesUploaded = yield* uploadParts(uploadFile, data, nextPartNumber, offset, uploadPartSize, minPartSize, maxConcurrentPartUploads, onProgress);
+            const newOffset = offset + bytesUploaded;
+            if (newOffset > maxConcurrentPartUploads)
+                if (uploadFile.size === newOffset) {
+                    yield* finishUpload(file_id, uploadFile, startTime);
+                }
+            return newOffset;
+        }).pipe(Effect.ensuring(activeUploadsGauge(Effect.succeed(0))))));
+        const getUpload = (id) => Effect.gen(function* () {
+            const uploadFile = yield* kvStore.get(id);
+            const { parts, uploadFound } = yield* retrieveParts(id);
+            if (!uploadFound) {
+                return {
+                    ...uploadFile,
+                    offset: uploadFile.size,
+                    size: uploadFile.size,
+                };
+            }
+            const offset = calcOffsetFromParts(parts);
+            const incompletePartSize = yield* getIncompletePartSize(id);
+            return {
+                ...uploadFile,
+                offset: offset + (incompletePartSize ?? 0),
+                size: uploadFile.size,
+                storage: uploadFile.storage,
+            };
+        });
+        // const read = (id: string) =>
+        //   Effect.gen(function* () {
+        //     return yield* s3Client.getObject(id);
+        //   });
+        // Helper functions
+        const prepareUpload = (fileId, initialOffset, initialData) => Effect.gen(function* () {
+            const uploadFile = yield* kvStore.get(fileId);
+            const { parts } = yield* retrieveParts(fileId);
+            const partNumber = parts.length > 0 && parts[parts.length - 1].PartNumber
+                ? (parts[parts.length - 1].PartNumber ?? 0)
+                : 0;
+            const nextPartNumber = partNumber + 1;
+            // Detect existing part size to maintain consistency
+            // We check the first part's size to ensure all subsequent parts match
+            const existingPartSize = parts.length > 0 && parts[0].Size ? parts[0].Size : null;
+            // Validate that all existing parts (except potentially the last one) have the same size
+            if (existingPartSize && parts.length > 1) {
+                const inconsistentPart = parts
+                    .slice(0, -1)
+                    .find((part) => part.Size !== existingPartSize);
+                if (inconsistentPart) {
+                    yield* Effect.logWarning("Inconsistent part sizes detected in existing upload").pipe(Effect.annotateLogs({
+                        upload_id: fileId,
+                        expected_size: existingPartSize,
+                        inconsistent_part: inconsistentPart.PartNumber,
+                        inconsistent_size: inconsistentPart.Size,
+                    }));
+                }
+            }
+            const incompletePart = yield* downloadIncompletePart(fileId);
+            if (incompletePart) {
+                yield* deleteIncompletePart(fileId);
+                const offset = initialOffset - incompletePart.size;
+                const data = incompletePart.stream.pipe(Stream.concat(initialData));
+                return {
+                    uploadFile,
+                    nextPartNumber,
+                    offset,
+                    incompletePartSize: incompletePart.size,
+                    data,
+                    existingPartSize,
+                };
+            }
+            else {
+                return {
+                    uploadFile,
+                    nextPartNumber,
+                    offset: initialOffset,
+                    incompletePartSize: 0,
+                    data: initialData,
+                    existingPartSize,
+                };
+            }
+        });
+        const finishUpload = (fileId, uploadFile, startTime) => Effect.gen(function* () {
+            const { parts } = yield* retrieveParts(fileId);
+            // Log all parts for debugging S3 multipart upload requirements
+            yield* Effect.logInfo("Attempting to complete multipart upload").pipe(Effect.annotateLogs({
+                upload_id: fileId,
+                parts_count: parts.length,
+                parts_info: parts.map((part, index) => ({
+                    part_number: part.PartNumber,
+                    size: part.Size,
+                    etag: part.ETag,
+                    is_final_part: index === parts.length - 1,
+                })),
+            }));
+            yield* complete(uploadFile, parts);
+            yield* completeMetadata(uploadFile, useTags);
+            // yield* clearCache(fileId);
+            // Log upload completion metrics
+            const endTime = Date.now();
+            const totalDurationMs = endTime - startTime;
+            const fileSize = uploadFile.size || 0;
+            const throughputBps = totalDurationMs > 0 ? (fileSize * 1000) / totalDurationMs : 0;
+            // Calculate average part size if we have parts
+            const averagePartSize = parts.length > 0
+                ? parts.reduce((sum, part) => sum + (part.Size || 0), 0) /
+                    parts.length
+                : undefined;
+            yield* logS3UploadCompletion(fileId, {
+                fileSize,
+                totalDurationMs,
+                partsCount: parts.length,
+                averagePartSize,
+                throughputBps,
+            });
+        }).pipe(Effect.tapError((error) => Effect.gen(function* () {
+            yield* uploadErrorsTotal(Effect.succeed(1));
+            yield* Effect.logError("Failed to finish upload").pipe(Effect.annotateLogs({
+                upload_id: fileId,
+                error: String(error),
+            }));
+        })));
+        const deleteExpired = Effect.gen(function* () {
+            if (expirationPeriodInMilliseconds === 0) {
+                return 0;
+            }
+            let keyMarker;
+            let uploadIdMarker;
+            let isTruncated = true;
+            let deleted = 0;
+            while (isTruncated) {
+                const listResponse = yield* s3Client.listMultipartUploads(keyMarker, uploadIdMarker);
+                const expiredUploads = listResponse.Uploads?.filter((multiPartUpload) => {
+                    const initiatedDate = multiPartUpload.Initiated;
+                    return (initiatedDate &&
+                        Date.now() >
+                            getExpirationDate(initiatedDate.toISOString(), expirationPeriodInMilliseconds).getTime());
+                }) || [];
+                const objectsToDelete = expiredUploads
+                    .filter((upload) => {
+                    return !!upload.Key;
+                })
+                    .map((upload) => upload.Key);
+                if (objectsToDelete.length > 0) {
+                    yield* s3Client.deleteObjects(objectsToDelete);
+                    // Abort multipart uploads
+                    yield* Effect.forEach(expiredUploads, (upload) => {
+                        return Effect.gen(function* () {
+                            if (!upload.Key || !upload.UploadId) {
+                                return;
+                            }
+                            yield* s3Client.abortMultipartUpload({
+                                bucket,
+                                key: upload.Key,
+                                uploadId: upload.UploadId,
+                            });
+                            return;
+                        });
+                    });
+                    deleted += objectsToDelete.length;
+                }
+                isTruncated = listResponse.IsTruncated ?? false;
+                if (isTruncated) {
+                    keyMarker = listResponse.NextKeyMarker;
+                    uploadIdMarker = listResponse.NextUploadIdMarker;
+                }
+            }
+            return deleted;
+        });
+        // Proper single-pass chunking using Effect's async stream constructor
+        // Ensures all parts except the final part are exactly the same size (S3 requirement)
+        const createChunkedStream = (chunkSize) => (stream) => {
+            return Stream.async((emit) => {
+                let buffer = new Uint8Array(0);
+                let partNumber = 1;
+                let totalBytesProcessed = 0;
+                const emitChunk = (data, isFinalChunk = false) => {
+                    // Log chunk information for debugging - use INFO level to see in logs
+                    Effect.runSync(Effect.logInfo("Creating chunk").pipe(Effect.annotateLogs({
+                        part_number: partNumber,
+                        chunk_size: data.length,
+                        expected_size: chunkSize,
+                        is_final_chunk: isFinalChunk,
+                        total_bytes_processed: totalBytesProcessed + data.length,
+                    })));
+                    emit.single({
+                        partNumber: partNumber++,
+                        data,
+                        size: data.length,
+                    });
+                };
+                const processChunk = (newData) => {
+                    // Combine buffer with new data
+                    const combined = new Uint8Array(buffer.length + newData.length);
+                    combined.set(buffer);
+                    combined.set(newData, buffer.length);
+                    buffer = combined;
+                    totalBytesProcessed += newData.length;
+                    // Emit full chunks of exactly chunkSize bytes
+                    // This ensures S3 multipart upload rule: all parts except last must be same size
+                    while (buffer.length >= chunkSize) {
+                        const chunk = buffer.slice(0, chunkSize);
+                        buffer = buffer.slice(chunkSize);
+                        emitChunk(chunk, false);
+                    }
+                };
+                // Process the stream
+                Effect.runFork(stream.pipe(Stream.runForEach((chunk) => Effect.sync(() => processChunk(chunk))), Effect.andThen(() => Effect.sync(() => {
+                    // Emit final chunk if there's remaining data
+                    // The final chunk can be any size < chunkSize (S3 allows this)
+                    if (buffer.length > 0) {
+                        emitChunk(buffer, true);
+                    }
+                    emit.end();
+                })), Effect.catchAll((error) => Effect.sync(() => emit.fail(error)))));
+            });
+        };
+        // Byte-level progress tracking during streaming
+        // This provides smooth, immediate progress feedback by tracking bytes as they
+        // flow through the stream, before they reach S3. This solves the issue where
+        // small files (< 5MB) would jump from 0% to 100% instantly.
+        const withByteProgressTracking = (onProgress, initialOffset = 0) => (stream) => {
+            if (!onProgress)
+                return stream;
+            return Effect.gen(function* () {
+                const totalBytesProcessedRef = yield* Ref.make(initialOffset);
+                return stream.pipe(Stream.tap((chunk) => Effect.gen(function* () {
+                    const newTotal = yield* Ref.updateAndGet(totalBytesProcessedRef, (total) => total + chunk.length);
+                    onProgress(newTotal);
+                })));
+            }).pipe(Stream.unwrap);
+        };
+        const uploadParts = (uploadFile, readStream, initCurrentPartNumber, initOffset, uploadPartSize, minPartSize, maxConcurrentPartUploads, onProgress) => Effect.gen(function* () {
+            yield* Effect.logInfo("Starting part uploads").pipe(Effect.annotateLogs({
+                upload_id: uploadFile.id,
+                init_offset: initOffset,
+                file_size: uploadFile.size,
+                part_size: uploadPartSize,
+                min_part_size: minPartSize,
+            }));
+            // Enhanced Progress Tracking Strategy:
+            // 1. Byte-level progress during streaming - provides immediate, smooth feedback
+            //    as data flows through the pipeline (even for small files)
+            // 2. This tracks progress BEFORE S3 upload, giving users immediate feedback
+            // 3. For large files with multiple parts, this provides granular updates
+            // 4. For small files (single part), this prevents 0%->100% jumps
+            const chunkStream = readStream.pipe(
+            // Add byte-level progress tracking during streaming (immediate feedback)
+            withByteProgressTracking(onProgress, initOffset), 
+            // Create chunks for S3 multipart upload with uniform part sizes
+            createChunkedStream(uploadPartSize));
+            // Track cumulative offset and total bytes with Effect Refs
+            const cumulativeOffsetRef = yield* Ref.make(initOffset);
+            const totalBytesUploadedRef = yield* Ref.make(0);
+            // Create a chunk upload function for the sink
+            const uploadChunk = (chunkInfo) => Effect.gen(function* () {
+                // Calculate cumulative bytes to determine if this is the final part
+                const cumulativeOffset = yield* Ref.updateAndGet(cumulativeOffsetRef, (offset) => offset + chunkInfo.size);
+                const isFinalPart = cumulativeOffset >= (uploadFile.size || 0);
+                yield* Effect.logDebug("Processing chunk").pipe(Effect.annotateLogs({
+                    upload_id: uploadFile.id,
+                    cumulative_offset: cumulativeOffset,
+                    file_size: uploadFile.size,
+                    chunk_size: chunkInfo.size,
+                    is_final_part: isFinalPart,
+                }));
+                const actualPartNumber = initCurrentPartNumber + chunkInfo.partNumber - 1;
+                if (chunkInfo.size > uploadPartSize) {
+                    yield* Effect.fail(UploadistaError.fromCode("FILE_WRITE_ERROR", new Error(`Part size ${chunkInfo.size} exceeds upload part size ${uploadPartSize}`)));
+                }
+                // For parts that meet the minimum part size (5MB) or are the final part,
+                // upload them as regular multipart parts
+                if (chunkInfo.size >= minPartSize || isFinalPart) {
+                    yield* Effect.logDebug("Uploading multipart chunk").pipe(Effect.annotateLogs({
+                        upload_id: uploadFile.id,
+                        part_number: actualPartNumber,
+                        chunk_size: chunkInfo.size,
+                        min_part_size: minPartSize,
+                        is_final_part: isFinalPart,
+                    }));
+                    yield* uploadPart(uploadFile, chunkInfo.data, actualPartNumber);
+                    yield* partSizeHistogram(Effect.succeed(chunkInfo.size));
+                }
+                else {
+                    // Only upload as incomplete part if it's smaller than minimum and not final
+                    yield* uploadIncompletePart(uploadFile.id, chunkInfo.data);
+                }
+                yield* Ref.update(totalBytesUploadedRef, (total) => total + chunkInfo.size);
+                // Note: Byte-level progress is now tracked during streaming phase
+                // This ensures smooth progress updates regardless of part size
+                // S3 upload completion is tracked via totalBytesUploadedRef for accuracy
+            });
+            // Process chunks concurrently with controlled concurrency
+            yield* chunkStream.pipe(Stream.runForEach((chunkInfo) => uploadChunk(chunkInfo)), Effect.withConcurrency(maxConcurrentPartUploads));
+            return yield* Ref.get(totalBytesUploadedRef);
+        });
+        const getCapabilities = () => ({
+            supportsParallelUploads: true,
+            supportsConcatenation: true,
+            supportsDeferredLength: true,
+            supportsResumableUploads: true,
+            supportsTransactionalUploads: true,
+            maxConcurrentUploads: maxConcurrentPartUploads,
+            minChunkSize: minPartSize,
+            maxChunkSize: 5_368_709_120, // 5GiB S3 limit
+            maxParts: maxMultipartParts,
+            optimalChunkSize: preferredPartSize,
+            requiresOrderedChunks: false,
+            requiresMimeTypeValidation: true,
+            maxValidationSize: undefined, // no size limit
+        });
+        const getChunkerConstraints = () => ({
+            minChunkSize: minPartSize,
+            maxChunkSize: 5_368_709_120, // 5GiB S3 limit
+            optimalChunkSize: preferredPartSize,
+            requiresOrderedChunks: false,
+        });
+        const validateUploadStrategy = (strategy) => {
+            const capabilities = getCapabilities();
+            const result = (() => {
+                switch (strategy) {
+                    case "parallel":
+                        return capabilities.supportsParallelUploads;
+                    case "single":
+                        return true;
+                    default:
+                        return false;
+                }
+            })();
+            return Effect.succeed(result);
+        };
+        const concatArrayBuffers = (chunks) => {
+            const result = new Uint8Array(chunks.reduce((a, c) => a + c.length, 0));
+            let offset = 0;
+            for (const chunk of chunks) {
+                result.set(chunk, offset);
+                offset += chunk.length;
+            }
+            return result;
+        };
+        const streamToArray = async (stream) => {
+            const reader = stream.getReader();
+            const chunks = [];
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                chunks.push(value);
+            }
+            return concatArrayBuffers(chunks);
+        };
+        const read = (id) => Effect.gen(function* () {
+            const upload = yield* kvStore.get(id);
+            console.log(upload);
+            if (!upload.id) {
+                return yield* Effect.fail(UploadistaError.fromCode("FILE_READ_ERROR", new Error("Upload Key is undefined")));
+            }
+            const s3Key = getS3Key(upload);
+            const stream = yield* s3Client.getObject(s3Key);
+            return yield* Effect.promise(() => streamToArray(stream));
+        });
+        return {
+            bucket,
+            create,
+            remove,
+            write,
+            getUpload,
+            read,
+            deleteExpired,
+            getCapabilities,
+            getChunkerConstraints,
+            validateUploadStrategy,
+        };
+    });
+}
+// Effect-based factory that uses services
+export const createS3Store = (options) => Effect.gen(function* () {
+    const kvStore = yield* UploadFileKVStore;
+    const { s3ClientConfig: { bucket, ...restS3ClientConfig }, } = options;
+    return yield* createS3StoreImplementation({
+        ...options,
+        kvStore,
+    }).pipe(Effect.provide(S3ClientLayer(restS3ClientConfig, bucket)));
+});
+// Backward compatibility: keep the original function for existing code
+export const s3Store = (config) => {
+    const { s3ClientConfig: { bucket, ...restS3ClientConfig }, } = config;
+    return Effect.runPromise(createS3StoreImplementation(config).pipe(Effect.provide(S3ClientLayer(restS3ClientConfig, bucket))));
+};