@uploadista/data-store-s3 0.0.3

package/dist/s3-store-old.js

@@ -0,0 +1,765 @@
+import { NoSuchKey, NotFound, S3 } from "@aws-sdk/client-s3";
+import { UploadistaError } from "@uploadista/core/errors";
+import { UploadFileDataStore, UploadFileKVStore } from "@uploadista/core/types";
+import { s3ActiveUploadsGauge as activeUploadsGauge, s3FileSizeHistogram as fileSizeHistogram, trackS3Error as logS3Error, s3PartSizeHistogram as partSizeHistogram, s3PartUploadDurationHistogram as partUploadDurationHistogram, s3UploadDurationHistogram as uploadDurationHistogram, s3UploadErrorsTotal as uploadErrorsTotal, s3UploadPartsTotal as uploadPartsTotal, s3UploadRequestsTotal as uploadRequestsTotal, s3UploadSuccessTotal as uploadSuccessTotal, withS3ApiMetrics, withS3TimingMetrics as withTimingMetrics, withS3UploadMetrics as withUploadMetrics, } from "@uploadista/observability";
+import { Effect, Layer, Ref, Schedule, Stream } from "effect";
+// Proper single-pass chunking using Effect's async stream constructor
+const createChunkedStream = (chunkSize) => (stream) => {
+    return Stream.async((emit) => {
+        let buffer = new Uint8Array(0);
+        let partNumber = 1;
+        const emitChunk = (data) => {
+            // Log chunk information for debugging
+            Effect.runSync(Effect.logDebug("Creating chunk").pipe(Effect.annotateLogs({
+                part_number: partNumber,
+                chunk_size: data.length,
+                expected_size: chunkSize,
+            })));
+            emit.single({
+                partNumber: partNumber++,
+                data,
+                size: data.length,
+            });
+        };
+        const processChunk = (newData) => {
+            // Combine buffer with new data
+            const combined = new Uint8Array(buffer.length + newData.length);
+            combined.set(buffer);
+            combined.set(newData, buffer.length);
+            buffer = combined;
+            // Emit full chunks
+            while (buffer.length >= chunkSize) {
+                const chunk = buffer.slice(0, chunkSize);
+                buffer = buffer.slice(chunkSize);
+                emitChunk(chunk);
+            }
+        };
+        // Process the stream
+        Effect.runFork(stream.pipe(Stream.runForEach((chunk) => Effect.sync(() => processChunk(chunk))), Effect.andThen(() => Effect.sync(() => {
+            // Emit final chunk if there's remaining data
+            if (buffer.length > 0) {
+                emitChunk(buffer);
+            }
+            emit.end();
+        })), Effect.catchAll((error) => Effect.sync(() => emit.fail(error)))));
+    });
+};
+// Progress tracking with side effects
+const withProgressTracking = (onProgress) => (stream) => onProgress
+    ? stream.pipe(Stream.tap((chunkInfo) => Effect.sync(() => onProgress(chunkInfo.size))))
+    : stream;
+function calcOffsetFromParts(parts) {
+    return parts && parts.length > 0
+        ? parts.reduce((a, b) => a + (b?.Size ?? 0), 0)
+        : 0;
+}
+export function createS3Storexx(options) {
+    return Effect.gen(function* () {
+        const kvStore = yield* UploadFileKVStore;
+        return createS3StoreImplementation({ ...options, kvStore });
+    });
+}
+// Internal s3Store implementation (keeping existing logic)
+function createS3StoreImplementation({ deliveryUrl, partSize, minPartSize = 5_242_880, useTags = true, maxMultipartParts = 10_000, kvStore, maxConcurrentPartUploads = 60, expirationPeriodInMilliseconds = 1000 * 60 * 60 * 24 * 7, // 1 week
+s3ClientConfig: { bucket, ...restS3ClientConfig }, }) {
+    const preferredPartSize = partSize || 8 * 1024 * 1024;
+    const maxUploadSize = 5_497_558_138_880; // 5TiB
+    const client = new S3(restS3ClientConfig);
+    const shouldUseExpirationTags = () => {
+        return expirationPeriodInMilliseconds !== 0 && useTags;
+    };
+    const completeMetadata = (upload) => {
+        return Effect.gen(function* () {
+            if (!shouldUseExpirationTags()) {
+                return 0;
+            }
+            const uploadFile = yield* kvStore.get(upload.id);
+            const uploadId = uploadFile.storage.uploadId;
+            if (!uploadId) {
+                return 0;
+            }
+            yield* kvStore.set(upload.id, {
+                ...uploadFile,
+                storage: { ...uploadFile.storage, uploadId },
+            });
+            return 0;
+        });
+    };
+    const partKey = (id) => {
+        return `${id}.part`;
+    };
+    const uploadPart = (uploadFile, data, partNumber) => withS3ApiMetrics("uploadPart", withTimingMetrics(partUploadDurationHistogram, Effect.tryPromise({
+        try: () => client.uploadPart({
+            Bucket: bucket,
+            Key: uploadFile.id,
+            UploadId: uploadFile.storage.uploadId,
+            PartNumber: partNumber,
+            Body: data,
+        }),
+        catch: (error) => {
+            // Use enhanced error tracking
+            Effect.runSync(logS3Error("uploadPart", error, {
+                upload_id: uploadFile.id,
+                part_number: partNumber,
+                part_size: data.length,
+                s3_bucket: bucket,
+            }));
+            return UploadistaError.fromCode("FILE_WRITE_ERROR", error);
+        },
+    }).pipe(Effect.retry(Schedule.exponential("1 second", 2.0).pipe(Schedule.intersect(Schedule.recurs(3)))), Effect.tapError((error) => Effect.logWarning("Retrying part upload").pipe(Effect.annotateLogs({
+        upload_id: uploadFile.id,
+        part_number: partNumber,
+        error_message: error.message,
+        retry_attempt: "unknown", // Will be overridden by the retry schedule
+        part_size: data.length,
+        s3_bucket: bucket,
+    }))), Effect.map((response) => response.ETag), Effect.tap(() => uploadPartsTotal(Effect.succeed(1))), Effect.tap(() => Effect.logInfo("Part uploaded successfully").pipe(Effect.annotateLogs({
+        upload_id: uploadFile.id,
+        part_number: partNumber,
+    })))))).pipe(Effect.withSpan(`s3-upload-part-${partNumber}`, {
+        attributes: {
+            "upload.id": uploadFile.id,
+            "upload.part_number": partNumber,
+            "upload.part_size": data.length,
+            "s3.bucket": bucket,
+            "s3.key": uploadFile.id,
+        },
+    }));
+    const uploadIncompletePart = (id, data) => Effect.tryPromise({
+        try: () => client.putObject({
+            Bucket: bucket,
+            Key: partKey(id),
+            Body: data,
+        }),
+        catch: (error) => UploadistaError.fromCode("FILE_WRITE_ERROR", error),
+    }).pipe(Effect.flatMap((response) => response.ETag
+        ? Effect.succeed(response.ETag)
+        : Effect.fail(UploadistaError.fromCode("FILE_WRITE_ERROR", new Error("ETag is undefined when uploading incomplete part")))), Effect.tap(() => Effect.logInfo("Incomplete part uploaded").pipe(Effect.annotateLogs({ upload_id: id }))));
+    const getIncompletePart = (id) => {
+        return Effect.tryPromise({
+            try: async () => {
+                try {
+                    const data = await client.getObject({
+                        Bucket: bucket,
+                        Key: partKey(id),
+                    });
+                    return data.Body;
+                }
+                catch (error) {
+                    if (error instanceof NoSuchKey) {
+                        return undefined;
+                    }
+                    throw error;
+                }
+            },
+            catch: (error) => UploadistaError.fromCode("FILE_WRITE_ERROR", error),
+        });
+    };
+    const getIncompletePartSize = (id) => {
+        return Effect.tryPromise({
+            try: async () => {
+                try {
+                    const data = await client.headObject({
+                        Bucket: bucket,
+                        Key: partKey(id),
+                    });
+                    return data.ContentLength;
+                }
+                catch (error) {
+                    if (error instanceof NotFound) {
+                        return undefined;
+                    }
+                    throw error;
+                }
+            },
+            catch: (error) => UploadistaError.fromCode("FILE_WRITE_ERROR", error),
+        });
+    };
+    const deleteIncompletePart = (id) => {
+        return Effect.tryPromise({
+            try: async () => {
+                await client.deleteObject({
+                    Bucket: bucket,
+                    Key: partKey(id),
+                });
+            },
+            catch: (error) => UploadistaError.fromCode("FILE_WRITE_ERROR", error),
+        });
+    };
+    const downloadIncompletePart = (id) => {
+        return Effect.gen(function* () {
+            const incompletePart = yield* getIncompletePart(id);
+            if (!incompletePart) {
+                return;
+            }
+            // Read the stream and collect all chunks to calculate size
+            const reader = incompletePart.getReader();
+            const chunks = [];
+            let incompletePartSize = 0;
+            try {
+                while (true) {
+                    const { done, value } = yield* Effect.promise(() => reader.read());
+                    if (done)
+                        break;
+                    chunks.push(value);
+                    incompletePartSize += value.length;
+                }
+            }
+            finally {
+                reader.releaseLock();
+            }
+            const stream = Stream.fromIterable(chunks);
+            return {
+                size: incompletePartSize,
+                stream,
+            };
+        });
+    };
+    const calcOptimalPartSize = (initSize) => {
+        const size = initSize ?? maxUploadSize;
+        let optimalPartSize;
+        if (size <= preferredPartSize) {
+            optimalPartSize = size;
+        }
+        else if (size <= preferredPartSize * maxMultipartParts) {
+            optimalPartSize = preferredPartSize;
+        }
+        else {
+            // Calculate the minimum part size needed to fit within the max parts limit
+            optimalPartSize = Math.ceil(size / maxMultipartParts);
+        }
+        // For small files (smaller than minPartSize), use the actual file size
+        // to avoid unnecessary chunking. Only enforce minPartSize for larger files.
+        const finalPartSize = initSize && initSize < minPartSize
+            ? optimalPartSize // Use actual file size for small files
+            : Math.max(optimalPartSize, minPartSize);
+        // Round up to ensure consistent part sizes (align to chunk boundaries)
+        // This ensures all parts except the last one will have exactly the same size
+        return Math.ceil(finalPartSize / 1024) * 1024; // Align to 1KB boundaries
+    };
+    /**
+     * Uploads a stream to S3 using multiple parts with streaming sink pattern
+     */
+    const uploadParts = (uploadFile, readStream, initCurrentPartNumber, initOffset, onProgress) => Effect.gen(function* () {
+        const uploadPartSize = calcOptimalPartSize(uploadFile.size);
+        yield* Effect.logInfo("Starting part uploads").pipe(Effect.annotateLogs({
+            upload_id: uploadFile.id,
+            init_offset: initOffset,
+            file_size: uploadFile.size,
+            part_size: uploadPartSize,
+            min_part_size: minPartSize,
+        }));
+        // Run the streaming pipeline
+        const chunkStream = readStream.pipe(createChunkedStream(uploadPartSize), withProgressTracking(onProgress
+            ? (_chunkSize) => {
+                // Progress tracking handled via the sink's Ref updates
+                // The actual progress reporting happens in uploadChunk
+            }
+            : undefined));
+        // Track cumulative offset and total bytes with Effect Refs
+        const cumulativeOffsetRef = yield* Ref.make(initOffset);
+        const totalBytesUploadedRef = yield* Ref.make(0);
+        // Create a chunk upload function for the sink
+        const uploadChunk = (chunkInfo) => Effect.gen(function* () {
+            // Calculate cumulative bytes to determine if this is the final part
+            const cumulativeOffset = yield* Ref.updateAndGet(cumulativeOffsetRef, (offset) => offset + chunkInfo.size);
+            const isFinalPart = cumulativeOffset >= (uploadFile.size || 0);
+            yield* Effect.logDebug("Processing chunk").pipe(Effect.annotateLogs({
+                upload_id: uploadFile.id,
+                cumulative_offset: cumulativeOffset,
+                file_size: uploadFile.size,
+                chunk_size: chunkInfo.size,
+                is_final_part: isFinalPart,
+            }));
+            const actualPartNumber = initCurrentPartNumber + chunkInfo.partNumber - 1;
+            if (chunkInfo.size > uploadPartSize) {
+                yield* Effect.fail(UploadistaError.fromCode("FILE_WRITE_ERROR", new Error(`Part size ${chunkInfo.size} exceeds upload part size ${uploadPartSize}`)));
+            }
+            // For parts that meet the minimum part size (5MB) or are the final part,
+            // upload them as regular multipart parts
+            if (chunkInfo.size >= minPartSize || isFinalPart) {
+                yield* Effect.logDebug("Uploading multipart chunk").pipe(Effect.annotateLogs({
+                    upload_id: uploadFile.id,
+                    part_number: actualPartNumber,
+                    chunk_size: chunkInfo.size,
+                    min_part_size: minPartSize,
+                    is_final_part: isFinalPart,
+                }));
+                yield* uploadPart(uploadFile, chunkInfo.data, actualPartNumber);
+                yield* partSizeHistogram(Effect.succeed(chunkInfo.size));
+            }
+            else {
+                // Only upload as incomplete part if it's smaller than minimum and not final
+                yield* uploadIncompletePart(uploadFile.id, chunkInfo.data);
+            }
+            yield* Ref.update(totalBytesUploadedRef, (total) => total + chunkInfo.size);
+            // Report progress if callback provided
+            if (onProgress && cumulativeOffset > initOffset) {
+                onProgress(cumulativeOffset);
+            }
+        });
+        // Process chunks concurrently with controlled concurrency
+        yield* chunkStream.pipe(Stream.runForEach((chunkInfo) => uploadChunk(chunkInfo)), Effect.withConcurrency(maxConcurrentPartUploads));
+        return yield* Ref.get(totalBytesUploadedRef);
+    });
+    /**
+     * Completes a multipart upload on S3.
+     * This is where S3 concatenates all the uploaded parts.
+     */
+    const finishMultipartUpload = (uploadFile, parts) => {
+        return withS3ApiMetrics("completeMultipartUpload", Effect.tryPromise({
+            try: () => client
+                .completeMultipartUpload({
+                Bucket: bucket,
+                Key: uploadFile.id,
+                UploadId: uploadFile.storage.uploadId,
+                MultipartUpload: {
+                    Parts: parts.map((part) => {
+                        return {
+                            ETag: part.ETag,
+                            PartNumber: part.PartNumber,
+                        };
+                    }),
+                },
+            })
+                .then((response) => response.Location),
+            catch: (error) => UploadistaError.fromCode("FILE_WRITE_ERROR", error),
+        })).pipe(Effect.tap(() => uploadSuccessTotal(Effect.succeed(1))), Effect.withSpan("s3-complete-multipart-upload", {
+            attributes: {
+                "upload.id": uploadFile.id,
+                "upload.parts_count": parts.length,
+                "s3.bucket": bucket,
+                "s3.key": uploadFile.id,
+            },
+        }));
+    };
+    const retrievePartsS3 = async ({ id, uploadId, partNumberMarker, }) => {
+        try {
+            const params = {
+                Bucket: bucket,
+                Key: id,
+                UploadId: uploadId,
+                PartNumberMarker: partNumberMarker,
+            };
+            const data = await client.listParts(params);
+            let parts = data.Parts ?? [];
+            if (data.IsTruncated) {
+                const rest = await retrievePartsS3({
+                    id,
+                    uploadId,
+                    partNumberMarker: data.NextPartNumberMarker,
+                });
+                parts = [...parts, ...rest.parts];
+            }
+            if (!partNumberMarker) {
+                parts.sort((a, b) => (a.PartNumber ?? 0) - (b.PartNumber ?? 0));
+            }
+            return { uploadFound: true, parts };
+        }
+        catch (error) {
+            // Check if the error is caused by the upload not being found. This happens
+            // when the multipart upload has already been completed or aborted. Since
+            // we already found the info object, we know that the upload has been
+            // completed and therefore can ensure the the offset is the size.
+            // AWS S3 returns NoSuchUpload, but other implementations, such as DigitalOcean
+            // Spaces, can also return NoSuchKey.
+            if (typeof error === "object" &&
+                error !== null &&
+                "code" in error &&
+                typeof error.code === "string" &&
+                (error.code === "NoSuchUpload" || error.code === "NoSuchKey")) {
+                Effect.runSync(Effect.logWarning("S3 upload not found during listParts").pipe(Effect.annotateLogs({
+                    upload_id: id,
+                    error_code: error.code,
+                })));
+                return { uploadFound: false, parts: [] };
+            }
+            throw error;
+        }
+    };
+    const getUploadId = (uploadFile) => {
+        const uploadId = uploadFile.storage.uploadId;
+        if (!uploadId) {
+            return Effect.fail(UploadistaError.fromCode("FILE_WRITE_ERROR", new Error("Upload ID is undefined")));
+        }
+        return Effect.succeed(uploadId);
+    };
+    /**
+     * Gets the number of complete parts/chunks already uploaded to S3.
+     * Retrieves only consecutive parts.
+     */
+    const retrieveParts = (id, partNumberMarker) => {
+        return Effect.gen(function* () {
+            const metadata = yield* kvStore.get(id);
+            const uploadId = yield* getUploadId(metadata);
+            const { parts, uploadFound } = yield* Effect.promise(() => retrievePartsS3({ id, uploadId, partNumberMarker }));
+            return { uploadFound, parts };
+        });
+    };
+    /**
+     * Removes cached data for a given file.
+     */
+    const clearCache = (id) => {
+        return Effect.gen(function* () {
+            yield* Effect.logInfo("Clearing cache").pipe(Effect.annotateLogs({ upload_id: id }));
+            yield* kvStore.delete(id);
+        });
+    };
+    const createMultipartUpload = (upload) => {
+        return withS3ApiMetrics("createMultipartUpload", Effect.tryPromise({
+            try: async () => {
+                const request = {
+                    Bucket: bucket,
+                    Key: upload.id,
+                };
+                if (upload.metadata?.contentType) {
+                    request.ContentType = upload.metadata.contentType;
+                }
+                if (upload.metadata?.cacheControl) {
+                    request.CacheControl = upload.metadata.cacheControl;
+                }
+                const res = await client.createMultipartUpload(request);
+                upload.storage = {
+                    id: upload.storage.id,
+                    type: upload.storage.type,
+                    path: res.Key,
+                    uploadId: res.UploadId,
+                    bucket,
+                };
+                upload.url = `${deliveryUrl}/${upload.id}`;
+                return upload;
+            },
+            catch: (error) => {
+                Effect.runSync(logS3Error("createMultipartUpload", error, {
+                    upload_id: upload.id,
+                    s3_bucket: bucket,
+                    content_type: upload.metadata?.contentType,
+                    file_size: upload.size,
+                }));
+                return UploadistaError.fromCode("FILE_WRITE_ERROR", error);
+            },
+        })).pipe(Effect.tap(() => fileSizeHistogram(Effect.succeed(upload.size || 0))), Effect.withSpan("s3-create-multipart-upload", {
+            attributes: {
+                "upload.id": upload.id,
+                "upload.size": upload.size || 0,
+                "upload.content_type": upload.metadata?.contentType || "unknown",
+                "s3.bucket": bucket,
+                "s3.key": upload.id,
+            },
+        }));
+    };
+    /**
+     * Creates a multipart upload on S3 attaching any metadata to it.
+     * Also, a `${file_id}.info` file is created which holds some information
+     * about the upload itself like: `upload-id`, `upload-length`, etc.
+     */
+    const create = (upload) => {
+        return Effect.gen(function* () {
+            yield* Effect.logInfo("Initializing multipart upload").pipe(Effect.annotateLogs({ upload_id: upload.id }));
+            const uploadCreated = yield* createMultipartUpload(upload);
+            yield* kvStore.set(upload.id, uploadCreated);
+            yield* Effect.logInfo("Multipart upload created").pipe(Effect.annotateLogs({
+                upload_id: upload.id,
+                s3_upload_id: uploadCreated.storage.uploadId,
+            }));
+            yield* uploadRequestsTotal(Effect.succeed(1));
+            return uploadCreated;
+        }).pipe(Effect.withSpan("s3-create-upload", {
+            attributes: {
+                "upload.id": upload.id,
+                "upload.size": upload.size || 0,
+                "s3.bucket": bucket,
+            },
+        }));
+    };
+    const read = (id) => {
+        return Effect.tryPromise({
+            try: async () => {
+                const data = await client.getObject({
+                    Bucket: bucket,
+                    Key: id,
+                });
+                return data.Body;
+            },
+            catch: (error) => UploadistaError.fromCode("FILE_WRITE_ERROR", error),
+        });
+    };
+    const prepareUpload = (file_id, initialOffset, initialData) => {
+        return Effect.gen(function* () {
+            const uploadFile = yield* kvStore.get(file_id);
+            yield* Effect.logDebug("Retrieved upload metadata").pipe(Effect.annotateLogs({
+                upload_id: file_id,
+                metadata: uploadFile,
+            }));
+            const { parts } = yield* retrieveParts(file_id);
+            yield* Effect.logDebug("Retrieved existing parts").pipe(Effect.annotateLogs({
+                upload_id: file_id,
+                parts_count: parts.length,
+                parts: parts.map((p) => ({ part: p.PartNumber, size: p.Size })),
+            }));
+            const partNumber = parts.length > 0 && parts[parts.length - 1].PartNumber
+                ? (parts[parts.length - 1].PartNumber ?? 0)
+                : 0;
+            const nextPartNumber = partNumber + 1;
+            const incompletePart = yield* downloadIncompletePart(file_id);
+            if (incompletePart) {
+                yield* Effect.logDebug("Found incomplete part").pipe(Effect.annotateLogs({
+                    upload_id: file_id,
+                    incomplete_part_size: incompletePart.size,
+                }));
+                yield* deleteIncompletePart(file_id);
+                const offset = initialOffset - incompletePart.size;
+                const data = incompletePart.stream.pipe(Stream.concat(initialData));
+                return {
+                    uploadFile,
+                    nextPartNumber, // Use the current next part number - incomplete part merges with new data
+                    offset,
+                    incompletePartSize: incompletePart.size,
+                    data,
+                };
+            }
+            else {
+                return {
+                    uploadFile,
+                    nextPartNumber,
+                    offset: initialOffset,
+                    incompletePartSize: 0,
+                    data: initialData,
+                };
+            }
+        });
+    };
+    /**
+     * Write to the file, starting at the provided offset
+     */
+    const write = (options, dependencies) => {
+        return withUploadMetrics(options.file_id, withTimingMetrics(uploadDurationHistogram, Effect.gen(function* () {
+            const { stream: initialData, file_id, offset: initialOffset, } = options;
+            const { onProgress } = dependencies;
+            // Track active upload
+            yield* activeUploadsGauge(Effect.succeed(1));
+            const prepareResult = yield* prepareUpload(file_id, initialOffset, initialData);
+            const { uploadFile, nextPartNumber, offset, data } = prepareResult;
+            const bytesUploaded = yield* uploadParts(uploadFile, data, nextPartNumber, offset, onProgress);
+            const newOffset = offset + bytesUploaded;
+            if (uploadFile.size === newOffset) {
+                const finishUploadEffect = Effect.gen(function* () {
+                    const { parts } = yield* retrieveParts(file_id);
+                    yield* finishMultipartUpload(uploadFile, parts);
+                    yield* completeMetadata(uploadFile);
+                    yield* clearCache(file_id);
+                });
+                yield* finishUploadEffect.pipe(Effect.tapError((error) => {
+                    return Effect.gen(function* () {
+                        yield* uploadErrorsTotal(Effect.succeed(1));
+                        yield* Effect.logError("Failed to finish upload").pipe(Effect.annotateLogs({
+                            upload_id: file_id,
+                            error: String(error),
+                        }));
+                    });
+                }));
+            }
+            return newOffset;
+        }).pipe(Effect.ensuring(activeUploadsGauge(Effect.succeed(0)))))).pipe(Effect.withSpan("s3-upload-write", {
+            attributes: {
+                "upload.id": options.file_id,
+                "upload.offset": options.offset,
+                "s3.bucket": bucket,
+            },
+        }));
+    };
+    const getUpload = (id) => {
+        return Effect.gen(function* () {
+            const uploadFile = yield* kvStore.get(id);
+            let offset = 0;
+            const { parts, uploadFound } = yield* retrieveParts(id);
+            if (!uploadFound) {
+                return {
+                    ...uploadFile,
+                    offset: uploadFile.size,
+                    size: uploadFile.size,
+                    metadata: uploadFile.metadata,
+                    storage: uploadFile.storage,
+                };
+            }
+            offset = calcOffsetFromParts(parts);
+            const incompletePartSize = yield* getIncompletePartSize(id);
+            return {
+                ...uploadFile,
+                offset: offset + (incompletePartSize ?? 0),
+                size: uploadFile.size,
+                storage: uploadFile.storage,
+            };
+        });
+    };
+    const abortMultipartUploadAndDeleteObjects = (id, uploadId) => {
+        return Effect.tryPromise({
+            try: async () => {
+                await client.abortMultipartUpload({
+                    Bucket: bucket,
+                    Key: id,
+                    UploadId: uploadId,
+                });
+                await client.deleteObjects({
+                    Bucket: bucket,
+                    Delete: {
+                        Objects: [{ Key: id }],
+                    },
+                });
+            },
+            catch: (error) => {
+                if (typeof error === "object" &&
+                    error !== null &&
+                    "code" in error &&
+                    typeof error.code === "string" &&
+                    ["NotFound", "NoSuchKey", "NoSuchUpload"].includes(error.code)) {
+                    Effect.runSync(Effect.logWarning("File not found during remove operation").pipe(Effect.annotateLogs({
+                        upload_id: id,
+                        error_code: error.code,
+                    })));
+                    return UploadistaError.fromCode("FILE_NOT_FOUND");
+                }
+                return UploadistaError.fromCode("FILE_WRITE_ERROR", error);
+            },
+        });
+    };
+    const remove = (id) => {
+        return Effect.gen(function* () {
+            const uploadFile = yield* kvStore.get(id);
+            const uploadId = yield* getUploadId(uploadFile);
+            yield* abortMultipartUploadAndDeleteObjects(id, uploadId);
+            yield* clearCache(id);
+        });
+    };
+    const getExpiration = () => {
+        return expirationPeriodInMilliseconds;
+    };
+    const getExpirationDate = (created_at) => {
+        const date = new Date(created_at);
+        return new Date(date.getTime() + getExpiration());
+    };
+    const deleteExpired = () => {
+        return Effect.tryPromise({
+            try: async () => {
+                if (getExpiration() === 0) {
+                    return 0;
+                }
+                let keyMarker;
+                let uploadIdMarker;
+                let isTruncated = true;
+                let deleted = 0;
+                while (isTruncated) {
+                    const listResponse = await client.listMultipartUploads({
+                        Bucket: bucket,
+                        KeyMarker: keyMarker,
+                        UploadIdMarker: uploadIdMarker,
+                    });
+                    const expiredUploads = listResponse.Uploads?.filter((multiPartUpload) => {
+                        const initiatedDate = multiPartUpload.Initiated;
+                        return (initiatedDate &&
+                            Date.now() >
+                                getExpirationDate(initiatedDate.toISOString()).getTime());
+                    }) || [];
+                    const objectsToDelete = expiredUploads.reduce((all, expiredUpload) => {
+                        all.push({
+                            key: partKey(expiredUpload.Key),
+                        });
+                        return all;
+                    }, []);
+                    const deletions = [];
+                    // Batch delete 1000 items at a time
+                    while (objectsToDelete.length > 0) {
+                        const objects = objectsToDelete.splice(0, 1000);
+                        deletions.push(client.deleteObjects({
+                            Bucket: bucket,
+                            Delete: {
+                                Objects: objects.map((object) => ({
+                                    Key: object.key,
+                                })),
+                            },
+                        }));
+                    }
+                    const [objectsDeleted] = await Promise.all([
+                        Promise.all(deletions),
+                        ...expiredUploads.map((expiredUpload) => {
+                            return client.abortMultipartUpload({
+                                Bucket: bucket,
+                                Key: expiredUpload.Key,
+                                UploadId: expiredUpload.UploadId,
+                            });
+                        }),
+                    ]);
+                    deleted += objectsDeleted.reduce((all, acc) => all + (acc.Deleted?.length ?? 0), 0);
+                    isTruncated = listResponse.IsTruncated ?? false;
+                    if (isTruncated) {
+                        keyMarker = listResponse.NextKeyMarker;
+                        uploadIdMarker = listResponse.NextUploadIdMarker;
+                    }
+                }
+                return deleted;
+            },
+            catch: (error) => UploadistaError.fromCode("FILE_WRITE_ERROR", error),
+        });
+    };
+    const getCapabilities = () => {
+        return {
+            supportsParallelUploads: true,
+            supportsConcatenation: true,
+            supportsDeferredLength: true,
+            supportsResumableUploads: true,
+            supportsTransactionalUploads: true,
+            maxConcurrentUploads: maxConcurrentPartUploads,
+            minChunkSize: minPartSize,
+            maxChunkSize: 5_368_709_120, // 5GiB S3 limit
+            maxParts: maxMultipartParts,
+            optimalChunkSize: preferredPartSize,
+            requiresOrderedChunks: false,
+        };
+    };
+    const getChunkerConstraints = () => {
+        return {
+            minChunkSize: minPartSize,
+            maxChunkSize: 5_368_709_120, // 5GiB S3 limit
+            optimalChunkSize: preferredPartSize,
+            requiresOrderedChunks: false,
+        };
+    };
+    const validateUploadStrategy = (strategy) => {
+        const capabilities = getCapabilities();
+        const result = (() => {
+            switch (strategy) {
+                case "parallel":
+                    return capabilities.supportsParallelUploads;
+                case "single":
+                    return true;
+                default:
+                    return false;
+            }
+        })();
+        return Effect.succeed(result);
+    };
+    return {
+        bucket,
+        create,
+        remove,
+        write,
+        getUpload,
+        read,
+        deleteExpired: deleteExpired(),
+        getCapabilities,
+        getChunkerConstraints,
+        validateUploadStrategy,
+    };
+}
+// Effect-based factory that uses services
+export const createS3Store = (options) => Effect.gen(function* () {
+    const kvStore = yield* UploadFileKVStore;
+    return createS3StoreImplementation({
+        ...options,
+        kvStore,
+    });
+});
+// Layer for providing the S3Store service
+export const S3StoreLayer = (options) => Layer.effect(UploadFileDataStore, createS3Store(options));
+// Backward compatibility: keep the original function for existing code
+export const s3Store = createS3StoreImplementation;