@uploadista/data-store-gcs 0.0.3

package/dist/gcs-store.js

@@ -0,0 +1,282 @@
+import { PassThrough, pipeline, Readable, Transform } from "node:stream";
+import { Storage } from "@google-cloud/storage";
+import { UploadistaError } from "@uploadista/core/errors";
+import { UploadFileDataStore, UploadFileKVStore, } from "@uploadista/core/types";
+import { gcsActiveUploadsGauge as activeUploadsGauge, gcsFileSizeHistogram as fileSizeHistogram, logGCSUploadCompletion, trackGCSError, gcsUploadDurationHistogram as uploadDurationHistogram, gcsUploadErrorsTotal as uploadErrorsTotal, gcsUploadRequestsTotal as uploadRequestsTotal, gcsUploadSuccessTotal as uploadSuccessTotal, withGCSTimingMetrics as withTimingMetrics, withGCSUploadMetrics as withUploadMetrics, } from "@uploadista/observability";
+import { Effect, Layer, Stream } from "effect";
+/**
+ * Convert the Upload object to a format that can be stored in GCS metadata.
+ */
+function stringifyUploadKeys(upload) {
+    return {
+        size: upload.size ?? null,
+        sizeIsDeferred: `${upload.sizeIsDeferred}`,
+        offset: upload.offset,
+        metadata: JSON.stringify(upload.metadata),
+        storage: JSON.stringify(upload.storage),
+    };
+}
+const getUpload = (bucket, id, kvStore) => {
+    return Effect.gen(function* () {
+        try {
+            const [metadata] = yield* Effect.promise(() => bucket.file(id).getMetadata());
+            const { size, metadata: meta } = metadata;
+            const file = yield* kvStore.get(id);
+            return {
+                id,
+                size: size ? Number.parseInt(`${size}`, 10) : undefined,
+                offset: metadata.size ? Number.parseInt(`${metadata.size}`, 10) : 0, // `size` is set by GCS
+                metadata: meta ? meta : undefined,
+                storage: {
+                    id: file.storage.id,
+                    type: file.storage.type,
+                    path: id,
+                    bucket: bucket.name,
+                },
+            };
+        }
+        catch (error) {
+            if (error &&
+                typeof error === "object" &&
+                "code" in error &&
+                error.code === 404) {
+                return yield* Effect.fail(UploadistaError.fromCode("FILE_NOT_FOUND"));
+            }
+            throw error;
+        }
+    });
+};
+export function createGCSStore({ keyFilename, credentials, bucketName, }) {
+    return Effect.gen(function* () {
+        const kvStore = yield* UploadFileKVStore;
+        return gcsStore({ keyFilename, credentials, bucketName, kvStore });
+    });
+}
+export function gcsStore({ keyFilename, credentials, bucketName, kvStore, }) {
+    const storage = new Storage(keyFilename ? { keyFilename } : credentials ? { credentials } : {});
+    const bucket = storage.bucket(bucketName);
+    const getCapabilities = () => {
+        return {
+            supportsParallelUploads: false, // GCS doesn't have native multipart upload like S3
+            supportsConcatenation: true, // Can combine files using bucket.combine
+            supportsDeferredLength: true,
+            supportsResumableUploads: true, // Through patch files
+            supportsTransactionalUploads: false,
+            maxConcurrentUploads: 1, // Sequential operations
+            minChunkSize: undefined,
+            maxChunkSize: undefined,
+            maxParts: undefined,
+            optimalChunkSize: 8 * 1024 * 1024, // 8MB default
+            requiresOrderedChunks: true, // Due to combine operation
+            requiresMimeTypeValidation: true,
+            maxValidationSize: undefined, // no size limit
+        };
+    };
+    const validateUploadStrategy = (strategy) => {
+        const capabilities = getCapabilities();
+        const result = (() => {
+            switch (strategy) {
+                case "parallel":
+                    return capabilities.supportsParallelUploads;
+                case "single":
+                    return true;
+                default:
+                    return false;
+            }
+        })();
+        return Effect.succeed(result);
+    };
+    return {
+        bucket: bucket.name,
+        create: (file) => {
+            return Effect.gen(function* () {
+                yield* uploadRequestsTotal(Effect.succeed(1));
+                yield* activeUploadsGauge(Effect.succeed(1));
+                yield* fileSizeHistogram(Effect.succeed(file.size || 0));
+                if (!file.id) {
+                    yield* uploadErrorsTotal(Effect.succeed(1));
+                    return yield* Effect.fail(UploadistaError.fromCode("FILE_NOT_FOUND"));
+                }
+                const gcs_file = bucket.file(file.id);
+                file.storage = {
+                    id: file.storage.id,
+                    type: file.storage.type,
+                    path: file.id,
+                    bucket: bucket.name,
+                };
+                console.log("file", gcs_file.id);
+                const options = {
+                    metadata: {
+                        metadata: {
+                            ...stringifyUploadKeys(file),
+                        },
+                    },
+                };
+                if (file.metadata?.contentType) {
+                    options.contentType = file.metadata.contentType.toString();
+                }
+                return yield* Effect.tryPromise({
+                    try: () => {
+                        console.log("creating file", gcs_file.id);
+                        return new Promise((resolve, reject) => {
+                            const fake_stream = new PassThrough();
+                            fake_stream.end();
+                            fake_stream
+                                .pipe(gcs_file.createWriteStream(options))
+                                .on("error", reject)
+                                .on("finish", () => {
+                                resolve(file);
+                            });
+                        });
+                    },
+                    catch: (error) => {
+                        console.error("error creating file", error);
+                        Effect.runSync(trackGCSError("create", error, {
+                            upload_id: file.id,
+                            bucket: bucket.name,
+                        }));
+                        return UploadistaError.fromCode("FILE_WRITE_ERROR", {
+                            cause: error,
+                        });
+                    },
+                });
+            });
+        },
+        read: (file_id) => {
+            return Effect.tryPromise({
+                try: async () => {
+                    const [buffer] = await bucket.file(file_id).download();
+                    return new Uint8Array(buffer);
+                },
+                catch: (error) => {
+                    Effect.runSync(trackGCSError("read", error, {
+                        upload_id: file_id,
+                        bucket: bucket.name,
+                    }));
+                    if (error &&
+                        typeof error === "object" &&
+                        "code" in error &&
+                        error.code === 404) {
+                        return UploadistaError.fromCode("FILE_NOT_FOUND");
+                    }
+                    return UploadistaError.fromCode("FILE_READ_ERROR", {
+                        cause: error,
+                    });
+                },
+            });
+        },
+        remove: (file_id) => {
+            return Effect.gen(function* () {
+                try {
+                    yield* Effect.promise(() => bucket.file(file_id).delete());
+                    yield* activeUploadsGauge(Effect.succeed(-1));
+                }
+                catch (error) {
+                    Effect.runSync(trackGCSError("remove", error, {
+                        upload_id: file_id,
+                        bucket: bucket.name,
+                    }));
+                    throw error;
+                }
+            });
+        },
+        /**
+         * Get the file metatata from the object in GCS, then upload a new version
+         * passing through the metadata to the new version.
+         */
+        write: (options, dependencies) => {
+            return withUploadMetrics(options.file_id, withTimingMetrics(uploadDurationHistogram, Effect.gen(function* () {
+                const startTime = Date.now();
+                const { file_id, offset, stream: effectStream } = options;
+                console.log("write", file_id, offset);
+                const { onProgress } = dependencies;
+                // GCS Doesn't persist metadata within versions,
+                // get that metadata first
+                const upload = yield* getUpload(bucket, file_id, kvStore);
+                console.log("upload", upload);
+                return yield* Effect.promise(() => new Promise((resolve, reject) => {
+                    const file = bucket.file(file_id);
+                    const destination = upload.offset === 0
+                        ? file
+                        : bucket.file(`${file_id}_patch`);
+                    upload.offset = offset;
+                    const gcsOptions = {
+                        metadata: {
+                            metadata: {
+                                ...stringifyUploadKeys(upload),
+                            },
+                        },
+                    };
+                    const write_stream = destination.createWriteStream(gcsOptions);
+                    if (!write_stream) {
+                        Effect.runSync(uploadErrorsTotal(Effect.succeed(1)));
+                        reject(UploadistaError.fromCode("FILE_WRITE_ERROR"));
+                        return;
+                    }
+                    let bytes_received = upload.offset;
+                    // Convert Effect Stream to ReadableStream
+                    const readableStream = Stream.toReadableStream(effectStream);
+                    const transform = new Transform({
+                        transform(chunk, _, callback) {
+                            bytes_received += chunk.length;
+                            onProgress?.(bytes_received);
+                            callback(null, chunk);
+                        },
+                    });
+                    const nodeReadable = Readable.fromWeb(readableStream);
+                    pipeline(nodeReadable, transform, write_stream, async (e) => {
+                        if (e) {
+                            console.error("error writing file", e);
+                            Effect.runSync(trackGCSError("write", e, {
+                                upload_id: file_id,
+                                bucket: bucket.name,
+                                offset,
+                            }));
+                            try {
+                                await destination.delete({ ignoreNotFound: true });
+                            }
+                            finally {
+                                reject(UploadistaError.fromCode("FILE_WRITE_ERROR"));
+                            }
+                        }
+                        else {
+                            try {
+                                if (file !== destination) {
+                                    await bucket.combine([file, destination], file);
+                                    await Promise.all([
+                                        file.setMetadata(gcsOptions.metadata),
+                                        destination.delete({ ignoreNotFound: true }),
+                                    ]);
+                                }
+                                // Log completion
+                                Effect.runSync(logGCSUploadCompletion(file_id, {
+                                    fileSize: upload.size || 0,
+                                    totalDurationMs: Date.now() - startTime,
+                                    partsCount: 1,
+                                    averagePartSize: upload.size,
+                                    throughputBps: (upload.size || 0) / (Date.now() - startTime),
+                                    retryCount: 0,
+                                }));
+                                Effect.runSync(uploadSuccessTotal(Effect.succeed(1)));
+                                Effect.runSync(activeUploadsGauge(Effect.succeed(-1)));
+                                resolve(bytes_received);
+                            }
+                            catch (error) {
+                                console.error(error);
+                                Effect.runSync(trackGCSError("write", error, {
+                                    upload_id: file_id,
+                                    bucket: bucket.name,
+                                    operation: "combine",
+                                }));
+                                reject(UploadistaError.fromCode("FILE_WRITE_ERROR"));
+                            }
+                        }
+                    });
+                }));
+            })));
+        },
+        getCapabilities,
+        validateUploadStrategy,
+    };
+}
+export const GCSStoreLayer = (options) => Layer.effect(UploadFileDataStore, createGCSStore(options));

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./gcs-store";
+export { gcsStoreNodejs, gcsStoreRest } from "./gcs-store-v2";
+export * from "./services";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,aAAa,CAAC;AAG5B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9D,cAAc,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,5 @@
+// Legacy Node.js implementation (for Node.js environments only)
+export * from "./gcs-store";
+// New service-based implementations (recommended)
+export { gcsStoreNodejs, gcsStoreRest } from "./gcs-store-v2";
+export * from "./services";

package/dist/services/gcs-client-nodejs.service.d.ts

@@ -0,0 +1,4 @@
+import { Layer } from "effect";
+import { type GCSClientConfig, GCSClientService } from "./gcs-client.service";
+export declare const GCSClientNodeJSLayer: (config: GCSClientConfig) => Layer.Layer<GCSClientService, never, never>;
+//# sourceMappingURL=gcs-client-nodejs.service.d.ts.map

package/dist/services/gcs-client-nodejs.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gcs-client-nodejs.service.d.ts","sourceRoot":"","sources":["../../src/services/gcs-client-nodejs.service.ts"],"names":[],"mappings":"AAGA,OAAO,EAAU,KAAK,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EACL,KAAK,eAAe,EACpB,gBAAgB,EAGjB,MAAM,sBAAsB,CAAC;AAwa9B,eAAO,MAAM,oBAAoB,GAAI,QAAQ,eAAe,gDACI,CAAC"}

package/dist/services/gcs-client-nodejs.service.js

@@ -0,0 +1,312 @@
+import { pipeline, Readable, Transform } from "node:stream";
+import { Storage } from "@google-cloud/storage";
+import { UploadistaError } from "@uploadista/core/errors";
+import { Effect, Layer } from "effect";
+import { GCSClientService, } from "./gcs-client.service";
+function createNodeJSGCSClient(config) {
+    // Dynamic import to avoid issues in non-Node environments
+    const storage = new Storage({
+        keyFilename: config.keyFilename,
+        credentials: config.credentials,
+        projectId: config.projectId,
+    });
+    const bucket = storage.bucket(config.bucket);
+    const getObject = (key) => Effect.tryPromise({
+        try: async () => {
+            const file = bucket.file(key);
+            const stream = file.createReadStream();
+            // Convert Node.js stream to Web ReadableStream
+            return new ReadableStream({
+                start(controller) {
+                    stream.on("data", (chunk) => {
+                        controller.enqueue(new Uint8Array(chunk));
+                    });
+                    stream.on("end", () => {
+                        controller.close();
+                    });
+                    stream.on("error", (error) => {
+                        controller.error(error);
+                    });
+                },
+            });
+        },
+        catch: (error) => {
+            if (error &&
+                typeof error === "object" &&
+                "code" in error &&
+                error.code === 404) {
+                return UploadistaError.fromCode("FILE_NOT_FOUND");
+            }
+            return UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error });
+        },
+    });
+    const getObjectMetadata = (key) => Effect.tryPromise({
+        try: async () => {
+            const file = bucket.file(key);
+            const [metadata] = await file.getMetadata();
+            const parseMetadata = (meta) => {
+                if (!meta)
+                    return {};
+                if (typeof meta.metadata === "string") {
+                    try {
+                        return JSON.parse(meta.metadata);
+                    }
+                    catch {
+                        return meta;
+                    }
+                }
+                return meta;
+            };
+            return {
+                name: metadata.name,
+                bucket: metadata.bucket,
+                size: metadata.size
+                    ? Number.parseInt(`${metadata.size}`, 10)
+                    : undefined,
+                contentType: metadata.contentType,
+                metadata: parseMetadata(metadata.metadata),
+                generation: metadata.generation,
+                timeCreated: metadata.timeCreated,
+                updated: metadata.updated,
+            };
+        },
+        catch: (error) => {
+            if (error &&
+                typeof error === "object" &&
+                "code" in error &&
+                error.code === 404) {
+                return UploadistaError.fromCode("FILE_NOT_FOUND");
+            }
+            return UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error });
+        },
+    });
+    const objectExists = (key) => Effect.tryPromise({
+        try: async () => {
+            const file = bucket.file(key);
+            const [exists] = await file.exists();
+            return exists;
+        },
+        catch: (error) => {
+            return UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error });
+        },
+    });
+    const putObject = (key, body, context) => Effect.tryPromise({
+        try: async () => {
+            return new Promise((resolve, reject) => {
+                const file = bucket.file(key);
+                const options = {
+                    metadata: {
+                        contentType: context?.contentType || "application/octet-stream",
+                        metadata: context?.metadata || {},
+                    },
+                };
+                const stream = file.createWriteStream(options);
+                stream.on("error", reject);
+                stream.on("finish", () => {
+                    resolve(file.name);
+                });
+                stream.end(Buffer.from(body));
+            });
+        },
+        catch: (error) => {
+            return UploadistaError.fromCode("FILE_WRITE_ERROR", { cause: error });
+        },
+    });
+    const putObjectFromStream = (key, offset, readableStream, context, onProgress) => Effect.tryPromise({
+        try: async () => {
+            return new Promise((resolve, reject) => {
+                const file = bucket.file(key);
+                const options = {
+                    metadata: {
+                        contentType: context?.contentType || "application/octet-stream",
+                        metadata: context?.metadata || {},
+                    },
+                };
+                const writeStream = file.createWriteStream(options);
+                let bytesWritten = offset;
+                const transform = new Transform({
+                    transform(chunk, _, callback) {
+                        bytesWritten += chunk.length;
+                        onProgress?.(bytesWritten);
+                        callback(null, chunk);
+                    },
+                });
+                const nodeReadable = Readable.fromWeb(readableStream);
+                pipeline(nodeReadable, transform, writeStream, (error) => {
+                    if (error) {
+                        reject(UploadistaError.fromCode("FILE_WRITE_ERROR", {
+                            cause: error,
+                        }));
+                    }
+                    else {
+                        resolve(bytesWritten);
+                    }
+                });
+            });
+        },
+        catch: (error) => {
+            console.error("error putting object from stream", error);
+            return UploadistaError.fromCode("FILE_WRITE_ERROR", { cause: error });
+        },
+    });
+    const deleteObject = (key) => Effect.tryPromise({
+        try: async () => {
+            const file = bucket.file(key);
+            await file.delete({ ignoreNotFound: true });
+        },
+        catch: (error) => {
+            return UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error });
+        },
+    });
+    const createResumableUpload = (context) => Effect.tryPromise({
+        try: async () => {
+            // For Node.js, we'll use a simplified approach
+            // In production, you'd want to implement proper resumable uploads
+            // Return a pseudo-URL that we can use to identify this upload
+            return `resumable://nodejs/${context.bucket}/${context.key}`;
+        },
+        catch: (error) => {
+            return UploadistaError.fromCode("FILE_WRITE_ERROR", { cause: error });
+        },
+    });
+    const uploadChunk = (uploadUrl, chunk, start, total) => Effect.tryPromise({
+        try: async () => {
+            // Extract key from pseudo-URL
+            const key = uploadUrl.split("/").pop();
+            if (!key) {
+                throw new Error("Invalid upload URL");
+            }
+            const file = bucket.file(key);
+            return new Promise((resolve, reject) => {
+                const stream = file.createWriteStream({
+                    resumable: true,
+                    offset: start,
+                });
+                stream.on("error", reject);
+                stream.on("finish", () => {
+                    resolve({
+                        completed: total ? start + chunk.length >= total : false,
+                        bytesUploaded: start + chunk.length,
+                    });
+                });
+                stream.end(Buffer.from(chunk));
+            });
+        },
+        catch: (error) => {
+            return UploadistaError.fromCode("FILE_WRITE_ERROR", { cause: error });
+        },
+    });
+    const getUploadStatus = (uploadUrl) => Effect.promise(async () => {
+        try {
+            const key = uploadUrl.split("/").pop();
+            if (!key) {
+                throw new Error("Invalid upload URL");
+            }
+            const file = bucket.file(key);
+            const [metadata] = await file.getMetadata();
+            return {
+                bytesUploaded: metadata.size
+                    ? Number.parseInt(`${metadata.size}`, 10)
+                    : 0,
+                completed: true, // Simplified for now
+            };
+        }
+        catch (_error) {
+            // If file doesn't exist, upload hasn't started
+            return { bytesUploaded: 0, completed: false };
+        }
+    });
+    const cancelUpload = (uploadUrl) => Effect.tryPromise({
+        try: async () => {
+            const key = uploadUrl.split("/").pop();
+            if (!key) {
+                throw new Error("Invalid upload URL");
+            }
+            const file = bucket.file(key);
+            await file.delete({ ignoreNotFound: true });
+        },
+        catch: (error) => {
+            return UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error });
+        },
+    });
+    const composeObjects = (sourceKeys, destinationKey, context) => Effect.tryPromise({
+        try: async () => {
+            const sources = sourceKeys.map((key) => bucket.file(key));
+            const destination = bucket.file(destinationKey);
+            await bucket.combine(sources, destination);
+            if (context?.metadata) {
+                await destination.setMetadata({
+                    metadata: context.metadata,
+                });
+            }
+            return destinationKey;
+        },
+        catch: (error) => {
+            return UploadistaError.fromCode("FILE_WRITE_ERROR", { cause: error });
+        },
+    });
+    const putObjectFromStreamWithPatching = (key, offset, readableStream, context, onProgress, // Called with incremental bytes per chunk
+    isAppend = false) => Effect.gen(function* () {
+        if (!isAppend) {
+            // Direct upload for new files
+            return yield* putObjectFromStream(key, offset, readableStream, context, onProgress);
+        }
+        // For append operations, create a patch file and then combine
+        const patchKey = `${key}_patch`;
+        const bytesWritten = yield* putObjectFromStream(patchKey, offset, readableStream, context, onProgress);
+        // Combine original with patch
+        yield* composeObjects([key, patchKey], key, context);
+        // Clean up patch file
+        yield* deleteObject(patchKey);
+        return bytesWritten;
+    });
+    const putTemporaryObject = (key, body, context) => putObject(`${key}_tmp`, body, context);
+    const getTemporaryObject = (key) => Effect.gen(function* () {
+        try {
+            return yield* getObject(`${key}_tmp`);
+        }
+        catch {
+            return undefined;
+        }
+    });
+    const deleteTemporaryObject = (key) => deleteObject(`${key}_tmp`);
+    const getObjectBuffer = (key) => {
+        return Effect.tryPromise({
+            try: async () => {
+                const [buffer] = await bucket.file(key).download();
+                return new Uint8Array(buffer);
+            },
+            catch: (error) => {
+                if (error &&
+                    typeof error === "object" &&
+                    "code" in error &&
+                    error.code === 404) {
+                    return UploadistaError.fromCode("FILE_NOT_FOUND");
+                }
+                return UploadistaError.fromCode("FILE_READ_ERROR", {
+                    cause: error,
+                });
+            },
+        });
+    };
+    return {
+        bucket: config.bucket,
+        getObject,
+        getObjectBuffer,
+        getObjectMetadata,
+        objectExists,
+        putObject,
+        putObjectFromStream,
+        putObjectFromStreamWithPatching,
+        deleteObject,
+        createResumableUpload,
+        uploadChunk,
+        getUploadStatus,
+        cancelUpload,
+        composeObjects,
+        putTemporaryObject,
+        getTemporaryObject,
+        deleteTemporaryObject,
+    };
+}
+export const GCSClientNodeJSLayer = (config) => Layer.succeed(GCSClientService, createNodeJSGCSClient(config));

package/dist/services/gcs-client-rest.service.d.ts

@@ -0,0 +1,4 @@
+import { Layer } from "effect";
+import { type GCSClientConfig, GCSClientService } from "./gcs-client.service";
+export declare const GCSClientRESTLayer: (config: GCSClientConfig) => Layer.Layer<GCSClientService, never, never>;
+//# sourceMappingURL=gcs-client-rest.service.d.ts.map

package/dist/services/gcs-client-rest.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gcs-client-rest.service.d.ts","sourceRoot":"","sources":["../../src/services/gcs-client-rest.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAU,KAAK,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EACL,KAAK,eAAe,EACpB,gBAAgB,EAGjB,MAAM,sBAAsB,CAAC;AA6Y9B,eAAO,MAAM,kBAAkB,GAAI,QAAQ,eAAe,gDACI,CAAC"}