@uploadista/data-store-gcs 0.0.3

package/dist/gcs-store-rest.js

@@ -0,0 +1,188 @@
+import { UploadistaError } from "@uploadista/core/errors";
+import { UploadFileDataStore, UploadFileKVStore, } from "@uploadista/core/types";
+import { Effect, Layer, Stream } from "effect";
+/**
+ * Google Cloud Storage implementation using REST API
+ * Compatible with Cloudflare Workers
+ */
+export function gcsStoreRest({ bucketName, accessToken, projectId, kvStore, }) {
+    const baseUrl = `https://storage.googleapis.com/storage/v1/b/${bucketName}`;
+    const uploadUrl = `https://storage.googleapis.com/upload/storage/v1/b/${bucketName}/o`;
+    const getCapabilities = () => {
+        return {
+            supportsParallelUploads: false,
+            supportsConcatenation: false, // REST API doesn't support combine easily
+            supportsDeferredLength: true,
+            supportsResumableUploads: true, // Via resumable uploads
+            supportsTransactionalUploads: false,
+            maxConcurrentUploads: 1,
+            minChunkSize: undefined,
+            maxChunkSize: undefined,
+            maxParts: undefined,
+            optimalChunkSize: 8 * 1024 * 1024, // 8MB
+            requiresOrderedChunks: true,
+        };
+    };
+    const validateUploadStrategy = (strategy) => {
+        const capabilities = getCapabilities();
+        const result = strategy === "single" || capabilities.supportsParallelUploads;
+        return Effect.succeed(result);
+    };
+    const getUpload = (id) => {
+        return Effect.gen(function* () {
+            try {
+                const response = yield* Effect.promise(() => fetch(`${baseUrl}/o/${encodeURIComponent(id)}?alt=json`, {
+                    headers: {
+                        Authorization: `Bearer ${accessToken}`,
+                    },
+                }));
+                if (!response.ok) {
+                    if (response.status === 404) {
+                        return yield* Effect.fail(UploadistaError.fromCode("FILE_NOT_FOUND"));
+                    }
+                    return yield* Effect.fail(UploadistaError.fromCode("FILE_READ_ERROR", {
+                        cause: new Error(`HTTP ${response.status}: ${response.statusText}`),
+                    }));
+                }
+                const metadata = yield* Effect.promise(() => response.json());
+                const file = yield* kvStore.get(id);
+                return {
+                    id,
+                    size: metadata.size ? Number.parseInt(metadata.size, 10) : undefined,
+                    offset: metadata.size ? Number.parseInt(metadata.size, 10) : 0,
+                    metadata: metadata.metadata || {},
+                    storage: {
+                        id: file.storage.id,
+                        type: file.storage.type,
+                        path: id,
+                        bucket: bucketName,
+                    },
+                };
+            }
+            catch (error) {
+                throw error;
+            }
+        });
+    };
+    return {
+        bucket: bucketName,
+        create: (file) => {
+            return Effect.gen(function* () {
+                if (!file.id) {
+                    return yield* Effect.fail(UploadistaError.fromCode("FILE_NOT_FOUND"));
+                }
+                file.storage = {
+                    id: file.storage.id,
+                    type: file.storage.type,
+                    path: file.id,
+                    bucket: bucketName,
+                };
+                // Create empty file using REST API
+                const metadata = {
+                    name: file.id,
+                    metadata: {
+                        contentType: file.metadata?.contentType || "application/octet-stream",
+                        uploadId: file.id,
+                        originalName: file.metadata?.originalName,
+                    },
+                };
+                return yield* Effect.tryPromise({
+                    try: async () => {
+                        const response = await fetch(`${uploadUrl}?uploadType=media&name=${encodeURIComponent(file.id)}`, {
+                            method: "POST",
+                            headers: {
+                                Authorization: `Bearer ${accessToken}`,
+                                "Content-Type": "application/octet-stream",
+                                "Content-Length": "0",
+                            },
+                            body: new Uint8Array(0),
+                        });
+                        if (!response.ok) {
+                            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                        }
+                        return file;
+                    },
+                    catch: (error) => {
+                        console.error("error creating file", error);
+                        return UploadistaError.fromCode("FILE_WRITE_ERROR", {
+                            cause: error,
+                        });
+                    },
+                });
+            });
+        },
+        remove: (file_id) => {
+            return Effect.promise(() => fetch(`${baseUrl}/o/${encodeURIComponent(file_id)}`, {
+                method: "DELETE",
+                headers: {
+                    Authorization: `Bearer ${accessToken}`,
+                },
+            }).then((response) => {
+                if (!response.ok && response.status !== 404) {
+                    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                }
+            }));
+        },
+        write: (options, dependencies) => {
+            return Effect.gen(function* () {
+                const { file_id, offset, stream: effectStream } = options;
+                const { onProgress } = dependencies;
+                // For simplicity, we'll use simple upload for now
+                // For production, you'd want to implement resumable uploads
+                return yield* Effect.promise(() => new Promise(async (resolve, reject) => {
+                    try {
+                        const readableStream = Stream.toReadableStream(effectStream);
+                        const reader = readableStream.getReader();
+                        const chunks = [];
+                        let totalBytes = 0;
+                        // Read all chunks
+                        while (true) {
+                            const { done, value } = await reader.read();
+                            if (done)
+                                break;
+                            chunks.push(value);
+                            totalBytes += value.byteLength;
+                            onProgress?.(totalBytes);
+                        }
+                        // Combine all chunks
+                        const combinedArray = new Uint8Array(totalBytes);
+                        let position = 0;
+                        for (const chunk of chunks) {
+                            combinedArray.set(chunk, position);
+                            position += chunk.byteLength;
+                        }
+                        // Upload to GCS
+                        const response = await fetch(`${uploadUrl}?uploadType=media&name=${encodeURIComponent(file_id)}`, {
+                            method: "POST",
+                            headers: {
+                                Authorization: `Bearer ${accessToken}`,
+                                "Content-Type": "application/octet-stream",
+                                "Content-Length": totalBytes.toString(),
+                            },
+                            body: combinedArray,
+                        });
+                        if (!response.ok) {
+                            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                        }
+                        resolve(totalBytes);
+                    }
+                    catch (error) {
+                        console.error("error writing file", error);
+                        reject(UploadistaError.fromCode("FILE_WRITE_ERROR", {
+                            cause: error,
+                        }));
+                    }
+                }));
+            });
+        },
+        getCapabilities,
+        validateUploadStrategy,
+    };
+}
+export function createGCSStoreRest({ bucketName, accessToken, projectId, }) {
+    return Effect.gen(function* () {
+        const kvStore = yield* UploadFileKVStore;
+        return gcsStoreRest({ bucketName, accessToken, projectId, kvStore });
+    });
+}
+export const GCSStoreRestLayer = (options) => Layer.effect(UploadFileDataStore, createGCSStoreRest(options));

package/dist/gcs-store-v2.d.ts

@@ -0,0 +1,13 @@
+import { type DataStore, type KvStore, type UploadFile, UploadFileDataStore, UploadFileKVStore } from "@uploadista/core/types";
+import { Effect, Layer } from "effect";
+import type { GCSClientConfig } from "./services";
+import { GCSClientService } from "./services";
+export type GCSStoreOptions = {
+    kvStore: KvStore<UploadFile>;
+} & GCSClientConfig;
+export declare function createGCSStore(options: Omit<GCSStoreOptions, "kvStore">): Effect.Effect<DataStore<UploadFile>, never, GCSClientService | UploadFileKVStore>;
+export declare function createGCSStoreImplementation(config: GCSStoreOptions): Effect.Effect<DataStore<UploadFile>, never, GCSClientService>;
+export declare const GCSStoreLayer: (options: Omit<GCSStoreOptions, "kvStore">) => Layer.Layer<UploadFileDataStore, never, GCSClientService | UploadFileKVStore>;
+export declare const gcsStoreRest: (config: GCSStoreOptions) => Promise<DataStore<UploadFile>>;
+export declare const gcsStoreNodejs: (config: GCSStoreOptions) => Promise<DataStore<UploadFile>>;
+//# sourceMappingURL=gcs-store-v2.d.ts.map

package/dist/gcs-store-v2.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gcs-store-v2.d.ts","sourceRoot":"","sources":["../src/gcs-store-v2.ts"],"names":[],"mappings":"AACA,OAAO,EACL,KAAK,SAAS,EAGd,KAAK,OAAO,EACZ,KAAK,UAAU,EACf,mBAAmB,EACnB,iBAAiB,EAElB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAU,MAAM,QAAQ,CAAC;AAC/C,OAAO,KAAK,EAEV,eAAe,EAEhB,MAAM,YAAY,CAAC;AACpB,OAAO,EAGL,gBAAgB,EACjB,MAAM,YAAY,CAAC;AAEpB,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAC9B,GAAG,eAAe,CAAC;AAgDpB,wBAAgB,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,qFAKvE;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,eAAe,GACtB,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,gBAAgB,CAAC,CAwL/D;AAED,eAAO,MAAM,aAAa,GAAI,SAAS,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,kFACX,CAAC;AAE7D,eAAO,MAAM,YAAY,GAAI,QAAQ,eAAe,mCAMnD,CAAC;AAEF,eAAO,MAAM,cAAc,GAAI,QAAQ,eAAe,mCAMrD,CAAC"}

package/dist/gcs-store-v2.js

@@ -0,0 +1,190 @@
+import { UploadistaError } from "@uploadista/core/errors";
+import { UploadFileDataStore, UploadFileKVStore, } from "@uploadista/core/types";
+import { Effect, Layer, Stream } from "effect";
+import { GCSClientNodeJSLayer, GCSClientRESTLayer, GCSClientService, } from "./services";
+/**
+ * Convert the Upload object to a format that can be stored in GCS metadata.
+ */
+function stringifyUploadKeys(upload) {
+    return {
+        size: upload.size?.toString() ?? null,
+        sizeIsDeferred: `${upload.sizeIsDeferred}`,
+        offset: upload.offset?.toString() ?? "0",
+        metadata: JSON.stringify(upload.metadata),
+        storage: JSON.stringify(upload.storage),
+    };
+}
+const getUpload = (id, kvStore, gcsClient) => {
+    return Effect.gen(function* () {
+        try {
+            const metadata = yield* gcsClient.getObjectMetadata(id);
+            const file = yield* kvStore.get(id);
+            return {
+                id,
+                size: metadata.size,
+                offset: metadata.size || 0,
+                metadata: metadata.metadata,
+                storage: {
+                    id: file.storage.id,
+                    type: file.storage.type,
+                    path: id,
+                    bucket: gcsClient.bucket,
+                },
+            };
+        }
+        catch (error) {
+            if (error instanceof UploadistaError && error.code === "FILE_NOT_FOUND") {
+                return yield* Effect.fail(error);
+            }
+            throw error;
+        }
+    });
+};
+export function createGCSStore(options) {
+    return Effect.gen(function* () {
+        const kvStore = yield* UploadFileKVStore;
+        return yield* createGCSStoreImplementation({ ...options, kvStore });
+    });
+}
+export function createGCSStoreImplementation(config) {
+    return Effect.gen(function* () {
+        const gcsClient = yield* GCSClientService;
+        const { kvStore } = config;
+        const getCapabilities = () => {
+            return {
+                supportsParallelUploads: false, // GCS doesn't have native multipart upload like S3
+                supportsConcatenation: true, // Can combine files using compose
+                supportsDeferredLength: true,
+                supportsResumableUploads: true, // Through resumable uploads
+                supportsTransactionalUploads: false,
+                maxConcurrentUploads: 1, // Sequential operations
+                minChunkSize: undefined,
+                maxChunkSize: undefined,
+                maxParts: undefined,
+                optimalChunkSize: 8 * 1024 * 1024, // 8MB default
+                requiresOrderedChunks: true, // Due to compose operation
+            };
+        };
+        const validateUploadStrategy = (strategy) => {
+            const capabilities = getCapabilities();
+            const result = (() => {
+                switch (strategy) {
+                    case "parallel":
+                        return capabilities.supportsParallelUploads;
+                    case "single":
+                        return true;
+                    default:
+                        return false;
+                }
+            })();
+            return Effect.succeed(result);
+        };
+        return {
+            bucket: gcsClient.bucket,
+            create: (file) => {
+                return Effect.gen(function* () {
+                    if (!file.id) {
+                        return yield* Effect.fail(UploadistaError.fromCode("FILE_NOT_FOUND"));
+                    }
+                    file.storage = {
+                        id: file.storage.id,
+                        type: file.storage.type,
+                        path: file.id,
+                        bucket: gcsClient.bucket,
+                    };
+                    // Create empty file
+                    const context = {
+                        bucket: gcsClient.bucket,
+                        key: file.id,
+                        contentType: file.metadata?.contentType?.toString() ||
+                            "application/octet-stream",
+                        metadata: stringifyUploadKeys(file),
+                    };
+                    yield* gcsClient.putObject(file.id, new Uint8Array(0), context);
+                    return file;
+                });
+            },
+            remove: (file_id) => {
+                return gcsClient.deleteObject(file_id);
+            },
+            write: (options, dependencies) => {
+                return Effect.gen(function* () {
+                    const { file_id, offset, stream: effectStream } = options;
+                    const { onProgress } = dependencies;
+                    // Get current upload metadata
+                    const upload = yield* getUpload(file_id, kvStore, gcsClient);
+                    upload.offset = offset;
+                    // Persist the updated offset
+                    yield* kvStore.set(file_id, upload);
+                    const context = {
+                        bucket: gcsClient.bucket,
+                        key: file_id,
+                        contentType: upload.metadata?.contentType || "application/octet-stream",
+                        metadata: stringifyUploadKeys(upload),
+                    };
+                    // Convert Effect Stream to ReadableStream
+                    const readableStream = Stream.toReadableStream(effectStream);
+                    // Use native streams if available (Node.js implementation)
+                    if (gcsClient.putObjectFromStreamWithPatching) {
+                        const isAppend = upload.offset > 0; // Check original file size, not write offset
+                        return yield* gcsClient.putObjectFromStreamWithPatching(file_id, upload.offset, readableStream, context, onProgress, isAppend);
+                    }
+                    else {
+                        // Fallback to chunk-based approach for REST implementation
+                        const reader = readableStream.getReader();
+                        const chunks = [];
+                        let totalBytes = 0;
+                        // Read all chunks
+                        while (true) {
+                            const { done, value } = yield* Effect.promise(() => reader.read());
+                            if (done)
+                                break;
+                            chunks.push(value);
+                            const chunkSize = value.byteLength;
+                            totalBytes += chunkSize;
+                            onProgress?.(totalBytes);
+                        }
+                        // Combine all chunks
+                        const combinedArray = new Uint8Array(totalBytes);
+                        let position = 0;
+                        for (const chunk of chunks) {
+                            combinedArray.set(chunk, position);
+                            position += chunk.byteLength;
+                        }
+                        // Check if we need to handle patches (append data)
+                        if (upload.offset === 0) {
+                            // Direct upload
+                            yield* gcsClient.putObject(file_id, combinedArray, context);
+                        }
+                        else {
+                            // We need to combine with existing data
+                            const patchKey = `${file_id}_patch`;
+                            // Upload patch data
+                            yield* gcsClient.putTemporaryObject(patchKey, combinedArray, context);
+                            // Combine original file with patch
+                            yield* gcsClient.composeObjects([file_id, patchKey], file_id, context);
+                            // Clean up patch file
+                            yield* gcsClient.deleteTemporaryObject(patchKey);
+                        }
+                        return totalBytes;
+                    }
+                });
+            },
+            getCapabilities,
+            validateUploadStrategy,
+            read: (file_id) => {
+                return Effect.gen(function* () {
+                    const buffer = yield* gcsClient.getObjectBuffer(file_id);
+                    return buffer;
+                });
+            },
+        };
+    });
+}
+export const GCSStoreLayer = (options) => Layer.effect(UploadFileDataStore, createGCSStore(options));
+export const gcsStoreRest = (config) => {
+    return Effect.runPromise(createGCSStoreImplementation(config).pipe(Effect.provide(GCSClientRESTLayer(config))));
+};
+export const gcsStoreNodejs = (config) => {
+    return Effect.runPromise(createGCSStoreImplementation(config).pipe(Effect.provide(GCSClientNodeJSLayer(config))));
+};

package/dist/gcs-store.d.ts

@@ -0,0 +1,12 @@
+import { type DataStore, type KvStore, type UploadFile, UploadFileDataStore, UploadFileKVStore } from "@uploadista/core/types";
+import { Effect, Layer } from "effect";
+export type GCSStoreOptions = {
+    keyFilename?: string;
+    credentials?: object;
+    bucketName: string;
+    kvStore: KvStore<UploadFile>;
+};
+export declare function createGCSStore({ keyFilename, credentials, bucketName, }: Omit<GCSStoreOptions, "kvStore">): Effect.Effect<DataStore<UploadFile>, never, UploadFileKVStore>;
+export declare function gcsStore({ keyFilename, credentials, bucketName, kvStore, }: GCSStoreOptions): DataStore<UploadFile>;
+export declare const GCSStoreLayer: (options: Omit<GCSStoreOptions, "kvStore">) => Layer.Layer<UploadFileDataStore, never, UploadFileKVStore>;
+//# sourceMappingURL=gcs-store.d.ts.map

package/dist/gcs-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gcs-store.d.ts","sourceRoot":"","sources":["../src/gcs-store.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,SAAS,EAGd,KAAK,OAAO,EACZ,KAAK,UAAU,EACf,mBAAmB,EACnB,iBAAiB,EAElB,MAAM,wBAAwB,CAAC;AAahC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAU,MAAM,QAAQ,CAAC;AAE/C,MAAM,MAAM,eAAe,GAAG;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAC9B,CAAC;AAsDF,wBAAgB,cAAc,CAAC,EAC7B,WAAW,EACX,WAAW,EACX,UAAU,GACX,EAAE,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,kEAKlC;AAED,wBAAgB,QAAQ,CAAC,EACvB,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,GACR,EAAE,eAAe,GAAG,SAAS,CAAC,UAAU,CAAC,CAmSzC;AAED,eAAO,MAAM,aAAa,GAAI,SAAS,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,+DACX,CAAC"}