@webda/gcp 2.0.0-beta.0

package/lib/services/storage.d.ts

@@ -0,0 +1,131 @@
+/// <reference types="node" />
+import { Bucket } from "@google-cloud/storage";
+import { CloudBinary, BinaryMap, BinaryParameters, CoreModel, BinaryFile, DeepPartial, Context } from "@webda/core";
+import { Readable } from "stream";
+export declare type StorageObject = {
+    key: string;
+    /**
+     * bucket is an option,
+     * if not provided: it will get the DEFAULT_BUCKET set from environment variable
+     */
+    bucket?: string;
+};
+export declare type DownloadParams = StorageObject & {
+    localPath: string;
+};
+export declare type PutObjectParams = StorageObject & {
+    localPath?: string;
+    content?: string | Buffer;
+};
+export declare type GetSignedUrlParams = StorageObject & {
+    expires?: number;
+    action: "read" | "write" | "delete";
+};
+export declare type StorageObjectMeta = {
+    size: number;
+    contentType: string;
+};
+export declare class StorageParameters extends BinaryParameters {
+    prefix?: string;
+    bucket: string;
+    constructor(params: any, service: Storage);
+}
+/**
+ * Storage handles the storage of binary on a Google Cloud Storage bucket
+ *
+ * See Binary the general interface
+ * @WebdaModda GoogleCloudStorage
+ */
+export default class Storage<T extends StorageParameters = StorageParameters> extends CloudBinary<T> {
+    private _storage?;
+    private get storage();
+    /**
+     * Load the parameters
+     *
+     * @param params
+     */
+    loadParameters(params: DeepPartial<T>): StorageParameters;
+    /**
+     * @override
+     */
+    _get(info: BinaryMap): Promise<Readable>;
+    /**
+     * @override
+     */
+    getSignedUrlFromMap(map: BinaryMap, expires: number, _context: Context): Promise<string>;
+    /**
+     * Get object content in Buffer
+     * @param {StorageObject} params
+     */
+    getContent({ bucket, key }: StorageObject): Promise<Readable>;
+    /**
+     * Upload a local file to GCS bucket destination
+     *
+     * @param key to add to
+     * @param body content of the object
+     * @param metadatas to put along the object
+     * @param bucket to use
+     */
+    putObject(key: string, body: Readable | Buffer | string, metadata?: {}, bucket?: string): Promise<void>;
+    /**
+     * Return the Google Bucket directly
+     *
+     * @param bucket name of the bucket default to the one predefined
+     * @returns
+     */
+    getStorageBucket(bucket?: string): Bucket;
+    /**
+     * @inheritdoc
+     */
+    _cleanUsage(hash: string, uuid: string): Promise<void>;
+    /**
+     * @inheritdoc
+     */
+    getUsageCount(hash: string): Promise<number>;
+    /**
+     * @inheritdoc
+     */
+    store(object: CoreModel, property: string, file: BinaryFile): Promise<void>;
+    /**
+     * Delete a file inside an existing bucket
+     * @param {DeleteObjectParams} params
+     */
+    deleteObject({ bucket, key }: StorageObject): Promise<void>;
+    /**
+     * Move a file from one bucket to an other
+     * @param {StorageObject} source
+     * @param {StorageObject} destination
+     */
+    moveObject(source: StorageObject, destination: StorageObject): Promise<void>;
+    /**
+     * Retrieve one signed URL to download the file in parameter
+     * @param {GetSignedUrlParams} params
+     * @returns {string} URL in order to download the file
+     */
+    getSignedUrl({ bucket, key, action, expires }: GetSignedUrlParams): Promise<string>;
+    /**
+     * Retrieve mandatory headers if needed by the cloud provider (ie. Azure)
+     * Returns empty object if no headers are mandatory
+     * @returns {[key:string]: string} mandatory headers
+     */
+    getSignedUrlHeaders(): {
+        [key: string]: string;
+    };
+    /**
+     * Retrieve public URL of this object
+     * @param {StorageModel} store
+     * @returns {string} Public URL in order to download the file
+     */
+    getPublicUrl({ bucket, key }: StorageObject): Promise<string>;
+    /**
+     * Fetch an object's metadata
+     * @param {StorageModel} store
+     * @returns {any} Metadata
+     */
+    getMeta({ bucket, key }: StorageObject): Promise<StorageObjectMeta>;
+    /**
+     * @inheritdoc
+     */
+    putMarker(hash: string, uuid: string, storeName: string): Promise<void>;
+}
+export { Storage };

package/lib/services/storage.js

@@ -0,0 +1,244 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Storage = exports.StorageParameters = void 0;
+const storage_1 = require("@google-cloud/storage");
+const core_1 = require("@webda/core");
+const stream_1 = require("stream");
+const fs_1 = require("fs");
+const mime = require("mime-types");
+class StorageParameters extends core_1.BinaryParameters {
+    constructor(params, service) {
+        super(params, service);
+        this.prefix = "";
+    }
+}
+exports.StorageParameters = StorageParameters;
+/**
+ * Storage handles the storage of binary on a Google Cloud Storage bucket
+ *
+ * See Binary the general interface
+ * @WebdaModda GoogleCloudStorage
+ */
+class Storage extends core_1.CloudBinary {
+    get storage() {
+        if (!this._storage) {
+            this._storage = new storage_1.Storage();
+        }
+        return this._storage;
+    }
+    /**
+     * Load the parameters
+     *
+     * @param params
+     */
+    loadParameters(params) {
+        return new StorageParameters(params, this);
+    }
+    /**
+     * @override
+     */
+    _get(info) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const key = this._getKey(info.hash);
+            return this.getContent({ key });
+        });
+    }
+    /**
+     * @override
+     */
+    getSignedUrlFromMap(map, expires, _context) {
+        return this.getSignedUrl({ key: this._getKey(map.hash), expires, action: "read" });
+    }
+    /**
+     * Get object content in Buffer
+     * @param {StorageObject} params
+     */
+    getContent({ bucket, key }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.getStorageBucket(bucket).file(key).createReadStream();
+        });
+    }
+    /**
+     * Upload a local file to GCS bucket destination
+     *
+     * @param key to add to
+     * @param body content of the object
+     * @param metadatas to put along the object
+     * @param bucket to use
+     */
+    putObject(key, body, metadata = {}, bucket = this.parameters.bucket
+    //{ bucket, localPath, content, key }: PutObjectParams
+    ) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const file = this.getStorageBucket(bucket).file(key);
+            let contentType = "application/octet-stream";
+            let stream;
+            if (body instanceof stream_1.Readable) {
+                stream = body;
+            }
+            else if (body instanceof Buffer) {
+                const bufferStream = new stream_1.Stream.PassThrough();
+                bufferStream.end(body);
+                stream = bufferStream;
+            }
+            else {
+                contentType = mime.lookup(body, "application/octet-stream");
+                stream = (0, fs_1.createReadStream)(body);
+            }
+            // NODE16-REFACTOR: const { pipeline } = require('stream/promises');
+            yield new Promise((resolve, reject) => {
+                stream
+                    .pipe(file.createWriteStream({
+                    contentType,
+                }))
+                    .on("error", reject)
+                    .on("finish", resolve);
+            });
+            // Save metadata now
+            if (metadata) {
+                yield file.setMetadata({ metadata: metadata });
+            }
+        });
+    }
+    /**
+     * Return the Google Bucket directly
+     *
+     * @param bucket name of the bucket default to the one predefined
+     * @returns
+     */
+    getStorageBucket(bucket = this.parameters.bucket) {
+        return this.storage.bucket(bucket);
+    }
+    /**
+     * @inheritdoc
+     */
+    _cleanUsage(hash, uuid) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.getStorageBucket().file(this._getKey(hash, uuid)).delete();
+        });
+    }
+    /**
+     * @inheritdoc
+     */
+    getUsageCount(hash) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const [files] = yield this.getStorageBucket().getFiles({
+                prefix: this._getKey(hash, ""),
+            });
+            return files.filter(n => !n.name.endsWith("/data")).length;
+        });
+    }
+    /**
+     * @inheritdoc
+     */
+    store(object, property, file) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.checkMap(object.getStore().getName(), property);
+            yield file.getHashes();
+            const [exists] = yield this.getStorageBucket().file(this._getKey(file.hash)).exists();
+            // If data already exist no need to upload
+            if (!exists) {
+                yield this.putObject(this._getKey(file.hash), yield file.get(), Object.assign(Object.assign({}, file.metadata), { challenge: file.challenge }));
+            }
+            yield this.putMarker(file.hash, object.getUuid(), object.getStore().getName());
+            yield this.uploadSuccess(object, property, file);
+        });
+    }
+    /**
+     * Delete a file inside an existing bucket
+     * @param {DeleteObjectParams} params
+     */
+    deleteObject({ bucket, key }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.getStorageBucket(bucket).file(key).delete();
+            this._webda.log("DEBUG", `gs://${bucket}/${key} deleted`);
+        });
+    }
+    /**
+     * Move a file from one bucket to an other
+     * @param {StorageObject} source
+     * @param {StorageObject} destination
+     */
+    moveObject(source, destination) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const newFile = this.getStorageBucket(destination.bucket).file(destination.key);
+            this._webda.log("DEBUG", `moveObject gs://${source.bucket}/${source.key} => gs://${destination.bucket}/${destination.key}`);
+            yield this.getStorageBucket(source.bucket).file(source.key).move(newFile);
+        });
+    }
+    /**
+     * Retrieve one signed URL to download the file in parameter
+     * @param {GetSignedUrlParams} params
+     * @returns {string} URL in order to download the file
+     */
+    getSignedUrl({ bucket, key, action, expires = 3600 }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const options = {
+                version: "v4",
+                action,
+                expires: Date.now() + expires * 1000,
+            };
+            const [url] = yield this.getStorageBucket(bucket).file(key).getSignedUrl(options);
+            this._webda.log("TRACE", `The signed url for ${bucket}/${key} is ${url}.`);
+            return url;
+        });
+    }
+    /**
+     * Retrieve mandatory headers if needed by the cloud provider (ie. Azure)
+     * Returns empty object if no headers are mandatory
+     * @returns {[key:string]: string} mandatory headers
+     */
+    getSignedUrlHeaders() {
+        return {};
+    }
+    /**
+     * Retrieve public URL of this object
+     * @param {StorageModel} store
+     * @returns {string} Public URL in order to download the file
+     */
+    getPublicUrl({ bucket, key }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.getStorageBucket(bucket).file(key).publicUrl();
+        });
+    }
+    /**
+     * Fetch an object's metadata
+     * @param {StorageModel} store
+     * @returns {any} Metadata
+     */
+    getMeta({ bucket, key }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const [metadata] = yield this.getStorageBucket(bucket).file(key).getMetadata();
+            return {
+                size: parseInt(metadata.size),
+                contentType: metadata.contentType,
+            };
+        });
+    }
+    /**
+     * @inheritdoc
+     */
+    putMarker(hash, uuid, storeName) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.getStorageBucket()
+                .file(this._getKey(hash, uuid))
+                .save("", {
+                metadata: {
+                    webdaStore: storeName,
+                },
+            });
+        });
+    }
+}
+exports.default = Storage;
+exports.Storage = Storage;
+//# sourceMappingURL=storage.js.map

package/lib/services/storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.js","sourceRoot":"","sources":["../../src/services/storage.ts"],"names":[],"mappings":"AAAA,YAAY,CAAC;;;;;;;;;;;;AACb,mDAAmF;AACnF,sCAAoH;AACpH,mCAA0C;AAC1C,2BAAsC;AACtC,mCAAmC;AA+BnC,MAAa,iBAAkB,SAAQ,uBAAgB;IAIrD,YAAY,MAAW,EAAE,OAAgB;QACvC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;IACnB,CAAC;CACF;AARD,8CAQC;AAED;;;;;GAKG;AACH,MAAqB,OAAyD,SAAQ,kBAAc;IAElG,IAAY,OAAO;QACjB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;YAClB,IAAI,CAAC,QAAQ,GAAG,IAAI,iBAAG,EAAE,CAAC;SAC3B;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACH,cAAc,CAAC,MAAsB;QACnC,OAAO,IAAI,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACG,IAAI,CAAC,IAAe;;YACxB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,OAAO,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAClC,CAAC;KAAA;IAED;;OAEG;IACH,mBAAmB,CAAC,GAAc,EAAE,OAAe,EAAE,QAAiB;QACpE,OAAO,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACrF,CAAC;IAED;;;OAGG;IACG,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,EAAiB;;YAC7C,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACpE,CAAC;KAAA;IAED;;;;;;;OAOG;IACG,SAAS,CACb,GAAW,EACX,IAAgC,EAChC,QAAQ,GAAG,EAAE,EACb,SAAiB,IAAI,CAAC,UAAU,CAAC,MAAM;IACvC,sDAAsD;;;YAEtD,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrD,IAAI,WAAW,GAAG,0BAA0B,CAAC;YAC7C,IAAI,MAAgB,CAAC;YACrB,IAAI,IAAI,YAAY,iBAAQ,EAAE;gBAC5B,MAAM,GAAG,IAAI,CAAC;aACf;iBAAM,IAAI,IAAI,YAAY,MAAM,EAAE;gBACjC,MAAM,YAAY,GAAG,IAAI,eAAM,CAAC,WAAW,EAAE,CAAC;gBAC9C,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACvB,MAAM,GAAG,YAAY,CAAC;aACvB;iBAAM;gBACL,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;gBAC5D,MAAM,GAAG,IAAA,qBAAgB,EAAC,IAAI,CAAC,CAAC;aACjC;YACD,oEAAoE;YACpE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACpC,MAAM;qBACH,IAAI,CACH,IAAI,CAAC,iBAAiB,CAAC;oBACrB,WAAW;iBACZ,CAAC,CACH;qBACA,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;qBACnB,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC3B,CAAC,CAAC,CAAC;YACH,oBAAoB;YACpB,IAAI,QAAQ,EAAE;gBACZ,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;aAChD;QACH,CAAC;KAAA;IAED;;;;;OAKG;IACH,gBAAgB,CAAC,SAAiB,IAAI,CAAC,UAAU,CAAC,MAAM;QACtD,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACG,WAAW,CAAC,IAAY,EAAE,IAAY;;YAC1C,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QACxE,CAAC;KAAA;IAED;;OAEG;IACG,aAAa,CAAC,IAAY;;YAC9B,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC,QAAQ,CAAC;gBACrD,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;aAC/B,CAAC,CAAC;YACH,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QAC7D,CAAC;KAAA;IAED;;OAEG;IACG,KAAK,CAAC,MAAiB,EAAE,QAAgB,EAAE,IAAgB;;YAC/D,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YAEvB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACtF,0CAA0C;YAC1C,IAAI,CAAC,MAAM,EAAE;gBACX,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,IAAI,CAAC,GAAG,EAAE,kCACzD,IAAI,CAAC,QAAQ,KAChB,SAAS,EAAE,IAAI,CAAC,SAAS,IACzB,CAAC;aACJ;YACD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/E,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACnD,CAAC;KAAA;IAED;;;OAGG;IACG,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,EAAiB;;YAC/C,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;YACvD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC;QAC5D,CAAC;KAAA;IAED;;;;OAIG;IACG,UAAU,CAAC,MAAqB,EAAE,WAA0B;;YAChE,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAChF,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,OAAO,EACP,mBAAmB,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,YAAY,WAAW,CAAC,MAAM,IAAI,WAAW,CAAC,GAAG,EAAE,CAClG,CAAC;YACF,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5E,CAAC;KAAA;IAED;;;;OAIG;IACG,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,GAAG,IAAI,EAAsB;;YAC5E,MAAM,OAAO,GAAuB;gBAClC,OAAO,EAAE,IAAI;gBACb,MAAM;gBACN,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,GAAG,IAAI;aACrC,CAAC;YACF,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YAClF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,sBAAsB,MAAM,IAAI,GAAG,OAAO,GAAG,GAAG,CAAC,CAAC;YAC3E,OAAO,GAAG,CAAC;QACb,CAAC;KAAA;IAED;;;;OAIG;IACH,mBAAmB;QACjB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;;;OAIG;IACG,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,EAAiB;;YAC/C,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC;QAC7D,CAAC;KAAA;IAED;;;;OAIG;IACG,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,EAAiB;;YAC1C,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/E,OAAO;gBACL,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAC7B,WAAW,EAAE,QAAQ,CAAC,WAAW;aAClC,CAAC;QACJ,CAAC;KAAA;IAED;;OAEG;IACG,SAAS,CAAC,IAAY,EAAE,IAAY,EAAE,SAAiB;;YAC3D,MAAM,IAAI,CAAC,gBAAgB,EAAE;iBAC1B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;iBAC9B,IAAI,CAAC,EAAE,EAAE;gBACR,QAAQ,EAAE;oBACR,UAAU,EAAE,SAAS;iBACtB;aACF,CAAC,CAAC;QACP,CAAC;KAAA;CACF;AAtND,0BAsNC;AAEQ,0BAAO"}

package/lib/services/storage.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/services/storage.spec.js

@@ -0,0 +1,213 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const binary_spec_1 = require("@webda/core/lib/services/binary.spec");
+const assert = require("assert");
+const mocha_1 = require("@testdeck/mocha");
+const sinon = require("sinon");
+const storage_1 = require("@google-cloud/storage");
+const path = require("path");
+class MockFile {
+    constructor(path) { }
+    delete() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+    getFiles() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+    exists() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+    save() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+    setMetadata() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+    createWriteStream() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+    getSignedUrl() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+    publicUrl() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+    getMetadata() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+    createReadStream() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+    move() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+}
+class MockBucket {
+    file(path) {
+        return new MockFile(path);
+    }
+}
+const BUCKET = "webda-dev";
+let StorageTest = class StorageTest extends binary_spec_1.BinaryTest {
+    before() {
+        const _super = Object.create(null, {
+            before: { get: () => super.before }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            this.buildWebda();
+            yield this.install();
+            yield _super.before.call(this);
+            this.prefix = this.getBinary().getWebda().getUuid();
+            this.getBinary().getParameters().prefix = this.prefix;
+            // Prepare for Mocked Version
+            if (!process.env.NO_MOCK && false) {
+                sinon.stub(this.getBinary(), "getStorageBucket").callsFake(() => {
+                    return new MockBucket();
+                });
+            }
+        });
+    }
+    after() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.cleanData();
+        });
+    }
+    install() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var storage = new storage_1.Storage();
+            try {
+                let [metadata] = yield storage.bucket(BUCKET).getMetadata();
+                if (!metadata) {
+                    yield storage.createBucket(BUCKET, { location: "US-WEST2", storageClass: "COLDLINE" });
+                    yield storage.bucket(BUCKET).getMetadata();
+                }
+                //this.webda.log("INFO", `Google cloud storage bucket '${BUCKET}' exists`);
+            }
+            catch (e) {
+                this.webda.log("ERROR", `Google cloud storage error (${e.message})`);
+            }
+        });
+    }
+    cleanData() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var storage = new storage_1.Storage();
+            const [files] = yield storage.bucket(BUCKET).getFiles({
+                prefix: this.prefix,
+            });
+            for (let file of files) {
+                yield file.delete();
+            }
+        });
+    }
+    // Override getNotFound as exception is raised after
+    // @test
+    // async getNotFound() {}
+    putObject() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const body = `RAW Body: ${new Date()}`;
+            const name = `${this.prefix}/plop/test`;
+            yield this.getBinary().putObject(name, Buffer.from(body), { meta1: "meta1" }, BUCKET);
+            let getFile = this.getBinary().getStorageBucket().file(name);
+            assert.deepStrictEqual((yield getFile.getMetadata())[0].metadata, { meta1: "meta1" });
+        });
+    }
+    // Override getNotFound as exception is raised after
+    getNotFound() {
+        return __awaiter(this, void 0, void 0, function* () { });
+    }
+    httpGetError() {
+        return __awaiter(this, void 0, void 0, function* () {
+            // GET is not through classic binary
+            // Skip it
+        });
+    }
+    redirectUrl() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { user1, ctx } = yield this.setupDefault();
+            // Making sure we are redirected on GET
+            let executor = this.getExecutor(ctx, "test.webda.io", "GET", `/binary/users/${user1.getUuid()}/images/0`, {});
+            yield executor.execute(ctx);
+            assert.ok(ctx.getResponseHeaders().Location !== undefined);
+        });
+    }
+    redirectUrlInfo() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { user1, ctx } = yield this.setupDefault();
+            // Making sure we are redirected on GET
+            let executor = this.getExecutor(ctx, "test.webda.io", "GET", `/binary/users/${user1.getUuid()}/images/0/url`, {});
+            yield executor.execute(ctx);
+            assert.ok(ctx.getResponseHeaders().Location === undefined);
+            assert.notStrictEqual(JSON.parse(ctx.getResponseBody()).Location, undefined);
+        });
+    }
+    cascadeDelete() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let stubDelete = sinon.stub(this.getBinary(), "_cleanUsage").callsFake(() => {
+                throw new Error();
+            });
+            try {
+                // @ts-ignore
+                yield this.getBinary().cascadeDelete({ hash: "pp" }, "pp");
+            }
+            finally {
+                stubDelete.restore();
+            }
+        });
+    }
+    defaultGCS() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { user1, ctx } = yield this.setupDefault();
+            // COV for double
+            const binary = this.getBinary();
+            const from = `${this.prefix}/rawAccess`;
+            const to = `${this.prefix}/movedAccess`;
+            yield binary.putObject(from, path.join(__dirname, "..", "..", "test", "Dockerfile.txt"));
+            yield binary.moveObject({ key: from }, { key: to });
+            assert.deepStrictEqual(binary.getSignedUrlHeaders(), {});
+            const meta = yield binary.getMeta({ bucket: "webda-dev", key: to });
+            assert.deepStrictEqual(meta, { size: 311, contentType: "text/plain" });
+            assert.ok(/https:\/\/storage\.googleapis\.com\/webda-dev\/webda-dev/.exec(yield binary.getPublicUrl({ key: "webda-dev" })));
+            yield binary.deleteObject({ key: to });
+        });
+    }
+};
+__decorate([
+    mocha_1.test
+], StorageTest.prototype, "putObject", null);
+__decorate([
+    mocha_1.test
+], StorageTest.prototype, "getNotFound", null);
+__decorate([
+    mocha_1.test
+], StorageTest.prototype, "httpGetError", null);
+__decorate([
+    mocha_1.test
+], StorageTest.prototype, "redirectUrl", null);
+__decorate([
+    mocha_1.test
+], StorageTest.prototype, "redirectUrlInfo", null);
+__decorate([
+    mocha_1.test
+], StorageTest.prototype, "cascadeDelete", null);
+__decorate([
+    mocha_1.test
+], StorageTest.prototype, "defaultGCS", null);
+StorageTest = __decorate([
+    mocha_1.suite
+], StorageTest);
+//# sourceMappingURL=storage.spec.js.map

package/lib/services/storage.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.spec.js","sourceRoot":"","sources":["../../src/services/storage.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,sEAAkE;AAClE,iCAAiC;AACjC,2CAA8C;AAE9C,+BAA+B;AAC/B,mDAAuD;AACvD,6BAA6B;AAE7B,MAAM,QAAQ;IACZ,YAAY,IAAI,IAAG,CAAC;IACd,MAAM;8DAAI,CAAC;KAAA;IACX,QAAQ;8DAAI,CAAC;KAAA;IACb,MAAM;8DAAI,CAAC;KAAA;IACX,IAAI;8DAAI,CAAC;KAAA;IACT,WAAW;8DAAI,CAAC;KAAA;IAChB,iBAAiB;8DAAI,CAAC;KAAA;IACtB,YAAY;8DAAI,CAAC;KAAA;IACjB,SAAS;8DAAI,CAAC;KAAA;IACd,WAAW;8DAAI,CAAC;KAAA;IAChB,gBAAgB;8DAAI,CAAC;KAAA;IACrB,IAAI;8DAAI,CAAC;KAAA;CAChB;AAED,MAAM,UAAU;IACd,IAAI,CAAC,IAAY;QACf,OAAO,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;CACF;AACD,MAAM,MAAM,GAAG,WAAW,CAAC;AAE3B,IAAM,WAAW,GAAjB,MAAM,WAAY,SAAQ,wBAAmB;IAErC,MAAM;;;;;YACV,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,OAAM,MAAM,WAAE,CAAC;YACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,CAAC;YACpD,IAAI,CAAC,SAAS,EAAE,CAAC,aAAa,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YACtD,6BAA6B;YAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,KAAK,EAAE;gBACjC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,kBAAkB,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE;oBAC9D,OAAY,IAAI,UAAU,EAAE,CAAC;gBAC/B,CAAC,CAAC,CAAC;aACJ;QACH,CAAC;KAAA;IAEK,KAAK;;YACT,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACzB,CAAC;KAAA;IAEK,OAAO;;YACX,IAAI,OAAO,GAAG,IAAI,iBAAG,EAAE,CAAC;YACxB,IAAI;gBACF,IAAI,CAAC,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;gBAC5D,IAAI,CAAC,QAAQ,EAAE;oBACb,MAAM,OAAO,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC;oBACvF,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;iBAC5C;gBACD,2EAA2E;aAC5E;YAAC,OAAO,CAAM,EAAE;gBACf,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,+BAA+B,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;aACtE;QACH,CAAC;KAAA;IAEK,SAAS;;YACb,IAAI,OAAO,GAAG,IAAI,iBAAG,EAAE,CAAC;YACxB,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;gBACpD,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC,CAAC;YACH,KAAK,IAAI,IAAI,IAAI,KAAK,EAAE;gBACtB,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;aACrB;QACH,CAAC;KAAA;IAED,oDAAoD;IACpD,QAAQ;IACR,yBAAyB;IAGnB,SAAS;;YACb,MAAM,IAAI,GAAG,aAAa,IAAI,IAAI,EAAE,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,YAAY,CAAC;YACxC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;YACtF,IAAI,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,gBAAgB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7D,MAAM,CAAC,eAAe,CAAC,CAAC,MAAM,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACxF,CAAC;KAAA;IAED,oDAAoD;IAE9C,WAAW;8DAAI,CAAC;KAAA;IAGhB,YAAY;;YAChB,oCAAoC;YACpC,UAAU;QACZ,CAAC;KAAA;IAGK,WAAW;;YACf,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC/C,uCAAuC;YACvC,IAAI,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;YAC9G,MAAM,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;QAC7D,CAAC;KAAA;IAGK,eAAe;;YACnB,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC/C,uCAAuC;YACvC,IAAI,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAClH,MAAM,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;YAC3D,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC/E,CAAC;KAAA;IAGK,aAAa;;YACjB,IAAI,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,aAAa,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE;gBAC1E,MAAM,IAAI,KAAK,EAAE,CAAC;YACpB,CAAC,CAAC,CAAC;YACH,IAAI;gBACF,aAAa;gBACb,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;aAC5D;oBAAS;gBACR,UAAU,CAAC,OAAO,EAAE,CAAC;aACtB;QACH,CAAC;KAAA;IAGK,UAAU;;YACd,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC/C,iBAAiB;YACjB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,YAAY,CAAC;YACxC,MAAM,EAAE,GAAG,GAAG,IAAI,CAAC,MAAM,cAAc,CAAC;YACxC,MAAM,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;YACzF,MAAM,MAAM,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,mBAAmB,EAAE,EAAE,EAAE,CAAC,CAAC;YACzD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;YACpE,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC;YACvE,MAAM,CAAC,EAAE,CACP,0DAA0D,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC,CACjH,CAAC;YACF,MAAM,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;QACzC,CAAC;KAAA;CACF,CAAA;AApEC;IADC,YAAI;4CAQJ;AAID;IADC,YAAI;8CACiB;AAGtB;IADC,YAAI;+CAIJ;AAGD;IADC,YAAI;8CAOJ;AAGD;IADC,YAAI;kDAQJ;AAGD;IADC,YAAI;gDAWJ;AAGD;IADC,YAAI;6CAgBJ;AApHG,WAAW;IADhB,aAAK;GACA,WAAW,CAqHhB"}

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@webda/gcp",
+  "version": "2.0.0-beta.0",
+  "description": "Webda GCP Services implementation",
+  "main": "lib/index.js",
+  "scripts": {
+    "build": "webda build",
+    "pretest": "npm run build",
+    "lint": "prettier --check src/**/*",
+    "lint:fix": "prettier --write src/**/*",
+    "test": "nyc mocha --recursive --exit --timeout=100000 -r ../../node_modules/ts-node/register src/**/*.spec.ts"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/loopingz/webda.io.git"
+  },
+  "keywords": [
+    "webda",
+    "modda"
+  ],
+  "files": [
+    "lib",
+    "webda.module.json",
+    "package.json"
+  ],
+  "author": "abalam666",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/loopingz/webda.io/issues"
+  },
+  "homepage": "https://github.com/loopingz/webda.io#readme",
+  "dependencies": {
+    "@google-cloud/firestore": "^5.0.2",
+    "@google-cloud/pubsub": "^2.19.0",
+    "@google-cloud/storage": "^5.16.0",
+    "@webda/core": "^2.0.0-beta.0"
+  },
+  "nyc": {
+    "extends": "@istanbuljs/nyc-config-typescript",
+    "report-dir": "./reports",
+    "reporter": [
+      "html",
+      "lcov",
+      "json",
+      "text"
+    ],
+    "exclude": [
+      "**/*.spec.ts",
+      "test/**/*"
+    ]
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "@testdeck/mocha": "^0.2.0",
+    "@types/node": "14.0.0",
+    "@webda/async": "^2.0.0-beta.0",
+    "@webda/shell": "^2.0.0-beta.0",
+    "sinon": "^13.0.1"
+  },
+  "gitHead": "5a559bca55832e9a6526794117e2f78188eba3d0"
+}