@webda/gcp 4.0.0-beta.2 → 4.0.0-beta.3

package/lib/services/queue.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue.service.js","sourceRoot":"","sources":["../../src/services/queue.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,MAAM,EAAgB,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAkB,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE5D;;GAEG;AACH,MAAM,OAAO,kBAAmB,SAAQ,eAAe;IAyBrD;;;;OAIG;IACH,IAAI,CAAC,MAAM,GAAQ,EAAE;QACnB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,IAAI,CAAC,IAAI,KAAT,IAAI,CAAC,IAAI,GAAK,UAAU,EAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,CAAC,OAAO,OAAO,QAAqE,SAAQ,KAAW;IAA7G;;QAgBE;;WAEG;QACH,aAAQ,GAA+B,EAAE,CAAC;IAoK5C,CAAC;IAlKC;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,YAAY,CAAC,kBAAkB,EAAE,CAAC;YACvC,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;gBACzC,oBAAoB;YACtB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YAC3B,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;YACxB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;gBAC9B,oBAAoB;YACtB,CAAC,CAAC,CAAC;QACL,CAAC;QACD,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACrB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,OAAO,CAAC,CAAC;IACX,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,aAAa,CAAC,EAAU;QAC5B,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,eAAe,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,GAAM;QACtB,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC;QACjC,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IACjH,CAAC;IAED;;;OAGG;IACK,YAAY;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC;QACrC,IAAI,CAAC,eAAe,KAApB,IAAI,CAAC,eAAe,GAAK,IAAI,OAAO,CAAsB,GAAG,CAAC,EAAE;YAC9D,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;QAC7B,CAAC,CAAC,EAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAAI,KAAmB;QACzC,IAAI,SAAyB,CAAC;QAC9B,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,CAAC,YAAY,KAAjB,IAAI,CAAC,YAAY,GAAK,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;YAC3E,WAAW,EAAE;gBACX,WAAW,EAAE,CAAC;gBACd,mBAAmB,EAAE,KAAK;aAC3B;SACF,CAAC,EAAC;QACH,IAAI,CAAC,eAAe,KAApB,IAAI,CAAC,eAAe,GAAK,IAAI,OAAO,CAAsB,OAAO,CAAC,EAAE;YAClE,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;QACjC,CAAC,CAAC,EAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC;QACpC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACrD,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,EAAE;gBACtC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;gBACvC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;gBACrC,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB;wBACE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAChC,aAAa,EAAE,OAAO,CAAC,KAAK;qBAC7B;iBACF,CAAC,CAAC;YACL,CAAC,CAAC;YACF,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE;gBAClC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,mBAAmB,EAAE,GAAG,CAAC,CAAC;YAC9C,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YAC5B,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC1B,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM;aACV,OAAO,CAAC,GAAG,EAAE;YACZ,IAAI,SAAS,EAAE,CAAC;gBACd,YAAY,CAAC,SAAS,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC;aACD,IAAI,CAAC,GAAG,CAAC,EAAE;YACV,0BAA0B;YAC1B,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gBACjB,OAAO;oBACL,GAAG,CAAC;oBACJ,OAAO,EAAE,IAAI,CAAC,WAAW,CAAmB,CAAC,CAAC,OAAQ,EAAE,KAAK,CAAC;iBAC/D,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;OAMG;IACH,OAAO,CAAC,QAAqC,EAAE,cAA8B;QAC3E,IAAI,CAAC,YAAY,KAAjB,IAAI,CAAC,YAAY,GAAK,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;YAC3E,WAAW,EAAE;gBACX,WAAW,EAAE,IAAI,CAAC,eAAe,EAAE;aACpC;SACF,CAAC,EAAC;QACH,MAAM,UAAU,GAAG,KAAK,EAAE,OAAgB,EAAE,EAAE;YAC5C,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;gBAC1E,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC;YAClC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAC1B,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC;YACrD,CAAC;QACH,CAAC,CAAC;QACF,OAAO,IAAI,iBAAiB,CAC1B,KAAK,IAAI,EAAE;YACT,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAC9C,CAAC,EACD,KAAK,IAAI,EAAE;YACT,IAAI,CAAC,YAAY,EAAE,cAAc,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAC3D,CAAC,CACF,CAAC;IACJ,CAAC;CACF;AAED,OAAO,EAAE,QAAQ,EAAE,CAAC"}

package/lib/services/storage.service.d.ts

@@ -0,0 +1,268 @@
+import { Bucket, StorageOptions } from "@google-cloud/storage";
+import { BinaryFile, BinaryFileInfo, BinaryMap, CloudBinary, CloudBinaryParameters, CoreModel, OperationContext } from "@webda/core";
+import { StorageFinder } from "@webda/utils";
+import { Readable, Writable } from "node:stream";
+export type StorageObject = {
+    key: string;
+    /**
+     * bucket is an option,
+     * if not provided: it will get the DEFAULT_BUCKET set from environment variable
+     */
+    bucket?: string;
+};
+export type DownloadParams = StorageObject & {
+    localPath: string;
+};
+export type PutObjectParams = StorageObject & {
+    localPath?: string;
+    content?: string | Buffer;
+};
+export type SignedUrlParams = StorageObject & {
+    expires?: number;
+    action: "read" | "write" | "delete";
+    contentMd5?: string;
+    contentType?: string;
+    extensionHeaders?: {
+        [key: string]: string;
+    };
+    responseDisposition?: string;
+    queryParams?: {
+        [key: string]: string;
+    };
+};
+export type StorageObjectMeta = {
+    size: number;
+    contentType: string;
+};
+/**
+ * Configuration for {@link Storage}
+ */
+export declare class StorageParameters extends CloudBinaryParameters {
+    /**
+     * Bucket to store binaries in
+     */
+    bucket: string;
+    /**
+     * Specify endpoint for GCS
+     */
+    endpoint?: string;
+}
+/**
+ * Google Storage Finder implementation
+ */
+export declare class GCSFinder implements StorageFinder {
+    private _storage;
+    /**
+     * @param options - GCS client options
+     */
+    constructor(options?: StorageOptions);
+    /**
+     * Get info from url
+     * @param path - the gs:// url
+     * @returns the bucket and key
+     */
+    getInfo(path: string): {
+        bucket: string;
+        key: string;
+    };
+    /**
+     * Depth or options is ignored
+     * @override
+     * @param path - the gs:// url to walk
+     * @param processor - called for each file found
+     * @param _options - ignored
+     * @param _depth - ignored
+     */
+    walk(path: string, processor: (filepath: string) => void, _options?: {
+        followSymlinks?: boolean;
+        resolveSymlink?: boolean;
+        includeDir?: boolean;
+        maxDepth?: number;
+    }, _depth?: number): Promise<void>;
+    /**
+     * Find files under a gs:// url
+     * @param currentPath - the gs:// url to search
+     * @param options - only `filterPattern` and `processor` are used
+     * @returns the files found
+     */
+    find(currentPath: string, options?: {
+        includeDir?: boolean;
+        maxDepth?: number;
+    } & {
+        filterPattern?: RegExp;
+        processor?: (filepath: string) => void;
+    }): Promise<string[]>;
+    /**
+     * Get a write stream
+     * @param path - the gs:// url
+     * @returns the write stream
+     */
+    getWriteStream(path: string): Promise<Writable>;
+    /**
+     * Get a read stream
+     * @param path - the gs:// url
+     * @returns the read stream
+     */
+    getReadStream(path: string): Promise<Readable>;
+}
+/**
+ * Storage handles the storage of binary on a Google Cloud Storage bucket
+ *
+ * See Binary the general interface
+ * @WebdaModda GoogleCloudStorage
+ */
+export default class Storage<T extends StorageParameters = StorageParameters> extends CloudBinary<T> {
+    private _storage?;
+    /**
+     * GCS client, created on first use
+     * @returns the GCS client
+     */
+    private get storage();
+    /**
+     * @override
+     * @param info - the binary map
+     * @returns the content stream
+     */
+    _get(info: BinaryMap): Promise<Readable>;
+    /**
+     * @override
+     * @param map - the binary map
+     * @param expires - expiration in seconds
+     * @param _context - the operation context
+     * @returns the signed url
+     */
+    getSignedUrlFromMap(map: BinaryMap, expires: number, _context: OperationContext): Promise<string>;
+    /**
+     * Get object content as a stream
+     * @param params - the object to read
+     * @param params.bucket - the bucket, default to the one predefined
+     * @param params.key - the object key
+     * @returns the content stream
+     */
+    getContent({ bucket, key }: StorageObject): Promise<Readable>;
+    /**
+     * Upload a local file to GCS bucket destination
+     *
+     * @param key - key to add to
+     * @param body - content of the object, a string is a local file path
+     * @param metadata - to put along the object
+     * @param bucket - to use
+     */
+    putObject(key: string, body: Readable | Buffer | string, metadata?: {}, bucket?: string): Promise<void>;
+    /**
+     * Return the Google Bucket directly
+     *
+     * @param bucket - name of the bucket default to the one predefined
+     * @returns the bucket
+     */
+    getStorageBucket(bucket?: string): Bucket;
+    /**
+     * @inheritdoc
+     * @param hash - the binary hash
+     * @param uuid - the object uuid
+     * @param property - the object property
+     */
+    _cleanUsage(hash: string, uuid: string, property?: string): Promise<void>;
+    /**
+     * @inheritdoc
+     * @param hash - the binary hash
+     * @returns the number of objects using the binary
+     */
+    getUsageCount(hash: string): Promise<number>;
+    /**
+     * @inheritdoc
+     * @param object - the object to attach the binary to
+     * @param property - the object property
+     * @param file - the binary to store
+     */
+    store(object: CoreModel, property: string, file: BinaryFile): Promise<void>;
+    /**
+     * Delete a file inside an existing bucket
+     * @param params - the object to delete
+     * @param params.bucket - the bucket, default to the one predefined
+     * @param params.key - the object key
+     */
+    deleteObject({ bucket, key }: StorageObject): Promise<void>;
+    /**
+     * Move a file from one bucket to an other
+     * @param source - the object to move
+     * @param destination - the new location
+     */
+    moveObject(source: StorageObject, destination: StorageObject): Promise<void>;
+    /**
+     * Get a signed url for the client to upload the binary directly to GCS
+     * @param object - the object to attach the binary to
+     * @param property - the object property
+     * @param info - the binary information (hash, challenge), read from the context if not provided
+     * @param context - the operation context
+     * @returns the upload url, undefined if the binary is already uploaded
+     */
+    putRedirectUrl(object: CoreModel, property: string, info?: BinaryFileInfo & {
+        hash: string;
+        challenge: string;
+    }, context?: OperationContext<any>): Promise<{
+        url: string;
+        method: string;
+        headers: {
+            [key: string]: string;
+        };
+    }>;
+    /**
+     * Retrieve one signed URL to download the file in parameter
+     * @param params.bucket - the bucket, default to the one predefined
+     * @param params.key - the object key
+     * @param params.expires - expiration in seconds
+     * @param params.action - the action allowed by the url
+     * @param params - the object and signature options
+     * @returns URL in order to download the file
+     */
+    getSignedUrl({ bucket, key, expires, action, ...params }: SignedUrlParams): Promise<string>;
+    /**
+     * Retrieve mandatory headers if needed by the cloud provider (ie. Azure)
+     * Returns empty object if no headers are mandatory
+     * @returns mandatory headers
+     */
+    getSignedUrlHeaders(): {
+        [key: string]: string;
+    };
+    /**
+     * Retrieve public URL of this object
+     * @param params - the object
+     * @param params.bucket - the bucket, default to the one predefined
+     * @param params.key - the object key
+     * @returns Public URL in order to download the file
+     */
+    getPublicUrl({ bucket, key }: StorageObject): Promise<string>;
+    /**
+     * Fetch an object's metadata
+     * @param params - the object
+     * @param params.bucket - the bucket, default to the one predefined
+     * @param params.key - the object key
+     * @returns size and content type
+     */
+    getMeta({ bucket, key }: StorageObject): Promise<StorageObjectMeta>;
+    /**
+     * Get a bucket size with a prefix
+     * @param bucket - the bucket, default to the one predefined
+     * @param prefix - on the bucket
+     * @param regex - to filter files
+     * @returns the total size and count of files
+     */
+    getBucketSize(bucket?: string, prefix?: string, regex?: RegExp): Promise<{
+        size: number;
+        count: number;
+    }>;
+    /**
+     * Add a usage marker for a binary
+     * @param hash - the binary hash
+     * @param uuid - the marker name: `${property}_${uuid}`
+     * @param model - the model identifier of the object
+     */
+    putMarker(hash: string, uuid: string, model: string): Promise<void>;
+    /**
+     * Delete every object under the prefix, used by tests
+     */
+    __clean(): Promise<void>;
+}
+export { Storage };
+//# sourceMappingURL=storage.service.d.ts.map

package/lib/services/storage.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.service.d.ts","sourceRoot":"","sources":["../../src/services/storage.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAA4C,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACzG,OAAO,EACL,UAAU,EACV,cAAc,EACd,SAAS,EACT,WAAW,EACX,qBAAqB,EACrB,SAAS,EACT,gBAAgB,EAEjB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,aAAa,EAAa,MAAM,cAAc,CAAC;AAGxD,OAAO,EAAE,QAAQ,EAAU,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGzD,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,aAAa,GAAG;IAC3C,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,aAAa,GAAG;IAC5C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,aAAa,GAAG;IAC5C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAC7C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,qBAAqB;IAC1D;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,qBAAa,SAAU,YAAW,aAAa;IAC7C,OAAO,CAAC,QAAQ,CAAM;IACtB;;OAEG;IACH,YAAY,OAAO,CAAC,EAAE,cAAc,EAEnC;IAED;;;;OAIG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAarD;IAED;;;;;;;OAOG;IACG,IAAI,CACR,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACrC,QAAQ,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,OAAO,CAAC;QAAC,cAAc,CAAC,EAAE,OAAO,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAC1G,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAUf;IAED;;;;;OAKG;IACG,IAAI,CACR,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG;QACtD,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;KACxC,GACA,OAAO,CAAC,MAAM,EAAE,CAAC,CAYnB;IAED;;;;OAIG;IACG,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAGpD;IAED;;;;OAIG;IACG,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAGnD;CACF;AAED;;;;;GAKG;AACH,MAAM,CAAC,OAAO,OAAO,OAAO,CAAC,CAAC,SAAS,iBAAiB,GAAG,iBAAiB,CAAE,SAAQ,WAAW,CAAC,CAAC,CAAC;IAClG,OAAO,CAAC,QAAQ,CAAC,CAAM;IACvB;;;OAGG;IACH,OAAO,KAAK,OAAO,GAKlB;IAED;;;;OAIG;IACG,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAG7C;IAED;;;;;;OAMG;IACH,mBAAmB,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAMhG;IAED;;;;;;OAMG;IACG,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,CAElE;IAED;;;;;;;OAOG;IACG,SAAS,CACb,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,MAAM,EAChC,QAAQ,KAAK,EACb,MAAM,GAAE,MAA+B,GACtC,OAAO,CAAC,IAAI,CAAC,CAgBf;IAED;;;;;OAKG;IACH,gBAAgB,CAAC,MAAM,GAAE,MAA+B,GAAG,MAAM,CAEhE;IAED;;;;;OAKG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,iBAS9D;IAED;;;;OAIG;IACG,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAKjD;IAED;;;;;OAKG;IACG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAchF;IAED;;;;;OAKG;IACG,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAGhE;IAED;;;;OAIG;IACG,UAAU,CAAC,MAAM,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjF;IAED;;;;;;;OAOG;IACG,cAAc,CAClB,MAAM,EAAE,SAAS,EACjB,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,cAAc,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,EAC3D,OAAO,CAAC,EAAE,gBAAgB,CAAC,GAAG,CAAC,GAC9B,OAAO,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAyC9E;IAED;;;;;;;;OAQG;IACG,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,OAAc,EAAE,MAAe,EAAE,GAAG,MAAM,EAAE,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAUhH;IAED;;;;OAIG;IACH,mBAAmB,IAAI;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAE/C;IAED;;;;;;OAMG;IACG,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAElE;IAED;;;;;;OAMG;IACG,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAMxE;IAED;;;;;;OAMG;IACG,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM;;;OA2BnE;IAED;;;;;OAKG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,iBAUxD;IAED;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAG7B;CACF;AAED,OAAO,EAAE,OAAO,EAAE,CAAC"}

package/lib/services/{storage.js → storage.service.js}

@@ -1,26 +1,29 @@
 import { Storage as GCS } from "@google-cloud/storage";
-import { BinaryParameters, CloudBinary, Throttler } from "@webda/core";
+import { CloudBinary, CloudBinaryParameters, useModelId } from "@webda/core";
+import { Throttler } from "@webda/utils";
 import { createReadStream } from "fs";
 import * as mime from "mime-types";
 import { Readable, Stream } from "node:stream";
 import { pipeline } from "node:stream/promises";
-export class StorageParameters extends BinaryParameters {
-    constructor(params, service) {
-        super(params, service);
-        this.prefix = "";
-    }
+/**
+ * Configuration for {@link Storage}
+ */
+export class StorageParameters extends CloudBinaryParameters {
 }
 /**
  * Google Storage Finder implementation
  */
 export class GCSFinder {
+    /**
+     * @param options - GCS client options
+     */
     constructor(options) {
         this._storage = new GCS(options);
     }
     /**
      * Get info from url
-     * @param path
-     * @returns
+     * @param path - the gs:// url
+     * @returns the bucket and key
      */
     getInfo(path) {
         const url = new URL(path);
@@ -39,8 +42,12 @@ export class GCSFinder {
     /**
      * Depth or options is ignored
      * @override
+     * @param path - the gs:// url to walk
+     * @param processor - called for each file found
+     * @param _options - ignored
+     * @param _depth - ignored
      */
-    async walk(path, processor, options, depth) {
+    async walk(path, processor, _options, _depth) {
         const { bucket, key } = this.getInfo(path);
         let pageToken;
         do {
@@ -52,10 +59,10 @@ export class GCSFinder {
         } while (pageToken);
     }
     /**
-     *
-     * @param currentPath
-     * @param options is ignored
-     * @returns
+     * Find files under a gs:// url
+     * @param currentPath - the gs:// url to search
+     * @param options - only `filterPattern` and `processor` are used
+     * @returns the files found
      */
     async find(currentPath, options) {
         const res = [];
@@ -72,8 +79,8 @@ export class GCSFinder {
     }
     /**
      * Get a write stream
-     * @param path
-     * @returns
+     * @param path - the gs:// url
+     * @returns the write stream
      */
     async getWriteStream(path) {
         const { bucket, key } = this.getInfo(path);
@@ -81,8 +88,8 @@ export class GCSFinder {
     }
     /**
      * Get a read stream
-     * @param path
-     * @returns
+     * @param path - the gs:// url
+     * @returns the read stream
      */
     async getReadStream(path) {
         const { bucket, key } = this.getInfo(path);
@@ -96,22 +103,20 @@ export class GCSFinder {
  * @WebdaModda GoogleCloudStorage
  */
 export default class Storage extends CloudBinary {
+    /**
+     * GCS client, created on first use
+     * @returns the GCS client
+     */
     get storage() {
         if (!this._storage) {
             this._storage = new GCS({ apiEndpoint: this.parameters.endpoint });
         }
         return this._storage;
     }
-    /**
-     * Load the parameters
-     *
-     * @param params
-     */
-    loadParameters(params) {
-        return new StorageParameters(params, this);
-    }
     /**
      * @override
+     * @param info - the binary map
+     * @returns the content stream
      */
     async _get(info) {
         const key = this._getKey(info.hash);
@@ -119,6 +124,10 @@ export default class Storage extends CloudBinary {
     }
     /**
      * @override
+     * @param map - the binary map
+     * @param expires - expiration in seconds
+     * @param _context - the operation context
+     * @returns the signed url
      */
     getSignedUrlFromMap(map, expires, _context) {
         return this.getSignedUrl({
@@ -128,8 +137,11 @@ export default class Storage extends CloudBinary {
         });
     }
     /**
-     * Get object content in Buffer
-     * @param {StorageObject} params
+     * Get object content as a stream
+     * @param params - the object to read
+     * @param params.bucket - the bucket, default to the one predefined
+     * @param params.key - the object key
+     * @returns the content stream
      */
     async getContent({ bucket, key }) {
         return this.getStorageBucket(bucket).file(key).createReadStream();
@@ -137,14 +149,12 @@ export default class Storage extends CloudBinary {
     /**
      * Upload a local file to GCS bucket destination
      *
-     * @param key to add to
-     * @param body content of the object
-     * @param metadatas to put along the object
-     * @param bucket to use
+     * @param key - key to add to
+     * @param body - content of the object, a string is a local file path
+     * @param metadata - to put along the object
+     * @param bucket - to use
      */
-    async putObject(key, body, metadata = {}, bucket = this.parameters.bucket
-    //{ bucket, localPath, content, key }: PutObjectParams
-    ) {
+    async putObject(key, body, metadata = {}, bucket = this.parameters.bucket) {
         const file = this.getStorageBucket(bucket).file(key);
         let contentType = "application/octet-stream";
         let stream;
@@ -166,14 +176,17 @@ export default class Storage extends CloudBinary {
     /**
      * Return the Google Bucket directly
      *
-     * @param bucket name of the bucket default to the one predefined
-     * @returns
+     * @param bucket - name of the bucket default to the one predefined
+     * @returns the bucket
      */
     getStorageBucket(bucket = this.parameters.bucket) {
         return this.storage.bucket(bucket);
     }
     /**
      * @inheritdoc
+     * @param hash - the binary hash
+     * @param uuid - the object uuid
+     * @param property - the object property
      */
     async _cleanUsage(hash, uuid, property) {
         const suffix = property ? `${property}_${uuid}` : `_${uuid}`;
@@ -187,6 +200,8 @@ export default class Storage extends CloudBinary {
     }
     /**
      * @inheritdoc
+     * @param hash - the binary hash
+     * @returns the number of objects using the binary
      */
     async getUsageCount(hash) {
         const [files] = await this.getStorageBucket().getFiles({
@@ -196,6 +211,9 @@ export default class Storage extends CloudBinary {
     }
     /**
      * @inheritdoc
+     * @param object - the object to attach the binary to
+     * @param property - the object property
+     * @param file - the binary to store
      */
     async store(object, property, file) {
         this.checkMap(object, property);
@@ -208,35 +226,41 @@ export default class Storage extends CloudBinary {
                 challenge: file.challenge
             });
         }
-        await this.putMarker(file.hash, `${property}_${object.getUuid()}`, object.getStore().getName());
-        await this.uploadSuccess(object, property, file);
+        await this.putMarker(file.hash, `${property}_${object.getUUID()}`, useModelId(object.constructor));
+        await this.uploadSuccess(object, property, file.toBinaryFileInfo());
     }
     /**
      * Delete a file inside an existing bucket
-     * @param {DeleteObjectParams} params
+     * @param params - the object to delete
+     * @param params.bucket - the bucket, default to the one predefined
+     * @param params.key - the object key
      */
     async deleteObject({ bucket, key }) {
         await this.getStorageBucket(bucket).file(key).delete();
-        this._webda.log("DEBUG", `gs://${bucket}/${key} deleted`);
+        this.log("DEBUG", `gs://${bucket}/${key} deleted`);
     }
     /**
      * Move a file from one bucket to an other
-     * @param {StorageObject} source
-     * @param {StorageObject} destination
+     * @param source - the object to move
+     * @param destination - the new location
      */
     async moveObject(source, destination) {
         const newFile = this.getStorageBucket(destination.bucket).file(destination.key);
-        this._webda.log("DEBUG", `moveObject gs://${source.bucket}/${source.key} => gs://${destination.bucket}/${destination.key}`);
+        this.log("DEBUG", `moveObject gs://${source.bucket}/${source.key} => gs://${destination.bucket}/${destination.key}`);
         await this.getStorageBucket(source.bucket).file(source.key).move(newFile);
     }
     /**
-     * @inheritdoc
+     * Get a signed url for the client to upload the binary directly to GCS
+     * @param object - the object to attach the binary to
+     * @param property - the object property
+     * @param info - the binary information (hash, challenge), read from the context if not provided
+     * @param context - the operation context
+     * @returns the upload url, undefined if the binary is already uploaded
      */
-    async putRedirectUrl(ctx) {
-        const body = await ctx.getRequestBody();
-        const { uuid, store, property } = ctx.getParameters();
-        const targetStore = this.verifyMapAndStore(ctx);
-        const object = await targetStore.get(uuid);
+    async putRedirectUrl(object, property, info, context) {
+        const body = info ?? (await context.getInput());
+        this.checkMap(object, property);
+        const uuid = object.getUUID();
         const base64String = Buffer.from(body.hash, "hex").toString("base64");
         const params = {
             bucket: this.parameters.bucket,
@@ -254,11 +278,11 @@ export default class Storage extends CloudBinary {
             const res = await this.getStorageBucket().file(params.key).getMetadata();
             challenge = res[0].metadata.challenge;
         }
-        catch (err) {
-            // Ignore error
+        catch {
+            // Ignore error: the data does not exist yet
         }
         await this.uploadSuccess(object, property, body);
-        await this.putMarker(body.hash, `${property}_${uuid}`, store);
+        await this.putMarker(body.hash, `${property}_${uuid}`, useModelId(object.constructor));
         // If the challenge is the same, no need to upload
         if (challenge && challenge === body.challenge) {
             return;
@@ -278,8 +302,12 @@ export default class Storage extends CloudBinary {
     }
     /**
      * Retrieve one signed URL to download the file in parameter
-     * @param {SignedUrlParams} params
-     * @returns {string} URL in order to download the file
+     * @param params.bucket - the bucket, default to the one predefined
+     * @param params.key - the object key
+     * @param params.expires - expiration in seconds
+     * @param params.action - the action allowed by the url
+     * @param params - the object and signature options
+     * @returns URL in order to download the file
      */
     async getSignedUrl({ bucket, key, expires = 3600, action = "read", ...params }) {
         const options = {
@@ -289,29 +317,33 @@ export default class Storage extends CloudBinary {
             expires: Date.now() + expires * 1000
         };
         const [url] = await this.getStorageBucket(bucket).file(key).getSignedUrl(options);
-        this._webda.log("TRACE", `The signed url for ${bucket}/${key} is ${url}.`);
+        this.log("TRACE", `The signed url for ${bucket}/${key} is ${url}.`);
         return url;
     }
     /**
      * Retrieve mandatory headers if needed by the cloud provider (ie. Azure)
      * Returns empty object if no headers are mandatory
-     * @returns {[key:string]: string} mandatory headers
+     * @returns mandatory headers
      */
     getSignedUrlHeaders() {
         return {};
     }
     /**
      * Retrieve public URL of this object
-     * @param {StorageModel} store
-     * @returns {string} Public URL in order to download the file
+     * @param params - the object
+     * @param params.bucket - the bucket, default to the one predefined
+     * @param params.key - the object key
+     * @returns Public URL in order to download the file
      */
     async getPublicUrl({ bucket, key }) {
         return this.getStorageBucket(bucket).file(key).publicUrl();
     }
     /**
      * Fetch an object's metadata
-     * @param {StorageModel} store
-     * @returns {any} Metadata
+     * @param params - the object
+     * @param params.bucket - the bucket, default to the one predefined
+     * @param params.key - the object key
+     * @returns size and content type
      */
     async getMeta({ bucket, key }) {
         const [metadata] = await this.getStorageBucket(bucket).file(key).getMetadata();
@@ -322,9 +354,10 @@ export default class Storage extends CloudBinary {
     }
     /**
      * Get a bucket size with a prefix
-     * @param bucket
-     * @param prefix on the bucket
-     * @param regex to filter files
+     * @param bucket - the bucket, default to the one predefined
+     * @param prefix - on the bucket
+     * @param regex - to filter files
+     * @returns the total size and count of files
      */
     async getBucketSize(bucket, prefix, regex) {
         let pageToken;
@@ -339,11 +372,13 @@ export default class Storage extends CloudBinary {
                     size += Number.parseInt(metadata.size);
                     count++;
                 }
-                catch (err) { }
+                catch {
+                    // Ignore files removed while computing
+                }
             };
         };
         do {
-            const [files, page, _] = await this.storage.bucket(bucket).getFiles({ maxResults: 1000, pageToken, prefix });
+            const [files, page] = await this.storage.bucket(bucket).getFiles({ maxResults: 1000, pageToken, prefix });
             files.filter(f => (regex ? f.name.match(regex) : true)).forEach(f => throttler.queue(dwl(f)));
             await throttler.wait();
             pageToken = page?.pageToken;
@@ -354,17 +389,29 @@ export default class Storage extends CloudBinary {
         };
     }
     /**
-     * @inheritdoc
+     * Add a usage marker for a binary
+     * @param hash - the binary hash
+     * @param uuid - the marker name: `${property}_${uuid}`
+     * @param model - the model identifier of the object
      */
-    async putMarker(hash, uuid, storeName) {
+    async putMarker(hash, uuid, model) {
         await this.getStorageBucket()
             .file(this._getKey(hash, uuid))
             .save("", {
             metadata: {
-                webdaStore: storeName
+                metadata: {
+                    webdaModel: model
+                }
             }
         });
     }
+    /**
+     * Delete every object under the prefix, used by tests
+     */
+    async __clean() {
+        const [files] = await this.getStorageBucket().getFiles({ prefix: this.parameters.prefix });
+        await Promise.all(files.map(f => f.delete()));
+    }
 }
 export { Storage };
-//# sourceMappingURL=storage.js.map
+//# sourceMappingURL=storage.service.js.map