@naturalcycles/cloud-storage-lib 1.9.1 → 1.10.0

package/dist/cloudStorage.d.ts

@@ -2,7 +2,7 @@
 /// <reference types="node" />
 import { Readable, Writable } from 'node:stream';
 import { Storage, StorageOptions } from '@google-cloud/storage';
-import { LocalTimeInput } from '@naturalcycles/js-lib';
+import { CommonLogger, LocalTimeInput } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
 import { GCPServiceAccount } from './model';
@@ -18,9 +18,13 @@ export { Storage, type StorageOptions, };
  */
 export interface CloudStorageCfg {
     /**
-     * It's optional, to allow automatic credentials in AppEngine, or GOOGLE_APPLICATION_CREDENTIALS.
+     * Default is console
      */
-    credentials?: GCPServiceAccount;
+    logger?: CommonLogger;
+    /**
+     * Pass true for extra debugging
+     */
+    debug?: boolean;
 }
 /**
  * CloudStorage implementation of CommonStorage API.
@@ -29,13 +33,17 @@ export interface CloudStorageCfg {
  */
 export declare class CloudStorage implements CommonStorage {
     storage: Storage;
+    private constructor();
+    cfg: CloudStorageCfg & {
+        logger: CommonLogger;
+    };
+    static createFromGCPServiceAccount(credentials?: GCPServiceAccount, cfg?: CloudStorageCfg): CloudStorage;
+    static createFromStorageOptions(storageOptions?: StorageOptions, cfg?: CloudStorageCfg): CloudStorage;
     /**
      * Passing the pre-created Storage allows to instantiate it from both
      * GCP Storage and FirebaseStorage.
      */
-    constructor(storage: Storage);
-    static createFromGCPServiceAccount(cfg: CloudStorageCfg): CloudStorage;
-    static createFromStorageOptions(storageOptions?: StorageOptions): CloudStorage;
+    static createFromStorage(storage: Storage, cfg?: CloudStorageCfg): CloudStorage;
     ping(bucketName?: string): Promise<void>;
     deletePath(bucketName: string, prefix: string): Promise<void>;
     deletePaths(bucketName: string, prefixes: string[]): Promise<void>;
@@ -57,7 +65,9 @@ export declare class CloudStorage implements CommonStorage {
     copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     movePath(fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string): Promise<void>;
-    combine(bucketName: string, filePaths: string[], toPath: string, toBucket?: string): Promise<void>;
+    deleteFiles(bucketName: string, filePaths: string[]): Promise<void>;
+    combineFiles(bucketName: string, filePaths: string[], toPath: string, toBucket?: string, currentRecursionDepth?: number): Promise<void>;
+    combine(bucketName: string, prefix: string, toPath: string, toBucket?: string): Promise<void>;
     /**
      * Acquires a "signed url", which allows bearer to use it to download ('read') the file.
      *

package/dist/cloudStorage.js

@@ -4,34 +4,43 @@ exports.CloudStorage = exports.Storage = void 0;
 const storage_1 = require("@google-cloud/storage");
 Object.defineProperty(exports, "Storage", { enumerable: true, get: function () { return storage_1.Storage; } });
 const js_lib_1 = require("@naturalcycles/js-lib");
+const MAX_RECURSION_DEPTH = 10;
+const BATCH_SIZE = 32;
 /**
  * CloudStorage implementation of CommonStorage API.
  *
  * API: https://googleapis.dev/nodejs/storage/latest/index.html
  */
 class CloudStorage {
-    /**
-     * Passing the pre-created Storage allows to instantiate it from both
-     * GCP Storage and FirebaseStorage.
-     */
-    constructor(storage) {
+    constructor(storage, cfg = {}) {
         this.storage = storage;
+        this.cfg = {
+            logger: console,
+            ...cfg,
+        };
     }
-    static createFromGCPServiceAccount(cfg) {
+    static createFromGCPServiceAccount(credentials, cfg) {
         const storage = new storage_1.Storage({
-            credentials: cfg.credentials,
+            credentials,
             // Explicitly passing it here to fix this error:
             // Error: Unable to detect a Project Id in the current environment.
             // To learn more about authentication and Google APIs, visit:
             // https://cloud.google.com/docs/authentication/getting-started
             //     at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
-            projectId: cfg.credentials?.project_id,
+            projectId: credentials?.project_id,
         });
-        return new CloudStorage(storage);
+        return new CloudStorage(storage, cfg);
     }
-    static createFromStorageOptions(storageOptions) {
+    static createFromStorageOptions(storageOptions, cfg) {
         const storage = new storage_1.Storage(storageOptions);
-        return new CloudStorage(storage);
+        return new CloudStorage(storage, cfg);
+    }
+    /**
+     * Passing the pre-created Storage allows to instantiate it from both
+     * GCP Storage and FirebaseStorage.
+     */
+    static createFromStorage(storage, cfg) {
+        return new CloudStorage(storage, cfg);
     }
     async ping(bucketName) {
         await this.storage.bucket(bucketName || 'non-existing-for-sure').exists();
@@ -88,6 +97,8 @@ class CloudStorage {
                 return [];
             const [content] = await f.download();
             return [{ filePath, content }];
+        }, {
+            concurrency: 16,
         });
     }
     async getFile(bucketName, filePath) {
@@ -153,14 +164,48 @@ class CloudStorage {
             await file.move(this.storage.bucket(toBucket || fromBucket).file(newName));
         });
     }
-    async combine(bucketName, filePaths, toPath, toBucket) {
-        // todo: if (filePaths.length > 32) - use recursive algorithm
-        (0, js_lib_1._assert)(filePaths.length <= 32, 'combine supports up to 32 input files');
-        await this.storage
-            .bucket(bucketName)
-            .combine(filePaths, this.storage.bucket(toBucket || bucketName).file(toPath));
-        // Delete original files
-        await this.deletePaths(bucketName, filePaths);
+    async deleteFiles(bucketName, filePaths) {
+        await (0, js_lib_1.pMap)(filePaths, async (filePath) => {
+            await this.storage.bucket(bucketName).file(filePath).delete();
+        });
+    }
+    async combineFiles(bucketName, filePaths, toPath, toBucket, currentRecursionDepth = 0) {
+        (0, js_lib_1._assert)(currentRecursionDepth <= MAX_RECURSION_DEPTH, `combineFiles reached max recursion depth of ${MAX_RECURSION_DEPTH}`);
+        const { logger, debug } = this.cfg;
+        if (debug) {
+            logger.log(`[${currentRecursionDepth}] Will compose ${filePaths.length} files, by batches of ${BATCH_SIZE}`);
+        }
+        const intermediateFiles = [];
+        if (filePaths.length <= BATCH_SIZE) {
+            await this.storage
+                .bucket(bucketName)
+                .combine(filePaths, this.storage.bucket(toBucket || bucketName).file(toPath));
+            if (debug) {
+                logger.log(`[${currentRecursionDepth}] Composed into ${toPath}!`);
+            }
+            await this.deleteFiles(bucketName, filePaths);
+            return;
+        }
+        const started = Date.now();
+        await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(filePaths, BATCH_SIZE), async (fileBatch, i) => {
+            if (debug) {
+                logger.log(`[${currentRecursionDepth}] Composing batch ${i + 1}...`);
+            }
+            const intermediateFile = `temp_${currentRecursionDepth}_${i}`;
+            await this.storage
+                .bucket(bucketName)
+                .combine(fileBatch, this.storage.bucket(toBucket || bucketName).file(intermediateFile));
+            intermediateFiles.push(intermediateFile);
+            await this.deleteFiles(bucketName, fileBatch);
+        });
+        if (debug) {
+            logger.log(`[${currentRecursionDepth}] Batch composed into ${intermediateFiles.length} files, in ${(0, js_lib_1._since)(started)}`);
+        }
+        await this.combineFiles(toBucket || bucketName, intermediateFiles, toPath, toBucket, currentRecursionDepth + 1);
+    }
+    async combine(bucketName, prefix, toPath, toBucket) {
+        const filePaths = await this.getFileNames(bucketName, { prefix });
+        await this.combineFiles(bucketName, filePaths, toPath, toBucket);
     }
     /**
      * Acquires a "signed url", which allows bearer to use it to download ('read') the file.

package/dist/commonStorage.d.ts

@@ -56,6 +56,10 @@ export interface CommonStorage {
      */
     deletePath: (bucketName: string, prefix: string) => Promise<void>;
     deletePaths: (bucketName: string, prefixes: string[]) => Promise<void>;
+    /**
+     * Should delete all files by their paths.
+     */
+    deleteFiles: (bucketName: string, filePaths: string[]) => Promise<void>;
     /**
      * Returns an array of strings which are file paths.
      * Files that are not found by the path are not present in the map.
@@ -95,7 +99,11 @@ export interface CommonStorage {
      *
      * @experimental
      */
-    combine: (bucketName: string, filePaths: string[], toPath: string, toBucket?: string) => Promise<void>;
+    combineFiles: (bucketName: string, filePaths: string[], toPath: string, toBucket?: string) => Promise<void>;
+    /**
+     * Like `combineFiles`, but for a `prefix`.
+     */
+    combine: (bucketName: string, prefix: string, toPath: string, toBucket?: string) => Promise<void>;
     /**
      * Acquire a "signed url", which allows bearer to use it to download ('read') the file.
      *

package/dist/inMemoryCommonStorage.d.ts

@@ -18,6 +18,7 @@ export declare class InMemoryCommonStorage implements CommonStorage {
     saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
     deletePath(bucketName: string, prefix: string): Promise<void>;
     deletePaths(bucketName: string, prefixes: string[]): Promise<void>;
+    deleteFiles(bucketName: string, filePaths: string[]): Promise<void>;
     getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>;
     getFileNamesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
     getFilesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
@@ -29,6 +30,7 @@ export declare class InMemoryCommonStorage implements CommonStorage {
     copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     movePath(fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string): Promise<void>;
-    combine(bucketName: string, filePaths: string[], toPath: string, toBucket?: string): Promise<void>;
+    combine(bucketName: string, prefix: string, toPath: string, toBucket?: string): Promise<void>;
+    combineFiles(bucketName: string, filePaths: string[], toPath: string, toBucket?: string): Promise<void>;
     getSignedUrl(bucketName: string, filePath: string, expires: LocalTimeInput): Promise<string>;
 }

package/dist/inMemoryCommonStorage.js

@@ -39,6 +39,11 @@ class InMemoryCommonStorage {
             }
         });
     }
+    async deleteFiles(bucketName, filePaths) {
+        if (!this.data[bucketName])
+            return;
+        filePaths.forEach(filePath => delete this.data[bucketName][filePath]);
+    }
     async getFileNames(bucketName, opt = {}) {
         const { prefix = '', fullPaths = true } = opt;
         return Object.keys(this.data[bucketName] || {})
@@ -104,7 +109,11 @@ class InMemoryCommonStorage {
             delete this.data[fromBucket][filePath];
         });
     }
-    async combine(bucketName, filePaths, toPath, toBucket) {
+    async combine(bucketName, prefix, toPath, toBucket) {
+        const filePaths = await this.getFileNames(bucketName, { prefix });
+        await this.combineFiles(bucketName, filePaths, toPath, toBucket);
+    }
+    async combineFiles(bucketName, filePaths, toPath, toBucket) {
         if (!this.data[bucketName])
             return;
         const tob = toBucket || bucketName;

package/package.json

@@ -35,7 +35,7 @@
   "engines": {
     "node": ">=18.12.0"
   },
-  "version": "1.9.1",
+  "version": "1.10.0",
   "description": "CommonStorage implementation based on Google Cloud Storage",
   "author": "Natural Cycles Team",
   "license": "MIT"

package/src/cloudStorage.ts

@@ -2,7 +2,10 @@ import { Readable, Writable } from 'node:stream'
 import { File, Storage, StorageOptions } from '@google-cloud/storage'
 import {
   _assert,
+  _chunk,
+  _since,
   _substringAfterLast,
+  CommonLogger,
   localTime,
   LocalTimeInput,
   pMap,
@@ -18,6 +21,9 @@ export {
   type StorageOptions,
 }
 
+const MAX_RECURSION_DEPTH = 10
+const BATCH_SIZE = 32
+
 /**
  * This object is intentionally made to NOT extend StorageOptions,
  * because StorageOptions is complicated and provides just too many ways
@@ -29,9 +35,14 @@ export {
  */
 export interface CloudStorageCfg {
   /**
-   * It's optional, to allow automatic credentials in AppEngine, or GOOGLE_APPLICATION_CREDENTIALS.
+   * Default is console
+   */
+  logger?: CommonLogger
+
+  /**
+   * Pass true for extra debugging
    */
-  credentials?: GCPServiceAccount
+  debug?: boolean
 }
 
 /**
@@ -40,29 +51,51 @@ export interface CloudStorageCfg {
  * API: https://googleapis.dev/nodejs/storage/latest/index.html
  */
 export class CloudStorage implements CommonStorage {
-  /**
-   * Passing the pre-created Storage allows to instantiate it from both
-   * GCP Storage and FirebaseStorage.
-   */
-  constructor(public storage: Storage) {}
+  private constructor(
+    public storage: Storage,
+    cfg: CloudStorageCfg = {},
+  ) {
+    this.cfg = {
+      logger: console,
+      ...cfg,
+    }
+  }
 
-  static createFromGCPServiceAccount(cfg: CloudStorageCfg): CloudStorage {
+  cfg: CloudStorageCfg & {
+    logger: CommonLogger
+  }
+
+  static createFromGCPServiceAccount(
+    credentials?: GCPServiceAccount,
+    cfg?: CloudStorageCfg,
+  ): CloudStorage {
     const storage = new Storage({
-      credentials: cfg.credentials,
+      credentials,
       // Explicitly passing it here to fix this error:
       // Error: Unable to detect a Project Id in the current environment.
       // To learn more about authentication and Google APIs, visit:
       // https://cloud.google.com/docs/authentication/getting-started
       //     at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
-      projectId: cfg.credentials?.project_id,
+      projectId: credentials?.project_id,
     })
 
-    return new CloudStorage(storage)
+    return new CloudStorage(storage, cfg)
   }
 
-  static createFromStorageOptions(storageOptions?: StorageOptions): CloudStorage {
+  static createFromStorageOptions(
+    storageOptions?: StorageOptions,
+    cfg?: CloudStorageCfg,
+  ): CloudStorage {
     const storage = new Storage(storageOptions)
-    return new CloudStorage(storage)
+    return new CloudStorage(storage, cfg)
+  }
+
+  /**
+   * Passing the pre-created Storage allows to instantiate it from both
+   * GCP Storage and FirebaseStorage.
+   */
+  static createFromStorage(storage: Storage, cfg?: CloudStorageCfg): CloudStorage {
+    return new CloudStorage(storage, cfg)
   }
 
   async ping(bucketName?: string): Promise<void> {
@@ -128,13 +161,18 @@ export class CloudStorage implements CommonStorage {
         prefix,
         maxResults: opt.limit || undefined,
       }) as ReadableTyped<File>
-    ).flatMap(async f => {
-      const filePath = this.normalizeFilename(f.name, fullPaths)
-      if (filePath === SKIP) return []
-
-      const [content] = await f.download()
-      return [{ filePath, content }] as FileEntry[]
-    })
+    ).flatMap(
+      async f => {
+        const filePath = this.normalizeFilename(f.name, fullPaths)
+        if (filePath === SKIP) return []
+
+        const [content] = await f.download()
+        return [{ filePath, content }] as FileEntry[]
+      },
+      {
+        concurrency: 16,
+      },
+    )
   }
 
   async getFile(bucketName: string, filePath: string): Promise<Buffer | null> {
@@ -230,21 +268,81 @@ export class CloudStorage implements CommonStorage {
       })
   }
 
-  async combine(
+  async deleteFiles(bucketName: string, filePaths: string[]): Promise<void> {
+    await pMap(filePaths, async filePath => {
+      await this.storage.bucket(bucketName).file(filePath).delete()
+    })
+  }
+
+  async combineFiles(
     bucketName: string,
     filePaths: string[],
     toPath: string,
     toBucket?: string,
+    currentRecursionDepth = 0, // not to be set publicly, only used internally
   ): Promise<void> {
-    // todo: if (filePaths.length > 32) - use recursive algorithm
-    _assert(filePaths.length <= 32, 'combine supports up to 32 input files')
+    _assert(
+      currentRecursionDepth <= MAX_RECURSION_DEPTH,
+      `combineFiles reached max recursion depth of ${MAX_RECURSION_DEPTH}`,
+    )
+    const { logger, debug } = this.cfg
+
+    if (debug) {
+      logger.log(
+        `[${currentRecursionDepth}] Will compose ${filePaths.length} files, by batches of ${BATCH_SIZE}`,
+      )
+    }
 
-    await this.storage
-      .bucket(bucketName)
-      .combine(filePaths, this.storage.bucket(toBucket || bucketName).file(toPath))
+    const intermediateFiles: string[] = []
+
+    if (filePaths.length <= BATCH_SIZE) {
+      await this.storage
+        .bucket(bucketName)
+        .combine(filePaths, this.storage.bucket(toBucket || bucketName).file(toPath))
+
+      if (debug) {
+        logger.log(`[${currentRecursionDepth}] Composed into ${toPath}!`)
+      }
+
+      await this.deleteFiles(bucketName, filePaths)
+      return
+    }
 
-    // Delete original files
-    await this.deletePaths(bucketName, filePaths)
+    const started = Date.now()
+    await pMap(_chunk(filePaths, BATCH_SIZE), async (fileBatch, i) => {
+      if (debug) {
+        logger.log(`[${currentRecursionDepth}] Composing batch ${i + 1}...`)
+      }
+      const intermediateFile = `temp_${currentRecursionDepth}_${i}`
+      await this.storage
+        .bucket(bucketName)
+        .combine(fileBatch, this.storage.bucket(toBucket || bucketName).file(intermediateFile))
+      intermediateFiles.push(intermediateFile)
+      await this.deleteFiles(bucketName, fileBatch)
+    })
+    if (debug) {
+      logger.log(
+        `[${currentRecursionDepth}] Batch composed into ${intermediateFiles.length} files, in ${_since(started)}`,
+      )
+    }
+
+    await this.combineFiles(
+      toBucket || bucketName,
+      intermediateFiles,
+      toPath,
+      toBucket,
+      currentRecursionDepth + 1,
+    )
+  }
+
+  async combine(
+    bucketName: string,
+    prefix: string,
+    toPath: string,
+    toBucket?: string,
+  ): Promise<void> {
+    const filePaths = await this.getFileNames(bucketName, { prefix })
+    await this.combineFiles(bucketName, filePaths, toPath, toBucket)
   }
 
   /**

package/src/commonStorage.ts

@@ -69,6 +69,11 @@ export interface CommonStorage {
 
   deletePaths: (bucketName: string, prefixes: string[]) => Promise<void>
 
+  /**
+   * Should delete all files by their paths.
+   */
+  deleteFiles: (bucketName: string, filePaths: string[]) => Promise<void>
+
   /**
    * Returns an array of strings which are file paths.
    * Files that are not found by the path are not present in the map.
@@ -134,13 +139,18 @@ export interface CommonStorage {
    *
    * @experimental
    */
-  combine: (
+  combineFiles: (
     bucketName: string,
     filePaths: string[],
     toPath: string,
     toBucket?: string,
   ) => Promise<void>
 
+  /**
+   * Like `combineFiles`, but for a `prefix`.
+   */
+  combine: (bucketName: string, prefix: string, toPath: string, toBucket?: string) => Promise<void>
+
   /**
    * Acquire a "signed url", which allows bearer to use it to download ('read') the file.
    *

package/src/inMemoryCommonStorage.ts

@@ -54,6 +54,11 @@ export class InMemoryCommonStorage implements CommonStorage {
     })
   }
 
+  async deleteFiles(bucketName: string, filePaths: string[]): Promise<void> {
+    if (!this.data[bucketName]) return
+    filePaths.forEach(filePath => delete this.data[bucketName]![filePath])
+  }
+
   async getFileNames(bucketName: string, opt: CommonStorageGetOptions = {}): Promise<string[]> {
     const { prefix = '', fullPaths = true } = opt
     return Object.keys(this.data[bucketName] || {})
@@ -156,6 +161,16 @@ export class InMemoryCommonStorage implements CommonStorage {
   }
 
   async combine(
+    bucketName: string,
+    prefix: string,
+    toPath: string,
+    toBucket?: string,
+  ): Promise<void> {
+    const filePaths = await this.getFileNames(bucketName, { prefix })
+    await this.combineFiles(bucketName, filePaths, toPath, toBucket)
+  }
+
+  async combineFiles(
     bucketName: string,
     filePaths: string[],
     toPath: string,