@naturalcycles/cloud-storage-lib 1.9.2 → 1.11.0

package/dist/cloudStorage.d.ts

@@ -1,12 +1,10 @@
 /// <reference types="node" />
-/// <reference types="node" />
-import { Readable, Writable } from 'node:stream';
-import { Storage, StorageOptions } from '@google-cloud/storage';
-import { LocalTimeInput } from '@naturalcycles/js-lib';
-import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
-import { GCPServiceAccount } from './model';
-export { Storage, type StorageOptions, };
+import type { Storage, StorageOptions } from '@google-cloud/storage';
+import { CommonLogger, LocalTimeInput } from '@naturalcycles/js-lib';
+import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
+import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
+import type { GCPServiceAccount } from './model';
+export { type Storage, type StorageOptions, };
 /**
  * This object is intentionally made to NOT extend StorageOptions,
  * because StorageOptions is complicated and provides just too many ways
@@ -18,9 +16,13 @@ export { Storage, type StorageOptions, };
  */
 export interface CloudStorageCfg {
     /**
-     * It's optional, to allow automatic credentials in AppEngine, or GOOGLE_APPLICATION_CREDENTIALS.
+     * Default is console
+     */
+    logger?: CommonLogger;
+    /**
+     * Pass true for extra debugging
      */
-    credentials?: GCPServiceAccount;
+    debug?: boolean;
 }
 /**
  * CloudStorage implementation of CommonStorage API.
@@ -29,13 +31,17 @@ export interface CloudStorageCfg {
  */
 export declare class CloudStorage implements CommonStorage {
     storage: Storage;
+    private constructor();
+    cfg: CloudStorageCfg & {
+        logger: CommonLogger;
+    };
+    static createFromGCPServiceAccount(credentials?: GCPServiceAccount, cfg?: CloudStorageCfg): CloudStorage;
+    static createFromStorageOptions(storageOptions?: StorageOptions, cfg?: CloudStorageCfg): CloudStorage;
     /**
      * Passing the pre-created Storage allows to instantiate it from both
      * GCP Storage and FirebaseStorage.
      */
-    constructor(storage: Storage);
-    static createFromGCPServiceAccount(cfg: CloudStorageCfg): CloudStorage;
-    static createFromStorageOptions(storageOptions?: StorageOptions): CloudStorage;
+    static createFromStorage(storage: Storage, cfg?: CloudStorageCfg): CloudStorage;
     ping(bucketName?: string): Promise<void>;
     deletePath(bucketName: string, prefix: string): Promise<void>;
     deletePaths(bucketName: string, prefixes: string[]): Promise<void>;
@@ -48,16 +54,18 @@ export declare class CloudStorage implements CommonStorage {
      * Returns a Readable that is NOT object mode,
      * so you can e.g pipe it to fs.createWriteStream()
      */
-    getFileReadStream(bucketName: string, filePath: string): Readable;
+    getFileReadStream(bucketName: string, filePath: string): ReadableBinary;
     saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
-    getFileWriteStream(bucketName: string, filePath: string): Writable;
+    getFileWriteStream(bucketName: string, filePath: string): WritableBinary;
     uploadFile(localFilePath: string, bucketName: string, bucketFilePath: string): Promise<void>;
     setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
     getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;
     copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     movePath(fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string): Promise<void>;
-    combine(bucketName: string, filePaths: string[], toPath: string, toBucket?: string): Promise<void>;
+    deleteFiles(bucketName: string, filePaths: string[]): Promise<void>;
+    combineFiles(bucketName: string, filePaths: string[], toPath: string, toBucket?: string, currentRecursionDepth?: number): Promise<void>;
+    combine(bucketName: string, prefix: string, toPath: string, toBucket?: string): Promise<void>;
     /**
      * Acquires a "signed url", which allows bearer to use it to download ('read') the file.
      *

package/dist/cloudStorage.js

@@ -1,37 +1,46 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CloudStorage = exports.Storage = void 0;
-const storage_1 = require("@google-cloud/storage");
-Object.defineProperty(exports, "Storage", { enumerable: true, get: function () { return storage_1.Storage; } });
+exports.CloudStorage = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
+const MAX_RECURSION_DEPTH = 10;
+const BATCH_SIZE = 32;
 /**
  * CloudStorage implementation of CommonStorage API.
  *
  * API: https://googleapis.dev/nodejs/storage/latest/index.html
  */
 class CloudStorage {
-    /**
-     * Passing the pre-created Storage allows to instantiate it from both
-     * GCP Storage and FirebaseStorage.
-     */
-    constructor(storage) {
+    constructor(storage, cfg = {}) {
         this.storage = storage;
-    }
-    static createFromGCPServiceAccount(cfg) {
-        const storage = new storage_1.Storage({
-            credentials: cfg.credentials,
+        this.cfg = {
+            logger: console,
+            ...cfg,
+        };
+    }
+    static createFromGCPServiceAccount(credentials, cfg) {
+        const storageLib = require('@google-cloud/storage');
+        const storage = new storageLib.Storage({
+            credentials,
             // Explicitly passing it here to fix this error:
             // Error: Unable to detect a Project Id in the current environment.
             // To learn more about authentication and Google APIs, visit:
             // https://cloud.google.com/docs/authentication/getting-started
             //     at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
-            projectId: cfg.credentials?.project_id,
+            projectId: credentials?.project_id,
         });
-        return new CloudStorage(storage);
+        return new CloudStorage(storage, cfg);
     }
-    static createFromStorageOptions(storageOptions) {
-        const storage = new storage_1.Storage(storageOptions);
-        return new CloudStorage(storage);
+    static createFromStorageOptions(storageOptions, cfg) {
+        const storageLib = require('@google-cloud/storage');
+        const storage = new storageLib.Storage(storageOptions);
+        return new CloudStorage(storage, cfg);
+    }
+    /**
+     * Passing the pre-created Storage allows to instantiate it from both
+     * GCP Storage and FirebaseStorage.
+     */
+    static createFromStorage(storage, cfg) {
+        return new CloudStorage(storage, cfg);
     }
     async ping(bucketName) {
         await this.storage.bucket(bucketName || 'non-existing-for-sure').exists();
@@ -155,14 +164,48 @@ class CloudStorage {
             await file.move(this.storage.bucket(toBucket || fromBucket).file(newName));
         });
     }
-    async combine(bucketName, filePaths, toPath, toBucket) {
-        // todo: if (filePaths.length > 32) - use recursive algorithm
-        (0, js_lib_1._assert)(filePaths.length <= 32, 'combine supports up to 32 input files');
-        await this.storage
-            .bucket(bucketName)
-            .combine(filePaths, this.storage.bucket(toBucket || bucketName).file(toPath));
-        // Delete original files
-        await this.deletePaths(bucketName, filePaths);
+    async deleteFiles(bucketName, filePaths) {
+        await (0, js_lib_1.pMap)(filePaths, async (filePath) => {
+            await this.storage.bucket(bucketName).file(filePath).delete();
+        });
+    }
+    async combineFiles(bucketName, filePaths, toPath, toBucket, currentRecursionDepth = 0) {
+        (0, js_lib_1._assert)(currentRecursionDepth <= MAX_RECURSION_DEPTH, `combineFiles reached max recursion depth of ${MAX_RECURSION_DEPTH}`);
+        const { logger, debug } = this.cfg;
+        if (debug) {
+            logger.log(`[${currentRecursionDepth}] Will compose ${filePaths.length} files, by batches of ${BATCH_SIZE}`);
+        }
+        const intermediateFiles = [];
+        if (filePaths.length <= BATCH_SIZE) {
+            await this.storage
+                .bucket(bucketName)
+                .combine(filePaths, this.storage.bucket(toBucket || bucketName).file(toPath));
+            if (debug) {
+                logger.log(`[${currentRecursionDepth}] Composed into ${toPath}!`);
+            }
+            await this.deleteFiles(bucketName, filePaths);
+            return;
+        }
+        const started = Date.now();
+        await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(filePaths, BATCH_SIZE), async (fileBatch, i) => {
+            if (debug) {
+                logger.log(`[${currentRecursionDepth}] Composing batch ${i + 1}...`);
+            }
+            const intermediateFile = `temp_${currentRecursionDepth}_${i}`;
+            await this.storage
+                .bucket(bucketName)
+                .combine(fileBatch, this.storage.bucket(toBucket || bucketName).file(intermediateFile));
+            intermediateFiles.push(intermediateFile);
+            await this.deleteFiles(bucketName, fileBatch);
+        });
+        if (debug) {
+            logger.log(`[${currentRecursionDepth}] Batch composed into ${intermediateFiles.length} files, in ${(0, js_lib_1._since)(started)}`);
+        }
+        await this.combineFiles(toBucket || bucketName, intermediateFiles, toPath, toBucket, currentRecursionDepth + 1);
+    }
+    async combine(bucketName, prefix, toPath, toBucket) {
+        const filePaths = await this.getFileNames(bucketName, { prefix });
+        await this.combineFiles(bucketName, filePaths, toPath, toBucket);
     }
     /**
      * Acquires a "signed url", which allows bearer to use it to download ('read') the file.

package/dist/commonStorage.d.ts

@@ -1,8 +1,6 @@
 /// <reference types="node" />
-/// <reference types="node" />
-import { Readable, Writable } from 'node:stream';
-import { LocalTimeInput } from '@naturalcycles/js-lib';
-import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import type { LocalTimeInput } from '@naturalcycles/js-lib';
+import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
 export interface FileEntry {
     filePath: string;
     content: Buffer;
@@ -56,6 +54,10 @@ export interface CommonStorage {
      */
     deletePath: (bucketName: string, prefix: string) => Promise<void>;
     deletePaths: (bucketName: string, prefixes: string[]) => Promise<void>;
+    /**
+     * Should delete all files by their paths.
+     */
+    deleteFiles: (bucketName: string, filePaths: string[]) => Promise<void>;
     /**
      * Returns an array of strings which are file paths.
      * Files that are not found by the path are not present in the map.
@@ -70,8 +72,8 @@ export interface CommonStorage {
     getFileNames: (bucketName: string, opt?: CommonStorageGetOptions) => Promise<string[]>;
     getFileNamesStream: (bucketName: string, opt?: CommonStorageGetOptions) => ReadableTyped<string>;
     getFilesStream: (bucketName: string, opt?: CommonStorageGetOptions) => ReadableTyped<FileEntry>;
-    getFileReadStream: (bucketName: string, filePath: string) => Readable;
-    getFileWriteStream: (bucketName: string, filePath: string) => Writable;
+    getFileReadStream: (bucketName: string, filePath: string) => ReadableBinary;
+    getFileWriteStream: (bucketName: string, filePath: string) => WritableBinary;
     /**
      * Upload local file to the bucket (by streaming it).
      */
@@ -95,7 +97,11 @@ export interface CommonStorage {
      *
      * @experimental
      */
-    combine: (bucketName: string, filePaths: string[], toPath: string, toBucket?: string) => Promise<void>;
+    combineFiles: (bucketName: string, filePaths: string[], toPath: string, toBucket?: string) => Promise<void>;
+    /**
+     * Like `combineFiles`, but for a `prefix`.
+     */
+    combine: (bucketName: string, prefix: string, toPath: string, toBucket?: string) => Promise<void>;
     /**
      * Acquire a "signed url", which allows bearer to use it to download ('read') the file.
      *

package/dist/inMemoryCommonStorage.d.ts

@@ -1,8 +1,6 @@
 /// <reference types="node" />
-/// <reference types="node" />
-import { Readable, Writable } from 'node:stream';
 import { LocalTimeInput, StringMap } from '@naturalcycles/js-lib';
-import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
 export declare class InMemoryCommonStorage implements CommonStorage {
     /**
@@ -18,17 +16,19 @@ export declare class InMemoryCommonStorage implements CommonStorage {
     saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
     deletePath(bucketName: string, prefix: string): Promise<void>;
     deletePaths(bucketName: string, prefixes: string[]): Promise<void>;
+    deleteFiles(bucketName: string, filePaths: string[]): Promise<void>;
     getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>;
     getFileNamesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
     getFilesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
-    getFileReadStream(bucketName: string, filePath: string): Readable;
-    getFileWriteStream(_bucketName: string, _filePath: string): Writable;
+    getFileReadStream(bucketName: string, filePath: string): ReadableBinary;
+    getFileWriteStream(_bucketName: string, _filePath: string): WritableBinary;
     uploadFile(localFilePath: string, bucketName: string, bucketFilePath: string): Promise<void>;
     setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
     getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;
     copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     movePath(fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string): Promise<void>;
-    combine(bucketName: string, filePaths: string[], toPath: string, toBucket?: string): Promise<void>;
+    combine(bucketName: string, prefix: string, toPath: string, toBucket?: string): Promise<void>;
+    combineFiles(bucketName: string, filePaths: string[], toPath: string, toBucket?: string): Promise<void>;
     getSignedUrl(bucketName: string, filePath: string, expires: LocalTimeInput): Promise<string>;
 }

package/dist/inMemoryCommonStorage.js

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryCommonStorage = void 0;
-const node_stream_1 = require("node:stream");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 class InMemoryCommonStorage {
@@ -17,7 +16,7 @@ class InMemoryCommonStorage {
         return Object.keys(this.data);
     }
     getBucketNamesStream() {
-        return node_stream_1.Readable.from(Object.keys(this.data));
+        return (0, nodejs_lib_1.readableFrom)(Object.keys(this.data));
     }
     async fileExists(bucketName, filePath) {
         return !!this.data[bucketName]?.[filePath];
@@ -39,6 +38,11 @@ class InMemoryCommonStorage {
             }
         });
     }
+    async deleteFiles(bucketName, filePaths) {
+        if (!this.data[bucketName])
+            return;
+        filePaths.forEach(filePath => delete this.data[bucketName][filePath]);
+    }
     async getFileNames(bucketName, opt = {}) {
         const { prefix = '', fullPaths = true } = opt;
         return Object.keys(this.data[bucketName] || {})
@@ -47,14 +51,14 @@ class InMemoryCommonStorage {
     }
     getFileNamesStream(bucketName, opt = {}) {
         const { prefix = '', fullPaths = true } = opt;
-        return node_stream_1.Readable.from(Object.keys(this.data[bucketName] || {})
+        return (0, nodejs_lib_1.readableFrom)(Object.keys(this.data[bucketName] || {})
             .filter(filePath => filePath.startsWith(prefix))
             .slice(0, opt.limit)
             .map(n => (fullPaths ? n : (0, js_lib_1._substringAfterLast)(n, '/'))));
     }
     getFilesStream(bucketName, opt = {}) {
         const { prefix = '', fullPaths = true } = opt;
-        return node_stream_1.Readable.from(Object.entries(this.data[bucketName] || {})
+        return (0, nodejs_lib_1.readableFrom)((0, js_lib_1._stringMapEntries)(this.data[bucketName] || {})
             .map(([filePath, content]) => ({
             filePath,
             content,
@@ -64,7 +68,7 @@ class InMemoryCommonStorage {
             .map(f => (fullPaths ? f : { ...f, filePath: (0, js_lib_1._substringAfterLast)(f.filePath, '/') })));
     }
     getFileReadStream(bucketName, filePath) {
-        return node_stream_1.Readable.from(this.data[bucketName][filePath]);
+        return (0, nodejs_lib_1.readableFrom)(this.data[bucketName][filePath]);
     }
     getFileWriteStream(_bucketName, _filePath) {
         throw new Error('Method not implemented.');
@@ -104,7 +108,11 @@ class InMemoryCommonStorage {
             delete this.data[fromBucket][filePath];
         });
     }
-    async combine(bucketName, filePaths, toPath, toBucket) {
+    async combine(bucketName, prefix, toPath, toBucket) {
+        const filePaths = await this.getFileNames(bucketName, { prefix });
+        await this.combineFiles(bucketName, filePaths, toPath, toBucket);
+    }
+    async combineFiles(bucketName, filePaths, toPath, toBucket) {
         if (!this.data[bucketName])
             return;
         const tob = toBucket || bucketName;

package/package.json

@@ -35,7 +35,7 @@
   "engines": {
     "node": ">=18.12.0"
   },
-  "version": "1.9.2",
+  "version": "1.11.0",
   "description": "CommonStorage implementation based on Google Cloud Storage",
   "author": "Natural Cycles Team",
   "license": "MIT"

package/src/cloudStorage.ts

@@ -1,23 +1,31 @@
-import { Readable, Writable } from 'node:stream'
-import { File, Storage, StorageOptions } from '@google-cloud/storage'
+// eslint-disable-next-line import/no-duplicates
+import type { File, Storage, StorageOptions } from '@google-cloud/storage'
+// eslint-disable-next-line import/no-duplicates
+import type * as StorageLib from '@google-cloud/storage'
 import {
   _assert,
+  _chunk,
+  _since,
   _substringAfterLast,
+  CommonLogger,
   localTime,
   LocalTimeInput,
   pMap,
   SKIP,
 } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '@naturalcycles/nodejs-lib'
-import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
-import { GCPServiceAccount } from './model'
+import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib'
+import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
+import type { GCPServiceAccount } from './model'
 
 export {
   // This is the latest version, to be imported by consumers
-  Storage,
+  type Storage,
   type StorageOptions,
 }
 
+const MAX_RECURSION_DEPTH = 10
+const BATCH_SIZE = 32
+
 /**
  * This object is intentionally made to NOT extend StorageOptions,
  * because StorageOptions is complicated and provides just too many ways
@@ -29,9 +37,14 @@ export {
  */
 export interface CloudStorageCfg {
   /**
-   * It's optional, to allow automatic credentials in AppEngine, or GOOGLE_APPLICATION_CREDENTIALS.
+   * Default is console
+   */
+  logger?: CommonLogger
+
+  /**
+   * Pass true for extra debugging
    */
-  credentials?: GCPServiceAccount
+  debug?: boolean
 }
 
 /**
@@ -40,29 +53,54 @@ export interface CloudStorageCfg {
  * API: https://googleapis.dev/nodejs/storage/latest/index.html
  */
 export class CloudStorage implements CommonStorage {
-  /**
-   * Passing the pre-created Storage allows to instantiate it from both
-   * GCP Storage and FirebaseStorage.
-   */
-  constructor(public storage: Storage) {}
+  private constructor(
+    public storage: Storage,
+    cfg: CloudStorageCfg = {},
+  ) {
+    this.cfg = {
+      logger: console,
+      ...cfg,
+    }
+  }
+
+  cfg: CloudStorageCfg & {
+    logger: CommonLogger
+  }
+
+  static createFromGCPServiceAccount(
+    credentials?: GCPServiceAccount,
+    cfg?: CloudStorageCfg,
+  ): CloudStorage {
+    const storageLib = require('@google-cloud/storage') as typeof StorageLib
 
-  static createFromGCPServiceAccount(cfg: CloudStorageCfg): CloudStorage {
-    const storage = new Storage({
-      credentials: cfg.credentials,
+    const storage = new storageLib.Storage({
+      credentials,
       // Explicitly passing it here to fix this error:
       // Error: Unable to detect a Project Id in the current environment.
       // To learn more about authentication and Google APIs, visit:
       // https://cloud.google.com/docs/authentication/getting-started
       //     at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
-      projectId: cfg.credentials?.project_id,
+      projectId: credentials?.project_id,
     })
 
-    return new CloudStorage(storage)
+    return new CloudStorage(storage, cfg)
   }
 
-  static createFromStorageOptions(storageOptions?: StorageOptions): CloudStorage {
-    const storage = new Storage(storageOptions)
-    return new CloudStorage(storage)
+  static createFromStorageOptions(
+    storageOptions?: StorageOptions,
+    cfg?: CloudStorageCfg,
+  ): CloudStorage {
+    const storageLib = require('@google-cloud/storage') as typeof StorageLib
+    const storage = new storageLib.Storage(storageOptions)
+    return new CloudStorage(storage, cfg)
+  }
+
+  /**
+   * Passing the pre-created Storage allows to instantiate it from both
+   * GCP Storage and FirebaseStorage.
+   */
+  static createFromStorage(storage: Storage, cfg?: CloudStorageCfg): CloudStorage {
+    return new CloudStorage(storage, cfg)
   }
 
   async ping(bucketName?: string): Promise<void> {
@@ -159,7 +197,7 @@ export class CloudStorage implements CommonStorage {
    * Returns a Readable that is NOT object mode,
    * so you can e.g pipe it to fs.createWriteStream()
    */
-  getFileReadStream(bucketName: string, filePath: string): Readable {
+  getFileReadStream(bucketName: string, filePath: string): ReadableBinary {
     return this.storage.bucket(bucketName).file(filePath).createReadStream()
   }
 
@@ -167,7 +205,7 @@ export class CloudStorage implements CommonStorage {
     await this.storage.bucket(bucketName).file(filePath).save(content)
   }
 
-  getFileWriteStream(bucketName: string, filePath: string): Writable {
+  getFileWriteStream(bucketName: string, filePath: string): WritableBinary {
     return this.storage.bucket(bucketName).file(filePath).createWriteStream()
   }
 
@@ -235,21 +273,81 @@ export class CloudStorage implements CommonStorage {
       })
   }
 
-  async combine(
+  async deleteFiles(bucketName: string, filePaths: string[]): Promise<void> {
+    await pMap(filePaths, async filePath => {
+      await this.storage.bucket(bucketName).file(filePath).delete()
+    })
+  }
+
+  async combineFiles(
     bucketName: string,
     filePaths: string[],
     toPath: string,
     toBucket?: string,
+    currentRecursionDepth = 0, // not to be set publicly, only used internally
   ): Promise<void> {
-    // todo: if (filePaths.length > 32) - use recursive algorithm
-    _assert(filePaths.length <= 32, 'combine supports up to 32 input files')
+    _assert(
+      currentRecursionDepth <= MAX_RECURSION_DEPTH,
+      `combineFiles reached max recursion depth of ${MAX_RECURSION_DEPTH}`,
+    )
+    const { logger, debug } = this.cfg
 
-    await this.storage
-      .bucket(bucketName)
-      .combine(filePaths, this.storage.bucket(toBucket || bucketName).file(toPath))
+    if (debug) {
+      logger.log(
+        `[${currentRecursionDepth}] Will compose ${filePaths.length} files, by batches of ${BATCH_SIZE}`,
+      )
+    }
+
+    const intermediateFiles: string[] = []
 
-    // Delete original files
-    await this.deletePaths(bucketName, filePaths)
+    if (filePaths.length <= BATCH_SIZE) {
+      await this.storage
+        .bucket(bucketName)
+        .combine(filePaths, this.storage.bucket(toBucket || bucketName).file(toPath))
+
+      if (debug) {
+        logger.log(`[${currentRecursionDepth}] Composed into ${toPath}!`)
+      }
+
+      await this.deleteFiles(bucketName, filePaths)
+      return
+    }
+
+    const started = Date.now()
+    await pMap(_chunk(filePaths, BATCH_SIZE), async (fileBatch, i) => {
+      if (debug) {
+        logger.log(`[${currentRecursionDepth}] Composing batch ${i + 1}...`)
+      }
+      const intermediateFile = `temp_${currentRecursionDepth}_${i}`
+      await this.storage
+        .bucket(bucketName)
+        .combine(fileBatch, this.storage.bucket(toBucket || bucketName).file(intermediateFile))
+      intermediateFiles.push(intermediateFile)
+      await this.deleteFiles(bucketName, fileBatch)
+    })
+    if (debug) {
+      logger.log(
+        `[${currentRecursionDepth}] Batch composed into ${intermediateFiles.length} files, in ${_since(started)}`,
+      )
+    }
+
+    await this.combineFiles(
+      toBucket || bucketName,
+      intermediateFiles,
+      toPath,
+      toBucket,
+      currentRecursionDepth + 1,
+    )
+  }
+
+  async combine(
+    bucketName: string,
+    prefix: string,
+    toPath: string,
+    toBucket?: string,
+  ): Promise<void> {
+    const filePaths = await this.getFileNames(bucketName, { prefix })
+    await this.combineFiles(bucketName, filePaths, toPath, toBucket)
   }
 
   /**

package/src/commonStorage.ts

@@ -1,6 +1,5 @@
-import { Readable, Writable } from 'node:stream'
-import { LocalTimeInput } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import type { LocalTimeInput } from '@naturalcycles/js-lib'
+import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib'
 
 export interface FileEntry {
   filePath: string
@@ -69,6 +68,11 @@ export interface CommonStorage {
 
   deletePaths: (bucketName: string, prefixes: string[]) => Promise<void>
 
+  /**
+   * Should delete all files by their paths.
+   */
+  deleteFiles: (bucketName: string, filePaths: string[]) => Promise<void>
+
   /**
    * Returns an array of strings which are file paths.
    * Files that are not found by the path are not present in the map.
@@ -86,9 +90,9 @@ export interface CommonStorage {
 
   getFilesStream: (bucketName: string, opt?: CommonStorageGetOptions) => ReadableTyped<FileEntry>
 
-  getFileReadStream: (bucketName: string, filePath: string) => Readable
+  getFileReadStream: (bucketName: string, filePath: string) => ReadableBinary
 
-  getFileWriteStream: (bucketName: string, filePath: string) => Writable
+  getFileWriteStream: (bucketName: string, filePath: string) => WritableBinary
 
   /**
    * Upload local file to the bucket (by streaming it).
@@ -134,13 +138,18 @@ export interface CommonStorage {
    *
    * @experimental
    */
-  combine: (
+  combineFiles: (
     bucketName: string,
     filePaths: string[],
     toPath: string,
     toBucket?: string,
   ) => Promise<void>
 
+  /**
+   * Like `combineFiles`, but for a `prefix`.
+   */
+  combine: (bucketName: string, prefix: string, toPath: string, toBucket?: string) => Promise<void>
+
   /**
    * Acquire a "signed url", which allows bearer to use it to download ('read') the file.
    *

package/src/inMemoryCommonStorage.ts

@@ -1,4 +1,3 @@
-import { Readable, Writable } from 'node:stream'
 import {
   _assert,
   _isTruthy,
@@ -8,7 +7,14 @@ import {
   LocalTimeInput,
   StringMap,
 } from '@naturalcycles/js-lib'
-import { fs2, md5, ReadableTyped } from '@naturalcycles/nodejs-lib'
+import {
+  fs2,
+  md5,
+  ReadableBinary,
+  readableFrom,
+  ReadableTyped,
+  WritableBinary,
+} from '@naturalcycles/nodejs-lib'
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
 
 export class InMemoryCommonStorage implements CommonStorage {
@@ -26,7 +32,7 @@ export class InMemoryCommonStorage implements CommonStorage {
   }
 
   getBucketNamesStream(): ReadableTyped<string> {
-    return Readable.from(Object.keys(this.data))
+    return readableFrom(Object.keys(this.data))
   }
 
   async fileExists(bucketName: string, filePath: string): Promise<boolean> {
@@ -54,6 +60,11 @@ export class InMemoryCommonStorage implements CommonStorage {
     })
   }
 
+  async deleteFiles(bucketName: string, filePaths: string[]): Promise<void> {
+    if (!this.data[bucketName]) return
+    filePaths.forEach(filePath => delete this.data[bucketName]![filePath])
+  }
+
   async getFileNames(bucketName: string, opt: CommonStorageGetOptions = {}): Promise<string[]> {
     const { prefix = '', fullPaths = true } = opt
     return Object.keys(this.data[bucketName] || {})
@@ -64,7 +75,7 @@ export class InMemoryCommonStorage implements CommonStorage {
   getFileNamesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<string> {
     const { prefix = '', fullPaths = true } = opt
 
-    return Readable.from(
+    return readableFrom(
       Object.keys(this.data[bucketName] || {})
         .filter(filePath => filePath.startsWith(prefix))
         .slice(0, opt.limit)
@@ -75,8 +86,8 @@ export class InMemoryCommonStorage implements CommonStorage {
   getFilesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<FileEntry> {
     const { prefix = '', fullPaths = true } = opt
 
-    return Readable.from(
-      Object.entries(this.data[bucketName] || {})
+    return readableFrom(
+      _stringMapEntries(this.data[bucketName] || {})
         .map(([filePath, content]) => ({
           filePath,
           content,
@@ -87,11 +98,11 @@ export class InMemoryCommonStorage implements CommonStorage {
     )
   }
 
-  getFileReadStream(bucketName: string, filePath: string): Readable {
-    return Readable.from(this.data[bucketName]![filePath]!)
+  getFileReadStream(bucketName: string, filePath: string): ReadableBinary {
+    return readableFrom(this.data[bucketName]![filePath]!)
   }
 
-  getFileWriteStream(_bucketName: string, _filePath: string): Writable {
+  getFileWriteStream(_bucketName: string, _filePath: string): WritableBinary {
     throw new Error('Method not implemented.')
   }
 
@@ -156,6 +167,16 @@ export class InMemoryCommonStorage implements CommonStorage {
   }
 
   async combine(
+    bucketName: string,
+    prefix: string,
+    toPath: string,
+    toBucket?: string,
+  ): Promise<void> {
+    const filePaths = await this.getFileNames(bucketName, { prefix })
+    await this.combineFiles(bucketName, filePaths, toPath, toBucket)
+  }
+
+  async combineFiles(
     bucketName: string,
     filePaths: string[],
     toPath: string,