@naturalcycles/cloud-storage-lib 1.10.0 → 1.12.0

package/dist/cloudStorage.d.ts

@@ -1,12 +1,10 @@
 /// <reference types="node" />
-/// <reference types="node" />
-import { Readable, Writable } from 'node:stream';
-import { Storage, StorageOptions } from '@google-cloud/storage';
+import type { Storage, StorageOptions } from '@google-cloud/storage';
 import { CommonLogger, LocalTimeInput } from '@naturalcycles/js-lib';
-import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
-import { GCPServiceAccount } from './model';
-export { Storage, type StorageOptions, };
+import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
+import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
+import type { GCPServiceAccount } from './model';
+export { type Storage, type StorageOptions, };
 /**
  * This object is intentionally made to NOT extend StorageOptions,
  * because StorageOptions is complicated and provides just too many ways
@@ -56,9 +54,9 @@ export declare class CloudStorage implements CommonStorage {
      * Returns a Readable that is NOT object mode,
      * so you can e.g pipe it to fs.createWriteStream()
      */
-    getFileReadStream(bucketName: string, filePath: string): Readable;
+    getFileReadStream(bucketName: string, filePath: string): ReadableBinary;
     saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
-    getFileWriteStream(bucketName: string, filePath: string): Writable;
+    getFileWriteStream(bucketName: string, filePath: string): WritableBinary;
     uploadFile(localFilePath: string, bucketName: string, bucketFilePath: string): Promise<void>;
     setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
     getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;

package/dist/cloudStorage.js

@@ -1,8 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CloudStorage = exports.Storage = void 0;
-const storage_1 = require("@google-cloud/storage");
-Object.defineProperty(exports, "Storage", { enumerable: true, get: function () { return storage_1.Storage; } });
+exports.CloudStorage = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
 const MAX_RECURSION_DEPTH = 10;
 const BATCH_SIZE = 32;
@@ -20,7 +18,8 @@ class CloudStorage {
         };
     }
     static createFromGCPServiceAccount(credentials, cfg) {
-        const storage = new storage_1.Storage({
+        const storageLib = require('@google-cloud/storage');
+        const storage = new storageLib.Storage({
             credentials,
             // Explicitly passing it here to fix this error:
             // Error: Unable to detect a Project Id in the current environment.
@@ -32,7 +31,8 @@ class CloudStorage {
         return new CloudStorage(storage, cfg);
     }
     static createFromStorageOptions(storageOptions, cfg) {
-        const storage = new storage_1.Storage(storageOptions);
+        const storageLib = require('@google-cloud/storage');
+        const storage = new storageLib.Storage(storageOptions);
         return new CloudStorage(storage, cfg);
     }
     /**
@@ -172,6 +172,12 @@ class CloudStorage {
     async combineFiles(bucketName, filePaths, toPath, toBucket, currentRecursionDepth = 0) {
         (0, js_lib_1._assert)(currentRecursionDepth <= MAX_RECURSION_DEPTH, `combineFiles reached max recursion depth of ${MAX_RECURSION_DEPTH}`);
         const { logger, debug } = this.cfg;
+        if (filePaths.length === 0) {
+            if (debug) {
+                logger.log(`[${currentRecursionDepth}] Nothing to compose, returning early!`);
+            }
+            return;
+        }
         if (debug) {
             logger.log(`[${currentRecursionDepth}] Will compose ${filePaths.length} files, by batches of ${BATCH_SIZE}`);
         }

package/dist/commonStorage.d.ts

@@ -1,8 +1,6 @@
 /// <reference types="node" />
-/// <reference types="node" />
-import { Readable, Writable } from 'node:stream';
-import { LocalTimeInput } from '@naturalcycles/js-lib';
-import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import type { LocalTimeInput } from '@naturalcycles/js-lib';
+import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
 export interface FileEntry {
     filePath: string;
     content: Buffer;
@@ -74,8 +72,8 @@ export interface CommonStorage {
     getFileNames: (bucketName: string, opt?: CommonStorageGetOptions) => Promise<string[]>;
     getFileNamesStream: (bucketName: string, opt?: CommonStorageGetOptions) => ReadableTyped<string>;
     getFilesStream: (bucketName: string, opt?: CommonStorageGetOptions) => ReadableTyped<FileEntry>;
-    getFileReadStream: (bucketName: string, filePath: string) => Readable;
-    getFileWriteStream: (bucketName: string, filePath: string) => Writable;
+    getFileReadStream: (bucketName: string, filePath: string) => ReadableBinary;
+    getFileWriteStream: (bucketName: string, filePath: string) => WritableBinary;
     /**
      * Upload local file to the bucket (by streaming it).
      */

package/dist/inMemoryCommonStorage.d.ts

@@ -1,8 +1,6 @@
 /// <reference types="node" />
-/// <reference types="node" />
-import { Readable, Writable } from 'node:stream';
 import { LocalTimeInput, StringMap } from '@naturalcycles/js-lib';
-import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
 export declare class InMemoryCommonStorage implements CommonStorage {
     /**
@@ -22,8 +20,8 @@ export declare class InMemoryCommonStorage implements CommonStorage {
     getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>;
     getFileNamesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
     getFilesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
-    getFileReadStream(bucketName: string, filePath: string): Readable;
-    getFileWriteStream(_bucketName: string, _filePath: string): Writable;
+    getFileReadStream(bucketName: string, filePath: string): ReadableBinary;
+    getFileWriteStream(_bucketName: string, _filePath: string): WritableBinary;
     uploadFile(localFilePath: string, bucketName: string, bucketFilePath: string): Promise<void>;
     setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
     getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;

package/dist/inMemoryCommonStorage.js

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryCommonStorage = void 0;
-const node_stream_1 = require("node:stream");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 class InMemoryCommonStorage {
@@ -17,7 +16,7 @@ class InMemoryCommonStorage {
         return Object.keys(this.data);
     }
     getBucketNamesStream() {
-        return node_stream_1.Readable.from(Object.keys(this.data));
+        return (0, nodejs_lib_1.readableFrom)(Object.keys(this.data));
     }
     async fileExists(bucketName, filePath) {
         return !!this.data[bucketName]?.[filePath];
@@ -52,14 +51,14 @@ class InMemoryCommonStorage {
     }
     getFileNamesStream(bucketName, opt = {}) {
         const { prefix = '', fullPaths = true } = opt;
-        return node_stream_1.Readable.from(Object.keys(this.data[bucketName] || {})
+        return (0, nodejs_lib_1.readableFrom)(Object.keys(this.data[bucketName] || {})
             .filter(filePath => filePath.startsWith(prefix))
             .slice(0, opt.limit)
             .map(n => (fullPaths ? n : (0, js_lib_1._substringAfterLast)(n, '/'))));
     }
     getFilesStream(bucketName, opt = {}) {
         const { prefix = '', fullPaths = true } = opt;
-        return node_stream_1.Readable.from(Object.entries(this.data[bucketName] || {})
+        return (0, nodejs_lib_1.readableFrom)((0, js_lib_1._stringMapEntries)(this.data[bucketName] || {})
             .map(([filePath, content]) => ({
             filePath,
             content,
@@ -69,7 +68,7 @@ class InMemoryCommonStorage {
             .map(f => (fullPaths ? f : { ...f, filePath: (0, js_lib_1._substringAfterLast)(f.filePath, '/') })));
     }
     getFileReadStream(bucketName, filePath) {
-        return node_stream_1.Readable.from(this.data[bucketName][filePath]);
+        return (0, nodejs_lib_1.readableFrom)(this.data[bucketName][filePath]);
     }
     getFileWriteStream(_bucketName, _filePath) {
         throw new Error('Method not implemented.');

package/package.json

@@ -35,7 +35,7 @@
   "engines": {
     "node": ">=18.12.0"
   },
-  "version": "1.10.0",
+  "version": "1.12.0",
   "description": "CommonStorage implementation based on Google Cloud Storage",
   "author": "Natural Cycles Team",
   "license": "MIT"

package/src/cloudStorage.ts

@@ -1,5 +1,7 @@
-import { Readable, Writable } from 'node:stream'
-import { File, Storage, StorageOptions } from '@google-cloud/storage'
+// eslint-disable-next-line import/no-duplicates
+import type { File, Storage, StorageOptions } from '@google-cloud/storage'
+// eslint-disable-next-line import/no-duplicates
+import type * as StorageLib from '@google-cloud/storage'
 import {
   _assert,
   _chunk,
@@ -11,13 +13,13 @@ import {
   pMap,
   SKIP,
 } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '@naturalcycles/nodejs-lib'
-import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
-import { GCPServiceAccount } from './model'
+import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib'
+import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
+import type { GCPServiceAccount } from './model'
 
 export {
   // This is the latest version, to be imported by consumers
-  Storage,
+  type Storage,
   type StorageOptions,
 }
 
@@ -69,7 +71,9 @@ export class CloudStorage implements CommonStorage {
     credentials?: GCPServiceAccount,
     cfg?: CloudStorageCfg,
   ): CloudStorage {
-    const storage = new Storage({
+    const storageLib = require('@google-cloud/storage') as typeof StorageLib
+
+    const storage = new storageLib.Storage({
       credentials,
       // Explicitly passing it here to fix this error:
       // Error: Unable to detect a Project Id in the current environment.
@@ -86,7 +90,8 @@ export class CloudStorage implements CommonStorage {
     storageOptions?: StorageOptions,
     cfg?: CloudStorageCfg,
   ): CloudStorage {
-    const storage = new Storage(storageOptions)
+    const storageLib = require('@google-cloud/storage') as typeof StorageLib
+    const storage = new storageLib.Storage(storageOptions)
     return new CloudStorage(storage, cfg)
   }
 
@@ -192,7 +197,7 @@ export class CloudStorage implements CommonStorage {
    * Returns a Readable that is NOT object mode,
    * so you can e.g pipe it to fs.createWriteStream()
    */
-  getFileReadStream(bucketName: string, filePath: string): Readable {
+  getFileReadStream(bucketName: string, filePath: string): ReadableBinary {
     return this.storage.bucket(bucketName).file(filePath).createReadStream()
   }
 
@@ -200,7 +205,7 @@ export class CloudStorage implements CommonStorage {
     await this.storage.bucket(bucketName).file(filePath).save(content)
   }
 
-  getFileWriteStream(bucketName: string, filePath: string): Writable {
+  getFileWriteStream(bucketName: string, filePath: string): WritableBinary {
     return this.storage.bucket(bucketName).file(filePath).createWriteStream()
   }
 
@@ -286,6 +291,12 @@ export class CloudStorage implements CommonStorage {
       `combineFiles reached max recursion depth of ${MAX_RECURSION_DEPTH}`,
     )
     const { logger, debug } = this.cfg
+    if (filePaths.length === 0) {
+      if (debug) {
+        logger.log(`[${currentRecursionDepth}] Nothing to compose, returning early!`)
+      }
+      return
+    }
 
     if (debug) {
       logger.log(

package/src/commonStorage.ts

@@ -1,6 +1,5 @@
-import { Readable, Writable } from 'node:stream'
-import { LocalTimeInput } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import type { LocalTimeInput } from '@naturalcycles/js-lib'
+import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib'
 
 export interface FileEntry {
   filePath: string
@@ -91,9 +90,9 @@ export interface CommonStorage {
 
   getFilesStream: (bucketName: string, opt?: CommonStorageGetOptions) => ReadableTyped<FileEntry>
 
-  getFileReadStream: (bucketName: string, filePath: string) => Readable
+  getFileReadStream: (bucketName: string, filePath: string) => ReadableBinary
 
-  getFileWriteStream: (bucketName: string, filePath: string) => Writable
+  getFileWriteStream: (bucketName: string, filePath: string) => WritableBinary
 
   /**
    * Upload local file to the bucket (by streaming it).

package/src/inMemoryCommonStorage.ts

@@ -1,4 +1,3 @@
-import { Readable, Writable } from 'node:stream'
 import {
   _assert,
   _isTruthy,
@@ -8,7 +7,14 @@ import {
   LocalTimeInput,
   StringMap,
 } from '@naturalcycles/js-lib'
-import { fs2, md5, ReadableTyped } from '@naturalcycles/nodejs-lib'
+import {
+  fs2,
+  md5,
+  ReadableBinary,
+  readableFrom,
+  ReadableTyped,
+  WritableBinary,
+} from '@naturalcycles/nodejs-lib'
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
 
 export class InMemoryCommonStorage implements CommonStorage {
@@ -26,7 +32,7 @@ export class InMemoryCommonStorage implements CommonStorage {
   }
 
   getBucketNamesStream(): ReadableTyped<string> {
-    return Readable.from(Object.keys(this.data))
+    return readableFrom(Object.keys(this.data))
   }
 
   async fileExists(bucketName: string, filePath: string): Promise<boolean> {
@@ -69,7 +75,7 @@ export class InMemoryCommonStorage implements CommonStorage {
   getFileNamesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<string> {
     const { prefix = '', fullPaths = true } = opt
 
-    return Readable.from(
+    return readableFrom(
       Object.keys(this.data[bucketName] || {})
         .filter(filePath => filePath.startsWith(prefix))
         .slice(0, opt.limit)
@@ -80,8 +86,8 @@ export class InMemoryCommonStorage implements CommonStorage {
   getFilesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<FileEntry> {
     const { prefix = '', fullPaths = true } = opt
 
-    return Readable.from(
-      Object.entries(this.data[bucketName] || {})
+    return readableFrom(
+      _stringMapEntries(this.data[bucketName] || {})
         .map(([filePath, content]) => ({
           filePath,
           content,
@@ -92,11 +98,11 @@ export class InMemoryCommonStorage implements CommonStorage {
     )
   }
 
-  getFileReadStream(bucketName: string, filePath: string): Readable {
-    return Readable.from(this.data[bucketName]![filePath]!)
+  getFileReadStream(bucketName: string, filePath: string): ReadableBinary {
+    return readableFrom(this.data[bucketName]![filePath]!)
   }
 
-  getFileWriteStream(_bucketName: string, _filePath: string): Writable {
+  getFileWriteStream(_bucketName: string, _filePath: string): WritableBinary {
     throw new Error('Method not implemented.')
   }