@naturalcycles/cloud-storage-lib 1.7.0 → 1.9.0

package/dist/cloudStorage.d.ts

@@ -38,6 +38,7 @@ export declare class CloudStorage implements CommonStorage {
     static createFromStorageOptions(storageOptions?: StorageOptions): CloudStorage;
     ping(bucketName?: string): Promise<void>;
     deletePath(bucketName: string, prefix: string): Promise<void>;
+    deletePaths(bucketName: string, prefixes: string[]): Promise<void>;
     fileExists(bucketName: string, filePath: string): Promise<boolean>;
     getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>;
     getFileNamesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
@@ -50,11 +51,13 @@ export declare class CloudStorage implements CommonStorage {
     getFileReadStream(bucketName: string, filePath: string): Readable;
     saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
     getFileWriteStream(bucketName: string, filePath: string): Writable;
+    uploadFile(localFilePath: string, bucketName: string, bucketFilePath: string): Promise<void>;
     setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
     getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;
     copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     movePath(fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string): Promise<void>;
+    combine(bucketName: string, filePaths: string[], toPath: string, toBucket?: string): Promise<void>;
     /**
      * Acquires a "signed url", which allows bearer to use it to download ('read') the file.
      *

package/dist/cloudStorage.js

@@ -38,10 +38,16 @@ class CloudStorage {
         await this.storage.bucket(bucketName || 'non-existing-for-sure').exists();
     }
     async deletePath(bucketName, prefix) {
-        await this.storage.bucket(bucketName).deleteFiles({
-            prefix,
-            // to keep going in case error occurs, similar to THROW_AGGREGATED
-            force: true,
+        await this.deletePaths(bucketName, [prefix]);
+    }
+    async deletePaths(bucketName, prefixes) {
+        const bucket = this.storage.bucket(bucketName);
+        await (0, js_lib_1.pMap)(prefixes, async (prefix) => {
+            await bucket.deleteFiles({
+                prefix,
+                // to keep going in case error occurs, similar to THROW_AGGREGATED
+                force: true,
+            });
         });
     }
     async fileExists(bucketName, filePath) {
@@ -111,6 +117,11 @@ class CloudStorage {
     getFileWriteStream(bucketName, filePath) {
         return this.storage.bucket(bucketName).file(filePath).createWriteStream();
     }
+    async uploadFile(localFilePath, bucketName, bucketFilePath) {
+        await this.storage.bucket(bucketName).upload(localFilePath, {
+            destination: bucketFilePath,
+        });
+    }
     async setFileVisibility(bucketName, filePath, isPublic) {
         await this.storage.bucket(bucketName).file(filePath)[isPublic ? 'makePublic' : 'makePrivate']();
     }
@@ -144,6 +155,15 @@ class CloudStorage {
             }),
         ]);
     }
+    async combine(bucketName, filePaths, toPath, toBucket) {
+        // todo: if (filePaths.length > 32) - use recursive algorithm
+        (0, js_lib_1._assert)(filePaths.length <= 32, 'combine supports up to 32 input files');
+        await this.storage
+            .bucket(bucketName)
+            .combine(filePaths, this.storage.bucket(toBucket || bucketName).file(toPath));
+        // Delete original files
+        await this.deletePaths(bucketName, filePaths);
+    }
     /**
      * Acquires a "signed url", which allows bearer to use it to download ('read') the file.
      *
@@ -157,6 +177,7 @@ class CloudStorage {
             .file(filePath)
             .getSignedUrl({
             action: 'read',
+            version: 'v4',
             expires: (0, js_lib_1.localTime)(expires).unixMillis(),
         });
         return url;

package/dist/commonStorage.d.ts

@@ -1,6 +1,7 @@
 /// <reference types="node" />
 /// <reference types="node" />
 import { Readable, Writable } from 'node:stream';
+import { LocalTimeInput } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 export interface FileEntry {
     filePath: string;
@@ -54,6 +55,7 @@ export interface CommonStorage {
      * Should recursively delete all files in a folder, if path is a folder.
      */
     deletePath: (bucketName: string, prefix: string) => Promise<void>;
+    deletePaths: (bucketName: string, prefixes: string[]) => Promise<void>;
     /**
      * Returns an array of strings which are file paths.
      * Files that are not found by the path are not present in the map.
@@ -70,6 +72,10 @@ export interface CommonStorage {
     getFilesStream: (bucketName: string, opt?: CommonStorageGetOptions) => ReadableTyped<FileEntry>;
     getFileReadStream: (bucketName: string, filePath: string) => Readable;
     getFileWriteStream: (bucketName: string, filePath: string) => Writable;
+    /**
+     * Upload local file to the bucket (by streaming it).
+     */
+    uploadFile: (localFilePath: string, bucketName: string, bucketFilePath: string) => Promise<void>;
     setFileVisibility: (bucketName: string, filePath: string, isPublic: boolean) => Promise<void>;
     getFileVisibility: (bucketName: string, filePath: string) => Promise<boolean>;
     copyFile: (fromBucket: string, fromPath: string, toPath: string, toBucket?: string) => Promise<void>;
@@ -81,4 +87,19 @@ export interface CommonStorage {
      * otherwise some folder that starts with the same prefix will be included.
      */
     movePath: (fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string) => Promise<void>;
+    /**
+     * Combine (compose) multiple input files into a single output file.
+     * Should support unlimited number of input files, using recursive algorithm if necessary.
+     *
+     * After the output file is created, all input files should be deleted.
+     *
+     * @experimental
+     */
+    combine: (bucketName: string, filePaths: string[], toPath: string, toBucket?: string) => Promise<void>;
+    /**
+     * Acquire a "signed url", which allows bearer to use it to download ('read') the file.
+     *
+     * @experimental
+     */
+    getSignedUrl: (bucketName: string, filePath: string, expires: LocalTimeInput) => Promise<string>;
 }

package/dist/inMemoryCommonStorage.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="node" />
 /// <reference types="node" />
 import { Readable, Writable } from 'node:stream';
-import { StringMap } from '@naturalcycles/js-lib';
+import { LocalTimeInput, StringMap } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
 export declare class InMemoryCommonStorage implements CommonStorage {
@@ -17,14 +17,18 @@ export declare class InMemoryCommonStorage implements CommonStorage {
     getFile(bucketName: string, filePath: string): Promise<Buffer | null>;
     saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
     deletePath(bucketName: string, prefix: string): Promise<void>;
+    deletePaths(bucketName: string, prefixes: string[]): Promise<void>;
     getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>;
     getFileNamesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
     getFilesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
     getFileReadStream(bucketName: string, filePath: string): Readable;
     getFileWriteStream(_bucketName: string, _filePath: string): Writable;
+    uploadFile(localFilePath: string, bucketName: string, bucketFilePath: string): Promise<void>;
     setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
     getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;
     copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     movePath(fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string): Promise<void>;
+    combine(bucketName: string, filePaths: string[], toPath: string, toBucket?: string): Promise<void>;
+    getSignedUrl(bucketName: string, filePath: string, expires: LocalTimeInput): Promise<string>;
 }

package/dist/inMemoryCommonStorage.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryCommonStorage = void 0;
 const node_stream_1 = require("node:stream");
 const js_lib_1 = require("@naturalcycles/js-lib");
+const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 class InMemoryCommonStorage {
     constructor() {
         /**
@@ -29,8 +30,11 @@ class InMemoryCommonStorage {
         this.data[bucketName][filePath] = content;
     }
     async deletePath(bucketName, prefix) {
+        await this.deletePaths(bucketName, [prefix]);
+    }
+    async deletePaths(bucketName, prefixes) {
         Object.keys(this.data[bucketName] || {}).forEach(filePath => {
-            if (filePath.startsWith(prefix)) {
+            if (prefixes.some(prefix => filePath.startsWith(prefix))) {
                 delete this.data[bucketName][filePath];
             }
         });
@@ -65,6 +69,10 @@ class InMemoryCommonStorage {
     getFileWriteStream(_bucketName, _filePath) {
         throw new Error('Method not implemented.');
     }
+    async uploadFile(localFilePath, bucketName, bucketFilePath) {
+        this.data[bucketName] ||= {};
+        this.data[bucketName][bucketFilePath] = await nodejs_lib_1.fs2.readBufferAsync(localFilePath);
+    }
     async setFileVisibility(bucketName, filePath, isPublic) {
         this.publicMap[bucketName] ||= {};
         this.publicMap[bucketName][filePath] = isPublic;
@@ -96,5 +104,20 @@ class InMemoryCommonStorage {
             delete this.data[fromBucket][filePath];
         });
     }
+    async combine(bucketName, filePaths, toPath, toBucket) {
+        if (!this.data[bucketName])
+            return;
+        const tob = toBucket || bucketName;
+        this.data[tob] ||= {};
+        this.data[tob][toPath] = Buffer.concat(filePaths.map(p => this.data[bucketName][p]).filter(js_lib_1._isTruthy));
+        // delete source files
+        filePaths.forEach(p => delete this.data[bucketName][p]);
+    }
+    async getSignedUrl(bucketName, filePath, expires) {
+        const buf = this.data[bucketName]?.[filePath];
+        (0, js_lib_1._assert)(buf, `getSignedUrl file not found: ${bucketName}/${filePath}`);
+        const signature = (0, nodejs_lib_1.md5)(buf);
+        return `https://testurl.com/${bucketName}/${filePath}?expires=${(0, js_lib_1.localTime)(expires).unix()}&signature=${signature}`;
+    }
 }
 exports.InMemoryCommonStorage = InMemoryCommonStorage;

package/package.json

@@ -35,7 +35,7 @@
   "engines": {
     "node": ">=18.12.0"
   },
-  "version": "1.7.0",
+  "version": "1.9.0",
   "description": "CommonStorage implementation based on Google Cloud Storage",
   "author": "Natural Cycles Team",
   "license": "MIT"

package/src/cloudStorage.ts

@@ -5,6 +5,7 @@ import {
   _substringAfterLast,
   localTime,
   LocalTimeInput,
+  pMap,
   SKIP,
 } from '@naturalcycles/js-lib'
 import {
@@ -75,10 +76,18 @@ export class CloudStorage implements CommonStorage {
   }
 
   async deletePath(bucketName: string, prefix: string): Promise<void> {
-    await this.storage.bucket(bucketName).deleteFiles({
-      prefix,
-      // to keep going in case error occurs, similar to THROW_AGGREGATED
-      force: true,
+    await this.deletePaths(bucketName, [prefix])
+  }
+
+  async deletePaths(bucketName: string, prefixes: string[]): Promise<void> {
+    const bucket = this.storage.bucket(bucketName)
+
+    await pMap(prefixes, async prefix => {
+      await bucket.deleteFiles({
+        prefix,
+        // to keep going in case error occurs, similar to THROW_AGGREGATED
+        force: true,
+      })
     })
   }
 
@@ -163,6 +172,16 @@ export class CloudStorage implements CommonStorage {
     return this.storage.bucket(bucketName).file(filePath).createWriteStream()
   }
 
+  async uploadFile(
+    localFilePath: string,
+    bucketName: string,
+    bucketFilePath: string,
+  ): Promise<void> {
+    await this.storage.bucket(bucketName).upload(localFilePath, {
+      destination: bucketFilePath,
+    })
+  }
+
   async setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void> {
     await this.storage.bucket(bucketName).file(filePath)[isPublic ? 'makePublic' : 'makePrivate']()
   }
@@ -217,6 +236,23 @@ export class CloudStorage implements CommonStorage {
     ])
   }
 
+  async combine(
+    bucketName: string,
+    filePaths: string[],
+    toPath: string,
+    toBucket?: string,
+  ): Promise<void> {
+    // todo: if (filePaths.length > 32) - use recursive algorithm
+    _assert(filePaths.length <= 32, 'combine supports up to 32 input files')
+
+    await this.storage
+      .bucket(bucketName)
+      .combine(filePaths, this.storage.bucket(toBucket || bucketName).file(toPath))
+
+    // Delete original files
+    await this.deletePaths(bucketName, filePaths)
+  }
+
   /**
    * Acquires a "signed url", which allows bearer to use it to download ('read') the file.
    *
@@ -234,6 +270,7 @@ export class CloudStorage implements CommonStorage {
       .file(filePath)
       .getSignedUrl({
         action: 'read',
+        version: 'v4',
         expires: localTime(expires).unixMillis(),
       })
 

package/src/commonStorage.ts

@@ -1,4 +1,5 @@
 import { Readable, Writable } from 'node:stream'
+import { LocalTimeInput } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 
 export interface FileEntry {
@@ -66,6 +67,8 @@ export interface CommonStorage {
    */
   deletePath: (bucketName: string, prefix: string) => Promise<void>
 
+  deletePaths: (bucketName: string, prefixes: string[]) => Promise<void>
+
   /**
    * Returns an array of strings which are file paths.
    * Files that are not found by the path are not present in the map.
@@ -87,6 +90,11 @@ export interface CommonStorage {
 
   getFileWriteStream: (bucketName: string, filePath: string) => Writable
 
+  /**
+   * Upload local file to the bucket (by streaming it).
+   */
+  uploadFile: (localFilePath: string, bucketName: string, bucketFilePath: string) => Promise<void>
+
   setFileVisibility: (bucketName: string, filePath: string, isPublic: boolean) => Promise<void>
 
   getFileVisibility: (bucketName: string, filePath: string) => Promise<boolean>
@@ -117,4 +125,26 @@ export interface CommonStorage {
     toPrefix: string,
     toBucket?: string,
   ) => Promise<void>
+
+  /**
+   * Combine (compose) multiple input files into a single output file.
+   * Should support unlimited number of input files, using recursive algorithm if necessary.
+   *
+   * After the output file is created, all input files should be deleted.
+   *
+   * @experimental
+   */
+  combine: (
+    bucketName: string,
+    filePaths: string[],
+    toPath: string,
+    toBucket?: string,
+  ) => Promise<void>
+
+  /**
+   * Acquire a "signed url", which allows bearer to use it to download ('read') the file.
+   *
+   * @experimental
+   */
+  getSignedUrl: (bucketName: string, filePath: string, expires: LocalTimeInput) => Promise<string>
 }

package/src/inMemoryCommonStorage.ts

@@ -1,6 +1,14 @@
 import { Readable, Writable } from 'node:stream'
-import { _stringMapEntries, _substringAfterLast, StringMap } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import {
+  _assert,
+  _isTruthy,
+  _stringMapEntries,
+  _substringAfterLast,
+  localTime,
+  LocalTimeInput,
+  StringMap,
+} from '@naturalcycles/js-lib'
+import { fs2, md5, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
 
 export class InMemoryCommonStorage implements CommonStorage {
@@ -35,8 +43,12 @@ export class InMemoryCommonStorage implements CommonStorage {
   }
 
   async deletePath(bucketName: string, prefix: string): Promise<void> {
+    await this.deletePaths(bucketName, [prefix])
+  }
+
+  async deletePaths(bucketName: string, prefixes: string[]): Promise<void> {
     Object.keys(this.data[bucketName] || {}).forEach(filePath => {
-      if (filePath.startsWith(prefix)) {
+      if (prefixes.some(prefix => filePath.startsWith(prefix))) {
         delete this.data[bucketName]![filePath]
       }
     })
@@ -83,6 +95,15 @@ export class InMemoryCommonStorage implements CommonStorage {
     throw new Error('Method not implemented.')
   }
 
+  async uploadFile(
+    localFilePath: string,
+    bucketName: string,
+    bucketFilePath: string,
+  ): Promise<void> {
+    this.data[bucketName] ||= {}
+    this.data[bucketName]![bucketFilePath] = await fs2.readBufferAsync(localFilePath)
+  }
+
   async setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void> {
     this.publicMap[bucketName] ||= {}
     this.publicMap[bucketName]![filePath] = isPublic
@@ -133,4 +154,32 @@ export class InMemoryCommonStorage implements CommonStorage {
       delete this.data[fromBucket]![filePath]
     })
   }
+
+  async combine(
+    bucketName: string,
+    filePaths: string[],
+    toPath: string,
+    toBucket?: string,
+  ): Promise<void> {
+    if (!this.data[bucketName]) return
+    const tob = toBucket || bucketName
+    this.data[tob] ||= {}
+    this.data[tob]![toPath] = Buffer.concat(
+      filePaths.map(p => this.data[bucketName]![p]).filter(_isTruthy),
+    )
+
+    // delete source files
+    filePaths.forEach(p => delete this.data[bucketName]![p])
+  }
+
+  async getSignedUrl(
+    bucketName: string,
+    filePath: string,
+    expires: LocalTimeInput,
+  ): Promise<string> {
+    const buf = this.data[bucketName]?.[filePath]
+    _assert(buf, `getSignedUrl file not found: ${bucketName}/${filePath}`)
+    const signature = md5(buf)
+    return `https://testurl.com/${bucketName}/${filePath}?expires=${localTime(expires).unix()}&signature=${signature}`
+  }
 }