@naturalcycles/cloud-storage-lib 1.6.5 → 1.8.0

package/dist/cloudStorage.d.ts

@@ -2,6 +2,7 @@
 /// <reference types="node" />
 import { Readable, Writable } from 'node:stream';
 import { Storage, StorageOptions } from '@google-cloud/storage';
+import { LocalTimeInput } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
 import { GCPServiceAccount } from './model';
@@ -21,6 +22,11 @@ export interface CloudStorageCfg {
      */
     credentials?: GCPServiceAccount;
 }
+/**
+ * CloudStorage implementation of CommonStorage API.
+ *
+ * API: https://googleapis.dev/nodejs/storage/latest/index.html
+ */
 export declare class CloudStorage implements CommonStorage {
     storage: Storage;
     /**
@@ -44,8 +50,23 @@ export declare class CloudStorage implements CommonStorage {
     getFileReadStream(bucketName: string, filePath: string): Readable;
     saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
     getFileWriteStream(bucketName: string, filePath: string): Writable;
+    uploadFile(localFilePath: string, bucketName: string, bucketFilePath: string): Promise<void>;
     setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
     getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;
     copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
+    movePath(fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string): Promise<void>;
+    /**
+     * Acquires a "signed url", which allows bearer to use it to download ('read') the file.
+     *
+     * expires: 'v4' supports maximum duration of 7 days from now.
+     *
+     * @experimental - not tested yet
+     */
+    getSignedUrl(bucketName: string, filePath: string, expires: LocalTimeInput): Promise<string>;
+    /**
+     * Returns SKIP if fileName is a folder.
+     * If !fullPaths - strip away the folder prefix.
+     */
+    private normalizeFilename;
 }

package/dist/cloudStorage.js

@@ -5,6 +5,11 @@ const storage_1 = require("@google-cloud/storage");
 Object.defineProperty(exports, "Storage", { enumerable: true, get: function () { return storage_1.Storage; } });
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
+/**
+ * CloudStorage implementation of CommonStorage API.
+ *
+ * API: https://googleapis.dev/nodejs/storage/latest/index.html
+ */
 class CloudStorage {
     /**
      * Passing the pre-created Storage allows to instantiate it from both
@@ -49,9 +54,11 @@ class CloudStorage {
             prefix,
         });
         if (fullPaths) {
-            return files.map(f => f.name);
+            // Paths that end with `/` are "folders", which are "virtual" in CloudStorage
+            // It doesn't make sense to return or do anything with them
+            return files.map(f => f.name).filter(s => !s.endsWith('/'));
         }
-        return files.map(f => (0, js_lib_1._substringAfterLast)(f.name, '/'));
+        return files.map(f => (0, js_lib_1._substringAfterLast)(f.name, '/')).filter(Boolean);
     }
     getFileNamesStream(bucketName, opt = {}) {
         const { prefix, fullPaths = true } = opt;
@@ -61,7 +68,7 @@ class CloudStorage {
             prefix,
             maxResults: opt.limit || undefined,
         })
-            .pipe((0, nodejs_lib_1.transformMapSimple)(f => fullPaths ? f.name : (0, js_lib_1._substringAfterLast)(f.name, '/')));
+            .pipe((0, nodejs_lib_1.transformMapSync)(f => this.normalizeFilename(f.name, fullPaths)));
     }
     getFilesStream(bucketName, opt = {}) {
         const { prefix, fullPaths = true } = opt;
@@ -72,8 +79,11 @@ class CloudStorage {
             maxResults: opt.limit || undefined,
         })
             .pipe((0, nodejs_lib_1.transformMap)(async (f) => {
+            const filePath = this.normalizeFilename(f.name, fullPaths);
+            if (filePath === js_lib_1.SKIP)
+                return js_lib_1.SKIP;
             const [content] = await f.download();
-            return { filePath: fullPaths ? f.name : (0, js_lib_1._substringAfterLast)(f.name, '/'), content };
+            return { filePath, content };
         }));
     }
     async getFile(bucketName, filePath) {
@@ -101,6 +111,11 @@ class CloudStorage {
     getFileWriteStream(bucketName, filePath) {
         return this.storage.bucket(bucketName).file(filePath).createWriteStream();
     }
+    async uploadFile(localFilePath, bucketName, bucketFilePath) {
+        await this.storage.bucket(bucketName).upload(localFilePath, {
+            destination: bucketFilePath,
+        });
+    }
     async setFileVisibility(bucketName, filePath, isPublic) {
         await this.storage.bucket(bucketName).file(filePath)[isPublic ? 'makePublic' : 'makePrivate']();
     }
@@ -120,5 +135,49 @@ class CloudStorage {
             .file(fromPath)
             .move(this.storage.bucket(toBucket || fromBucket).file(toPath));
     }
+    async movePath(fromBucket, fromPrefix, toPrefix, toBucket) {
+        (0, js_lib_1._assert)(fromPrefix.endsWith('/'), 'fromPrefix should end with `/`');
+        (0, js_lib_1._assert)(toPrefix.endsWith('/'), 'toPrefix should end with `/`');
+        await (0, nodejs_lib_1._pipeline)([
+            this.storage.bucket(fromBucket).getFilesStream({
+                prefix: fromPrefix,
+            }),
+            (0, nodejs_lib_1.writableForEach)(async (file) => {
+                const { name } = file;
+                const newName = toPrefix + name.slice(fromPrefix.length);
+                await file.move(this.storage.bucket(toBucket || fromBucket).file(newName));
+            }),
+        ]);
+    }
+    /**
+     * Acquires a "signed url", which allows bearer to use it to download ('read') the file.
+     *
+     * expires: 'v4' supports maximum duration of 7 days from now.
+     *
+     * @experimental - not tested yet
+     */
+    async getSignedUrl(bucketName, filePath, expires) {
+        const [url] = await this.storage
+            .bucket(bucketName)
+            .file(filePath)
+            .getSignedUrl({
+            action: 'read',
+            expires: (0, js_lib_1.localTime)(expires).unixMillis(),
+        });
+        return url;
+    }
+    /**
+     * Returns SKIP if fileName is a folder.
+     * If !fullPaths - strip away the folder prefix.
+     */
+    normalizeFilename(fileName, fullPaths) {
+        if (fullPaths) {
+            if (fileName.endsWith('/'))
+                return js_lib_1.SKIP; // skip folders
+            return fileName;
+        }
+        fileName = (0, js_lib_1._substringAfterLast)(fileName, '/');
+        return fileName || js_lib_1.SKIP; // skip folders
+    }
 }
 exports.CloudStorage = CloudStorage;

package/dist/commonStorage.d.ts

@@ -70,8 +70,19 @@ export interface CommonStorage {
     getFilesStream: (bucketName: string, opt?: CommonStorageGetOptions) => ReadableTyped<FileEntry>;
     getFileReadStream: (bucketName: string, filePath: string) => Readable;
     getFileWriteStream: (bucketName: string, filePath: string) => Writable;
+    /**
+     * Upload local file to the bucket (by streaming it).
+     */
+    uploadFile: (localFilePath: string, bucketName: string, bucketFilePath: string) => Promise<void>;
     setFileVisibility: (bucketName: string, filePath: string, isPublic: boolean) => Promise<void>;
     getFileVisibility: (bucketName: string, filePath: string) => Promise<boolean>;
     copyFile: (fromBucket: string, fromPath: string, toPath: string, toBucket?: string) => Promise<void>;
     moveFile: (fromBucket: string, fromPath: string, toPath: string, toBucket?: string) => Promise<void>;
+    /**
+     * Allows to move "directory" with all its contents.
+     *
+     * Prefixes should end with `/` to work properly,
+     * otherwise some folder that starts with the same prefix will be included.
+     */
+    movePath: (fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string) => Promise<void>;
 }

package/dist/inMemoryCommonStorage.d.ts

@@ -22,8 +22,10 @@ export declare class InMemoryCommonStorage implements CommonStorage {
     getFilesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
     getFileReadStream(bucketName: string, filePath: string): Readable;
     getFileWriteStream(_bucketName: string, _filePath: string): Writable;
+    uploadFile(localFilePath: string, bucketName: string, bucketFilePath: string): Promise<void>;
     setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
     getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;
     copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
     moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
+    movePath(fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string): Promise<void>;
 }

package/dist/inMemoryCommonStorage.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryCommonStorage = void 0;
 const node_stream_1 = require("node:stream");
 const js_lib_1 = require("@naturalcycles/js-lib");
+const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 class InMemoryCommonStorage {
     constructor() {
         /**
@@ -65,6 +66,10 @@ class InMemoryCommonStorage {
     getFileWriteStream(_bucketName, _filePath) {
         throw new Error('Method not implemented.');
     }
+    async uploadFile(localFilePath, bucketName, bucketFilePath) {
+        this.data[bucketName] ||= {};
+        this.data[bucketName][bucketFilePath] = await nodejs_lib_1.fs2.readBufferAsync(localFilePath);
+    }
     async setFileVisibility(bucketName, filePath, isPublic) {
         this.publicMap[bucketName] ||= {};
         this.publicMap[bucketName][filePath] = isPublic;
@@ -85,5 +90,16 @@ class InMemoryCommonStorage {
         this.data[tob][toPath] = this.data[fromBucket][fromPath];
         delete this.data[fromBucket][fromPath];
     }
+    async movePath(fromBucket, fromPrefix, toPrefix, toBucket) {
+        const tob = toBucket || fromBucket;
+        this.data[fromBucket] ||= {};
+        this.data[tob] ||= {};
+        (0, js_lib_1._stringMapEntries)(this.data[fromBucket]).forEach(([filePath, v]) => {
+            if (!filePath.startsWith(fromPrefix))
+                return;
+            this.data[tob][toPrefix + filePath.slice(fromPrefix.length)] = v;
+            delete this.data[fromBucket][filePath];
+        });
+    }
 }
 exports.InMemoryCommonStorage = InMemoryCommonStorage;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/cloud-storage-lib",
   "scripts": {
-    "prepare": "husky install"
+    "prepare": "husky"
   },
   "dependencies": {
     "@google-cloud/storage": "^7.0.0",
@@ -35,8 +35,8 @@
   "engines": {
     "node": ">=18.12.0"
   },
-  "version": "1.6.5",
-  "description": "",
+  "version": "1.8.0",
+  "description": "CommonStorage implementation based on Google Cloud Storage",
   "author": "Natural Cycles Team",
   "license": "MIT"
 }

package/src/cloudStorage.ts

@@ -1,7 +1,19 @@
 import { Readable, Writable } from 'node:stream'
 import { File, Storage, StorageOptions } from '@google-cloud/storage'
-import { _substringAfterLast } from '@naturalcycles/js-lib'
-import { ReadableTyped, transformMap, transformMapSimple } from '@naturalcycles/nodejs-lib'
+import {
+  _assert,
+  _substringAfterLast,
+  localTime,
+  LocalTimeInput,
+  SKIP,
+} from '@naturalcycles/js-lib'
+import {
+  _pipeline,
+  ReadableTyped,
+  transformMap,
+  transformMapSync,
+  writableForEach,
+} from '@naturalcycles/nodejs-lib'
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
 import { GCPServiceAccount } from './model'
 
@@ -27,6 +39,11 @@ export interface CloudStorageCfg {
   credentials?: GCPServiceAccount
 }
 
+/**
+ * CloudStorage implementation of CommonStorage API.
+ *
+ * API: https://googleapis.dev/nodejs/storage/latest/index.html
+ */
 export class CloudStorage implements CommonStorage {
   /**
    * Passing the pre-created Storage allows to instantiate it from both
@@ -77,10 +94,12 @@ export class CloudStorage implements CommonStorage {
     })
 
     if (fullPaths) {
-      return files.map(f => f.name)
+      // Paths that end with `/` are "folders", which are "virtual" in CloudStorage
+      // It doesn't make sense to return or do anything with them
+      return files.map(f => f.name).filter(s => !s.endsWith('/'))
     }
 
-    return files.map(f => _substringAfterLast(f.name, '/'))
+    return files.map(f => _substringAfterLast(f.name, '/')).filter(Boolean)
   }
 
   getFileNamesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<string> {
@@ -92,11 +111,7 @@ export class CloudStorage implements CommonStorage {
         prefix,
         maxResults: opt.limit || undefined,
       })
-      .pipe(
-        transformMapSimple<File, string>(f =>
-          fullPaths ? f.name : _substringAfterLast(f.name, '/'),
-        ),
-      )
+      .pipe(transformMapSync<File, string>(f => this.normalizeFilename(f.name, fullPaths)))
   }
 
   getFilesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<FileEntry> {
@@ -110,8 +125,11 @@ export class CloudStorage implements CommonStorage {
       })
       .pipe(
         transformMap<File, FileEntry>(async f => {
+          const filePath = this.normalizeFilename(f.name, fullPaths)
+          if (filePath === SKIP) return SKIP
+
           const [content] = await f.download()
-          return { filePath: fullPaths ? f.name : _substringAfterLast(f.name, '/'), content }
+          return { filePath, content }
         }),
       )
   }
@@ -145,6 +163,16 @@ export class CloudStorage implements CommonStorage {
     return this.storage.bucket(bucketName).file(filePath).createWriteStream()
   }
 
+  async uploadFile(
+    localFilePath: string,
+    bucketName: string,
+    bucketFilePath: string,
+  ): Promise<void> {
+    await this.storage.bucket(bucketName).upload(localFilePath, {
+      destination: bucketFilePath,
+    })
+  }
+
   async setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void> {
     await this.storage.bucket(bucketName).file(filePath)[isPublic ? 'makePublic' : 'makePrivate']()
   }
@@ -177,4 +205,62 @@ export class CloudStorage implements CommonStorage {
       .file(fromPath)
       .move(this.storage.bucket(toBucket || fromBucket).file(toPath))
   }
+
+  async movePath(
+    fromBucket: string,
+    fromPrefix: string,
+    toPrefix: string,
+    toBucket?: string,
+  ): Promise<void> {
+    _assert(fromPrefix.endsWith('/'), 'fromPrefix should end with `/`')
+    _assert(toPrefix.endsWith('/'), 'toPrefix should end with `/`')
+
+    await _pipeline([
+      this.storage.bucket(fromBucket).getFilesStream({
+        prefix: fromPrefix,
+      }),
+      writableForEach<File>(async file => {
+        const { name } = file
+        const newName = toPrefix + name.slice(fromPrefix.length)
+        await file.move(this.storage.bucket(toBucket || fromBucket).file(newName))
+      }),
+    ])
+  }
+
+  /**
+   * Acquires a "signed url", which allows bearer to use it to download ('read') the file.
+   *
+   * expires: 'v4' supports maximum duration of 7 days from now.
+   *
+   * @experimental - not tested yet
+   */
+  async getSignedUrl(
+    bucketName: string,
+    filePath: string,
+    expires: LocalTimeInput,
+  ): Promise<string> {
+    const [url] = await this.storage
+      .bucket(bucketName)
+      .file(filePath)
+      .getSignedUrl({
+        action: 'read',
+        expires: localTime(expires).unixMillis(),
+      })
+
+    return url
+  }
+
+  /**
+   * Returns SKIP if fileName is a folder.
+   * If !fullPaths - strip away the folder prefix.
+   */
+  private normalizeFilename(fileName: string, fullPaths: boolean): string | typeof SKIP {
+    if (fullPaths) {
+      if (fileName.endsWith('/')) return SKIP // skip folders
+      return fileName
+    }
+
+    fileName = _substringAfterLast(fileName, '/')
+    return fileName || SKIP // skip folders
+  }
 }

package/src/commonStorage.ts

@@ -87,6 +87,11 @@ export interface CommonStorage {
 
   getFileWriteStream: (bucketName: string, filePath: string) => Writable
 
+  /**
+   * Upload local file to the bucket (by streaming it).
+   */
+  uploadFile: (localFilePath: string, bucketName: string, bucketFilePath: string) => Promise<void>
+
   setFileVisibility: (bucketName: string, filePath: string, isPublic: boolean) => Promise<void>
 
   getFileVisibility: (bucketName: string, filePath: string) => Promise<boolean>
@@ -97,10 +102,24 @@ export interface CommonStorage {
     toPath: string,
     toBucket?: string,
   ) => Promise<void>
+
   moveFile: (
     fromBucket: string,
     fromPath: string,
     toPath: string,
     toBucket?: string,
   ) => Promise<void>
+
+  /**
+   * Allows to move "directory" with all its contents.
+   *
+   * Prefixes should end with `/` to work properly,
+   * otherwise some folder that starts with the same prefix will be included.
+   */
+  movePath: (
+    fromBucket: string,
+    fromPrefix: string,
+    toPrefix: string,
+    toBucket?: string,
+  ) => Promise<void>
 }

package/src/inMemoryCommonStorage.ts

@@ -1,6 +1,6 @@
 import { Readable, Writable } from 'node:stream'
-import { _substringAfterLast, StringMap } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import { _stringMapEntries, _substringAfterLast, StringMap } from '@naturalcycles/js-lib'
+import { fs2, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
 
 export class InMemoryCommonStorage implements CommonStorage {
@@ -83,6 +83,15 @@ export class InMemoryCommonStorage implements CommonStorage {
     throw new Error('Method not implemented.')
   }
 
+  async uploadFile(
+    localFilePath: string,
+    bucketName: string,
+    bucketFilePath: string,
+  ): Promise<void> {
+    this.data[bucketName] ||= {}
+    this.data[bucketName]![bucketFilePath] = await fs2.readBufferAsync(localFilePath)
+  }
+
   async setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void> {
     this.publicMap[bucketName] ||= {}
     this.publicMap[bucketName]![filePath] = isPublic
@@ -116,4 +125,21 @@ export class InMemoryCommonStorage implements CommonStorage {
     this.data[tob]![toPath] = this.data[fromBucket]![fromPath]
     delete this.data[fromBucket]![fromPath]
   }
+
+  async movePath(
+    fromBucket: string,
+    fromPrefix: string,
+    toPrefix: string,
+    toBucket?: string,
+  ): Promise<void> {
+    const tob = toBucket || fromBucket
+    this.data[fromBucket] ||= {}
+    this.data[tob] ||= {}
+
+    _stringMapEntries(this.data[fromBucket]!).forEach(([filePath, v]) => {
+      if (!filePath.startsWith(fromPrefix)) return
+      this.data[tob]![toPrefix + filePath.slice(fromPrefix.length)] = v
+      delete this.data[fromBucket]![filePath]
+    })
+  }
 }