@naturalcycles/cloud-storage-lib 1.3.0 → 1.4.0

package/dist/cloudStorage.d.ts

@@ -17,12 +17,14 @@ export interface CloudStorageCfg {
     credentials: GCPServiceAccount;
 }
 export declare class CloudStorage implements CommonStorage {
-    cfg: CloudStorageCfg;
-    constructor(cfg: CloudStorageCfg);
     storage: Storage;
+    /**
+     * Passing the pre-created Storage allows to instantiate it from both
+     * GCP Storage and FirebaseStorage.
+     */
+    constructor(storage: Storage);
+    static createFromGCPServiceAccount(cfg: CloudStorageCfg): CloudStorage;
     ping(bucketName?: string): Promise<void>;
-    getBucketNames(opt?: CommonStorageGetOptions): Promise<string[]>;
-    getBucketNamesStream(): ReadableTyped<string>;
     deletePath(bucketName: string, prefix: string): Promise<void>;
     fileExists(bucketName: string, filePath: string): Promise<boolean>;
     getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>;

package/dist/cloudStorage.js

@@ -5,9 +5,15 @@ const js_lib_1 = require("@naturalcycles/js-lib");
 const storage_1 = require("@google-cloud/storage");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 class CloudStorage {
-    constructor(cfg) {
-        this.cfg = cfg;
-        this.storage = new storage_1.Storage({
+    /**
+     * Passing the pre-created Storage allows to instantiate it from both
+     * GCP Storage and FirebaseStorage.
+     */
+    constructor(storage) {
+        this.storage = storage;
+    }
+    static createFromGCPServiceAccount(cfg) {
+        const storage = new storage_1.Storage({
             credentials: cfg.credentials,
             // Explicitly passing it here to fix this error:
             // Error: Unable to detect a Project Id in the current environment.
@@ -16,23 +22,11 @@ class CloudStorage {
             //     at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
             projectId: cfg.credentials.project_id,
         });
+        return new CloudStorage(storage);
     }
-    // async createBucket(bucketName: string): Promise<void> {
-    //   const bucket = await this.storage.createBucket(bucketName)
-    //   console.log(bucket) // debugging
-    // }
     async ping(bucketName) {
         await this.storage.bucket(bucketName || 'non-existing-for-sure').exists();
     }
-    async getBucketNames(opt = {}) {
-        const [buckets] = await this.storage.getBuckets({
-            maxResults: opt.limit,
-        });
-        return buckets.map(b => b.name);
-    }
-    getBucketNamesStream() {
-        return this.storage.getBucketsStream().pipe((0, nodejs_lib_1.transformMapSimple)(b => b.name));
-    }
     async deletePath(bucketName, prefix) {
         await this.storage.bucket(bucketName).deleteFiles({
             prefix,

package/dist/commonStorage.d.ts

@@ -40,11 +40,6 @@ export interface CommonStorage {
      * Pass `bucketName` in case you only have permissions to operate on that bucket.
      */
     ping(bucketName?: string): Promise<void>;
-    /**
-     * Often needs a special permission.
-     */
-    getBucketNames(opt?: CommonStorageGetOptions): Promise<string[]>;
-    getBucketNamesStream(): ReadableTyped<string>;
     /**
      * Creates a new bucket by given name.
      * todo: check what to do if it already exists

package/dist/testing/commonStorageTest.js

@@ -33,29 +33,30 @@ function runCommonStorageTest(storage, bucketName) {
     //   await storage.createBucket(bucketName)
     // })
     test('ping', async () => {
-        await storage.ping();
-    });
-    test('listBuckets', async () => {
-        const buckets = await storage.getBucketNames();
-        console.log(buckets);
-    });
-    test('streamBuckets', async () => {
-        const buckets = await (0, nodejs_lib_1.readableToArray)(storage.getBucketNamesStream());
-        console.log(buckets);
+        await storage.ping(bucketName);
     });
+    // test('listBuckets', async () => {
+    //   const buckets = await storage.getBucketNames()
+    //   console.log(buckets)
+    // })
+    //
+    // test('streamBuckets', async () => {
+    //   const buckets = await readableToArray(storage.getBucketNamesStream())
+    //   console.log(buckets)
+    // })
     test('prepare: clear bucket', async () => {
         await (0, js_lib_1.pMap)(TEST_FILES.map(f => f.filePath), async (filePath) => await storage.deletePath(bucketName, filePath));
     });
-    test('listFileNames on root should return empty', async () => {
-        const fileNames = await storage.getFileNames(bucketName);
-        expect(fileNames).toEqual([]);
-    });
+    // test('listFileNames on root should return empty', async () => {
+    //   const fileNames = await storage.getFileNames(bucketName)
+    //   expect(fileNames).toEqual([])
+    // })
     test(`listFileNames on ${TEST_FOLDER} should return empty`, async () => {
         const fileNames = await storage.getFileNames(bucketName, { prefix: TEST_FOLDER });
         expect(fileNames).toEqual([]);
     });
-    test('streamFileNames on root should return empty', async () => {
-        const fileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName));
+    test(`streamFileNames on ${TEST_FOLDER} should return empty`, async () => {
+        const fileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName, { prefix: TEST_FOLDER }));
         expect(fileNames).toEqual([]);
     });
     test(`exists should return empty array`, async () => {
@@ -88,7 +89,7 @@ function runCommonStorageTest(storage, bucketName) {
         });
     });
     test('cleanup', async () => {
-        await storage.deletePath(bucketName, '');
+        await storage.deletePath(bucketName, TEST_FOLDER);
     });
     // Cannot update access control for an object when uniform bucket-level access is enabled. Read more at https://cloud.google.com/storage/docs/uniform-bucket-level-access
     /*

package/package.json

@@ -12,6 +12,7 @@
   "devDependencies": {
     "@naturalcycles/dev-lib": "^12.1.3",
     "@types/node": "^17.0.5",
+    "firebase-admin": "^10.0.1",
     "jest": "^27.1.0"
   },
   "files": [
@@ -34,7 +35,7 @@
   "engines": {
     "node": ">=14.16.0"
   },
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "",
   "author": "Natural Cycles Team",
   "license": "MIT"

package/src/cloudStorage.ts

@@ -1,7 +1,7 @@
 import * as Buffer from 'buffer'
 import { Readable, Writable } from 'stream'
 import { _substringAfterLast } from '@naturalcycles/js-lib'
-import { Bucket, File, Storage } from '@google-cloud/storage'
+import { File, Storage } from '@google-cloud/storage'
 import { ReadableTyped, transformMap, transformMapSimple } from '@naturalcycles/nodejs-lib'
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
 import { GCPServiceAccount } from './model'
@@ -20,8 +20,14 @@ export interface CloudStorageCfg {
 }
 
 export class CloudStorage implements CommonStorage {
-  constructor(public cfg: CloudStorageCfg) {
-    this.storage = new Storage({
+  /**
+   * Passing the pre-created Storage allows to instantiate it from both
+   * GCP Storage and FirebaseStorage.
+   */
+  constructor(public storage: Storage) {}
+
+  static createFromGCPServiceAccount(cfg: CloudStorageCfg): CloudStorage {
+    const storage = new Storage({
       credentials: cfg.credentials,
       // Explicitly passing it here to fix this error:
       // Error: Unable to detect a Project Id in the current environment.
@@ -30,31 +36,14 @@ export class CloudStorage implements CommonStorage {
       //     at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
       projectId: cfg.credentials.project_id,
     })
-  }
-
-  storage: Storage
 
-  // async createBucket(bucketName: string): Promise<void> {
-  //   const bucket = await this.storage.createBucket(bucketName)
-  //   console.log(bucket) // debugging
-  // }
+    return new CloudStorage(storage)
+  }
 
   async ping(bucketName?: string): Promise<void> {
     await this.storage.bucket(bucketName || 'non-existing-for-sure').exists()
   }
 
-  async getBucketNames(opt: CommonStorageGetOptions = {}): Promise<string[]> {
-    const [buckets] = await this.storage.getBuckets({
-      maxResults: opt.limit,
-    })
-
-    return buckets.map(b => b.name)
-  }
-
-  getBucketNamesStream(): ReadableTyped<string> {
-    return this.storage.getBucketsStream().pipe(transformMapSimple<Bucket, string>(b => b.name))
-  }
-
   async deletePath(bucketName: string, prefix: string): Promise<void> {
     await this.storage.bucket(bucketName).deleteFiles({
       prefix,

package/src/commonStorage.ts

@@ -47,13 +47,6 @@ export interface CommonStorage {
    */
   ping(bucketName?: string): Promise<void>
 
-  /**
-   * Often needs a special permission.
-   */
-  getBucketNames(opt?: CommonStorageGetOptions): Promise<string[]>
-
-  getBucketNamesStream(): ReadableTyped<string>
-
   /**
    * Creates a new bucket by given name.
    * todo: check what to do if it already exists

package/src/testing/commonStorageTest.ts

@@ -38,18 +38,18 @@ export function runCommonStorageTest(storage: CommonStorage, bucketName: string)
   // })
 
   test('ping', async () => {
-    await storage.ping()
+    await storage.ping(bucketName)
   })
 
-  test('listBuckets', async () => {
-    const buckets = await storage.getBucketNames()
-    console.log(buckets)
-  })
-
-  test('streamBuckets', async () => {
-    const buckets = await readableToArray(storage.getBucketNamesStream())
-    console.log(buckets)
-  })
+  // test('listBuckets', async () => {
+  //   const buckets = await storage.getBucketNames()
+  //   console.log(buckets)
+  // })
+  //
+  // test('streamBuckets', async () => {
+  //   const buckets = await readableToArray(storage.getBucketNamesStream())
+  //   console.log(buckets)
+  // })
 
   test('prepare: clear bucket', async () => {
     await pMap(
@@ -58,18 +58,20 @@ export function runCommonStorageTest(storage: CommonStorage, bucketName: string)
     )
   })
 
-  test('listFileNames on root should return empty', async () => {
-    const fileNames = await storage.getFileNames(bucketName)
-    expect(fileNames).toEqual([])
-  })
+  // test('listFileNames on root should return empty', async () => {
+  //   const fileNames = await storage.getFileNames(bucketName)
+  //   expect(fileNames).toEqual([])
+  // })
 
   test(`listFileNames on ${TEST_FOLDER} should return empty`, async () => {
     const fileNames = await storage.getFileNames(bucketName, { prefix: TEST_FOLDER })
     expect(fileNames).toEqual([])
   })
 
-  test('streamFileNames on root should return empty', async () => {
-    const fileNames = await readableToArray(storage.getFileNamesStream(bucketName))
+  test(`streamFileNames on ${TEST_FOLDER} should return empty`, async () => {
+    const fileNames = await readableToArray(
+      storage.getFileNamesStream(bucketName, { prefix: TEST_FOLDER }),
+    )
     expect(fileNames).toEqual([])
   })
 
@@ -117,7 +119,7 @@ export function runCommonStorageTest(storage: CommonStorage, bucketName: string)
   })
 
   test('cleanup', async () => {
-    await storage.deletePath(bucketName, '')
+    await storage.deletePath(bucketName, TEST_FOLDER)
   })
 
   // Cannot update access control for an object when uniform bucket-level access is enabled. Read more at https://cloud.google.com/storage/docs/uniform-bucket-level-access