@naturalcycles/cloud-storage-lib 1.2.0 → 1.5.0

package/dist/cloudStorage.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 import { Readable, Writable } from 'stream';
-import { Storage } from '@google-cloud/storage';
+import { Storage, StorageOptions } from '@google-cloud/storage';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
 import { GCPServiceAccount } from './model';
@@ -14,20 +14,26 @@ import { GCPServiceAccount } from './model';
  * (either personal one or non-personal).
  */
 export interface CloudStorageCfg {
-    credentials: GCPServiceAccount;
+    /**
+     * It's optional, to allow automatic credentials in AppEngine, or GOOGLE_APPLICATION_CREDENTIALS.
+     */
+    credentials?: GCPServiceAccount;
 }
 export declare class CloudStorage implements CommonStorage {
-    cfg: CloudStorageCfg;
-    constructor(cfg: CloudStorageCfg);
     storage: Storage;
+    /**
+     * Passing the pre-created Storage allows to instantiate it from both
+     * GCP Storage and FirebaseStorage.
+     */
+    constructor(storage: Storage);
+    static createFromGCPServiceAccount(cfg: CloudStorageCfg): CloudStorage;
+    static createFromStorageOptions(storageOptions?: StorageOptions): CloudStorage;
     ping(bucketName?: string): Promise<void>;
-    getBucketNames(opt?: CommonStorageGetOptions): Promise<string[]>;
-    getBucketNamesStream(): ReadableTyped<string>;
     deletePath(bucketName: string, prefix: string): Promise<void>;
     fileExists(bucketName: string, filePath: string): Promise<boolean>;
-    getFileNames(bucketName: string, prefix: string): Promise<string[]>;
-    getFileNamesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
-    getFilesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
+    getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>;
+    getFileNamesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
+    getFilesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
     getFile(bucketName: string, filePath: string): Promise<Buffer | null>;
     /**
      * Returns a Readable that is NOT object mode,

package/dist/cloudStorage.js

@@ -1,37 +1,36 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CloudStorage = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
 const storage_1 = require("@google-cloud/storage");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 class CloudStorage {
-    constructor(cfg) {
-        this.cfg = cfg;
-        this.storage = new storage_1.Storage({
+    /**
+     * Passing the pre-created Storage allows to instantiate it from both
+     * GCP Storage and FirebaseStorage.
+     */
+    constructor(storage) {
+        this.storage = storage;
+    }
+    static createFromGCPServiceAccount(cfg) {
+        const storage = new storage_1.Storage({
             credentials: cfg.credentials,
             // Explicitly passing it here to fix this error:
             // Error: Unable to detect a Project Id in the current environment.
             // To learn more about authentication and Google APIs, visit:
             // https://cloud.google.com/docs/authentication/getting-started
             //     at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
-            projectId: cfg.credentials.project_id,
+            projectId: cfg.credentials?.project_id,
         });
+        return new CloudStorage(storage);
+    }
+    static createFromStorageOptions(storageOptions) {
+        const storage = new storage_1.Storage(storageOptions);
+        return new CloudStorage(storage);
     }
-    // async createBucket(bucketName: string): Promise<void> {
-    //   const bucket = await this.storage.createBucket(bucketName)
-    //   console.log(bucket) // debugging
-    // }
     async ping(bucketName) {
         await this.storage.bucket(bucketName || 'non-existing-for-sure').exists();
     }
-    async getBucketNames(opt = {}) {
-        const [buckets] = await this.storage.getBuckets({
-            maxResults: opt.limit,
-        });
-        return buckets.map(b => b.name);
-    }
-    getBucketNamesStream() {
-        return this.storage.getBucketsStream().pipe((0, nodejs_lib_1.transformMapSimple)(b => b.name));
-    }
     async deletePath(bucketName, prefix) {
         await this.storage.bucket(bucketName).deleteFiles({
             prefix,
@@ -43,22 +42,28 @@ class CloudStorage {
         const [exists] = await this.storage.bucket(bucketName).file(filePath).exists();
         return exists;
     }
-    async getFileNames(bucketName, prefix) {
+    async getFileNames(bucketName, opt = {}) {
+        const { prefix, fullPaths = true } = opt;
         const [files] = await this.storage.bucket(bucketName).getFiles({
             prefix,
         });
-        return files.map(f => f.name);
+        if (fullPaths) {
+            return files.map(f => f.name);
+        }
+        return files.map(f => (0, js_lib_1._substringAfterLast)(f.name, '/'));
     }
-    getFileNamesStream(bucketName, prefix, opt = {}) {
+    getFileNamesStream(bucketName, opt = {}) {
+        const { prefix, fullPaths = true } = opt;
         return this.storage
             .bucket(bucketName)
             .getFilesStream({
             prefix,
             maxResults: opt.limit,
         })
-            .pipe((0, nodejs_lib_1.transformMapSimple)(f => f.name));
+            .pipe((0, nodejs_lib_1.transformMapSimple)(f => fullPaths ? f.name : (0, js_lib_1._substringAfterLast)(f.name, '/')));
     }
-    getFilesStream(bucketName, prefix, opt = {}) {
+    getFilesStream(bucketName, opt = {}) {
+        const { prefix, fullPaths = true } = opt;
         return this.storage
             .bucket(bucketName)
             .getFilesStream({
@@ -67,7 +72,7 @@ class CloudStorage {
         })
             .pipe((0, nodejs_lib_1.transformMap)(async (f) => {
             const [content] = await f.download();
-            return { filePath: f.name, content };
+            return { filePath: fullPaths ? f.name : (0, js_lib_1._substringAfterLast)(f.name, '/'), content };
         }));
     }
     async getFile(bucketName, filePath) {

package/dist/commonStorage.d.ts

@@ -10,6 +10,11 @@ export interface CommonStorageGetOptions {
      * Will filter resulting files based on `prefix`.
      */
     prefix?: string;
+    /**
+     * Defaults to true.
+     * Set to false to return file names instead of full paths.
+     */
+    fullPaths?: boolean;
     /**
      * Limits the number of results.
      *
@@ -35,11 +40,6 @@ export interface CommonStorage {
      * Pass `bucketName` in case you only have permissions to operate on that bucket.
      */
     ping(bucketName?: string): Promise<void>;
-    /**
-     * Often needs a special permission.
-     */
-    getBucketNames(opt?: CommonStorageGetOptions): Promise<string[]>;
-    getBucketNamesStream(): ReadableTyped<string>;
     /**
      * Creates a new bucket by given name.
      * todo: check what to do if it already exists
@@ -62,9 +62,9 @@ export interface CommonStorage {
      * Important difference between `prefix` and `path` is that `prefix` will
      * return all files from sub-directories too!
      */
-    getFileNames(bucketName: string, prefix: string): Promise<string[]>;
-    getFileNamesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
-    getFilesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
+    getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>;
+    getFileNamesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
+    getFilesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
     getFileReadStream(bucketName: string, filePath: string): Readable;
     getFileWriteStream(bucketName: string, filePath: string): Writable;
     setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;

package/dist/commonStorageBucket.d.ts

@@ -31,6 +31,12 @@ export declare class CommonStorageBucket {
         content: T;
     }[]>;
     saveFile(filePath: string, content: Buffer): Promise<void>;
+    /**
+     * Convenience method that does:
+     * await saveFile
+     * await setFileVisibility
+     */
+    savePublicFile(filePath: string, content: Buffer): Promise<void>;
     saveStringFile(filePath: string, content: string): Promise<void>;
     saveJsonFile(filePath: string, content: any): Promise<void>;
     saveFiles(entries: FileEntry[]): Promise<void>;
@@ -50,13 +56,9 @@ export declare class CommonStorageBucket {
      * Important difference between `prefix` and `path` is that `prefix` will
      * return all files from sub-directories too!
      */
-    getFileNames(prefix: string, opt?: {
-        fullPath?: boolean;
-    }): Promise<string[]>;
-    getFileNamesStream(prefix: string, opt?: CommonStorageGetOptions & {
-        fullPath?: boolean;
-    }): ReadableTyped<string>;
-    getFilesStream(prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
+    getFileNames(opt?: CommonStorageGetOptions): Promise<string[]>;
+    getFileNamesStream(opt?: CommonStorageGetOptions): ReadableTyped<string>;
+    getFilesStream(opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
     getFileReadStream(filePath: string): Readable;
     getFileWriteStream(filePath: string): Writable;
     setFileVisibility(filePath: string, isPublic: boolean): Promise<void>;

package/dist/commonStorageBucket.js

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CommonStorageBucket = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 /**
  * Convenience wrapper around CommonStorage for a given Bucket.
  *
@@ -74,6 +73,15 @@ class CommonStorageBucket {
     async saveFile(filePath, content) {
         await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, content);
     }
+    /**
+     * Convenience method that does:
+     * await saveFile
+     * await setFileVisibility
+     */
+    async savePublicFile(filePath, content) {
+        await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, content);
+        await this.cfg.storage.setFileVisibility(this.cfg.bucketName, filePath, true);
+    }
     async saveStringFile(filePath, content) {
         await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, Buffer.from(content));
     }
@@ -107,27 +115,14 @@ class CommonStorageBucket {
      * Important difference between `prefix` and `path` is that `prefix` will
      * return all files from sub-directories too!
      */
-    async getFileNames(prefix, opt = {}) {
-        const { fullPath = true } = opt;
-        const names = await this.cfg.storage.getFileNames(this.cfg.bucketName, prefix);
-        if (!fullPath && prefix) {
-            const start = `${prefix}/`.length;
-            return names.map(n => n.slice(start));
-        }
-        return names;
-    }
-    getFileNamesStream(prefix, opt = {}) {
-        const { fullPath = true } = opt;
-        if (fullPath || !prefix) {
-            return this.cfg.storage.getFileNamesStream(this.cfg.bucketName, prefix, opt);
-        }
-        const start = `${prefix}/`.length;
-        return this.cfg.storage
-            .getFileNamesStream(this.cfg.bucketName, prefix, opt)
-            .pipe((0, nodejs_lib_1.transformMapSimple)(f => f.slice(start)));
-    }
-    getFilesStream(prefix, opt) {
-        return this.cfg.storage.getFilesStream(this.cfg.bucketName, prefix, opt);
+    async getFileNames(opt) {
+        return await this.cfg.storage.getFileNames(this.cfg.bucketName, opt);
+    }
+    getFileNamesStream(opt) {
+        return this.cfg.storage.getFileNamesStream(this.cfg.bucketName, opt);
+    }
+    getFilesStream(opt) {
+        return this.cfg.storage.getFilesStream(this.cfg.bucketName, opt);
     }
     getFileReadStream(filePath) {
         return this.cfg.storage.getFileReadStream(this.cfg.bucketName, filePath);

package/dist/commonStorageKeyValueDB.js

@@ -62,30 +62,26 @@ class CommonStorageKeyValueDB {
     }
     streamIds(table, limit) {
         const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        const index = prefix.length + 1;
-        return this.cfg.storage
-            .getFileNamesStream(bucketName, prefix, { limit })
-            .pipe((0, nodejs_lib_1.transformMapSimple)(f => f.slice(index)));
+        return this.cfg.storage.getFileNamesStream(bucketName, { prefix, limit, fullPaths: false });
     }
     streamValues(table, limit) {
         const { bucketName, prefix } = this.getBucketAndPrefix(table);
         return this.cfg.storage
-            .getFilesStream(bucketName, prefix, { limit })
+            .getFilesStream(bucketName, { prefix, limit })
             .pipe((0, nodejs_lib_1.transformMapSimple)(f => f.content));
     }
     streamEntries(table, limit) {
         const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        const index = prefix.length + 1;
         return this.cfg.storage
-            .getFilesStream(bucketName, prefix, { limit })
+            .getFilesStream(bucketName, { prefix, limit, fullPaths: false })
             .pipe((0, nodejs_lib_1.transformMapSimple)(({ filePath, content }) => [
-            filePath.slice(index),
+            filePath,
             content,
         ]));
     }
     async count(table) {
         const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        return (await this.cfg.storage.getFileNames(bucketName, prefix)).length;
+        return (await this.cfg.storage.getFileNames(bucketName, { prefix })).length;
     }
 }
 exports.CommonStorageKeyValueDB = CommonStorageKeyValueDB;

package/dist/inMemoryCommonStorage.d.ts

@@ -16,9 +16,9 @@ export declare class InMemoryCommonStorage implements CommonStorage {
     getFile(bucketName: string, filePath: string): Promise<Buffer | null>;
     saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
     deletePath(bucketName: string, prefix: string): Promise<void>;
-    getFileNames(bucketName: string, prefix: string): Promise<string[]>;
-    getFileNamesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
-    getFilesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
+    getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>;
+    getFileNamesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
+    getFilesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
     getFileReadStream(bucketName: string, filePath: string): Readable;
     getFileWriteStream(_bucketName: string, _filePath: string): Writable;
     setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;

package/dist/inMemoryCommonStorage.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryCommonStorage = void 0;
 const stream_1 = require("stream");
+const js_lib_1 = require("@naturalcycles/js-lib");
 class InMemoryCommonStorage {
     constructor() {
         /**
@@ -35,19 +36,29 @@ class InMemoryCommonStorage {
             }
         });
     }
-    async getFileNames(bucketName, prefix) {
-        return Object.keys(this.data[bucketName] || {}).filter(filePath => filePath.startsWith(prefix));
+    async getFileNames(bucketName, opt = {}) {
+        const { prefix = '', fullPaths = true } = opt;
+        return Object.keys(this.data[bucketName] || {})
+            .filter(filePath => filePath.startsWith(prefix))
+            .map(f => (fullPaths ? f : (0, js_lib_1._substringAfterLast)(f, '/')));
     }
-    getFileNamesStream(bucketName, prefix, opt = {}) {
+    getFileNamesStream(bucketName, opt = {}) {
+        const { prefix = '', fullPaths = true } = opt;
         return stream_1.Readable.from(Object.keys(this.data[bucketName] || {})
             .filter(filePath => filePath.startsWith(prefix))
-            .slice(0, opt.limit));
+            .slice(0, opt.limit)
+            .map(n => (fullPaths ? n : (0, js_lib_1._substringAfterLast)(n, '/'))));
     }
-    getFilesStream(bucketName, prefix, opt = {}) {
+    getFilesStream(bucketName, opt = {}) {
+        const { prefix = '', fullPaths = true } = opt;
         return stream_1.Readable.from(Object.entries(this.data[bucketName] || {})
-            .map(([filePath, content]) => ({ filePath, content }))
+            .map(([filePath, content]) => ({
+            filePath,
+            content,
+        }))
             .filter(f => f.filePath.startsWith(prefix))
-            .slice(0, opt.limit));
+            .slice(0, opt.limit)
+            .map(f => (fullPaths ? f : { ...f, filePath: (0, js_lib_1._substringAfterLast)(f.filePath, '/') })));
     }
     getFileReadStream(bucketName, filePath) {
         return stream_1.Readable.from(this.data[bucketName][filePath]);

package/dist/testing/commonStorageTest.js

@@ -33,29 +33,30 @@ function runCommonStorageTest(storage, bucketName) {
     //   await storage.createBucket(bucketName)
     // })
     test('ping', async () => {
-        await storage.ping();
-    });
-    test('listBuckets', async () => {
-        const buckets = await storage.getBucketNames();
-        console.log(buckets);
-    });
-    test('streamBuckets', async () => {
-        const buckets = await (0, nodejs_lib_1.readableToArray)(storage.getBucketNamesStream());
-        console.log(buckets);
+        await storage.ping(bucketName);
     });
+    // test('listBuckets', async () => {
+    //   const buckets = await storage.getBucketNames()
+    //   console.log(buckets)
+    // })
+    //
+    // test('streamBuckets', async () => {
+    //   const buckets = await readableToArray(storage.getBucketNamesStream())
+    //   console.log(buckets)
+    // })
     test('prepare: clear bucket', async () => {
         await (0, js_lib_1.pMap)(TEST_FILES.map(f => f.filePath), async (filePath) => await storage.deletePath(bucketName, filePath));
     });
-    test('listFileNames on root should return empty', async () => {
-        const fileNames = await storage.getFileNames(bucketName, '');
-        expect(fileNames).toEqual([]);
-    });
+    // test('listFileNames on root should return empty', async () => {
+    //   const fileNames = await storage.getFileNames(bucketName)
+    //   expect(fileNames).toEqual([])
+    // })
     test(`listFileNames on ${TEST_FOLDER} should return empty`, async () => {
-        const fileNames = await storage.getFileNames(bucketName, TEST_FOLDER);
+        const fileNames = await storage.getFileNames(bucketName, { prefix: TEST_FOLDER });
         expect(fileNames).toEqual([]);
     });
-    test('streamFileNames on root should return empty', async () => {
-        const fileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName, ''));
+    test(`streamFileNames on ${TEST_FOLDER} should return empty`, async () => {
+        const fileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName, { prefix: TEST_FOLDER }));
         expect(fileNames).toEqual([]);
     });
     test(`exists should return empty array`, async () => {
@@ -68,9 +69,14 @@ function runCommonStorageTest(storage, bucketName) {
         const testFilesMap = Object.fromEntries(TEST_FILES.map(f => [f.filePath, f.content]));
         // It's done in the same test to ensure "strong consistency"
         await (0, js_lib_1.pMap)(TEST_FILES, async (f) => await storage.saveFile(bucketName, f.filePath, f.content));
-        const fileNames = await storage.getFileNames(bucketName, TEST_FOLDER);
+        const fileNamesShort = await storage.getFileNames(bucketName, {
+            prefix: TEST_FOLDER,
+            fullPaths: false,
+        });
+        expect(fileNamesShort.sort()).toEqual(TEST_FILES.map(f => (0, js_lib_1._substringAfterLast)(f.filePath, '/')).sort());
+        const fileNames = await storage.getFileNames(bucketName, { prefix: TEST_FOLDER });
         expect(fileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
-        const streamedFileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName, TEST_FOLDER));
+        const streamedFileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName, { prefix: TEST_FOLDER }));
         expect(streamedFileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
         const filesMap = {};
         await (0, js_lib_1.pMap)(fileNames, async (filePath) => {
@@ -83,7 +89,7 @@ function runCommonStorageTest(storage, bucketName) {
         });
     });
     test('cleanup', async () => {
-        await storage.deletePath(bucketName, '');
+        await storage.deletePath(bucketName, TEST_FOLDER);
     });
     // Cannot update access control for an object when uniform bucket-level access is enabled. Read more at https://cloud.google.com/storage/docs/uniform-bucket-level-access
     /*

package/package.json

@@ -11,7 +11,8 @@
   },
   "devDependencies": {
     "@naturalcycles/dev-lib": "^12.1.3",
-    "@types/node": "^17.0.5",
+    "@types/node": "^17.0.8",
+    "firebase-admin": "^10.0.1",
     "jest": "^27.1.0"
   },
   "files": [
@@ -34,7 +35,7 @@
   "engines": {
     "node": ">=14.16.0"
   },
-  "version": "1.2.0",
+  "version": "1.5.0",
   "description": "",
   "author": "Natural Cycles Team",
   "license": "MIT"

package/src/cloudStorage.ts

@@ -1,6 +1,6 @@
-import * as Buffer from 'buffer'
 import { Readable, Writable } from 'stream'
-import { Bucket, File, Storage } from '@google-cloud/storage'
+import { _substringAfterLast } from '@naturalcycles/js-lib'
+import { File, Storage, StorageOptions } from '@google-cloud/storage'
 import { ReadableTyped, transformMap, transformMapSimple } from '@naturalcycles/nodejs-lib'
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
 import { GCPServiceAccount } from './model'
@@ -15,43 +15,40 @@ import { GCPServiceAccount } from './model'
  * (either personal one or non-personal).
  */
 export interface CloudStorageCfg {
-  credentials: GCPServiceAccount
+  /**
+   * It's optional, to allow automatic credentials in AppEngine, or GOOGLE_APPLICATION_CREDENTIALS.
+   */
+  credentials?: GCPServiceAccount
 }
 
 export class CloudStorage implements CommonStorage {
-  constructor(public cfg: CloudStorageCfg) {
-    this.storage = new Storage({
+  /**
+   * Passing the pre-created Storage allows to instantiate it from both
+   * GCP Storage and FirebaseStorage.
+   */
+  constructor(public storage: Storage) {}
+
+  static createFromGCPServiceAccount(cfg: CloudStorageCfg): CloudStorage {
+    const storage = new Storage({
       credentials: cfg.credentials,
       // Explicitly passing it here to fix this error:
       // Error: Unable to detect a Project Id in the current environment.
       // To learn more about authentication and Google APIs, visit:
       // https://cloud.google.com/docs/authentication/getting-started
       //     at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
-      projectId: cfg.credentials.project_id,
+      projectId: cfg.credentials?.project_id,
     })
-  }
-
-  storage: Storage
-
-  // async createBucket(bucketName: string): Promise<void> {
-  //   const bucket = await this.storage.createBucket(bucketName)
-  //   console.log(bucket) // debugging
-  // }
 
-  async ping(bucketName?: string): Promise<void> {
-    await this.storage.bucket(bucketName || 'non-existing-for-sure').exists()
+    return new CloudStorage(storage)
   }
 
-  async getBucketNames(opt: CommonStorageGetOptions = {}): Promise<string[]> {
-    const [buckets] = await this.storage.getBuckets({
-      maxResults: opt.limit,
-    })
-
-    return buckets.map(b => b.name)
+  static createFromStorageOptions(storageOptions?: StorageOptions): CloudStorage {
+    const storage = new Storage(storageOptions)
+    return new CloudStorage(storage)
   }
 
-  getBucketNamesStream(): ReadableTyped<string> {
-    return this.storage.getBucketsStream().pipe(transformMapSimple<Bucket, string>(b => b.name))
+  async ping(bucketName?: string): Promise<void> {
+    await this.storage.bucket(bucketName || 'non-existing-for-sure').exists()
   }
 
   async deletePath(bucketName: string, prefix: string): Promise<void> {
@@ -67,32 +64,38 @@ export class CloudStorage implements CommonStorage {
     return exists
   }
 
-  async getFileNames(bucketName: string, prefix: string): Promise<string[]> {
+  async getFileNames(bucketName: string, opt: CommonStorageGetOptions = {}): Promise<string[]> {
+    const { prefix, fullPaths = true } = opt
     const [files] = await this.storage.bucket(bucketName).getFiles({
       prefix,
     })
-    return files.map(f => f.name)
+
+    if (fullPaths) {
+      return files.map(f => f.name)
+    }
+
+    return files.map(f => _substringAfterLast(f.name, '/'))
   }
 
-  getFileNamesStream(
-    bucketName: string,
-    prefix: string,
-    opt: CommonStorageGetOptions = {},
-  ): ReadableTyped<string> {
+  getFileNamesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<string> {
+    const { prefix, fullPaths = true } = opt
+
     return this.storage
       .bucket(bucketName)
       .getFilesStream({
         prefix,
         maxResults: opt.limit,
       })
-      .pipe(transformMapSimple<File, string>(f => f.name))
+      .pipe(
+        transformMapSimple<File, string>(f =>
+          fullPaths ? f.name : _substringAfterLast(f.name, '/'),
+        ),
+      )
   }
 
-  getFilesStream(
-    bucketName: string,
-    prefix: string,
-    opt: CommonStorageGetOptions = {},
-  ): ReadableTyped<FileEntry> {
+  getFilesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<FileEntry> {
+    const { prefix, fullPaths = true } = opt
+
     return this.storage
       .bucket(bucketName)
       .getFilesStream({
@@ -102,7 +105,7 @@ export class CloudStorage implements CommonStorage {
       .pipe(
         transformMap<File, FileEntry>(async f => {
           const [content] = await f.download()
-          return { filePath: f.name, content }
+          return { filePath: fullPaths ? f.name : _substringAfterLast(f.name, '/'), content }
         }),
       )
   }

package/src/commonStorage.ts

@@ -14,6 +14,12 @@ export interface CommonStorageGetOptions {
    */
   prefix?: string
 
+  /**
+   * Defaults to true.
+   * Set to false to return file names instead of full paths.
+   */
+  fullPaths?: boolean
+
   /**
    * Limits the number of results.
    *
@@ -41,13 +47,6 @@ export interface CommonStorage {
    */
   ping(bucketName?: string): Promise<void>
 
-  /**
-   * Often needs a special permission.
-   */
-  getBucketNames(opt?: CommonStorageGetOptions): Promise<string[]>
-
-  getBucketNamesStream(): ReadableTyped<string>
-
   /**
    * Creates a new bucket by given name.
    * todo: check what to do if it already exists
@@ -76,19 +75,11 @@ export interface CommonStorage {
    * Important difference between `prefix` and `path` is that `prefix` will
    * return all files from sub-directories too!
    */
-  getFileNames(bucketName: string, prefix: string): Promise<string[]>
-
-  getFileNamesStream(
-    bucketName: string,
-    prefix: string,
-    opt?: CommonStorageGetOptions,
-  ): ReadableTyped<string>
-
-  getFilesStream(
-    bucketName: string,
-    prefix: string,
-    opt?: CommonStorageGetOptions,
-  ): ReadableTyped<FileEntry>
+  getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>
+
+  getFileNamesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<string>
+
+  getFilesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>
 
   getFileReadStream(bucketName: string, filePath: string): Readable
 

package/src/commonStorageBucket.ts

@@ -1,6 +1,6 @@
 import { Readable, Writable } from 'stream'
 import { AppError, pMap } from '@naturalcycles/js-lib'
-import { ReadableTyped, transformMapSimple } from '@naturalcycles/nodejs-lib'
+import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
 
 export interface CommonStorageBucketCfg {
@@ -103,6 +103,16 @@ export class CommonStorageBucket {
     await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, content)
   }
 
+  /**
+   * Convenience method that does:
+   * await saveFile
+   * await setFileVisibility
+   */
+  async savePublicFile(filePath: string, content: Buffer): Promise<void> {
+    await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, content)
+    await this.cfg.storage.setFileVisibility(this.cfg.bucketName, filePath, true)
+  }
+
   async saveStringFile(filePath: string, content: string): Promise<void> {
     await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, Buffer.from(content))
   }
@@ -145,36 +155,16 @@ export class CommonStorageBucket {
    * Important difference between `prefix` and `path` is that `prefix` will
    * return all files from sub-directories too!
    */
-  async getFileNames(prefix: string, opt: { fullPath?: boolean } = {}): Promise<string[]> {
-    const { fullPath = true } = opt
-    const names = await this.cfg.storage.getFileNames(this.cfg.bucketName, prefix)
-
-    if (!fullPath && prefix) {
-      const start = `${prefix}/`.length
-      return names.map(n => n.slice(start))
-    }
-
-    return names
+  async getFileNames(opt?: CommonStorageGetOptions): Promise<string[]> {
+    return await this.cfg.storage.getFileNames(this.cfg.bucketName, opt)
   }
 
-  getFileNamesStream(
-    prefix: string,
-    opt: CommonStorageGetOptions & { fullPath?: boolean } = {},
-  ): ReadableTyped<string> {
-    const { fullPath = true } = opt
-
-    if (fullPath || !prefix) {
-      return this.cfg.storage.getFileNamesStream(this.cfg.bucketName, prefix, opt)
-    }
-
-    const start = `${prefix}/`.length
-    return this.cfg.storage
-      .getFileNamesStream(this.cfg.bucketName, prefix, opt)
-      .pipe(transformMapSimple<string, string>(f => f.slice(start)))
+  getFileNamesStream(opt?: CommonStorageGetOptions): ReadableTyped<string> {
+    return this.cfg.storage.getFileNamesStream(this.cfg.bucketName, opt)
   }
 
-  getFilesStream(prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry> {
-    return this.cfg.storage.getFilesStream(this.cfg.bucketName, prefix, opt)
+  getFilesStream(opt?: CommonStorageGetOptions): ReadableTyped<FileEntry> {
+    return this.cfg.storage.getFilesStream(this.cfg.bucketName, opt)
   }
 
   getFileReadStream(filePath: string): Readable {

package/src/commonStorageKeyValueDB.ts

@@ -1,4 +1,3 @@
-import * as Buffer from 'buffer'
 import { CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib'
 import { pMap, StringMap } from '@naturalcycles/js-lib'
 import { ReadableTyped, transformMapSimple } from '@naturalcycles/nodejs-lib'
@@ -78,30 +77,26 @@ export class CommonStorageKeyValueDB implements CommonKeyValueDB {
 
   streamIds(table: string, limit?: number): ReadableTyped<string> {
     const { bucketName, prefix } = this.getBucketAndPrefix(table)
-    const index = prefix.length + 1
 
-    return this.cfg.storage
-      .getFileNamesStream(bucketName, prefix, { limit })
-      .pipe(transformMapSimple<string, string>(f => f.slice(index)))
+    return this.cfg.storage.getFileNamesStream(bucketName, { prefix, limit, fullPaths: false })
   }
 
   streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
     const { bucketName, prefix } = this.getBucketAndPrefix(table)
 
     return this.cfg.storage
-      .getFilesStream(bucketName, prefix, { limit })
+      .getFilesStream(bucketName, { prefix, limit })
       .pipe(transformMapSimple<FileEntry, Buffer>(f => f.content))
   }
 
   streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
     const { bucketName, prefix } = this.getBucketAndPrefix(table)
-    const index = prefix.length + 1
 
     return this.cfg.storage
-      .getFilesStream(bucketName, prefix, { limit })
+      .getFilesStream(bucketName, { prefix, limit, fullPaths: false })
       .pipe(
         transformMapSimple<FileEntry, KeyValueDBTuple>(({ filePath, content }) => [
-          filePath.slice(index),
+          filePath,
           content,
         ]),
       )
@@ -110,6 +105,6 @@ export class CommonStorageKeyValueDB implements CommonKeyValueDB {
   async count(table: string): Promise<number> {
     const { bucketName, prefix } = this.getBucketAndPrefix(table)
 
-    return (await this.cfg.storage.getFileNames(bucketName, prefix)).length
+    return (await this.cfg.storage.getFileNames(bucketName, { prefix })).length
   }
 }

package/src/inMemoryCommonStorage.ts

@@ -1,5 +1,5 @@
 import { Readable, Writable } from 'stream'
-import { StringMap } from '@naturalcycles/js-lib'
+import { _substringAfterLast, StringMap } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
 
@@ -42,32 +42,36 @@ export class InMemoryCommonStorage implements CommonStorage {
     })
   }
 
-  async getFileNames(bucketName: string, prefix: string): Promise<string[]> {
-    return Object.keys(this.data[bucketName] || {}).filter(filePath => filePath.startsWith(prefix))
+  async getFileNames(bucketName: string, opt: CommonStorageGetOptions = {}): Promise<string[]> {
+    const { prefix = '', fullPaths = true } = opt
+    return Object.keys(this.data[bucketName] || {})
+      .filter(filePath => filePath.startsWith(prefix))
+      .map(f => (fullPaths ? f : _substringAfterLast(f, '/')))
   }
 
-  getFileNamesStream(
-    bucketName: string,
-    prefix: string,
-    opt: CommonStorageGetOptions = {},
-  ): ReadableTyped<string> {
+  getFileNamesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<string> {
+    const { prefix = '', fullPaths = true } = opt
+
     return Readable.from(
       Object.keys(this.data[bucketName] || {})
         .filter(filePath => filePath.startsWith(prefix))
-        .slice(0, opt.limit),
+        .slice(0, opt.limit)
+        .map(n => (fullPaths ? n : _substringAfterLast(n, '/'))),
     )
   }
 
-  getFilesStream(
-    bucketName: string,
-    prefix: string,
-    opt: CommonStorageGetOptions = {},
-  ): ReadableTyped<FileEntry> {
+  getFilesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<FileEntry> {
+    const { prefix = '', fullPaths = true } = opt
+
     return Readable.from(
       Object.entries(this.data[bucketName] || {})
-        .map(([filePath, content]) => ({ filePath, content }))
+        .map(([filePath, content]) => ({
+          filePath,
+          content,
+        }))
         .filter(f => f.filePath.startsWith(prefix))
-        .slice(0, opt.limit),
+        .slice(0, opt.limit)
+        .map(f => (fullPaths ? f : { ...f, filePath: _substringAfterLast(f.filePath, '/') })),
     )
   }
 

package/src/testing/commonStorageTest.ts

@@ -1,4 +1,4 @@
-import { _range, pMap, StringMap } from '@naturalcycles/js-lib'
+import { _range, _substringAfterLast, pMap, StringMap } from '@naturalcycles/js-lib'
 import { readableToArray } from '@naturalcycles/nodejs-lib'
 import { CommonStorage, FileEntry } from '../commonStorage'
 
@@ -38,18 +38,18 @@ export function runCommonStorageTest(storage: CommonStorage, bucketName: string)
   // })
 
   test('ping', async () => {
-    await storage.ping()
+    await storage.ping(bucketName)
   })
 
-  test('listBuckets', async () => {
-    const buckets = await storage.getBucketNames()
-    console.log(buckets)
-  })
-
-  test('streamBuckets', async () => {
-    const buckets = await readableToArray(storage.getBucketNamesStream())
-    console.log(buckets)
-  })
+  // test('listBuckets', async () => {
+  //   const buckets = await storage.getBucketNames()
+  //   console.log(buckets)
+  // })
+  //
+  // test('streamBuckets', async () => {
+  //   const buckets = await readableToArray(storage.getBucketNamesStream())
+  //   console.log(buckets)
+  // })
 
   test('prepare: clear bucket', async () => {
     await pMap(
@@ -58,18 +58,20 @@ export function runCommonStorageTest(storage: CommonStorage, bucketName: string)
     )
   })
 
-  test('listFileNames on root should return empty', async () => {
-    const fileNames = await storage.getFileNames(bucketName, '')
-    expect(fileNames).toEqual([])
-  })
+  // test('listFileNames on root should return empty', async () => {
+  //   const fileNames = await storage.getFileNames(bucketName)
+  //   expect(fileNames).toEqual([])
+  // })
 
   test(`listFileNames on ${TEST_FOLDER} should return empty`, async () => {
-    const fileNames = await storage.getFileNames(bucketName, TEST_FOLDER)
+    const fileNames = await storage.getFileNames(bucketName, { prefix: TEST_FOLDER })
     expect(fileNames).toEqual([])
   })
 
-  test('streamFileNames on root should return empty', async () => {
-    const fileNames = await readableToArray(storage.getFileNamesStream(bucketName, ''))
+  test(`streamFileNames on ${TEST_FOLDER} should return empty`, async () => {
+    const fileNames = await readableToArray(
+      storage.getFileNamesStream(bucketName, { prefix: TEST_FOLDER }),
+    )
     expect(fileNames).toEqual([])
   })
 
@@ -86,11 +88,19 @@ export function runCommonStorageTest(storage: CommonStorage, bucketName: string)
     // It's done in the same test to ensure "strong consistency"
     await pMap(TEST_FILES, async f => await storage.saveFile(bucketName, f.filePath, f.content))
 
-    const fileNames = await storage.getFileNames(bucketName, TEST_FOLDER)
+    const fileNamesShort = await storage.getFileNames(bucketName, {
+      prefix: TEST_FOLDER,
+      fullPaths: false,
+    })
+    expect(fileNamesShort.sort()).toEqual(
+      TEST_FILES.map(f => _substringAfterLast(f.filePath, '/')).sort(),
+    )
+
+    const fileNames = await storage.getFileNames(bucketName, { prefix: TEST_FOLDER })
     expect(fileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort())
 
     const streamedFileNames = await readableToArray(
-      storage.getFileNamesStream(bucketName, TEST_FOLDER),
+      storage.getFileNamesStream(bucketName, { prefix: TEST_FOLDER }),
     )
     expect(streamedFileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort())
 
@@ -109,7 +119,7 @@ export function runCommonStorageTest(storage: CommonStorage, bucketName: string)
   })
 
   test('cleanup', async () => {
-    await storage.deletePath(bucketName, '')
+    await storage.deletePath(bucketName, TEST_FOLDER)
   })
 
   // Cannot update access control for an object when uniform bucket-level access is enabled. Read more at https://cloud.google.com/storage/docs/uniform-bucket-level-access