@naturalcycles/cloud-storage-lib 1.0.0

package/dist/cloudStorage.d.ts

@@ -0,0 +1,43 @@
+/// <reference types="node" />
+import { Readable, Writable } from 'stream';
+import { Storage } from '@google-cloud/storage';
+import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
+import { GCPServiceAccount } from './model';
+/**
+ * This object is intentionally made to NOT extend StorageOptions,
+ * because StorageOptions is complicated and provides just too many ways
+ * to configure credentials.
+ *
+ * Here we define the minimum simple set of credentials needed.
+ * All of these properties are available from the "service account" json file
+ * (either personal one or non-personal).
+ */
+export interface CloudStorageCfg {
+    credentials: GCPServiceAccount;
+}
+export declare class CloudStorage implements CommonStorage {
+    cfg: CloudStorageCfg;
+    constructor(cfg: CloudStorageCfg);
+    storage: Storage;
+    ping(bucketName?: string): Promise<void>;
+    getBucketNames(opt?: CommonStorageGetOptions): Promise<string[]>;
+    getBucketNamesStream(): ReadableTyped<string>;
+    deletePath(bucketName: string, prefix: string): Promise<void>;
+    fileExists(bucketName: string, filePath: string): Promise<boolean>;
+    getFileNames(bucketName: string, prefix: string): Promise<string[]>;
+    getFileNamesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
+    getFilesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
+    getFile(bucketName: string, filePath: string): Promise<Buffer | null>;
+    /**
+     * Returns a Readable that is NOT object mode,
+     * so you can e.g pipe it to fs.createWriteStream()
+     */
+    getFileReadStream(bucketName: string, filePath: string): Readable;
+    saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
+    getFileWriteStream(bucketName: string, filePath: string): Writable;
+    setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
+    getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;
+    copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
+    moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
+}

package/dist/cloudStorage.js

@@ -0,0 +1,118 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CloudStorage = void 0;
+const storage_1 = require("@google-cloud/storage");
+const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
+class CloudStorage {
+    constructor(cfg) {
+        this.cfg = cfg;
+        this.storage = new storage_1.Storage({
+            credentials: cfg.credentials,
+            // Explicitly passing it here to fix this error:
+            // Error: Unable to detect a Project Id in the current environment.
+            // To learn more about authentication and Google APIs, visit:
+            // https://cloud.google.com/docs/authentication/getting-started
+            //     at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
+            projectId: cfg.credentials.project_id,
+        });
+    }
+    // async createBucket(bucketName: string): Promise<void> {
+    //   const bucket = await this.storage.createBucket(bucketName)
+    //   console.log(bucket) // debugging
+    // }
+    async ping(bucketName) {
+        await this.storage.bucket(bucketName || 'non-existing-for-sure').exists();
+    }
+    async getBucketNames(opt = {}) {
+        const [buckets] = await this.storage.getBuckets({
+            maxResults: opt.limit,
+        });
+        return buckets.map(b => b.name);
+    }
+    getBucketNamesStream() {
+        return this.storage.getBucketsStream().pipe((0, nodejs_lib_1.transformMapSimple)(b => b.name));
+    }
+    async deletePath(bucketName, prefix) {
+        await this.storage.bucket(bucketName).deleteFiles({
+            prefix,
+            // to keep going in case error occurs, similar to THROW_AGGREGATED
+            force: true,
+        });
+    }
+    async fileExists(bucketName, filePath) {
+        const [exists] = await this.storage.bucket(bucketName).file(filePath).exists();
+        return exists;
+    }
+    async getFileNames(bucketName, prefix) {
+        const [files] = await this.storage.bucket(bucketName).getFiles({
+            prefix,
+        });
+        return files.map(f => f.name);
+    }
+    getFileNamesStream(bucketName, prefix, opt = {}) {
+        return this.storage
+            .bucket(bucketName)
+            .getFilesStream({
+            prefix,
+            maxResults: opt.limit,
+        })
+            .pipe((0, nodejs_lib_1.transformMapSimple)(f => f.name));
+    }
+    getFilesStream(bucketName, prefix, opt = {}) {
+        return this.storage
+            .bucket(bucketName)
+            .getFilesStream({
+            prefix,
+            maxResults: opt.limit,
+        })
+            .pipe((0, nodejs_lib_1.transformMap)(async (f) => {
+            const [content] = await f.download();
+            return { filePath: f.name, content };
+        }));
+    }
+    async getFile(bucketName, filePath) {
+        const [buf] = await this.storage
+            .bucket(bucketName)
+            .file(filePath)
+            .download()
+            .catch(err => {
+            if (err?.code === 404)
+                return [null]; // file not found
+            throw err; // rethrow otherwise
+        });
+        return buf;
+    }
+    /**
+     * Returns a Readable that is NOT object mode,
+     * so you can e.g pipe it to fs.createWriteStream()
+     */
+    getFileReadStream(bucketName, filePath) {
+        return this.storage.bucket(bucketName).file(filePath).createReadStream();
+    }
+    async saveFile(bucketName, filePath, content) {
+        await this.storage.bucket(bucketName).file(filePath).save(content);
+    }
+    getFileWriteStream(bucketName, filePath) {
+        return this.storage.bucket(bucketName).file(filePath).createWriteStream();
+    }
+    async setFileVisibility(bucketName, filePath, isPublic) {
+        await this.storage.bucket(bucketName).file(filePath)[isPublic ? 'makePublic' : 'makePrivate']();
+    }
+    async getFileVisibility(bucketName, filePath) {
+        const [isPublic] = await this.storage.bucket(bucketName).file(filePath).isPublic();
+        return isPublic;
+    }
+    async copyFile(fromBucket, fromPath, toPath, toBucket) {
+        await this.storage
+            .bucket(fromBucket)
+            .file(fromPath)
+            .copy(this.storage.bucket(toBucket || fromBucket).file(toPath));
+    }
+    async moveFile(fromBucket, fromPath, toPath, toBucket) {
+        await this.storage
+            .bucket(fromBucket)
+            .file(fromPath)
+            .move(this.storage.bucket(toBucket || fromBucket).file(toPath));
+    }
+}
+exports.CloudStorage = CloudStorage;

package/dist/commonStorage.d.ts

@@ -0,0 +1,74 @@
+/// <reference types="node" />
+import { Readable, Writable } from 'stream';
+import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+export interface FileEntry {
+    filePath: string;
+    content: Buffer;
+}
+export interface CommonStorageGetOptions {
+    /**
+     * Will filter resulting files based on `prefix`.
+     */
+    prefix?: string;
+    /**
+     * Limits the number of results.
+     *
+     * By default it's unlimited.
+     */
+    limit?: number;
+}
+/**
+ * Common denominator interface for File Storage.
+ * Modelled after GCP Cloud Storage, Firebase Storage.
+ *
+ * Uses the concept of Bucket (identified by string name) and Path within the Bucket.
+ *
+ * Path MUST NOT start with a slash !
+ *
+ * Similarly to CommonDB, Bucket is like a Table, and Path is like an `id`.
+ */
+export interface CommonStorage {
+    /**
+     * Ensure that the credentials are correct and the connection is working.
+     * Idempotent.
+     *
+     * Pass `bucketName` in case you only have permissions to operate on that bucket.
+     */
+    ping(bucketName?: string): Promise<void>;
+    /**
+     * Often needs a special permission.
+     */
+    getBucketNames(opt?: CommonStorageGetOptions): Promise<string[]>;
+    getBucketNamesStream(): ReadableTyped<string>;
+    /**
+     * Creates a new bucket by given name.
+     * todo: check what to do if it already exists
+     */
+    fileExists(bucketName: string, filePath: string): Promise<boolean>;
+    getFile(bucketName: string, filePath: string): Promise<Buffer | null>;
+    saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
+    /**
+     * Should recursively delete all files in a folder, if path is a folder.
+     */
+    deletePath(bucketName: string, prefix: string): Promise<void>;
+    /**
+     * Returns an array of strings which are file paths.
+     * Files that are not found by the path are not present in the map.
+     *
+     * Second argument is called `prefix` (same as `path`) to explain how
+     * listing works (it filters all files by `startsWith`). Also, to match
+     * GCP Cloud Storage API.
+     *
+     * Important difference between `prefix` and `path` is that `prefix` will
+     * return all files from sub-directories too!
+     */
+    getFileNames(bucketName: string, prefix: string): Promise<string[]>;
+    getFileNamesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
+    getFilesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
+    getFileReadStream(bucketName: string, filePath: string): Readable;
+    getFileWriteStream(bucketName: string, filePath: string): Writable;
+    setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
+    getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;
+    copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
+    moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
+}

package/dist/commonStorage.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/commonStorageBucket.d.ts

@@ -0,0 +1,60 @@
+/// <reference types="node" />
+import { Readable, Writable } from 'stream';
+import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
+export interface CommonStorageBucketCfg {
+    storage: CommonStorage;
+    bucketName: string;
+}
+/**
+ * Convenience wrapper around CommonStorage for a given Bucket.
+ *
+ * Similar to what CommonDao is to CommonDB.
+ */
+export declare class CommonStorageBucket {
+    cfg: CommonStorageBucketCfg;
+    constructor(cfg: CommonStorageBucketCfg);
+    ping(bucketName?: string): Promise<void>;
+    fileExists(filePath: string): Promise<boolean>;
+    getFile(filePath: string): Promise<Buffer | null>;
+    getFileAsString(filePath: string): Promise<string | null>;
+    getFileAsJson<T = any>(filePath: string): Promise<T | null>;
+    requireFile(filePath: string): Promise<Buffer>;
+    requireFileAsString(filePath: string): Promise<string>;
+    requireFileAsJson<T = any>(filePath: string): Promise<T>;
+    private throwRequiredError;
+    getFileContents(paths: string[]): Promise<Buffer[]>;
+    getFileContentsAsJson<T = any>(paths: string[]): Promise<T[]>;
+    getFileEntries(paths: string[]): Promise<FileEntry[]>;
+    getFileEntriesAsJson<T = any>(paths: string[]): Promise<{
+        filePath: string;
+        content: T;
+    }[]>;
+    saveFile(filePath: string, content: Buffer): Promise<void>;
+    saveFiles(entries: FileEntry[]): Promise<void>;
+    /**
+     * Should recursively delete all files in a folder, if path is a folder.
+     */
+    deletePath(prefix: string): Promise<void>;
+    deletePaths(prefixes: string[]): Promise<void>;
+    /**
+     * Returns an array of strings which are file paths.
+     * Files that are not found by the path are not present in the map.
+     *
+     * Second argument is called `prefix` (same as `path`) to explain how
+     * listing works (it filters all files by `startsWith`). Also, to match
+     * GCP Cloud Storage API.
+     *
+     * Important difference between `prefix` and `path` is that `prefix` will
+     * return all files from sub-directories too!
+     */
+    getFileNames(prefix: string): Promise<string[]>;
+    getFileNamesStream(prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
+    getFilesStream(prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
+    getFileReadStream(filePath: string): Readable;
+    getFileWriteStream(filePath: string): Writable;
+    setFileVisibility(filePath: string, isPublic: boolean): Promise<void>;
+    getFileVisibility(filePath: string): Promise<boolean>;
+    copyFile(fromPath: string, toPath: string, toBucket?: string): Promise<void>;
+    moveFile(fromPath: string, toPath: string, toBucket?: string): Promise<void>;
+}

package/dist/commonStorageBucket.js

@@ -0,0 +1,131 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CommonStorageBucket = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
+/**
+ * Convenience wrapper around CommonStorage for a given Bucket.
+ *
+ * Similar to what CommonDao is to CommonDB.
+ */
+class CommonStorageBucket {
+    constructor(cfg) {
+        this.cfg = cfg;
+    }
+    async ping(bucketName) {
+        await this.cfg.storage.ping(bucketName);
+    }
+    async fileExists(filePath) {
+        return await this.cfg.storage.fileExists(this.cfg.bucketName, filePath);
+    }
+    async getFile(filePath) {
+        return await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
+    }
+    async getFileAsString(filePath) {
+        const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
+        return buf?.toString() || null;
+    }
+    async getFileAsJson(filePath) {
+        const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
+        if (!buf)
+            return null;
+        return JSON.parse(buf.toString());
+    }
+    async requireFile(filePath) {
+        const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
+        if (!buf)
+            this.throwRequiredError(filePath);
+        return buf;
+    }
+    async requireFileAsString(filePath) {
+        const s = await this.getFileAsString(filePath);
+        return s ?? this.throwRequiredError(filePath);
+    }
+    async requireFileAsJson(filePath) {
+        const v = await this.getFileAsJson(filePath);
+        return v ?? this.throwRequiredError(filePath);
+    }
+    throwRequiredError(filePath) {
+        throw new js_lib_1.AppError(`File required, but not found: ${this.cfg.bucketName}/${filePath}`, {
+            code: 'FILE_REQUIRED',
+        });
+    }
+    async getFileContents(paths) {
+        return (await (0, js_lib_1.pMap)(paths, async (filePath) => (await this.cfg.storage.getFile(this.cfg.bucketName, filePath)))).filter(Boolean);
+    }
+    async getFileContentsAsJson(paths) {
+        return (await (0, js_lib_1.pMap)(paths, async (filePath) => {
+            const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
+            return buf ? JSON.parse(buf.toString()) : null;
+        })).filter(Boolean);
+    }
+    async getFileEntries(paths) {
+        return (await (0, js_lib_1.pMap)(paths, async (filePath) => {
+            const content = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
+            return { filePath, content: content };
+        })).filter(f => f.content);
+    }
+    async getFileEntriesAsJson(paths) {
+        return (await (0, js_lib_1.pMap)(paths, async (filePath) => {
+            const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
+            return buf ? { filePath, content: JSON.parse(buf.toString()) } : null;
+        })).filter(Boolean);
+    }
+    async saveFile(filePath, content) {
+        await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, content);
+    }
+    async saveFiles(entries) {
+        await (0, js_lib_1.pMap)(entries, async (f) => {
+            await this.cfg.storage.saveFile(this.cfg.bucketName, f.filePath, f.content);
+        });
+    }
+    /**
+     * Should recursively delete all files in a folder, if path is a folder.
+     */
+    async deletePath(prefix) {
+        return await this.cfg.storage.deletePath(this.cfg.bucketName, prefix);
+    }
+    async deletePaths(prefixes) {
+        await (0, js_lib_1.pMap)(prefixes, async (prefix) => {
+            return await this.cfg.storage.deletePath(this.cfg.bucketName, prefix);
+        });
+    }
+    /**
+     * Returns an array of strings which are file paths.
+     * Files that are not found by the path are not present in the map.
+     *
+     * Second argument is called `prefix` (same as `path`) to explain how
+     * listing works (it filters all files by `startsWith`). Also, to match
+     * GCP Cloud Storage API.
+     *
+     * Important difference between `prefix` and `path` is that `prefix` will
+     * return all files from sub-directories too!
+     */
+    async getFileNames(prefix) {
+        return await this.cfg.storage.getFileNames(this.cfg.bucketName, prefix);
+    }
+    getFileNamesStream(prefix, opt) {
+        return this.cfg.storage.getFileNamesStream(this.cfg.bucketName, prefix, opt);
+    }
+    getFilesStream(prefix, opt) {
+        return this.cfg.storage.getFilesStream(this.cfg.bucketName, prefix, opt);
+    }
+    getFileReadStream(filePath) {
+        return this.cfg.storage.getFileReadStream(this.cfg.bucketName, filePath);
+    }
+    getFileWriteStream(filePath) {
+        return this.cfg.storage.getFileWriteStream(this.cfg.bucketName, filePath);
+    }
+    async setFileVisibility(filePath, isPublic) {
+        await this.cfg.storage.setFileVisibility(this.cfg.bucketName, filePath, isPublic);
+    }
+    async getFileVisibility(filePath) {
+        return await this.cfg.storage.getFileVisibility(this.cfg.bucketName, filePath);
+    }
+    async copyFile(fromPath, toPath, toBucket) {
+        await this.cfg.storage.copyFile(this.cfg.bucketName, fromPath, toPath, toBucket);
+    }
+    async moveFile(fromPath, toPath, toBucket) {
+        await this.cfg.storage.moveFile(this.cfg.bucketName, fromPath, toPath, toBucket);
+    }
+}
+exports.CommonStorageBucket = CommonStorageBucket;

package/dist/commonStorageKeyValueDB.d.ts

@@ -0,0 +1,32 @@
+/// <reference types="node" />
+import { CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib';
+import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import { CommonStorage } from './commonStorage';
+export interface CommonStorageKeyValueDBCfg {
+    storage: CommonStorage;
+    bucketName: string;
+}
+/**
+ * CommonKeyValueDB, backed up by a CommonStorage implementation.
+ *
+ * Each Table is represented as a Folder.
+ * Each Item is represented as a File:
+ * fileName is ${id} (without extension)
+ * file contents is ${v} (Buffer)
+ */
+export declare class CommonStorageKeyValueDB implements CommonKeyValueDB {
+    cfg: CommonStorageKeyValueDBCfg;
+    constructor(cfg: CommonStorageKeyValueDBCfg);
+    ping(): Promise<void>;
+    createTable(_table: string, _opt?: CommonDBCreateOptions): Promise<void>;
+    /**
+     * Allows to pass `SomeBucket.SomeTable` in `table`, to override a Bucket.
+     */
+    private getBucketAndPrefix;
+    deleteByIds(table: string, ids: string[]): Promise<void>;
+    getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
+    saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void>;
+    streamIds(table: string, limit?: number): ReadableTyped<string>;
+    streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
+    streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+}

package/dist/commonStorageKeyValueDB.js

@@ -0,0 +1,87 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CommonStorageKeyValueDB = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
+const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
+/**
+ * CommonKeyValueDB, backed up by a CommonStorage implementation.
+ *
+ * Each Table is represented as a Folder.
+ * Each Item is represented as a File:
+ * fileName is ${id} (without extension)
+ * file contents is ${v} (Buffer)
+ */
+class CommonStorageKeyValueDB {
+    constructor(cfg) {
+        this.cfg = cfg;
+    }
+    async ping() {
+        await this.cfg.storage.ping(this.cfg.bucketName);
+    }
+    async createTable(_table, _opt) {
+        // no-op
+    }
+    /**
+     * Allows to pass `SomeBucket.SomeTable` in `table`, to override a Bucket.
+     */
+    getBucketAndPrefix(table) {
+        const [part1, part2] = table.split('.');
+        if (part2) {
+            return {
+                bucketName: part1,
+                prefix: part2,
+            };
+        }
+        // As is
+        return {
+            bucketName: this.cfg.bucketName,
+            prefix: table,
+        };
+    }
+    async deleteByIds(table, ids) {
+        const { bucketName, prefix } = this.getBucketAndPrefix(table);
+        await (0, js_lib_1.pMap)(ids, async (id) => {
+            await this.cfg.storage.deletePath(bucketName, [prefix, id].join('/'));
+        });
+    }
+    async getByIds(table, ids) {
+        const { bucketName, prefix } = this.getBucketAndPrefix(table);
+        const map = {};
+        await (0, js_lib_1.pMap)(ids, async (id) => {
+            const buf = await this.cfg.storage.getFile(bucketName, [prefix, id].join('/'));
+            if (buf)
+                map[id] = buf;
+        });
+        return ids.map(id => [id, map[id]]).filter(t => t[1]);
+    }
+    async saveBatch(table, entries) {
+        const { bucketName, prefix } = this.getBucketAndPrefix(table);
+        await (0, js_lib_1.pMap)(entries, async ([id, content]) => {
+            await this.cfg.storage.saveFile(bucketName, [prefix, id].join('/'), content);
+        });
+    }
+    streamIds(table, limit) {
+        const { bucketName, prefix } = this.getBucketAndPrefix(table);
+        const index = prefix.length + 1;
+        return this.cfg.storage
+            .getFileNamesStream(bucketName, prefix, { limit })
+            .pipe((0, nodejs_lib_1.transformMapSimple)(f => f.slice(index)));
+    }
+    streamValues(table, limit) {
+        const { bucketName, prefix } = this.getBucketAndPrefix(table);
+        return this.cfg.storage
+            .getFilesStream(bucketName, prefix, { limit })
+            .pipe((0, nodejs_lib_1.transformMapSimple)(f => f.content));
+    }
+    streamEntries(table, limit) {
+        const { bucketName, prefix } = this.getBucketAndPrefix(table);
+        const index = prefix.length + 1;
+        return this.cfg.storage
+            .getFilesStream(bucketName, prefix, { limit })
+            .pipe((0, nodejs_lib_1.transformMapSimple)(({ filePath, content }) => [
+            filePath.slice(index),
+            content,
+        ]));
+    }
+}
+exports.CommonStorageKeyValueDB = CommonStorageKeyValueDB;

package/dist/inMemoryCommonStorage.d.ts

@@ -0,0 +1,28 @@
+/// <reference types="node" />
+import { Readable, Writable } from 'stream';
+import { StringMap } from '@naturalcycles/js-lib';
+import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
+export declare class InMemoryCommonStorage implements CommonStorage {
+    /**
+     * data[bucketName][filePath] = Buffer
+     */
+    data: StringMap<StringMap<Buffer>>;
+    publicMap: StringMap<StringMap<boolean>>;
+    ping(): Promise<void>;
+    getBucketNames(): Promise<string[]>;
+    getBucketNamesStream(): ReadableTyped<string>;
+    fileExists(bucketName: string, filePath: string): Promise<boolean>;
+    getFile(bucketName: string, filePath: string): Promise<Buffer | null>;
+    saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
+    deletePath(bucketName: string, prefix: string): Promise<void>;
+    getFileNames(bucketName: string, prefix: string): Promise<string[]>;
+    getFileNamesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
+    getFilesStream(bucketName: string, prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
+    getFileReadStream(bucketName: string, filePath: string): Readable;
+    getFileWriteStream(_bucketName: string, _filePath: string): Writable;
+    setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
+    getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;
+    copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
+    moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
+}

package/dist/inMemoryCommonStorage.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InMemoryCommonStorage = void 0;
+const stream_1 = require("stream");
+class InMemoryCommonStorage {
+    constructor() {
+        /**
+         * data[bucketName][filePath] = Buffer
+         */
+        this.data = {};
+        this.publicMap = {};
+    }
+    async ping() { }
+    async getBucketNames() {
+        return Object.keys(this.data);
+    }
+    getBucketNamesStream() {
+        return stream_1.Readable.from(Object.keys(this.data));
+    }
+    async fileExists(bucketName, filePath) {
+        return !!this.data[bucketName]?.[filePath];
+    }
+    async getFile(bucketName, filePath) {
+        return this.data[bucketName]?.[filePath] || null;
+    }
+    async saveFile(bucketName, filePath, content) {
+        var _a;
+        (_a = this.data)[bucketName] || (_a[bucketName] = {});
+        this.data[bucketName][filePath] = content;
+    }
+    async deletePath(bucketName, prefix) {
+        Object.keys(this.data[bucketName] || {}).forEach(filePath => {
+            if (filePath.startsWith(prefix)) {
+                delete this.data[bucketName][filePath];
+            }
+        });
+    }
+    async getFileNames(bucketName, prefix) {
+        return Object.keys(this.data[bucketName] || {}).filter(filePath => filePath.startsWith(prefix));
+    }
+    getFileNamesStream(bucketName, prefix, opt = {}) {
+        return stream_1.Readable.from(Object.keys(this.data[bucketName] || {})
+            .filter(filePath => filePath.startsWith(prefix))
+            .slice(0, opt.limit));
+    }
+    getFilesStream(bucketName, prefix, opt = {}) {
+        return stream_1.Readable.from(Object.entries(this.data[bucketName] || {})
+            .map(([filePath, content]) => ({ filePath, content }))
+            .filter(f => f.filePath.startsWith(prefix))
+            .slice(0, opt.limit));
+    }
+    getFileReadStream(bucketName, filePath) {
+        return stream_1.Readable.from(this.data[bucketName][filePath]);
+    }
+    getFileWriteStream(_bucketName, _filePath) {
+        throw new Error('Method not implemented.');
+    }
+    async setFileVisibility(bucketName, filePath, isPublic) {
+        var _a;
+        (_a = this.publicMap)[bucketName] || (_a[bucketName] = {});
+        this.publicMap[bucketName][filePath] = isPublic;
+    }
+    async getFileVisibility(bucketName, filePath) {
+        return !!this.publicMap[bucketName]?.[filePath];
+    }
+    async copyFile(fromBucket, fromPath, toPath, toBucket) {
+        var _a, _b;
+        const tob = toBucket || fromBucket;
+        (_a = this.data)[fromBucket] || (_a[fromBucket] = {});
+        (_b = this.data)[tob] || (_b[tob] = {});
+        this.data[tob][toPath] = this.data[fromBucket][fromPath];
+    }
+    async moveFile(fromBucket, fromPath, toPath, toBucket) {
+        var _a, _b;
+        const tob = toBucket || fromBucket;
+        (_a = this.data)[fromBucket] || (_a[fromBucket] = {});
+        (_b = this.data)[tob] || (_b[tob] = {});
+        this.data[tob][toPath] = this.data[fromBucket][fromPath];
+        delete this.data[fromBucket][fromPath];
+    }
+}
+exports.InMemoryCommonStorage = InMemoryCommonStorage;

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+import { CloudStorage, CloudStorageCfg } from './cloudStorage';
+import { CommonStorage, CommonStorageGetOptions } from './commonStorage';
+import { CommonStorageBucket, CommonStorageBucketCfg } from './commonStorageBucket';
+import { CommonStorageKeyValueDB, CommonStorageKeyValueDBCfg } from './commonStorageKeyValueDB';
+import { InMemoryCommonStorage } from './inMemoryCommonStorage';
+import { GCPServiceAccount } from './model';
+import { runCommonStorageTest } from './testing/commonStorageTest';
+export type { CommonStorage, CloudStorageCfg, CommonStorageGetOptions, GCPServiceAccount, CommonStorageBucketCfg, CommonStorageKeyValueDBCfg, };
+export { CloudStorage, CommonStorageKeyValueDB, CommonStorageBucket, InMemoryCommonStorage, runCommonStorageTest, };