@naturalcycles/cloud-storage-lib 1.12.0 → 1.13.0

package/package.json

@@ -10,7 +10,7 @@
     "@naturalcycles/nodejs-lib": "^13.1.0"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "^13.0.1",
+    "@naturalcycles/dev-lib": "^15.18.0",
     "@types/node": "^20.4.6",
     "firebase-admin": "^12.0.0",
     "jest": "^29.1.2"
@@ -35,7 +35,7 @@
   "engines": {
     "node": ">=18.12.0"
   },
-  "version": "1.12.0",
+  "version": "1.13.0",
   "description": "CommonStorage implementation based on Google Cloud Storage",
   "author": "Natural Cycles Team",
   "license": "MIT"

package/src/cloudStorage.ts

@@ -374,7 +374,7 @@ export class CloudStorage implements CommonStorage {
       .getSignedUrl({
         action: 'read',
         version: 'v4',
-        expires: localTime(expires).unixMillis(),
+        expires: localTime(expires).unixMillis,
       })
 
     return url

package/src/commonStorageKeyValueDB.ts

@@ -1,5 +1,5 @@
 import { CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib'
-import { pMap, StringMap } from '@naturalcycles/js-lib'
+import { AppError, pMap, StringMap } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { CommonStorage } from './commonStorage'
 
@@ -100,4 +100,8 @@ export class CommonStorageKeyValueDB implements CommonKeyValueDB {
 
     return (await this.cfg.storage.getFileNames(bucketName, { prefix })).length
   }
+
+  async increment(_table: string, _id: string, _by?: number): Promise<number> {
+    throw new AppError('CommonStorageKeyValueDB.increment() is not implemented')
+  }
 }

package/src/inMemoryCommonStorage.ts

@@ -45,7 +45,7 @@ export class InMemoryCommonStorage implements CommonStorage {
 
   async saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void> {
     this.data[bucketName] ||= {}
-    this.data[bucketName]![filePath] = content
+    this.data[bucketName][filePath] = content
   }
 
   async deletePath(bucketName: string, prefix: string): Promise<void> {
@@ -112,12 +112,12 @@ export class InMemoryCommonStorage implements CommonStorage {
     bucketFilePath: string,
   ): Promise<void> {
     this.data[bucketName] ||= {}
-    this.data[bucketName]![bucketFilePath] = await fs2.readBufferAsync(localFilePath)
+    this.data[bucketName][bucketFilePath] = await fs2.readBufferAsync(localFilePath)
   }
 
   async setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void> {
     this.publicMap[bucketName] ||= {}
-    this.publicMap[bucketName]![filePath] = isPublic
+    this.publicMap[bucketName][filePath] = isPublic
   }
 
   async getFileVisibility(bucketName: string, filePath: string): Promise<boolean> {
@@ -133,7 +133,7 @@ export class InMemoryCommonStorage implements CommonStorage {
     const tob = toBucket || fromBucket
     this.data[fromBucket] ||= {}
     this.data[tob] ||= {}
-    this.data[tob]![toPath] = this.data[fromBucket]![fromPath]
+    this.data[tob][toPath] = this.data[fromBucket][fromPath]
   }
 
   async moveFile(
@@ -145,8 +145,8 @@ export class InMemoryCommonStorage implements CommonStorage {
     const tob = toBucket || fromBucket
     this.data[fromBucket] ||= {}
     this.data[tob] ||= {}
-    this.data[tob]![toPath] = this.data[fromBucket]![fromPath]
-    delete this.data[fromBucket]![fromPath]
+    this.data[tob][toPath] = this.data[fromBucket][fromPath]
+    delete this.data[fromBucket][fromPath]
   }
 
   async movePath(
@@ -159,7 +159,7 @@ export class InMemoryCommonStorage implements CommonStorage {
     this.data[fromBucket] ||= {}
     this.data[tob] ||= {}
 
-    _stringMapEntries(this.data[fromBucket]!).forEach(([filePath, v]) => {
+    _stringMapEntries(this.data[fromBucket]).forEach(([filePath, v]) => {
       if (!filePath.startsWith(fromPrefix)) return
       this.data[tob]![toPrefix + filePath.slice(fromPrefix.length)] = v
       delete this.data[fromBucket]![filePath]
@@ -185,7 +185,7 @@ export class InMemoryCommonStorage implements CommonStorage {
     if (!this.data[bucketName]) return
     const tob = toBucket || bucketName
     this.data[tob] ||= {}
-    this.data[tob]![toPath] = Buffer.concat(
+    this.data[tob][toPath] = Buffer.concat(
       filePaths.map(p => this.data[bucketName]![p]).filter(_isTruthy),
     )
 
@@ -201,6 +201,6 @@ export class InMemoryCommonStorage implements CommonStorage {
     const buf = this.data[bucketName]?.[filePath]
     _assert(buf, `getSignedUrl file not found: ${bucketName}/${filePath}`)
     const signature = md5(buf)
-    return `https://testurl.com/${bucketName}/${filePath}?expires=${localTime(expires).unix()}&signature=${signature}`
+    return `https://testurl.com/${bucketName}/${filePath}?expires=${localTime(expires).unix}&signature=${signature}`
   }
 }

package/dist/cloudStorage.d.ts

@@ -1,82 +0,0 @@
-/// <reference types="node" />
-import type { Storage, StorageOptions } from '@google-cloud/storage';
-import { CommonLogger, LocalTimeInput } from '@naturalcycles/js-lib';
-import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
-import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
-import type { GCPServiceAccount } from './model';
-export { type Storage, type StorageOptions, };
-/**
- * This object is intentionally made to NOT extend StorageOptions,
- * because StorageOptions is complicated and provides just too many ways
- * to configure credentials.
- *
- * Here we define the minimum simple set of credentials needed.
- * All of these properties are available from the "service account" json file
- * (either personal one or non-personal).
- */
-export interface CloudStorageCfg {
-    /**
-     * Default is console
-     */
-    logger?: CommonLogger;
-    /**
-     * Pass true for extra debugging
-     */
-    debug?: boolean;
-}
-/**
- * CloudStorage implementation of CommonStorage API.
- *
- * API: https://googleapis.dev/nodejs/storage/latest/index.html
- */
-export declare class CloudStorage implements CommonStorage {
-    storage: Storage;
-    private constructor();
-    cfg: CloudStorageCfg & {
-        logger: CommonLogger;
-    };
-    static createFromGCPServiceAccount(credentials?: GCPServiceAccount, cfg?: CloudStorageCfg): CloudStorage;
-    static createFromStorageOptions(storageOptions?: StorageOptions, cfg?: CloudStorageCfg): CloudStorage;
-    /**
-     * Passing the pre-created Storage allows to instantiate it from both
-     * GCP Storage and FirebaseStorage.
-     */
-    static createFromStorage(storage: Storage, cfg?: CloudStorageCfg): CloudStorage;
-    ping(bucketName?: string): Promise<void>;
-    deletePath(bucketName: string, prefix: string): Promise<void>;
-    deletePaths(bucketName: string, prefixes: string[]): Promise<void>;
-    fileExists(bucketName: string, filePath: string): Promise<boolean>;
-    getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>;
-    getFileNamesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
-    getFilesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
-    getFile(bucketName: string, filePath: string): Promise<Buffer | null>;
-    /**
-     * Returns a Readable that is NOT object mode,
-     * so you can e.g pipe it to fs.createWriteStream()
-     */
-    getFileReadStream(bucketName: string, filePath: string): ReadableBinary;
-    saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
-    getFileWriteStream(bucketName: string, filePath: string): WritableBinary;
-    uploadFile(localFilePath: string, bucketName: string, bucketFilePath: string): Promise<void>;
-    setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
-    getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;
-    copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
-    moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
-    movePath(fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string): Promise<void>;
-    deleteFiles(bucketName: string, filePaths: string[]): Promise<void>;
-    combineFiles(bucketName: string, filePaths: string[], toPath: string, toBucket?: string, currentRecursionDepth?: number): Promise<void>;
-    combine(bucketName: string, prefix: string, toPath: string, toBucket?: string): Promise<void>;
-    /**
-     * Acquires a "signed url", which allows bearer to use it to download ('read') the file.
-     *
-     * expires: 'v4' supports maximum duration of 7 days from now.
-     *
-     * @experimental - not tested yet
-     */
-    getSignedUrl(bucketName: string, filePath: string, expires: LocalTimeInput): Promise<string>;
-    /**
-     * Returns SKIP if fileName is a folder.
-     * If !fullPaths - strip away the folder prefix.
-     */
-    private normalizeFilename;
-}

package/dist/cloudStorage.js

@@ -1,248 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CloudStorage = void 0;
-const js_lib_1 = require("@naturalcycles/js-lib");
-const MAX_RECURSION_DEPTH = 10;
-const BATCH_SIZE = 32;
-/**
- * CloudStorage implementation of CommonStorage API.
- *
- * API: https://googleapis.dev/nodejs/storage/latest/index.html
- */
-class CloudStorage {
-    constructor(storage, cfg = {}) {
-        this.storage = storage;
-        this.cfg = {
-            logger: console,
-            ...cfg,
-        };
-    }
-    static createFromGCPServiceAccount(credentials, cfg) {
-        const storageLib = require('@google-cloud/storage');
-        const storage = new storageLib.Storage({
-            credentials,
-            // Explicitly passing it here to fix this error:
-            // Error: Unable to detect a Project Id in the current environment.
-            // To learn more about authentication and Google APIs, visit:
-            // https://cloud.google.com/docs/authentication/getting-started
-            //     at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
-            projectId: credentials?.project_id,
-        });
-        return new CloudStorage(storage, cfg);
-    }
-    static createFromStorageOptions(storageOptions, cfg) {
-        const storageLib = require('@google-cloud/storage');
-        const storage = new storageLib.Storage(storageOptions);
-        return new CloudStorage(storage, cfg);
-    }
-    /**
-     * Passing the pre-created Storage allows to instantiate it from both
-     * GCP Storage and FirebaseStorage.
-     */
-    static createFromStorage(storage, cfg) {
-        return new CloudStorage(storage, cfg);
-    }
-    async ping(bucketName) {
-        await this.storage.bucket(bucketName || 'non-existing-for-sure').exists();
-    }
-    async deletePath(bucketName, prefix) {
-        await this.deletePaths(bucketName, [prefix]);
-    }
-    async deletePaths(bucketName, prefixes) {
-        const bucket = this.storage.bucket(bucketName);
-        await (0, js_lib_1.pMap)(prefixes, async (prefix) => {
-            await bucket.deleteFiles({
-                prefix,
-                // to keep going in case error occurs, similar to THROW_AGGREGATED
-                force: true,
-            });
-        });
-    }
-    async fileExists(bucketName, filePath) {
-        const [exists] = await this.storage.bucket(bucketName).file(filePath).exists();
-        return exists;
-    }
-    async getFileNames(bucketName, opt = {}) {
-        const { prefix, fullPaths = true } = opt;
-        const [files] = await this.storage.bucket(bucketName).getFiles({
-            prefix,
-        });
-        if (fullPaths) {
-            // Paths that end with `/` are "folders", which are "virtual" in CloudStorage
-            // It doesn't make sense to return or do anything with them
-            return files.map(f => f.name).filter(s => !s.endsWith('/'));
-        }
-        return files.map(f => (0, js_lib_1._substringAfterLast)(f.name, '/')).filter(Boolean);
-    }
-    getFileNamesStream(bucketName, opt = {}) {
-        const { prefix, fullPaths = true } = opt;
-        return this.storage.bucket(bucketName).getFilesStream({
-            prefix,
-            maxResults: opt.limit || undefined,
-        }).flatMap(f => {
-            const r = this.normalizeFilename(f.name, fullPaths);
-            if (r === js_lib_1.SKIP)
-                return [];
-            return [r];
-        });
-    }
-    getFilesStream(bucketName, opt = {}) {
-        const { prefix, fullPaths = true } = opt;
-        return this.storage.bucket(bucketName).getFilesStream({
-            prefix,
-            maxResults: opt.limit || undefined,
-        }).flatMap(async (f) => {
-            const filePath = this.normalizeFilename(f.name, fullPaths);
-            if (filePath === js_lib_1.SKIP)
-                return [];
-            const [content] = await f.download();
-            return [{ filePath, content }];
-        }, {
-            concurrency: 16,
-        });
-    }
-    async getFile(bucketName, filePath) {
-        const [buf] = await this.storage
-            .bucket(bucketName)
-            .file(filePath)
-            .download()
-            .catch(err => {
-            if (err?.code === 404)
-                return [null]; // file not found
-            throw err; // rethrow otherwise
-        });
-        return buf;
-    }
-    /**
-     * Returns a Readable that is NOT object mode,
-     * so you can e.g pipe it to fs.createWriteStream()
-     */
-    getFileReadStream(bucketName, filePath) {
-        return this.storage.bucket(bucketName).file(filePath).createReadStream();
-    }
-    async saveFile(bucketName, filePath, content) {
-        await this.storage.bucket(bucketName).file(filePath).save(content);
-    }
-    getFileWriteStream(bucketName, filePath) {
-        return this.storage.bucket(bucketName).file(filePath).createWriteStream();
-    }
-    async uploadFile(localFilePath, bucketName, bucketFilePath) {
-        await this.storage.bucket(bucketName).upload(localFilePath, {
-            destination: bucketFilePath,
-        });
-    }
-    async setFileVisibility(bucketName, filePath, isPublic) {
-        await this.storage.bucket(bucketName).file(filePath)[isPublic ? 'makePublic' : 'makePrivate']();
-    }
-    async getFileVisibility(bucketName, filePath) {
-        const [isPublic] = await this.storage.bucket(bucketName).file(filePath).isPublic();
-        return isPublic;
-    }
-    async copyFile(fromBucket, fromPath, toPath, toBucket) {
-        await this.storage
-            .bucket(fromBucket)
-            .file(fromPath)
-            .copy(this.storage.bucket(toBucket || fromBucket).file(toPath));
-    }
-    async moveFile(fromBucket, fromPath, toPath, toBucket) {
-        await this.storage
-            .bucket(fromBucket)
-            .file(fromPath)
-            .move(this.storage.bucket(toBucket || fromBucket).file(toPath));
-    }
-    async movePath(fromBucket, fromPrefix, toPrefix, toBucket) {
-        (0, js_lib_1._assert)(fromPrefix.endsWith('/'), 'fromPrefix should end with `/`');
-        (0, js_lib_1._assert)(toPrefix.endsWith('/'), 'toPrefix should end with `/`');
-        await this.storage
-            .bucket(fromBucket)
-            .getFilesStream({
-            prefix: fromPrefix,
-        })
-            .forEach(async (file) => {
-            const { name } = file;
-            const newName = toPrefix + name.slice(fromPrefix.length);
-            await file.move(this.storage.bucket(toBucket || fromBucket).file(newName));
-        });
-    }
-    async deleteFiles(bucketName, filePaths) {
-        await (0, js_lib_1.pMap)(filePaths, async (filePath) => {
-            await this.storage.bucket(bucketName).file(filePath).delete();
-        });
-    }
-    async combineFiles(bucketName, filePaths, toPath, toBucket, currentRecursionDepth = 0) {
-        (0, js_lib_1._assert)(currentRecursionDepth <= MAX_RECURSION_DEPTH, `combineFiles reached max recursion depth of ${MAX_RECURSION_DEPTH}`);
-        const { logger, debug } = this.cfg;
-        if (filePaths.length === 0) {
-            if (debug) {
-                logger.log(`[${currentRecursionDepth}] Nothing to compose, returning early!`);
-            }
-            return;
-        }
-        if (debug) {
-            logger.log(`[${currentRecursionDepth}] Will compose ${filePaths.length} files, by batches of ${BATCH_SIZE}`);
-        }
-        const intermediateFiles = [];
-        if (filePaths.length <= BATCH_SIZE) {
-            await this.storage
-                .bucket(bucketName)
-                .combine(filePaths, this.storage.bucket(toBucket || bucketName).file(toPath));
-            if (debug) {
-                logger.log(`[${currentRecursionDepth}] Composed into ${toPath}!`);
-            }
-            await this.deleteFiles(bucketName, filePaths);
-            return;
-        }
-        const started = Date.now();
-        await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(filePaths, BATCH_SIZE), async (fileBatch, i) => {
-            if (debug) {
-                logger.log(`[${currentRecursionDepth}] Composing batch ${i + 1}...`);
-            }
-            const intermediateFile = `temp_${currentRecursionDepth}_${i}`;
-            await this.storage
-                .bucket(bucketName)
-                .combine(fileBatch, this.storage.bucket(toBucket || bucketName).file(intermediateFile));
-            intermediateFiles.push(intermediateFile);
-            await this.deleteFiles(bucketName, fileBatch);
-        });
-        if (debug) {
-            logger.log(`[${currentRecursionDepth}] Batch composed into ${intermediateFiles.length} files, in ${(0, js_lib_1._since)(started)}`);
-        }
-        await this.combineFiles(toBucket || bucketName, intermediateFiles, toPath, toBucket, currentRecursionDepth + 1);
-    }
-    async combine(bucketName, prefix, toPath, toBucket) {
-        const filePaths = await this.getFileNames(bucketName, { prefix });
-        await this.combineFiles(bucketName, filePaths, toPath, toBucket);
-    }
-    /**
-     * Acquires a "signed url", which allows bearer to use it to download ('read') the file.
-     *
-     * expires: 'v4' supports maximum duration of 7 days from now.
-     *
-     * @experimental - not tested yet
-     */
-    async getSignedUrl(bucketName, filePath, expires) {
-        const [url] = await this.storage
-            .bucket(bucketName)
-            .file(filePath)
-            .getSignedUrl({
-            action: 'read',
-            version: 'v4',
-            expires: (0, js_lib_1.localTime)(expires).unixMillis(),
-        });
-        return url;
-    }
-    /**
-     * Returns SKIP if fileName is a folder.
-     * If !fullPaths - strip away the folder prefix.
-     */
-    normalizeFilename(fileName, fullPaths) {
-        if (fullPaths) {
-            if (fileName.endsWith('/'))
-                return js_lib_1.SKIP; // skip folders
-            return fileName;
-        }
-        fileName = (0, js_lib_1._substringAfterLast)(fileName, '/');
-        return fileName || js_lib_1.SKIP; // skip folders
-    }
-}
-exports.CloudStorage = CloudStorage;

package/dist/commonStorage.d.ts

@@ -1,111 +0,0 @@
-/// <reference types="node" />
-import type { LocalTimeInput } from '@naturalcycles/js-lib';
-import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
-export interface FileEntry {
-    filePath: string;
-    content: Buffer;
-}
-export interface CommonStorageGetOptions {
-    /**
-     * Will filter resulting files based on `prefix`.
-     */
-    prefix?: string;
-    /**
-     * Defaults to true.
-     * Set to false to return file names instead of full paths.
-     */
-    fullPaths?: boolean;
-    /**
-     * Limits the number of results.
-     *
-     * By default it's unlimited.
-     *
-     * 0 is treated as `undefined`.
-     */
-    limit?: number;
-}
-/**
- * Common denominator interface for File Storage.
- * Modelled after GCP Cloud Storage, Firebase Storage.
- *
- * Uses the concept of Bucket (identified by string name) and Path within the Bucket.
- *
- * Path MUST NOT start with a slash !
- *
- * Similarly to CommonDB, Bucket is like a Table, and Path is like an `id`.
- */
-export interface CommonStorage {
-    /**
-     * Ensure that the credentials are correct and the connection is working.
-     * Idempotent.
-     *
-     * Pass `bucketName` in case you only have permissions to operate on that bucket.
-     */
-    ping: (bucketName?: string) => Promise<void>;
-    /**
-     * Creates a new bucket by given name.
-     * todo: check what to do if it already exists
-     */
-    fileExists: (bucketName: string, filePath: string) => Promise<boolean>;
-    getFile: (bucketName: string, filePath: string) => Promise<Buffer | null>;
-    saveFile: (bucketName: string, filePath: string, content: Buffer) => Promise<void>;
-    /**
-     * Should recursively delete all files in a folder, if path is a folder.
-     */
-    deletePath: (bucketName: string, prefix: string) => Promise<void>;
-    deletePaths: (bucketName: string, prefixes: string[]) => Promise<void>;
-    /**
-     * Should delete all files by their paths.
-     */
-    deleteFiles: (bucketName: string, filePaths: string[]) => Promise<void>;
-    /**
-     * Returns an array of strings which are file paths.
-     * Files that are not found by the path are not present in the map.
-     *
-     * Second argument is called `prefix` (same as `path`) to explain how
-     * listing works (it filters all files by `startsWith`). Also, to match
-     * GCP Cloud Storage API.
-     *
-     * Important difference between `prefix` and `path` is that `prefix` will
-     * return all files from sub-directories too!
-     */
-    getFileNames: (bucketName: string, opt?: CommonStorageGetOptions) => Promise<string[]>;
-    getFileNamesStream: (bucketName: string, opt?: CommonStorageGetOptions) => ReadableTyped<string>;
-    getFilesStream: (bucketName: string, opt?: CommonStorageGetOptions) => ReadableTyped<FileEntry>;
-    getFileReadStream: (bucketName: string, filePath: string) => ReadableBinary;
-    getFileWriteStream: (bucketName: string, filePath: string) => WritableBinary;
-    /**
-     * Upload local file to the bucket (by streaming it).
-     */
-    uploadFile: (localFilePath: string, bucketName: string, bucketFilePath: string) => Promise<void>;
-    setFileVisibility: (bucketName: string, filePath: string, isPublic: boolean) => Promise<void>;
-    getFileVisibility: (bucketName: string, filePath: string) => Promise<boolean>;
-    copyFile: (fromBucket: string, fromPath: string, toPath: string, toBucket?: string) => Promise<void>;
-    moveFile: (fromBucket: string, fromPath: string, toPath: string, toBucket?: string) => Promise<void>;
-    /**
-     * Allows to move "directory" with all its contents.
-     *
-     * Prefixes should end with `/` to work properly,
-     * otherwise some folder that starts with the same prefix will be included.
-     */
-    movePath: (fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string) => Promise<void>;
-    /**
-     * Combine (compose) multiple input files into a single output file.
-     * Should support unlimited number of input files, using recursive algorithm if necessary.
-     *
-     * After the output file is created, all input files should be deleted.
-     *
-     * @experimental
-     */
-    combineFiles: (bucketName: string, filePaths: string[], toPath: string, toBucket?: string) => Promise<void>;
-    /**
-     * Like `combineFiles`, but for a `prefix`.
-     */
-    combine: (bucketName: string, prefix: string, toPath: string, toBucket?: string) => Promise<void>;
-    /**
-     * Acquire a "signed url", which allows bearer to use it to download ('read') the file.
-     *
-     * @experimental
-     */
-    getSignedUrl: (bucketName: string, filePath: string, expires: LocalTimeInput) => Promise<string>;
-}

package/dist/commonStorage.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/commonStorageBucket.d.ts

@@ -1,69 +0,0 @@
-/// <reference types="node" />
-/// <reference types="node" />
-import { Readable, Writable } from 'node:stream';
-import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
-export interface CommonStorageBucketCfg {
-    storage: CommonStorage;
-    bucketName: string;
-}
-/**
- * Convenience wrapper around CommonStorage for a given Bucket.
- *
- * Similar to what CommonDao is to CommonDB.
- */
-export declare class CommonStorageBucket {
-    cfg: CommonStorageBucketCfg;
-    constructor(cfg: CommonStorageBucketCfg);
-    ping(bucketName?: string): Promise<void>;
-    fileExists(filePath: string): Promise<boolean>;
-    getFile(filePath: string): Promise<Buffer | null>;
-    getFileAsString(filePath: string): Promise<string | null>;
-    getFileAsJson<T = any>(filePath: string): Promise<T | null>;
-    requireFile(filePath: string): Promise<Buffer>;
-    requireFileAsString(filePath: string): Promise<string>;
-    requireFileAsJson<T = any>(filePath: string): Promise<T>;
-    private throwRequiredError;
-    getFileContents(paths: string[]): Promise<Buffer[]>;
-    getFileContentsAsJson<T = any>(paths: string[]): Promise<T[]>;
-    getFileEntries(paths: string[]): Promise<FileEntry[]>;
-    getFileEntriesAsJson<T = any>(paths: string[]): Promise<{
-        filePath: string;
-        content: T;
-    }[]>;
-    saveFile(filePath: string, content: Buffer): Promise<void>;
-    /**
-     * Convenience method that does:
-     * await saveFile
-     * await setFileVisibility
-     */
-    savePublicFile(filePath: string, content: Buffer): Promise<void>;
-    saveStringFile(filePath: string, content: string): Promise<void>;
-    saveJsonFile(filePath: string, content: any): Promise<void>;
-    saveFiles(entries: FileEntry[]): Promise<void>;
-    /**
-     * Should recursively delete all files in a folder, if path is a folder.
-     */
-    deletePath(prefix: string): Promise<void>;
-    deletePaths(prefixes: string[]): Promise<void>;
-    /**
-     * Returns an array of strings which are file paths.
-     * Files that are not found by the path are not present in the map.
-     *
-     * Second argument is called `prefix` (same as `path`) to explain how
-     * listing works (it filters all files by `startsWith`). Also, to match
-     * GCP Cloud Storage API.
-     *
-     * Important difference between `prefix` and `path` is that `prefix` will
-     * return all files from sub-directories too!
-     */
-    getFileNames(opt?: CommonStorageGetOptions): Promise<string[]>;
-    getFileNamesStream(opt?: CommonStorageGetOptions): ReadableTyped<string>;
-    getFilesStream(opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
-    getFileReadStream(filePath: string): Readable;
-    getFileWriteStream(filePath: string): Writable;
-    setFileVisibility(filePath: string, isPublic: boolean): Promise<void>;
-    getFileVisibility(filePath: string): Promise<boolean>;
-    copyFile(fromPath: string, toPath: string, toBucket?: string): Promise<void>;
-    moveFile(fromPath: string, toPath: string, toBucket?: string): Promise<void>;
-}

package/dist/commonStorageBucket.js

@@ -1,146 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CommonStorageBucket = void 0;
-const js_lib_1 = require("@naturalcycles/js-lib");
-/**
- * Convenience wrapper around CommonStorage for a given Bucket.
- *
- * Similar to what CommonDao is to CommonDB.
- */
-class CommonStorageBucket {
-    constructor(cfg) {
-        this.cfg = cfg;
-    }
-    async ping(bucketName) {
-        await this.cfg.storage.ping(bucketName);
-    }
-    async fileExists(filePath) {
-        return await this.cfg.storage.fileExists(this.cfg.bucketName, filePath);
-    }
-    async getFile(filePath) {
-        return await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
-    }
-    async getFileAsString(filePath) {
-        const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
-        return buf?.toString() || null;
-    }
-    async getFileAsJson(filePath) {
-        const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
-        if (!buf)
-            return null;
-        return JSON.parse(buf.toString());
-    }
-    async requireFile(filePath) {
-        const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
-        if (!buf)
-            this.throwRequiredError(filePath);
-        return buf;
-    }
-    async requireFileAsString(filePath) {
-        const s = await this.getFileAsString(filePath);
-        return s ?? this.throwRequiredError(filePath);
-    }
-    async requireFileAsJson(filePath) {
-        const v = await this.getFileAsJson(filePath);
-        return v ?? this.throwRequiredError(filePath);
-    }
-    throwRequiredError(filePath) {
-        throw new js_lib_1.AppError(`File required, but not found: ${this.cfg.bucketName}/${filePath}`, {
-            code: 'FILE_REQUIRED',
-        });
-    }
-    async getFileContents(paths) {
-        return (await (0, js_lib_1.pMap)(paths, async (filePath) => (await this.cfg.storage.getFile(this.cfg.bucketName, filePath)))).filter(Boolean);
-    }
-    async getFileContentsAsJson(paths) {
-        return (await (0, js_lib_1.pMap)(paths, async (filePath) => {
-            const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
-            return buf ? JSON.parse(buf.toString()) : null;
-        })).filter(Boolean);
-    }
-    async getFileEntries(paths) {
-        return (await (0, js_lib_1.pMap)(paths, async (filePath) => {
-            const content = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
-            return { filePath, content: content };
-        })).filter(f => f.content);
-    }
-    async getFileEntriesAsJson(paths) {
-        return (await (0, js_lib_1.pMap)(paths, async (filePath) => {
-            const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
-            return buf ? { filePath, content: JSON.parse(buf.toString()) } : null;
-        })).filter(Boolean);
-    }
-    async saveFile(filePath, content) {
-        await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, content);
-    }
-    /**
-     * Convenience method that does:
-     * await saveFile
-     * await setFileVisibility
-     */
-    async savePublicFile(filePath, content) {
-        await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, content);
-        await this.cfg.storage.setFileVisibility(this.cfg.bucketName, filePath, true);
-    }
-    async saveStringFile(filePath, content) {
-        await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, Buffer.from(content));
-    }
-    async saveJsonFile(filePath, content) {
-        await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, Buffer.from(JSON.stringify(content)));
-    }
-    async saveFiles(entries) {
-        await (0, js_lib_1.pMap)(entries, async (f) => {
-            await this.cfg.storage.saveFile(this.cfg.bucketName, f.filePath, f.content);
-        });
-    }
-    /**
-     * Should recursively delete all files in a folder, if path is a folder.
-     */
-    async deletePath(prefix) {
-        return await this.cfg.storage.deletePath(this.cfg.bucketName, prefix);
-    }
-    async deletePaths(prefixes) {
-        await (0, js_lib_1.pMap)(prefixes, async (prefix) => {
-            return await this.cfg.storage.deletePath(this.cfg.bucketName, prefix);
-        });
-    }
-    /**
-     * Returns an array of strings which are file paths.
-     * Files that are not found by the path are not present in the map.
-     *
-     * Second argument is called `prefix` (same as `path`) to explain how
-     * listing works (it filters all files by `startsWith`). Also, to match
-     * GCP Cloud Storage API.
-     *
-     * Important difference between `prefix` and `path` is that `prefix` will
-     * return all files from sub-directories too!
-     */
-    async getFileNames(opt) {
-        return await this.cfg.storage.getFileNames(this.cfg.bucketName, opt);
-    }
-    getFileNamesStream(opt) {
-        return this.cfg.storage.getFileNamesStream(this.cfg.bucketName, opt);
-    }
-    getFilesStream(opt) {
-        return this.cfg.storage.getFilesStream(this.cfg.bucketName, opt);
-    }
-    getFileReadStream(filePath) {
-        return this.cfg.storage.getFileReadStream(this.cfg.bucketName, filePath);
-    }
-    getFileWriteStream(filePath) {
-        return this.cfg.storage.getFileWriteStream(this.cfg.bucketName, filePath);
-    }
-    async setFileVisibility(filePath, isPublic) {
-        await this.cfg.storage.setFileVisibility(this.cfg.bucketName, filePath, isPublic);
-    }
-    async getFileVisibility(filePath) {
-        return await this.cfg.storage.getFileVisibility(this.cfg.bucketName, filePath);
-    }
-    async copyFile(fromPath, toPath, toBucket) {
-        await this.cfg.storage.copyFile(this.cfg.bucketName, fromPath, toPath, toBucket);
-    }
-    async moveFile(fromPath, toPath, toBucket) {
-        await this.cfg.storage.moveFile(this.cfg.bucketName, fromPath, toPath, toBucket);
-    }
-}
-exports.CommonStorageBucket = CommonStorageBucket;

package/dist/commonStorageKeyValueDB.d.ts

@@ -1,33 +0,0 @@
-/// <reference types="node" />
-import { CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib';
-import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { CommonStorage } from './commonStorage';
-export interface CommonStorageKeyValueDBCfg {
-    storage: CommonStorage;
-    bucketName: string;
-}
-/**
- * CommonKeyValueDB, backed up by a CommonStorage implementation.
- *
- * Each Table is represented as a Folder.
- * Each Item is represented as a File:
- * fileName is ${id} (without extension)
- * file contents is ${v} (Buffer)
- */
-export declare class CommonStorageKeyValueDB implements CommonKeyValueDB {
-    cfg: CommonStorageKeyValueDBCfg;
-    constructor(cfg: CommonStorageKeyValueDBCfg);
-    ping(): Promise<void>;
-    createTable(_table: string, _opt?: CommonDBCreateOptions): Promise<void>;
-    /**
-     * Allows to pass `SomeBucket.SomeTable` in `table`, to override a Bucket.
-     */
-    private getBucketAndPrefix;
-    deleteByIds(table: string, ids: string[]): Promise<void>;
-    getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
-    saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void>;
-    streamIds(table: string, limit?: number): ReadableTyped<string>;
-    streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
-    streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
-    count(table: string): Promise<number>;
-}

package/dist/commonStorageKeyValueDB.js

@@ -1,81 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CommonStorageKeyValueDB = void 0;
-const js_lib_1 = require("@naturalcycles/js-lib");
-/**
- * CommonKeyValueDB, backed up by a CommonStorage implementation.
- *
- * Each Table is represented as a Folder.
- * Each Item is represented as a File:
- * fileName is ${id} (without extension)
- * file contents is ${v} (Buffer)
- */
-class CommonStorageKeyValueDB {
-    constructor(cfg) {
-        this.cfg = cfg;
-    }
-    async ping() {
-        await this.cfg.storage.ping(this.cfg.bucketName);
-    }
-    async createTable(_table, _opt) {
-        // no-op
-    }
-    /**
-     * Allows to pass `SomeBucket.SomeTable` in `table`, to override a Bucket.
-     */
-    getBucketAndPrefix(table) {
-        const [part1, part2] = table.split('.');
-        if (part2) {
-            return {
-                bucketName: part1,
-                prefix: part2,
-            };
-        }
-        // As is
-        return {
-            bucketName: this.cfg.bucketName,
-            prefix: table,
-        };
-    }
-    async deleteByIds(table, ids) {
-        const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        await (0, js_lib_1.pMap)(ids, async (id) => {
-            await this.cfg.storage.deletePath(bucketName, [prefix, id].join('/'));
-        });
-    }
-    async getByIds(table, ids) {
-        const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        const map = {};
-        await (0, js_lib_1.pMap)(ids, async (id) => {
-            const buf = await this.cfg.storage.getFile(bucketName, [prefix, id].join('/'));
-            if (buf)
-                map[id] = buf;
-        });
-        return ids.map(id => [id, map[id]]).filter(t => t[1]);
-    }
-    async saveBatch(table, entries) {
-        const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        await (0, js_lib_1.pMap)(entries, async ([id, content]) => {
-            await this.cfg.storage.saveFile(bucketName, [prefix, id].join('/'), content);
-        });
-    }
-    streamIds(table, limit) {
-        const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        return this.cfg.storage.getFileNamesStream(bucketName, { prefix, limit, fullPaths: false });
-    }
-    streamValues(table, limit) {
-        const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        return this.cfg.storage.getFilesStream(bucketName, { prefix, limit }).map(f => f.content);
-    }
-    streamEntries(table, limit) {
-        const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        return this.cfg.storage
-            .getFilesStream(bucketName, { prefix, limit, fullPaths: false })
-            .map(f => [f.filePath, f.content]);
-    }
-    async count(table) {
-        const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        return (await this.cfg.storage.getFileNames(bucketName, { prefix })).length;
-    }
-}
-exports.CommonStorageKeyValueDB = CommonStorageKeyValueDB;

package/dist/inMemoryCommonStorage.d.ts

@@ -1,34 +0,0 @@
-/// <reference types="node" />
-import { LocalTimeInput, StringMap } from '@naturalcycles/js-lib';
-import { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
-import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
-export declare class InMemoryCommonStorage implements CommonStorage {
-    /**
-     * data[bucketName][filePath] = Buffer
-     */
-    data: StringMap<StringMap<Buffer>>;
-    publicMap: StringMap<StringMap<boolean>>;
-    ping(): Promise<void>;
-    getBucketNames(): Promise<string[]>;
-    getBucketNamesStream(): ReadableTyped<string>;
-    fileExists(bucketName: string, filePath: string): Promise<boolean>;
-    getFile(bucketName: string, filePath: string): Promise<Buffer | null>;
-    saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>;
-    deletePath(bucketName: string, prefix: string): Promise<void>;
-    deletePaths(bucketName: string, prefixes: string[]): Promise<void>;
-    deleteFiles(bucketName: string, filePaths: string[]): Promise<void>;
-    getFileNames(bucketName: string, opt?: CommonStorageGetOptions): Promise<string[]>;
-    getFileNamesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<string>;
-    getFilesStream(bucketName: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry>;
-    getFileReadStream(bucketName: string, filePath: string): ReadableBinary;
-    getFileWriteStream(_bucketName: string, _filePath: string): WritableBinary;
-    uploadFile(localFilePath: string, bucketName: string, bucketFilePath: string): Promise<void>;
-    setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>;
-    getFileVisibility(bucketName: string, filePath: string): Promise<boolean>;
-    copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
-    moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>;
-    movePath(fromBucket: string, fromPrefix: string, toPrefix: string, toBucket?: string): Promise<void>;
-    combine(bucketName: string, prefix: string, toPath: string, toBucket?: string): Promise<void>;
-    combineFiles(bucketName: string, filePaths: string[], toPath: string, toBucket?: string): Promise<void>;
-    getSignedUrl(bucketName: string, filePath: string, expires: LocalTimeInput): Promise<string>;
-}

package/dist/inMemoryCommonStorage.js

@@ -1,131 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.InMemoryCommonStorage = void 0;
-const js_lib_1 = require("@naturalcycles/js-lib");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
-class InMemoryCommonStorage {
-    constructor() {
-        /**
-         * data[bucketName][filePath] = Buffer
-         */
-        this.data = {};
-        this.publicMap = {};
-    }
-    async ping() { }
-    async getBucketNames() {
-        return Object.keys(this.data);
-    }
-    getBucketNamesStream() {
-        return (0, nodejs_lib_1.readableFrom)(Object.keys(this.data));
-    }
-    async fileExists(bucketName, filePath) {
-        return !!this.data[bucketName]?.[filePath];
-    }
-    async getFile(bucketName, filePath) {
-        return this.data[bucketName]?.[filePath] || null;
-    }
-    async saveFile(bucketName, filePath, content) {
-        this.data[bucketName] ||= {};
-        this.data[bucketName][filePath] = content;
-    }
-    async deletePath(bucketName, prefix) {
-        await this.deletePaths(bucketName, [prefix]);
-    }
-    async deletePaths(bucketName, prefixes) {
-        Object.keys(this.data[bucketName] || {}).forEach(filePath => {
-            if (prefixes.some(prefix => filePath.startsWith(prefix))) {
-                delete this.data[bucketName][filePath];
-            }
-        });
-    }
-    async deleteFiles(bucketName, filePaths) {
-        if (!this.data[bucketName])
-            return;
-        filePaths.forEach(filePath => delete this.data[bucketName][filePath]);
-    }
-    async getFileNames(bucketName, opt = {}) {
-        const { prefix = '', fullPaths = true } = opt;
-        return Object.keys(this.data[bucketName] || {})
-            .filter(filePath => filePath.startsWith(prefix))
-            .map(f => (fullPaths ? f : (0, js_lib_1._substringAfterLast)(f, '/')));
-    }
-    getFileNamesStream(bucketName, opt = {}) {
-        const { prefix = '', fullPaths = true } = opt;
-        return (0, nodejs_lib_1.readableFrom)(Object.keys(this.data[bucketName] || {})
-            .filter(filePath => filePath.startsWith(prefix))
-            .slice(0, opt.limit)
-            .map(n => (fullPaths ? n : (0, js_lib_1._substringAfterLast)(n, '/'))));
-    }
-    getFilesStream(bucketName, opt = {}) {
-        const { prefix = '', fullPaths = true } = opt;
-        return (0, nodejs_lib_1.readableFrom)((0, js_lib_1._stringMapEntries)(this.data[bucketName] || {})
-            .map(([filePath, content]) => ({
-            filePath,
-            content,
-        }))
-            .filter(f => f.filePath.startsWith(prefix))
-            .slice(0, opt.limit)
-            .map(f => (fullPaths ? f : { ...f, filePath: (0, js_lib_1._substringAfterLast)(f.filePath, '/') })));
-    }
-    getFileReadStream(bucketName, filePath) {
-        return (0, nodejs_lib_1.readableFrom)(this.data[bucketName][filePath]);
-    }
-    getFileWriteStream(_bucketName, _filePath) {
-        throw new Error('Method not implemented.');
-    }
-    async uploadFile(localFilePath, bucketName, bucketFilePath) {
-        this.data[bucketName] ||= {};
-        this.data[bucketName][bucketFilePath] = await nodejs_lib_1.fs2.readBufferAsync(localFilePath);
-    }
-    async setFileVisibility(bucketName, filePath, isPublic) {
-        this.publicMap[bucketName] ||= {};
-        this.publicMap[bucketName][filePath] = isPublic;
-    }
-    async getFileVisibility(bucketName, filePath) {
-        return !!this.publicMap[bucketName]?.[filePath];
-    }
-    async copyFile(fromBucket, fromPath, toPath, toBucket) {
-        const tob = toBucket || fromBucket;
-        this.data[fromBucket] ||= {};
-        this.data[tob] ||= {};
-        this.data[tob][toPath] = this.data[fromBucket][fromPath];
-    }
-    async moveFile(fromBucket, fromPath, toPath, toBucket) {
-        const tob = toBucket || fromBucket;
-        this.data[fromBucket] ||= {};
-        this.data[tob] ||= {};
-        this.data[tob][toPath] = this.data[fromBucket][fromPath];
-        delete this.data[fromBucket][fromPath];
-    }
-    async movePath(fromBucket, fromPrefix, toPrefix, toBucket) {
-        const tob = toBucket || fromBucket;
-        this.data[fromBucket] ||= {};
-        this.data[tob] ||= {};
-        (0, js_lib_1._stringMapEntries)(this.data[fromBucket]).forEach(([filePath, v]) => {
-            if (!filePath.startsWith(fromPrefix))
-                return;
-            this.data[tob][toPrefix + filePath.slice(fromPrefix.length)] = v;
-            delete this.data[fromBucket][filePath];
-        });
-    }
-    async combine(bucketName, prefix, toPath, toBucket) {
-        const filePaths = await this.getFileNames(bucketName, { prefix });
-        await this.combineFiles(bucketName, filePaths, toPath, toBucket);
-    }
-    async combineFiles(bucketName, filePaths, toPath, toBucket) {
-        if (!this.data[bucketName])
-            return;
-        const tob = toBucket || bucketName;
-        this.data[tob] ||= {};
-        this.data[tob][toPath] = Buffer.concat(filePaths.map(p => this.data[bucketName][p]).filter(js_lib_1._isTruthy));
-        // delete source files
-        filePaths.forEach(p => delete this.data[bucketName][p]);
-    }
-    async getSignedUrl(bucketName, filePath, expires) {
-        const buf = this.data[bucketName]?.[filePath];
-        (0, js_lib_1._assert)(buf, `getSignedUrl file not found: ${bucketName}/${filePath}`);
-        const signature = (0, nodejs_lib_1.md5)(buf);
-        return `https://testurl.com/${bucketName}/${filePath}?expires=${(0, js_lib_1.localTime)(expires).unix()}&signature=${signature}`;
-    }
-}
-exports.InMemoryCommonStorage = InMemoryCommonStorage;

package/dist/index.d.ts

@@ -1,7 +0,0 @@
-export * from './cloudStorage';
-export * from './commonStorage';
-export * from './commonStorageBucket';
-export * from './commonStorageKeyValueDB';
-export * from './inMemoryCommonStorage';
-export * from './model';
-export * from './testing/commonStorageTest';

package/dist/index.js

@@ -1,10 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const tslib_1 = require("tslib");
-tslib_1.__exportStar(require("./cloudStorage"), exports);
-tslib_1.__exportStar(require("./commonStorage"), exports);
-tslib_1.__exportStar(require("./commonStorageBucket"), exports);
-tslib_1.__exportStar(require("./commonStorageKeyValueDB"), exports);
-tslib_1.__exportStar(require("./inMemoryCommonStorage"), exports);
-tslib_1.__exportStar(require("./model"), exports);
-tslib_1.__exportStar(require("./testing/commonStorageTest"), exports);

package/dist/model.d.ts

@@ -1,5 +0,0 @@
-export interface GCPServiceAccount {
-    client_email: string;
-    private_key: string;
-    project_id: string;
-}

package/dist/model.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/testing/commonStorageTest.d.ts

@@ -1,5 +0,0 @@
-import { CommonStorage } from '../commonStorage';
-/**
- * This test suite must be idempotent.
- */
-export declare function runCommonStorageTest(storage: CommonStorage, bucketName: string): void;

package/dist/testing/commonStorageTest.js

@@ -1,117 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runCommonStorageTest = void 0;
-const js_lib_1 = require("@naturalcycles/js-lib");
-const TEST_FOLDER = 'test/subdir';
-const TEST_ITEMS = (0, js_lib_1._range)(10).map(n => ({
-    id: `id_${n + 1}`,
-    n,
-    even: n % 2 === 0,
-}));
-const TEST_ITEMS2 = (0, js_lib_1._range)(10).map(n => ({
-    fileType: 2,
-    id: `id_${n + 1}`,
-    n,
-    even: n % 2 === 0,
-}));
-const TEST_ITEMS3 = (0, js_lib_1._range)(10).map(n => ({
-    fileType: 3,
-    id: `id_${n + 1}`,
-    n,
-    even: n % 2 === 0,
-}));
-const TEST_FILES = [TEST_ITEMS, TEST_ITEMS2, TEST_ITEMS3].map((obj, i) => ({
-    filePath: `${TEST_FOLDER}/file_${i + 1}.json`,
-    content: Buffer.from(JSON.stringify(obj)),
-}));
-/**
- * This test suite must be idempotent.
- */
-function runCommonStorageTest(storage, bucketName) {
-    // test('createBucket', async () => {
-    //   await storage.createBucket(bucketName)
-    // })
-    test('ping', async () => {
-        await storage.ping(bucketName);
-    });
-    // test('listBuckets', async () => {
-    //   const buckets = await storage.getBucketNames()
-    //   console.log(buckets)
-    // })
-    //
-    // test('streamBuckets', async () => {
-    //   const buckets = await readableToArray(storage.getBucketNamesStream())
-    //   console.log(buckets)
-    // })
-    test('prepare: clear bucket', async () => {
-        await (0, js_lib_1.pMap)(TEST_FILES.map(f => f.filePath), async (filePath) => await storage.deletePath(bucketName, filePath));
-    });
-    // test('listFileNames on root should return empty', async () => {
-    //   const fileNames = await storage.getFileNames(bucketName)
-    //   expect(fileNames).toEqual([])
-    // })
-    test(`listFileNames on ${TEST_FOLDER} should return empty`, async () => {
-        const fileNames = await storage.getFileNames(bucketName, { prefix: TEST_FOLDER });
-        expect(fileNames).toEqual([]);
-    });
-    test(`streamFileNames on ${TEST_FOLDER} should return empty`, async () => {
-        const fileNames = await storage
-            .getFileNamesStream(bucketName, { prefix: TEST_FOLDER })
-            .toArray();
-        expect(fileNames).toEqual([]);
-    });
-    test(`exists should return empty array`, async () => {
-        await (0, js_lib_1.pMap)(TEST_FILES, async (f) => {
-            const exists = await storage.fileExists(bucketName, f.filePath);
-            expect(exists).toBe(false);
-        });
-    });
-    test(`saveFiles, then listFileNames, streamFileNames and getFiles should return just saved files`, async () => {
-        const testFilesMap = Object.fromEntries(TEST_FILES.map(f => [f.filePath, f.content]));
-        // It's done in the same test to ensure "strong consistency"
-        await (0, js_lib_1.pMap)(TEST_FILES, async (f) => await storage.saveFile(bucketName, f.filePath, f.content));
-        const fileNamesShort = await storage.getFileNames(bucketName, {
-            prefix: TEST_FOLDER,
-            fullPaths: false,
-        });
-        expect(fileNamesShort.sort()).toEqual(TEST_FILES.map(f => (0, js_lib_1._substringAfterLast)(f.filePath, '/')).sort());
-        const fileNames = await storage.getFileNames(bucketName, { prefix: TEST_FOLDER });
-        expect(fileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
-        const streamedFileNames = await storage
-            .getFileNamesStream(bucketName, { prefix: TEST_FOLDER })
-            .toArray();
-        expect(streamedFileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
-        const filesMap = {};
-        await (0, js_lib_1.pMap)(fileNames, async (filePath) => {
-            filesMap[filePath] = (await storage.getFile(bucketName, filePath));
-        });
-        expect(filesMap).toEqual(testFilesMap);
-        await (0, js_lib_1.pMap)(fileNames, async (filePath) => {
-            const exists = await storage.fileExists(bucketName, filePath);
-            expect(exists).toBe(true);
-        });
-    });
-    test('cleanup', async () => {
-        await storage.deletePath(bucketName, TEST_FOLDER);
-    });
-    // Cannot update access control for an object when uniform bucket-level access is enabled. Read more at https://cloud.google.com/storage/docs/uniform-bucket-level-access
-    /*
-    test(`get/set FilesVisibility`, async () => {
-      const fileNames = TEST_FILES.map(f => f.filePath)
-  
-      let map = await storage.getFilesVisibility(bucketName, fileNames)
-      expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, false])))
-  
-      await storage.setFilesVisibility(bucketName, fileNames, true)
-  
-      map = await storage.getFilesVisibility(bucketName, fileNames)
-      expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, true])))
-  
-      await storage.setFilesVisibility(bucketName, fileNames, false)
-  
-      map = await storage.getFilesVisibility(bucketName, fileNames)
-      expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, false])))
-    })
-     */
-}
-exports.runCommonStorageTest = runCommonStorageTest;