@naturalcycles/cloud-storage-lib 1.13.3 → 2.0.0

package/dist/cloudStorage.d.ts

@@ -1,8 +1,8 @@
 import type { Storage, StorageOptions } from '@google-cloud/storage';
-import { CommonLogger, LocalTimeInput } from '@naturalcycles/js-lib';
+import type { CommonLogger, LocalTimeInput } from '@naturalcycles/js-lib';
 import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
-import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
-import type { GCPServiceAccount } from './model';
+import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage.js';
+import type { GCPServiceAccount } from './model.js';
 export type { Storage, StorageOptions, };
 /**
  * This object is intentionally made to NOT extend StorageOptions,
@@ -34,8 +34,8 @@ export declare class CloudStorage implements CommonStorage {
     cfg: CloudStorageCfg & {
         logger: CommonLogger;
     };
-    static createFromGCPServiceAccount(credentials?: GCPServiceAccount, cfg?: CloudStorageCfg): CloudStorage;
-    static createFromStorageOptions(storageOptions?: StorageOptions, cfg?: CloudStorageCfg): CloudStorage;
+    static createFromGCPServiceAccount(credentials?: GCPServiceAccount, cfg?: CloudStorageCfg): Promise<CloudStorage>;
+    static createFromStorageOptions(storageOptions?: StorageOptions, cfg?: CloudStorageCfg): Promise<CloudStorage>;
     /**
      * Passing the pre-created Storage allows to instantiate it from both
      * GCP Storage and FirebaseStorage.

package/dist/cloudStorage.js

@@ -1,7 +1,4 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CloudStorage = void 0;
-const js_lib_1 = require("@naturalcycles/js-lib");
+import { _assert, _chunk, _since, _substringAfterLast, localTime, pMap, SKIP, } from '@naturalcycles/js-lib';
 const MAX_RECURSION_DEPTH = 10;
 const BATCH_SIZE = 32;
 /**
@@ -9,7 +6,8 @@ const BATCH_SIZE = 32;
  *
  * API: https://googleapis.dev/nodejs/storage/latest/index.html
  */
-class CloudStorage {
+export class CloudStorage {
+    storage;
     constructor(storage, cfg = {}) {
         this.storage = storage;
         this.cfg = {
@@ -17,9 +15,10 @@ class CloudStorage {
             ...cfg,
         };
     }
-    static createFromGCPServiceAccount(credentials, cfg) {
-        const storageLib = require('@google-cloud/storage');
-        const storage = new storageLib.Storage({
+    cfg;
+    static async createFromGCPServiceAccount(credentials, cfg) {
+        const { Storage } = await import('@google-cloud/storage');
+        const storage = new Storage({
             credentials,
             // Explicitly passing it here to fix this error:
             // Error: Unable to detect a Project Id in the current environment.
@@ -30,9 +29,9 @@ class CloudStorage {
         });
         return new CloudStorage(storage, cfg);
     }
-    static createFromStorageOptions(storageOptions, cfg) {
-        const storageLib = require('@google-cloud/storage');
-        const storage = new storageLib.Storage(storageOptions);
+    static async createFromStorageOptions(storageOptions, cfg) {
+        const { Storage } = await import('@google-cloud/storage');
+        const storage = new Storage(storageOptions);
         return new CloudStorage(storage, cfg);
     }
     /**
@@ -50,7 +49,7 @@ class CloudStorage {
     }
     async deletePaths(bucketName, prefixes) {
         const bucket = this.storage.bucket(bucketName);
-        await (0, js_lib_1.pMap)(prefixes, async (prefix) => {
+        await pMap(prefixes, async (prefix) => {
             await bucket.deleteFiles({
                 prefix,
                 // to keep going in case error occurs, similar to THROW_AGGREGATED
@@ -72,7 +71,7 @@ class CloudStorage {
             // It doesn't make sense to return or do anything with them
             return files.map(f => f.name).filter(s => !s.endsWith('/'));
         }
-        return files.map(f => (0, js_lib_1._substringAfterLast)(f.name, '/')).filter(Boolean);
+        return files.map(f => _substringAfterLast(f.name, '/')).filter(Boolean);
     }
     getFileNamesStream(bucketName, opt = {}) {
         const { prefix, fullPaths = true } = opt;
@@ -81,7 +80,7 @@ class CloudStorage {
             maxResults: opt.limit || undefined,
         }).flatMap(f => {
             const r = this.normalizeFilename(f.name, fullPaths);
-            if (r === js_lib_1.SKIP)
+            if (r === SKIP)
                 return [];
             return [r];
         });
@@ -93,7 +92,7 @@ class CloudStorage {
             maxResults: opt.limit || undefined,
         }).flatMap(async (f) => {
             const filePath = this.normalizeFilename(f.name, fullPaths);
-            if (filePath === js_lib_1.SKIP)
+            if (filePath === SKIP)
                 return [];
             const [content] = await f.download();
             return [{ filePath, content }];
@@ -151,8 +150,8 @@ class CloudStorage {
             .move(this.storage.bucket(toBucket || fromBucket).file(toPath));
     }
     async movePath(fromBucket, fromPrefix, toPrefix, toBucket) {
-        (0, js_lib_1._assert)(fromPrefix.endsWith('/'), 'fromPrefix should end with `/`');
-        (0, js_lib_1._assert)(toPrefix.endsWith('/'), 'toPrefix should end with `/`');
+        _assert(fromPrefix.endsWith('/'), 'fromPrefix should end with `/`');
+        _assert(toPrefix.endsWith('/'), 'toPrefix should end with `/`');
         await this.storage
             .bucket(fromBucket)
             .getFilesStream({
@@ -165,12 +164,12 @@ class CloudStorage {
         });
     }
     async deleteFiles(bucketName, filePaths) {
-        await (0, js_lib_1.pMap)(filePaths, async (filePath) => {
+        await pMap(filePaths, async (filePath) => {
             await this.storage.bucket(bucketName).file(filePath).delete();
         });
     }
     async combineFiles(bucketName, filePaths, toPath, toBucket, currentRecursionDepth = 0) {
-        (0, js_lib_1._assert)(currentRecursionDepth <= MAX_RECURSION_DEPTH, `combineFiles reached max recursion depth of ${MAX_RECURSION_DEPTH}`);
+        _assert(currentRecursionDepth <= MAX_RECURSION_DEPTH, `combineFiles reached max recursion depth of ${MAX_RECURSION_DEPTH}`);
         const { logger, debug } = this.cfg;
         if (filePaths.length === 0) {
             if (debug) {
@@ -193,7 +192,7 @@ class CloudStorage {
             return;
         }
         const started = Date.now();
-        await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(filePaths, BATCH_SIZE), async (fileBatch, i) => {
+        await pMap(_chunk(filePaths, BATCH_SIZE), async (fileBatch, i) => {
             if (debug) {
                 logger.log(`[${currentRecursionDepth}] Composing batch ${i + 1}...`);
             }
@@ -205,7 +204,7 @@ class CloudStorage {
             await this.deleteFiles(bucketName, fileBatch);
         });
         if (debug) {
-            logger.log(`[${currentRecursionDepth}] Batch composed into ${intermediateFiles.length} files, in ${(0, js_lib_1._since)(started)}`);
+            logger.log(`[${currentRecursionDepth}] Batch composed into ${intermediateFiles.length} files, in ${_since(started)}`);
         }
         await this.combineFiles(toBucket || bucketName, intermediateFiles, toPath, toBucket, currentRecursionDepth + 1);
     }
@@ -227,7 +226,7 @@ class CloudStorage {
             .getSignedUrl({
             action: 'read',
             version: 'v4',
-            expires: (0, js_lib_1.localTime)(expires).unixMillis,
+            expires: localTime(expires).unixMillis,
         });
         return url;
     }
@@ -238,11 +237,10 @@ class CloudStorage {
     normalizeFilename(fileName, fullPaths) {
         if (fullPaths) {
             if (fileName.endsWith('/'))
-                return js_lib_1.SKIP; // skip folders
+                return SKIP; // skip folders
             return fileName;
         }
-        fileName = (0, js_lib_1._substringAfterLast)(fileName, '/');
-        return fileName || js_lib_1.SKIP; // skip folders
+        fileName = _substringAfterLast(fileName, '/');
+        return fileName || SKIP; // skip folders
     }
 }
-exports.CloudStorage = CloudStorage;

package/dist/commonStorage.js

@@ -1,2 +1 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
+export {};

package/dist/commonStorageBucket.d.ts

@@ -1,6 +1,6 @@
-import { Readable, Writable } from 'node:stream';
-import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
+import type { Readable, Writable } from 'node:stream';
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage.js';
 export interface CommonStorageBucketCfg {
     storage: CommonStorage;
     bucketName: string;

package/dist/commonStorageBucket.js

@@ -1,13 +1,11 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CommonStorageBucket = void 0;
-const js_lib_1 = require("@naturalcycles/js-lib");
+import { AppError, pMap } from '@naturalcycles/js-lib';
 /**
  * Convenience wrapper around CommonStorage for a given Bucket.
  *
  * Similar to what CommonDao is to CommonDB.
  */
-class CommonStorageBucket {
+export class CommonStorageBucket {
+    cfg;
     constructor(cfg) {
         this.cfg = cfg;
     }
@@ -45,27 +43,27 @@ class CommonStorageBucket {
         return v ?? this.throwRequiredError(filePath);
     }
     throwRequiredError(filePath) {
-        throw new js_lib_1.AppError(`File required, but not found: ${this.cfg.bucketName}/${filePath}`, {
+        throw new AppError(`File required, but not found: ${this.cfg.bucketName}/${filePath}`, {
             code: 'FILE_REQUIRED',
         });
     }
     async getFileContents(paths) {
-        return (await (0, js_lib_1.pMap)(paths, async (filePath) => (await this.cfg.storage.getFile(this.cfg.bucketName, filePath)))).filter(Boolean);
+        return (await pMap(paths, async (filePath) => (await this.cfg.storage.getFile(this.cfg.bucketName, filePath)))).filter(Boolean);
     }
     async getFileContentsAsJson(paths) {
-        return (await (0, js_lib_1.pMap)(paths, async (filePath) => {
+        return (await pMap(paths, async (filePath) => {
             const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
             return buf ? JSON.parse(buf.toString()) : null;
         })).filter(Boolean);
     }
     async getFileEntries(paths) {
-        return (await (0, js_lib_1.pMap)(paths, async (filePath) => {
+        return (await pMap(paths, async (filePath) => {
             const content = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
             return { filePath, content: content };
         })).filter(f => f.content);
     }
     async getFileEntriesAsJson(paths) {
-        return (await (0, js_lib_1.pMap)(paths, async (filePath) => {
+        return (await pMap(paths, async (filePath) => {
             const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath);
             return buf ? { filePath, content: JSON.parse(buf.toString()) } : null;
         })).filter(Boolean);
@@ -89,7 +87,7 @@ class CommonStorageBucket {
         await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, Buffer.from(JSON.stringify(content)));
     }
     async saveFiles(entries) {
-        await (0, js_lib_1.pMap)(entries, async (f) => {
+        await pMap(entries, async (f) => {
             await this.cfg.storage.saveFile(this.cfg.bucketName, f.filePath, f.content);
         });
     }
@@ -100,7 +98,7 @@ class CommonStorageBucket {
         return await this.cfg.storage.deletePath(this.cfg.bucketName, prefix);
     }
     async deletePaths(prefixes) {
-        await (0, js_lib_1.pMap)(prefixes, async (prefix) => {
+        await pMap(prefixes, async (prefix) => {
             return await this.cfg.storage.deletePath(this.cfg.bucketName, prefix);
         });
     }
@@ -143,4 +141,3 @@ class CommonStorageBucket {
         await this.cfg.storage.moveFile(this.cfg.bucketName, fromPath, toPath, toBucket);
     }
 }
-exports.CommonStorageBucket = CommonStorageBucket;

package/dist/commonStorageKeyValueDB.d.ts

@@ -1,8 +1,7 @@
-import { CommonDBCreateOptions, CommonKeyValueDB } from '@naturalcycles/db-lib';
-import { IncrementTuple } from '@naturalcycles/db-lib/dist/kv/commonKeyValueDB';
-import { KeyValueTuple } from '@naturalcycles/js-lib';
-import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { CommonStorage } from './commonStorage';
+import type { CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib';
+import type { IncrementTuple } from '@naturalcycles/db-lib/dist/kv/commonKeyValueDB.js';
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import type { CommonStorage } from './commonStorage.js';
 export interface CommonStorageKeyValueDBCfg {
     storage: CommonStorage;
     bucketName: string;
@@ -29,11 +28,11 @@ export declare class CommonStorageKeyValueDB implements CommonKeyValueDB {
      */
     private getBucketAndPrefix;
     deleteByIds(table: string, ids: string[]): Promise<void>;
-    getByIds<V>(table: string, ids: string[]): Promise<KeyValueTuple<string, V>[]>;
-    saveBatch<V>(table: string, entries: KeyValueTuple<string, V>[]): Promise<void>;
+    getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
+    saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void>;
     streamIds(table: string, limit?: number): ReadableTyped<string>;
-    streamValues<V>(table: string, limit?: number): ReadableTyped<V>;
-    streamEntries<V>(table: string, limit?: number): ReadableTyped<KeyValueTuple<string, V>>;
+    streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
+    streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
     count(table: string): Promise<number>;
     incrementBatch(_table: string, _entries: IncrementTuple[]): Promise<IncrementTuple[]>;
 }

package/dist/commonStorageKeyValueDB.js

@@ -1,8 +1,5 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CommonStorageKeyValueDB = void 0;
-const db_lib_1 = require("@naturalcycles/db-lib");
-const js_lib_1 = require("@naturalcycles/js-lib");
+import { commonKeyValueDBFullSupport } from '@naturalcycles/db-lib';
+import { AppError, pMap } from '@naturalcycles/js-lib';
 /**
  * CommonKeyValueDB, backed up by a CommonStorage implementation.
  *
@@ -11,14 +8,15 @@ const js_lib_1 = require("@naturalcycles/js-lib");
  * fileName is ${id} (without extension)
  * file contents is ${v} (Buffer)
  */
-class CommonStorageKeyValueDB {
+export class CommonStorageKeyValueDB {
+    cfg;
     constructor(cfg) {
         this.cfg = cfg;
-        this.support = {
-            ...db_lib_1.commonKeyValueDBFullSupport,
-            increment: false,
-        };
     }
+    support = {
+        ...commonKeyValueDBFullSupport,
+        increment: false,
+    };
     async ping() {
         await this.cfg.storage.ping(this.cfg.bucketName);
     }
@@ -44,14 +42,14 @@ class CommonStorageKeyValueDB {
     }
     async deleteByIds(table, ids) {
         const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        await (0, js_lib_1.pMap)(ids, async (id) => {
+        await pMap(ids, async (id) => {
             await this.cfg.storage.deletePath(bucketName, [prefix, id].join('/'));
         });
     }
     async getByIds(table, ids) {
         const { bucketName, prefix } = this.getBucketAndPrefix(table);
         const map = {};
-        await (0, js_lib_1.pMap)(ids, async (id) => {
+        await pMap(ids, async (id) => {
             const buf = await this.cfg.storage.getFile(bucketName, [prefix, id].join('/'));
             if (buf)
                 map[id] = buf;
@@ -60,7 +58,7 @@ class CommonStorageKeyValueDB {
     }
     async saveBatch(table, entries) {
         const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        await (0, js_lib_1.pMap)(entries, async ([id, content]) => {
+        await pMap(entries, async ([id, content]) => {
             await this.cfg.storage.saveFile(bucketName, [prefix, id].join('/'), content);
         });
     }
@@ -83,7 +81,6 @@ class CommonStorageKeyValueDB {
         return (await this.cfg.storage.getFileNames(bucketName, { prefix })).length;
     }
     async incrementBatch(_table, _entries) {
-        throw new js_lib_1.AppError('CommonStorageKeyValueDB.incrementBatch() is not implemented');
+        throw new AppError('CommonStorageKeyValueDB.incrementBatch() is not implemented');
     }
 }
-exports.CommonStorageKeyValueDB = CommonStorageKeyValueDB;

package/dist/inMemoryCommonStorage.d.ts

@@ -1,6 +1,6 @@
-import { LocalTimeInput, StringMap } from '@naturalcycles/js-lib';
-import { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
-import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage';
+import type { LocalTimeInput, StringMap } from '@naturalcycles/js-lib';
+import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib';
+import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage.js';
 export declare class InMemoryCommonStorage implements CommonStorage {
     /**
      * data[bucketName][filePath] = Buffer

package/dist/inMemoryCommonStorage.js

@@ -1,22 +1,17 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.InMemoryCommonStorage = void 0;
-const js_lib_1 = require("@naturalcycles/js-lib");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
-class InMemoryCommonStorage {
-    constructor() {
-        /**
-         * data[bucketName][filePath] = Buffer
-         */
-        this.data = {};
-        this.publicMap = {};
-    }
+import { _assert, _isTruthy, _stringMapEntries, _substringAfterLast, localTime, } from '@naturalcycles/js-lib';
+import { fs2, md5, readableFrom } from '@naturalcycles/nodejs-lib';
+export class InMemoryCommonStorage {
+    /**
+     * data[bucketName][filePath] = Buffer
+     */
+    data = {};
+    publicMap = {};
     async ping() { }
     async getBucketNames() {
         return Object.keys(this.data);
     }
     getBucketNamesStream() {
-        return (0, nodejs_lib_1.readableFrom)(Object.keys(this.data));
+        return readableFrom(Object.keys(this.data));
     }
     async fileExists(bucketName, filePath) {
         return !!this.data[bucketName]?.[filePath];
@@ -47,35 +42,35 @@ class InMemoryCommonStorage {
         const { prefix = '', fullPaths = true } = opt;
         return Object.keys(this.data[bucketName] || {})
             .filter(filePath => filePath.startsWith(prefix))
-            .map(f => (fullPaths ? f : (0, js_lib_1._substringAfterLast)(f, '/')));
+            .map(f => (fullPaths ? f : _substringAfterLast(f, '/')));
     }
     getFileNamesStream(bucketName, opt = {}) {
         const { prefix = '', fullPaths = true } = opt;
-        return (0, nodejs_lib_1.readableFrom)(Object.keys(this.data[bucketName] || {})
+        return readableFrom(Object.keys(this.data[bucketName] || {})
             .filter(filePath => filePath.startsWith(prefix))
             .slice(0, opt.limit)
-            .map(n => (fullPaths ? n : (0, js_lib_1._substringAfterLast)(n, '/'))));
+            .map(n => (fullPaths ? n : _substringAfterLast(n, '/'))));
     }
     getFilesStream(bucketName, opt = {}) {
         const { prefix = '', fullPaths = true } = opt;
-        return (0, nodejs_lib_1.readableFrom)((0, js_lib_1._stringMapEntries)(this.data[bucketName] || {})
+        return readableFrom(_stringMapEntries(this.data[bucketName] || {})
             .map(([filePath, content]) => ({
             filePath,
             content,
         }))
             .filter(f => f.filePath.startsWith(prefix))
             .slice(0, opt.limit)
-            .map(f => (fullPaths ? f : { ...f, filePath: (0, js_lib_1._substringAfterLast)(f.filePath, '/') })));
+            .map(f => (fullPaths ? f : { ...f, filePath: _substringAfterLast(f.filePath, '/') })));
     }
     getFileReadStream(bucketName, filePath) {
-        return (0, nodejs_lib_1.readableFrom)(this.data[bucketName][filePath]);
+        return readableFrom(this.data[bucketName][filePath]);
     }
     getFileWriteStream(_bucketName, _filePath) {
         throw new Error('Method not implemented.');
     }
     async uploadFile(localFilePath, bucketName, bucketFilePath) {
         this.data[bucketName] ||= {};
-        this.data[bucketName][bucketFilePath] = await nodejs_lib_1.fs2.readBufferAsync(localFilePath);
+        this.data[bucketName][bucketFilePath] = await fs2.readBufferAsync(localFilePath);
     }
     async setFileVisibility(bucketName, filePath, isPublic) {
         this.publicMap[bucketName] ||= {};
@@ -101,7 +96,7 @@ class InMemoryCommonStorage {
         const tob = toBucket || fromBucket;
         this.data[fromBucket] ||= {};
         this.data[tob] ||= {};
-        (0, js_lib_1._stringMapEntries)(this.data[fromBucket]).forEach(([filePath, v]) => {
+        _stringMapEntries(this.data[fromBucket]).forEach(([filePath, v]) => {
             if (!filePath.startsWith(fromPrefix))
                 return;
             this.data[tob][toPrefix + filePath.slice(fromPrefix.length)] = v;
@@ -117,15 +112,14 @@ class InMemoryCommonStorage {
             return;
         const tob = toBucket || bucketName;
         this.data[tob] ||= {};
-        this.data[tob][toPath] = Buffer.concat(filePaths.map(p => this.data[bucketName][p]).filter(js_lib_1._isTruthy));
+        this.data[tob][toPath] = Buffer.concat(filePaths.map(p => this.data[bucketName][p]).filter(_isTruthy));
         // delete source files
         filePaths.forEach(p => delete this.data[bucketName][p]);
     }
     async getSignedUrl(bucketName, filePath, expires) {
         const buf = this.data[bucketName]?.[filePath];
-        (0, js_lib_1._assert)(buf, `getSignedUrl file not found: ${bucketName}/${filePath}`);
-        const signature = (0, nodejs_lib_1.md5)(buf);
-        return `https://testurl.com/${bucketName}/${filePath}?expires=${(0, js_lib_1.localTime)(expires).unix}&signature=${signature}`;
+        _assert(buf, `getSignedUrl file not found: ${bucketName}/${filePath}`);
+        const signature = md5(buf);
+        return `https://testurl.com/${bucketName}/${filePath}?expires=${localTime(expires).unix}&signature=${signature}`;
     }
 }
-exports.InMemoryCommonStorage = InMemoryCommonStorage;

package/dist/index.d.ts

@@ -1,7 +1,7 @@
-export * from './cloudStorage';
-export * from './commonStorage';
-export * from './commonStorageBucket';
-export * from './commonStorageKeyValueDB';
-export * from './inMemoryCommonStorage';
-export * from './model';
-export * from './testing/commonStorageTest';
+export * from './cloudStorage.js';
+export * from './commonStorage.js';
+export * from './commonStorageBucket.js';
+export * from './commonStorageKeyValueDB.js';
+export * from './inMemoryCommonStorage.js';
+export * from './model.js';
+export * from './testing/commonStorageTest.js';

package/dist/index.js

@@ -1,10 +1,7 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const tslib_1 = require("tslib");
-tslib_1.__exportStar(require("./cloudStorage"), exports);
-tslib_1.__exportStar(require("./commonStorage"), exports);
-tslib_1.__exportStar(require("./commonStorageBucket"), exports);
-tslib_1.__exportStar(require("./commonStorageKeyValueDB"), exports);
-tslib_1.__exportStar(require("./inMemoryCommonStorage"), exports);
-tslib_1.__exportStar(require("./model"), exports);
-tslib_1.__exportStar(require("./testing/commonStorageTest"), exports);
+export * from './cloudStorage.js';
+export * from './commonStorage.js';
+export * from './commonStorageBucket.js';
+export * from './commonStorageKeyValueDB.js';
+export * from './inMemoryCommonStorage.js';
+export * from './model.js';
+export * from './testing/commonStorageTest.js';

package/dist/model.js

@@ -1,2 +1 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
+export {};

package/dist/testing/commonStorageTest.d.ts

@@ -1,4 +1,4 @@
-import { CommonStorage } from '../commonStorage';
+import type { CommonStorage } from '../commonStorage.js';
 /**
  * This test suite must be idempotent.
  */

package/dist/testing/commonStorageTest.js

@@ -1,20 +1,18 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runCommonStorageTest = runCommonStorageTest;
-const js_lib_1 = require("@naturalcycles/js-lib");
+import { _range, _substringAfterLast, pMap } from '@naturalcycles/js-lib';
+import { expect, test } from 'vitest';
 const TEST_FOLDER = 'test/subdir';
-const TEST_ITEMS = (0, js_lib_1._range)(10).map(n => ({
+const TEST_ITEMS = _range(10).map(n => ({
     id: `id_${n + 1}`,
     n,
     even: n % 2 === 0,
 }));
-const TEST_ITEMS2 = (0, js_lib_1._range)(10).map(n => ({
+const TEST_ITEMS2 = _range(10).map(n => ({
     fileType: 2,
     id: `id_${n + 1}`,
     n,
     even: n % 2 === 0,
 }));
-const TEST_ITEMS3 = (0, js_lib_1._range)(10).map(n => ({
+const TEST_ITEMS3 = _range(10).map(n => ({
     fileType: 3,
     id: `id_${n + 1}`,
     n,
@@ -27,7 +25,7 @@ const TEST_FILES = [TEST_ITEMS, TEST_ITEMS2, TEST_ITEMS3].map((obj, i) => ({
 /**
  * This test suite must be idempotent.
  */
-function runCommonStorageTest(storage, bucketName) {
+export function runCommonStorageTest(storage, bucketName) {
     // test('createBucket', async () => {
     //   await storage.createBucket(bucketName)
     // })
@@ -44,7 +42,7 @@ function runCommonStorageTest(storage, bucketName) {
     //   console.log(buckets)
     // })
     test('prepare: clear bucket', async () => {
-        await (0, js_lib_1.pMap)(TEST_FILES.map(f => f.filePath), async (filePath) => await storage.deletePath(bucketName, filePath));
+        await pMap(TEST_FILES.map(f => f.filePath), async (filePath) => await storage.deletePath(bucketName, filePath));
     });
     // test('listFileNames on root should return empty', async () => {
     //   const fileNames = await storage.getFileNames(bucketName)
@@ -61,7 +59,7 @@ function runCommonStorageTest(storage, bucketName) {
         expect(fileNames).toEqual([]);
     });
     test(`exists should return empty array`, async () => {
-        await (0, js_lib_1.pMap)(TEST_FILES, async (f) => {
+        await pMap(TEST_FILES, async (f) => {
             const exists = await storage.fileExists(bucketName, f.filePath);
             expect(exists).toBe(false);
         });
@@ -69,12 +67,12 @@ function runCommonStorageTest(storage, bucketName) {
     test(`saveFiles, then listFileNames, streamFileNames and getFiles should return just saved files`, async () => {
         const testFilesMap = Object.fromEntries(TEST_FILES.map(f => [f.filePath, f.content]));
         // It's done in the same test to ensure "strong consistency"
-        await (0, js_lib_1.pMap)(TEST_FILES, async (f) => await storage.saveFile(bucketName, f.filePath, f.content));
+        await pMap(TEST_FILES, async (f) => await storage.saveFile(bucketName, f.filePath, f.content));
         const fileNamesShort = await storage.getFileNames(bucketName, {
             prefix: TEST_FOLDER,
             fullPaths: false,
         });
-        expect(fileNamesShort.sort()).toEqual(TEST_FILES.map(f => (0, js_lib_1._substringAfterLast)(f.filePath, '/')).sort());
+        expect(fileNamesShort.sort()).toEqual(TEST_FILES.map(f => _substringAfterLast(f.filePath, '/')).sort());
         const fileNames = await storage.getFileNames(bucketName, { prefix: TEST_FOLDER });
         expect(fileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
         const streamedFileNames = await storage
@@ -82,11 +80,11 @@ function runCommonStorageTest(storage, bucketName) {
             .toArray();
         expect(streamedFileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
         const filesMap = {};
-        await (0, js_lib_1.pMap)(fileNames, async (filePath) => {
+        await pMap(fileNames, async (filePath) => {
             filesMap[filePath] = (await storage.getFile(bucketName, filePath));
         });
         expect(filesMap).toEqual(testFilesMap);
-        await (0, js_lib_1.pMap)(fileNames, async (filePath) => {
+        await pMap(fileNames, async (filePath) => {
             const exists = await storage.fileExists(bucketName, filePath);
             expect(exists).toBe(true);
         });

package/package.json

@@ -1,5 +1,6 @@
 {
   "name": "@naturalcycles/cloud-storage-lib",
+  "type": "module",
   "scripts": {
     "prepare": "husky",
     "build": "dev-lib build",
@@ -9,16 +10,18 @@
     "lbt": "dev-lib lbt"
   },
   "dependencies": {
-    "@google-cloud/storage": "^7.0.0",
-    "@naturalcycles/db-lib": "^9.1.0",
-    "@naturalcycles/js-lib": "^14.41.0",
-    "@naturalcycles/nodejs-lib": "^13.1.0"
+    "@google-cloud/storage": "^7",
+    "@naturalcycles/db-lib": "^10",
+    "@naturalcycles/js-lib": "^14",
+    "@naturalcycles/nodejs-lib": "^13"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "^15.18.0",
-    "@types/node": "^22.7.4",
-    "firebase-admin": "^12.0.0",
-    "jest": "^29.1.2"
+    "@naturalcycles/dev-lib": "^17",
+    "@types/node": "^22",
+    "@vitest/coverage-v8": "^3",
+    "firebase-admin": "^13",
+    "tsx": "^4",
+    "vitest": "^3"
   },
   "files": [
     "dist",
@@ -38,9 +41,9 @@
     "url": "https://github.com/NaturalCycles/cloud-storage-lib"
   },
   "engines": {
-    "node": ">=20.13.0"
+    "node": ">=22.12.0"
   },
-  "version": "1.13.3",
+  "version": "2.0.0",
   "description": "CommonStorage implementation based on Google Cloud Storage",
   "author": "Natural Cycles Team",
   "license": "MIT"

package/src/cloudStorage.ts

@@ -1,21 +1,17 @@
-// eslint-disable-next-line import-x/no-duplicates
 import type { File, Storage, StorageOptions } from '@google-cloud/storage'
-// eslint-disable-next-line import-x/no-duplicates
-import type * as StorageLib from '@google-cloud/storage'
+import type { CommonLogger, LocalTimeInput, UnixTimestampMillis } from '@naturalcycles/js-lib'
 import {
   _assert,
   _chunk,
   _since,
   _substringAfterLast,
-  CommonLogger,
   localTime,
-  LocalTimeInput,
   pMap,
   SKIP,
 } from '@naturalcycles/js-lib'
 import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib'
-import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
-import type { GCPServiceAccount } from './model'
+import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage.js'
+import type { GCPServiceAccount } from './model.js'
 
 export type {
   // This is the latest version, to be imported by consumers
@@ -67,13 +63,13 @@ export class CloudStorage implements CommonStorage {
     logger: CommonLogger
   }
 
-  static createFromGCPServiceAccount(
+  static async createFromGCPServiceAccount(
     credentials?: GCPServiceAccount,
     cfg?: CloudStorageCfg,
-  ): CloudStorage {
-    const storageLib = require('@google-cloud/storage') as typeof StorageLib
+  ): Promise<CloudStorage> {
+    const { Storage } = await import('@google-cloud/storage')
 
-    const storage = new storageLib.Storage({
+    const storage = new Storage({
       credentials,
       // Explicitly passing it here to fix this error:
       // Error: Unable to detect a Project Id in the current environment.
@@ -86,12 +82,12 @@ export class CloudStorage implements CommonStorage {
     return new CloudStorage(storage, cfg)
   }
 
-  static createFromStorageOptions(
+  static async createFromStorageOptions(
     storageOptions?: StorageOptions,
     cfg?: CloudStorageCfg,
-  ): CloudStorage {
-    const storageLib = require('@google-cloud/storage') as typeof StorageLib
-    const storage = new storageLib.Storage(storageOptions)
+  ): Promise<CloudStorage> {
+    const { Storage } = await import('@google-cloud/storage')
+    const storage = new Storage(storageOptions)
     return new CloudStorage(storage, cfg)
   }
 
@@ -319,7 +315,7 @@ export class CloudStorage implements CommonStorage {
       return
     }
 
-    const started = Date.now()
+    const started = Date.now() as UnixTimestampMillis
     await pMap(_chunk(filePaths, BATCH_SIZE), async (fileBatch, i) => {
       if (debug) {
         logger.log(`[${currentRecursionDepth}] Composing batch ${i + 1}...`)

package/src/commonStorageBucket.ts

@@ -1,7 +1,7 @@
-import { Readable, Writable } from 'node:stream'
+import type { Readable, Writable } from 'node:stream'
 import { AppError, pMap } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '@naturalcycles/nodejs-lib'
-import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage.js'
 
 export interface CommonStorageBucketCfg {
   storage: CommonStorage

package/src/commonStorageKeyValueDB.ts

@@ -1,12 +1,14 @@
-import {
+import type {
   CommonDBCreateOptions,
   CommonKeyValueDB,
-  commonKeyValueDBFullSupport,
+  KeyValueDBTuple,
 } from '@naturalcycles/db-lib'
-import { IncrementTuple } from '@naturalcycles/db-lib/dist/kv/commonKeyValueDB'
-import { AppError, KeyValueTuple, pMap, StringMap } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '@naturalcycles/nodejs-lib'
-import { CommonStorage } from './commonStorage'
+import { commonKeyValueDBFullSupport } from '@naturalcycles/db-lib'
+import type { IncrementTuple } from '@naturalcycles/db-lib/dist/kv/commonKeyValueDB.js'
+import type { StringMap } from '@naturalcycles/js-lib'
+import { AppError, pMap } from '@naturalcycles/js-lib'
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import type { CommonStorage } from './commonStorage.js'
 
 export interface CommonStorageKeyValueDBCfg {
   storage: CommonStorage
@@ -64,7 +66,7 @@ export class CommonStorageKeyValueDB implements CommonKeyValueDB {
     })
   }
 
-  async getByIds<V>(table: string, ids: string[]): Promise<KeyValueTuple<string, V>[]> {
+  async getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]> {
     const { bucketName, prefix } = this.getBucketAndPrefix(table)
 
     const map: StringMap<Buffer> = {}
@@ -74,14 +76,14 @@ export class CommonStorageKeyValueDB implements CommonKeyValueDB {
       if (buf) map[id] = buf
     })
 
-    return ids.map(id => [id, map[id]] as KeyValueTuple<string, V>).filter(t => t[1])
+    return ids.map(id => [id, map[id]] as KeyValueDBTuple).filter(t => t[1])
   }
 
-  async saveBatch<V>(table: string, entries: KeyValueTuple<string, V>[]): Promise<void> {
+  async saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void> {
     const { bucketName, prefix } = this.getBucketAndPrefix(table)
 
     await pMap(entries, async ([id, content]) => {
-      await this.cfg.storage.saveFile(bucketName, [prefix, id].join('/'), content as Buffer)
+      await this.cfg.storage.saveFile(bucketName, [prefix, id].join('/'), content)
     })
   }
 
@@ -91,18 +93,18 @@ export class CommonStorageKeyValueDB implements CommonKeyValueDB {
     return this.cfg.storage.getFileNamesStream(bucketName, { prefix, limit, fullPaths: false })
   }
 
-  streamValues<V>(table: string, limit?: number): ReadableTyped<V> {
+  streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
     const { bucketName, prefix } = this.getBucketAndPrefix(table)
 
-    return this.cfg.storage.getFilesStream(bucketName, { prefix, limit }).map(f => f.content as V)
+    return this.cfg.storage.getFilesStream(bucketName, { prefix, limit }).map(f => f.content)
   }
 
-  streamEntries<V>(table: string, limit?: number): ReadableTyped<KeyValueTuple<string, V>> {
+  streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
     const { bucketName, prefix } = this.getBucketAndPrefix(table)
 
     return this.cfg.storage
       .getFilesStream(bucketName, { prefix, limit, fullPaths: false })
-      .map(f => [f.filePath, f.content as V])
+      .map(f => [f.filePath, f.content])
   }
 
   async count(table: string): Promise<number> {

package/src/inMemoryCommonStorage.ts

@@ -1,21 +1,14 @@
+import type { LocalTimeInput, StringMap } from '@naturalcycles/js-lib'
 import {
   _assert,
   _isTruthy,
   _stringMapEntries,
   _substringAfterLast,
   localTime,
-  LocalTimeInput,
-  StringMap,
 } from '@naturalcycles/js-lib'
-import {
-  fs2,
-  md5,
-  ReadableBinary,
-  readableFrom,
-  ReadableTyped,
-  WritableBinary,
-} from '@naturalcycles/nodejs-lib'
-import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
+import type { ReadableBinary, ReadableTyped, WritableBinary } from '@naturalcycles/nodejs-lib'
+import { fs2, md5, readableFrom } from '@naturalcycles/nodejs-lib'
+import type { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage.js'
 
 export class InMemoryCommonStorage implements CommonStorage {
   /**

package/src/index.ts

@@ -1,7 +1,7 @@
-export * from './cloudStorage'
-export * from './commonStorage'
-export * from './commonStorageBucket'
-export * from './commonStorageKeyValueDB'
-export * from './inMemoryCommonStorage'
-export * from './model'
-export * from './testing/commonStorageTest'
+export * from './cloudStorage.js'
+export * from './commonStorage.js'
+export * from './commonStorageBucket.js'
+export * from './commonStorageKeyValueDB.js'
+export * from './inMemoryCommonStorage.js'
+export * from './model.js'
+export * from './testing/commonStorageTest.js'

package/src/testing/commonStorageTest.ts

@@ -1,5 +1,7 @@
-import { _range, _substringAfterLast, pMap, StringMap } from '@naturalcycles/js-lib'
-import { CommonStorage, FileEntry } from '../commonStorage'
+import type { StringMap } from '@naturalcycles/js-lib'
+import { _range, _substringAfterLast, pMap } from '@naturalcycles/js-lib'
+import { expect, test } from 'vitest'
+import type { CommonStorage, FileEntry } from '../commonStorage.js'
 
 const TEST_FOLDER = 'test/subdir'