@naturalcycles/cloud-storage-lib 1.9.0 → 1.9.2

package/dist/cloudStorage.js

@@ -4,7 +4,6 @@ exports.CloudStorage = exports.Storage = void 0;
 const storage_1 = require("@google-cloud/storage");
 Object.defineProperty(exports, "Storage", { enumerable: true, get: function () { return storage_1.Storage; } });
 const js_lib_1 = require("@naturalcycles/js-lib");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 /**
  * CloudStorage implementation of CommonStorage API.
  *
@@ -68,29 +67,30 @@ class CloudStorage {
     }
     getFileNamesStream(bucketName, opt = {}) {
         const { prefix, fullPaths = true } = opt;
-        return this.storage
-            .bucket(bucketName)
-            .getFilesStream({
+        return this.storage.bucket(bucketName).getFilesStream({
             prefix,
             maxResults: opt.limit || undefined,
-        })
-            .pipe((0, nodejs_lib_1.transformMapSync)(f => this.normalizeFilename(f.name, fullPaths)));
+        }).flatMap(f => {
+            const r = this.normalizeFilename(f.name, fullPaths);
+            if (r === js_lib_1.SKIP)
+                return [];
+            return [r];
+        });
     }
     getFilesStream(bucketName, opt = {}) {
         const { prefix, fullPaths = true } = opt;
-        return this.storage
-            .bucket(bucketName)
-            .getFilesStream({
+        return this.storage.bucket(bucketName).getFilesStream({
             prefix,
             maxResults: opt.limit || undefined,
-        })
-            .pipe((0, nodejs_lib_1.transformMap)(async (f) => {
+        }).flatMap(async (f) => {
             const filePath = this.normalizeFilename(f.name, fullPaths);
             if (filePath === js_lib_1.SKIP)
-                return js_lib_1.SKIP;
+                return [];
             const [content] = await f.download();
-            return { filePath, content };
-        }));
+            return [{ filePath, content }];
+        }, {
+            concurrency: 16,
+        });
     }
     async getFile(bucketName, filePath) {
         const [buf] = await this.storage
@@ -144,16 +144,16 @@ class CloudStorage {
     async movePath(fromBucket, fromPrefix, toPrefix, toBucket) {
         (0, js_lib_1._assert)(fromPrefix.endsWith('/'), 'fromPrefix should end with `/`');
         (0, js_lib_1._assert)(toPrefix.endsWith('/'), 'toPrefix should end with `/`');
-        await (0, nodejs_lib_1._pipeline)([
-            this.storage.bucket(fromBucket).getFilesStream({
-                prefix: fromPrefix,
-            }),
-            (0, nodejs_lib_1.writableForEach)(async (file) => {
-                const { name } = file;
-                const newName = toPrefix + name.slice(fromPrefix.length);
-                await file.move(this.storage.bucket(toBucket || fromBucket).file(newName));
-            }),
-        ]);
+        await this.storage
+            .bucket(fromBucket)
+            .getFilesStream({
+            prefix: fromPrefix,
+        })
+            .forEach(async (file) => {
+            const { name } = file;
+            const newName = toPrefix + name.slice(fromPrefix.length);
+            await file.move(this.storage.bucket(toBucket || fromBucket).file(newName));
+        });
     }
     async combine(bucketName, filePaths, toPath, toBucket) {
         // todo: if (filePaths.length > 32) - use recursive algorithm

package/dist/commonStorageKeyValueDB.js

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CommonStorageKeyValueDB = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 /**
  * CommonKeyValueDB, backed up by a CommonStorage implementation.
  *
@@ -66,18 +65,13 @@ class CommonStorageKeyValueDB {
     }
     streamValues(table, limit) {
         const { bucketName, prefix } = this.getBucketAndPrefix(table);
-        return this.cfg.storage
-            .getFilesStream(bucketName, { prefix, limit })
-            .pipe((0, nodejs_lib_1.transformMapSimple)(f => f.content));
+        return this.cfg.storage.getFilesStream(bucketName, { prefix, limit }).map(f => f.content);
     }
     streamEntries(table, limit) {
         const { bucketName, prefix } = this.getBucketAndPrefix(table);
         return this.cfg.storage
             .getFilesStream(bucketName, { prefix, limit, fullPaths: false })
-            .pipe((0, nodejs_lib_1.transformMapSimple)(({ filePath, content }) => [
-            filePath,
-            content,
-        ]));
+            .map(f => [f.filePath, f.content]);
     }
     async count(table) {
         const { bucketName, prefix } = this.getBucketAndPrefix(table);

package/dist/testing/commonStorageTest.js

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runCommonStorageTest = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const TEST_FOLDER = 'test/subdir';
 const TEST_ITEMS = (0, js_lib_1._range)(10).map(n => ({
     id: `id_${n + 1}`,
@@ -56,7 +55,9 @@ function runCommonStorageTest(storage, bucketName) {
         expect(fileNames).toEqual([]);
     });
     test(`streamFileNames on ${TEST_FOLDER} should return empty`, async () => {
-        const fileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName, { prefix: TEST_FOLDER }));
+        const fileNames = await storage
+            .getFileNamesStream(bucketName, { prefix: TEST_FOLDER })
+            .toArray();
         expect(fileNames).toEqual([]);
     });
     test(`exists should return empty array`, async () => {
@@ -76,7 +77,9 @@ function runCommonStorageTest(storage, bucketName) {
         expect(fileNamesShort.sort()).toEqual(TEST_FILES.map(f => (0, js_lib_1._substringAfterLast)(f.filePath, '/')).sort());
         const fileNames = await storage.getFileNames(bucketName, { prefix: TEST_FOLDER });
         expect(fileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
-        const streamedFileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName, { prefix: TEST_FOLDER }));
+        const streamedFileNames = await storage
+            .getFileNamesStream(bucketName, { prefix: TEST_FOLDER })
+            .toArray();
         expect(streamedFileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
         const filesMap = {};
         await (0, js_lib_1.pMap)(fileNames, async (filePath) => {

package/package.json

@@ -35,7 +35,7 @@
   "engines": {
     "node": ">=18.12.0"
   },
-  "version": "1.9.0",
+  "version": "1.9.2",
   "description": "CommonStorage implementation based on Google Cloud Storage",
   "author": "Natural Cycles Team",
   "license": "MIT"

package/src/cloudStorage.ts

@@ -8,13 +8,7 @@ import {
   pMap,
   SKIP,
 } from '@naturalcycles/js-lib'
-import {
-  _pipeline,
-  ReadableTyped,
-  transformMap,
-  transformMapSync,
-  writableForEach,
-} from '@naturalcycles/nodejs-lib'
+import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
 import { GCPServiceAccount } from './model'
 
@@ -114,33 +108,38 @@ export class CloudStorage implements CommonStorage {
   getFileNamesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<string> {
     const { prefix, fullPaths = true } = opt
 
-    return this.storage
-      .bucket(bucketName)
-      .getFilesStream({
+    return (
+      this.storage.bucket(bucketName).getFilesStream({
         prefix,
         maxResults: opt.limit || undefined,
-      })
-      .pipe(transformMapSync<File, string>(f => this.normalizeFilename(f.name, fullPaths)))
+      }) as ReadableTyped<File>
+    ).flatMap(f => {
+      const r = this.normalizeFilename(f.name, fullPaths)
+      if (r === SKIP) return []
+      return [r]
+    })
   }
 
   getFilesStream(bucketName: string, opt: CommonStorageGetOptions = {}): ReadableTyped<FileEntry> {
     const { prefix, fullPaths = true } = opt
 
-    return this.storage
-      .bucket(bucketName)
-      .getFilesStream({
+    return (
+      this.storage.bucket(bucketName).getFilesStream({
         prefix,
         maxResults: opt.limit || undefined,
-      })
-      .pipe(
-        transformMap<File, FileEntry>(async f => {
-          const filePath = this.normalizeFilename(f.name, fullPaths)
-          if (filePath === SKIP) return SKIP
-
-          const [content] = await f.download()
-          return { filePath, content }
-        }),
-      )
+      }) as ReadableTyped<File>
+    ).flatMap(
+      async f => {
+        const filePath = this.normalizeFilename(f.name, fullPaths)
+        if (filePath === SKIP) return []
+
+        const [content] = await f.download()
+        return [{ filePath, content }] as FileEntry[]
+      },
+      {
+        concurrency: 16,
+      },
+    )
   }
 
   async getFile(bucketName: string, filePath: string): Promise<Buffer | null> {
@@ -224,16 +223,16 @@ export class CloudStorage implements CommonStorage {
     _assert(fromPrefix.endsWith('/'), 'fromPrefix should end with `/`')
     _assert(toPrefix.endsWith('/'), 'toPrefix should end with `/`')
 
-    await _pipeline([
-      this.storage.bucket(fromBucket).getFilesStream({
+    await this.storage
+      .bucket(fromBucket)
+      .getFilesStream({
         prefix: fromPrefix,
-      }),
-      writableForEach<File>(async file => {
+      })
+      .forEach(async file => {
         const { name } = file
         const newName = toPrefix + name.slice(fromPrefix.length)
         await file.move(this.storage.bucket(toBucket || fromBucket).file(newName))
-      }),
-    ])
+      })
   }
 
   async combine(

package/src/commonStorageKeyValueDB.ts

@@ -1,7 +1,7 @@
 import { CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib'
 import { pMap, StringMap } from '@naturalcycles/js-lib'
-import { ReadableTyped, transformMapSimple } from '@naturalcycles/nodejs-lib'
-import { CommonStorage, FileEntry } from './commonStorage'
+import { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import { CommonStorage } from './commonStorage'
 
 export interface CommonStorageKeyValueDBCfg {
   storage: CommonStorage
@@ -84,9 +84,7 @@ export class CommonStorageKeyValueDB implements CommonKeyValueDB {
   streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
     const { bucketName, prefix } = this.getBucketAndPrefix(table)
 
-    return this.cfg.storage
-      .getFilesStream(bucketName, { prefix, limit })
-      .pipe(transformMapSimple<FileEntry, Buffer>(f => f.content))
+    return this.cfg.storage.getFilesStream(bucketName, { prefix, limit }).map(f => f.content)
   }
 
   streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
@@ -94,12 +92,7 @@ export class CommonStorageKeyValueDB implements CommonKeyValueDB {
 
     return this.cfg.storage
       .getFilesStream(bucketName, { prefix, limit, fullPaths: false })
-      .pipe(
-        transformMapSimple<FileEntry, KeyValueDBTuple>(({ filePath, content }) => [
-          filePath,
-          content,
-        ]),
-      )
+      .map(f => [f.filePath, f.content] satisfies KeyValueDBTuple)
   }
 
   async count(table: string): Promise<number> {

package/src/testing/commonStorageTest.ts

@@ -1,5 +1,4 @@
 import { _range, _substringAfterLast, pMap, StringMap } from '@naturalcycles/js-lib'
-import { readableToArray } from '@naturalcycles/nodejs-lib'
 import { CommonStorage, FileEntry } from '../commonStorage'
 
 const TEST_FOLDER = 'test/subdir'
@@ -69,9 +68,9 @@ export function runCommonStorageTest(storage: CommonStorage, bucketName: string)
   })
 
   test(`streamFileNames on ${TEST_FOLDER} should return empty`, async () => {
-    const fileNames = await readableToArray(
-      storage.getFileNamesStream(bucketName, { prefix: TEST_FOLDER }),
-    )
+    const fileNames = await storage
+      .getFileNamesStream(bucketName, { prefix: TEST_FOLDER })
+      .toArray()
     expect(fileNames).toEqual([])
   })
 
@@ -99,9 +98,9 @@ export function runCommonStorageTest(storage: CommonStorage, bucketName: string)
     const fileNames = await storage.getFileNames(bucketName, { prefix: TEST_FOLDER })
     expect(fileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort())
 
-    const streamedFileNames = await readableToArray(
-      storage.getFileNamesStream(bucketName, { prefix: TEST_FOLDER }),
-    )
+    const streamedFileNames = await storage
+      .getFileNamesStream(bucketName, { prefix: TEST_FOLDER })
+      .toArray()
     expect(streamedFileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort())
 
     const filesMap: StringMap<Buffer> = {}