@naturalcycles/db-lib 10.5.0 → 10.6.0

package/dist/adapter/file/file.db.js

@@ -1,5 +1,5 @@
 import { _assert, _by, _deepEquals, _filterUndefinedValues, _since, _sortBy, _sortObjectDeep, _stringMapValues, generateJsonSchemaFromData, localTime, } from '@naturalcycles/js-lib';
-import { dimGrey } from '@naturalcycles/nodejs-lib';
+import { dimGrey } from '@naturalcycles/nodejs-lib/colors';
 import { readableCreate } from '@naturalcycles/nodejs-lib/stream';
 import { BaseCommonDB } from '../../base.common.db.js';
 import { commonDBFullSupport } from '../../common.db.js';

package/dist/adapter/file/localFile.persistence.plugin.js

@@ -1,7 +1,7 @@
 import { Readable } from 'node:stream';
 import { pMap } from '@naturalcycles/js-lib';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs';
-import { _pipeline } from '@naturalcycles/nodejs-lib/stream';
+import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
+import { _pipeline, createReadStreamAsNDJSON, createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
 /**
  * Persists in local filesystem as ndjson.
  */
@@ -26,7 +26,7 @@ export class LocalFilePersistencePlugin {
         const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
         if (!(await fs2.pathExistsAsync(filePath)))
             return [];
-        return await fs2.createReadStreamAsNDJSON(filePath).toArray();
+        return await createReadStreamAsNDJSON(filePath).toArray();
     }
     async saveFiles(ops) {
         await pMap(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 32 });
@@ -35,6 +35,6 @@ export class LocalFilePersistencePlugin {
         await fs2.ensureDirAsync(this.cfg.storagePath);
         const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
         const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
-        await _pipeline([Readable.from(rows), ...fs2.createWriteStreamAsNDJSON(filePath)]);
+        await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(filePath)]);
     }
 }

package/dist/adapter/inmemory/inMemory.db.js

@@ -1,8 +1,8 @@
 import { Readable } from 'node:stream';
 import { _assert, _by, _deepCopy, _isEmptyObject, _since, _sortObjectDeep, _stringMapEntries, _stringMapValues, generateJsonSchemaFromData, localTime, pMap, } from '@naturalcycles/js-lib';
-import { dimGrey, yellow } from '@naturalcycles/nodejs-lib';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs';
-import { _pipeline, bufferReviver } from '@naturalcycles/nodejs-lib/stream';
+import { dimGrey, yellow } from '@naturalcycles/nodejs-lib/colors';
+import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
+import { _pipeline, bufferReviver, createReadStreamAsNDJSON, createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
 import { commonDBFullSupport, CommonDBType } from '../../common.db.js';
 import { queryInMemory } from './queryInMemory.js';
 export class InMemoryDB {
@@ -183,7 +183,7 @@ export class InMemoryDB {
                 return; // 0 rows
             tables++;
             const fname = `${persistentStoragePath}/${table}.ndjson${persistZip ? '.gz' : ''}`;
-            await _pipeline([Readable.from(rows), ...fs2.createWriteStreamAsNDJSON(fname)]);
+            await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(fname)]);
         });
         this.cfg.logger.log(`flushToDisk took ${dimGrey(_since(started))} to save ${yellow(tables)} tables`);
     }
@@ -201,7 +201,7 @@ export class InMemoryDB {
         await pMap(files, async (file) => {
             const fname = `${persistentStoragePath}/${file}`;
             const table = file.split('.ndjson')[0];
-            const rows = await fs2.createReadStreamAsNDJSON(fname).toArray();
+            const rows = await createReadStreamAsNDJSON(fname).toArray();
             this.data[table] = _by(rows, r => r.id);
         });
         this.cfg.logger.log(`restoreFromDisk took ${dimGrey(_since(started))} to read ${yellow(files.length)} tables`);

package/dist/kv/commonKeyValueDao.js

@@ -1,5 +1,5 @@
 import { AppError, pMap } from '@naturalcycles/js-lib';
-import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib';
+import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib/zip';
 export const commonKeyValueDaoDeflatedJsonTransformer = {
     valueToBuffer: async (v) => await deflateString(JSON.stringify(v)),
     bufferToValue: async (buf) => JSON.parse(await inflateToString(buf)),

package/dist/pipeline/dbPipelineBackup.d.ts

@@ -1,6 +1,6 @@
 import type { AsyncMapper, StringMap, UnixTimestamp } from '@naturalcycles/js-lib';
 import { ErrorMode } from '@naturalcycles/js-lib';
-import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
+import { type TransformLogProgressOptions, type TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDB } from '../common.db.js';
 import { DBQuery } from '../query/dbQuery.js';

package/dist/pipeline/dbPipelineBackup.js

@@ -1,6 +1,7 @@
 import { _passthroughMapper, AppError, ErrorMode, localTime, pMap } from '@naturalcycles/js-lib';
-import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs';
+import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
+import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
+import { createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
 import { _pipeline, NDJsonStats, transformLogProgress, transformMap, transformTap, } from '@naturalcycles/nodejs-lib/stream';
 import { DBQuery } from '../query/dbQuery.js';
 /**
@@ -68,7 +69,7 @@ export async function dbPipelineBackup(opt) {
             transformTap(() => {
                 rows++;
             }),
-            ...fs2.createWriteStreamAsNDJSON(filePath),
+            ...createWriteStreamAsNDJSON(filePath),
         ]);
         const { size: sizeBytes } = await fs2.statAsync(filePath);
         const stats = NDJsonStats.create({

package/dist/pipeline/dbPipelineCopy.js

@@ -1,5 +1,5 @@
 import { _passthroughMapper, ErrorMode, localTime, pMap } from '@naturalcycles/js-lib';
-import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib';
+import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
 import { _pipeline, NDJsonStats, transformChunk, transformLogProgress, transformMap, transformTap, writableForEach, } from '@naturalcycles/nodejs-lib/stream';
 import { DBQuery } from '../query/dbQuery.js';
 /**

package/dist/pipeline/dbPipelineRestore.d.ts

@@ -1,6 +1,6 @@
 import type { AsyncMapper, UnixTimestamp } from '@naturalcycles/js-lib';
 import { ErrorMode } from '@naturalcycles/js-lib';
-import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
+import { type TransformLogProgressOptions, type TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDB } from '../common.db.js';
 import type { CommonDBSaveOptions } from '../db.model.js';

package/dist/pipeline/dbPipelineRestore.js

@@ -1,6 +1,7 @@
 import { _hb, _mapValues, _passthroughMapper, ErrorMode, localTime, pMap, } from '@naturalcycles/js-lib';
-import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs';
+import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
+import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
+import { createReadStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
 import { _pipeline, NDJsonStats, transformChunk, transformFilterSync, transformLogProgress, transformMap, transformTap, writableForEach, } from '@naturalcycles/nodejs-lib/stream';
 /**
  * Pipeline from NDJSON files in a folder (optionally gzipped) to CommonDB.
@@ -62,7 +63,7 @@ export async function dbPipelineRestore(opt) {
         const sizeBytes = sizeByTable[table];
         console.log(`<< ${grey(filePath)} ${dimWhite(_hb(sizeBytes))} started...`);
         await _pipeline([
-            fs2.createReadStreamAsNDJSON(filePath).take(limit || Number.POSITIVE_INFINITY),
+            createReadStreamAsNDJSON(filePath).take(limit || Number.POSITIVE_INFINITY),
             transformTap(() => rows++),
             transformLogProgress({
                 logEvery: 1000,

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@naturalcycles/db-lib",
   "type": "module",
-  "version": "10.5.0",
+  "version": "10.6.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@naturalcycles/nodejs-lib": "^15"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "19.5.0"
+    "@naturalcycles/dev-lib": "19.6.0"
   },
   "files": [
     "dist",

package/src/adapter/file/file.db.ts

@@ -11,7 +11,7 @@ import {
   generateJsonSchemaFromData,
   localTime,
 } from '@naturalcycles/js-lib'
-import { dimGrey } from '@naturalcycles/nodejs-lib'
+import { dimGrey } from '@naturalcycles/nodejs-lib/colors'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
 import { readableCreate } from '@naturalcycles/nodejs-lib/stream'
 import { BaseCommonDB } from '../../base.common.db.js'

package/src/adapter/file/localFile.persistence.plugin.ts

@@ -1,8 +1,12 @@
 import { Readable } from 'node:stream'
 import type { ObjectWithId } from '@naturalcycles/js-lib'
 import { pMap } from '@naturalcycles/js-lib'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs'
-import { _pipeline } from '@naturalcycles/nodejs-lib/stream'
+import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
+import {
+  _pipeline,
+  createReadStreamAsNDJSON,
+  createWriteStreamAsNDJSON,
+} from '@naturalcycles/nodejs-lib/stream'
 import type { DBSaveBatchOperation } from '../../db.model.js'
 import type { FileDBPersistencePlugin } from './file.db.model.js'
 
@@ -47,7 +51,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
 
     if (!(await fs2.pathExistsAsync(filePath))) return []
 
-    return await fs2.createReadStreamAsNDJSON(filePath).toArray()
+    return await createReadStreamAsNDJSON(filePath).toArray()
   }
 
   async saveFiles(ops: DBSaveBatchOperation<any>[]): Promise<void> {
@@ -59,6 +63,6 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
     const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
     const filePath = `${this.cfg.storagePath}/${table}.${ext}`
 
-    await _pipeline([Readable.from(rows), ...fs2.createWriteStreamAsNDJSON(filePath)])
+    await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(filePath)])
   }
 }

package/src/adapter/inmemory/inMemory.db.ts

@@ -20,9 +20,15 @@ import {
   localTime,
   pMap,
 } from '@naturalcycles/js-lib'
-import { dimGrey, yellow } from '@naturalcycles/nodejs-lib'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs'
-import { _pipeline, bufferReviver, type ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { dimGrey, yellow } from '@naturalcycles/nodejs-lib/colors'
+import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
+import {
+  _pipeline,
+  bufferReviver,
+  createReadStreamAsNDJSON,
+  createWriteStreamAsNDJSON,
+  type ReadableTyped,
+} from '@naturalcycles/nodejs-lib/stream'
 import type { CommonDB, CommonDBSupport } from '../../common.db.js'
 import { commonDBFullSupport, CommonDBType } from '../../common.db.js'
 import type {
@@ -329,7 +335,7 @@ export class InMemoryDB implements CommonDB {
       tables++
       const fname = `${persistentStoragePath}/${table}.ndjson${persistZip ? '.gz' : ''}`
 
-      await _pipeline([Readable.from(rows), ...fs2.createWriteStreamAsNDJSON(fname)])
+      await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(fname)])
     })
 
     this.cfg.logger!.log(
@@ -357,7 +363,7 @@ export class InMemoryDB implements CommonDB {
       const fname = `${persistentStoragePath}/${file}`
       const table = file.split('.ndjson')[0]!
 
-      const rows = await fs2.createReadStreamAsNDJSON(fname).toArray()
+      const rows = await createReadStreamAsNDJSON(fname).toArray()
 
       this.data[table] = _by(rows, r => r.id)
     })

package/src/kv/commonKeyValueDao.ts

@@ -1,7 +1,7 @@
 import type { CommonLogger, KeyValueTuple } from '@naturalcycles/js-lib'
 import { AppError, pMap } from '@naturalcycles/js-lib'
-import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib/zip'
 import type { CommonDaoLogLevel } from '../commondao/common.dao.model.js'
 import type { CommonDBCreateOptions } from '../db.model.js'
 import type {

package/src/pipeline/dbPipelineBackup.ts

@@ -1,10 +1,11 @@
 import type { AsyncMapper, StringMap, UnixTimestamp } from '@naturalcycles/js-lib'
 import { _passthroughMapper, AppError, ErrorMode, localTime, pMap } from '@naturalcycles/js-lib'
-import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs'
-import type {
-  TransformLogProgressOptions,
-  TransformMapOptions,
+import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
+import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
+import {
+  createWriteStreamAsNDJSON,
+  type TransformLogProgressOptions,
+  type TransformMapOptions,
 } from '@naturalcycles/nodejs-lib/stream'
 import {
   _pipeline,
@@ -223,7 +224,7 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
         transformTap(() => {
           rows++
         }),
-        ...fs2.createWriteStreamAsNDJSON(filePath),
+        ...createWriteStreamAsNDJSON(filePath),
       ])
 
       const { size: sizeBytes } = await fs2.statAsync(filePath)

package/src/pipeline/dbPipelineCopy.ts

@@ -1,6 +1,6 @@
 import type { AsyncMapper, BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib'
 import { _passthroughMapper, ErrorMode, localTime, pMap } from '@naturalcycles/js-lib'
-import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
+import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
 import type {
   TransformLogProgressOptions,
   TransformMapOptions,

package/src/pipeline/dbPipelineRestore.ts

@@ -12,11 +12,12 @@ import {
   localTime,
   pMap,
 } from '@naturalcycles/js-lib'
-import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs'
-import type {
-  TransformLogProgressOptions,
-  TransformMapOptions,
+import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
+import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
+import {
+  createReadStreamAsNDJSON,
+  type TransformLogProgressOptions,
+  type TransformMapOptions,
 } from '@naturalcycles/nodejs-lib/stream'
 import {
   _pipeline,
@@ -200,7 +201,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
       console.log(`<< ${grey(filePath)} ${dimWhite(_hb(sizeBytes))} started...`)
 
       await _pipeline([
-        fs2.createReadStreamAsNDJSON(filePath).take(limit || Number.POSITIVE_INFINITY),
+        createReadStreamAsNDJSON(filePath).take(limit || Number.POSITIVE_INFINITY),
         transformTap(() => rows++),
         transformLogProgress({
           logEvery: 1000,