@naturalcycles/db-lib 8.54.0 → 8.54.1

package/dist/adapter/file/localFile.persistence.plugin.js

@@ -1,11 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LocalFilePersistencePlugin = void 0;
+const fs = require("node:fs");
 const node_stream_1 = require("node:stream");
+const fsp = require("node:fs/promises");
 const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
-const fs = require("fs-extra");
 /**
  * Persists in local filesystem as ndjson.
  */
@@ -19,15 +20,15 @@ class LocalFilePersistencePlugin {
     }
     async ping() { }
     async getTables() {
-        return (await fs.readdir(this.cfg.storagePath))
+        return (await fsp.readdir(this.cfg.storagePath))
             .filter(f => f.includes('.ndjson'))
             .map(f => f.split('.ndjson')[0]);
     }
     async loadFile(table) {
-        await fs.ensureDir(this.cfg.storagePath);
+        await (0, nodejs_lib_1._ensureDir)(this.cfg.storagePath);
         const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
         const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
-        if (!(await fs.pathExists(filePath)))
+        if (!(await (0, nodejs_lib_1._pathExists)(filePath)))
             return [];
         const transformUnzip = this.cfg.gzip ? [(0, node_zlib_1.createUnzip)()] : [];
         const rows = [];
@@ -44,7 +45,7 @@ class LocalFilePersistencePlugin {
         await (0, js_lib_1.pMap)(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 16 });
     }
     async saveFile(table, rows) {
-        await fs.ensureDir(this.cfg.storagePath);
+        await (0, nodejs_lib_1._ensureDir)(this.cfg.storagePath);
         const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
         const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
         const transformZip = this.cfg.gzip ? [(0, node_zlib_1.createGzip)()] : [];

package/dist/adapter/inmemory/inMemory.db.js

@@ -1,12 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryDB = void 0;
+const fs = require("node:fs");
+const fsp = require("node:fs/promises");
 const node_stream_1 = require("node:stream");
 const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
-const fs = require("fs-extra");
 const __1 = require("../..");
 const dbQuery_1 = require("../../query/dbQuery");
 class InMemoryDB {
@@ -166,7 +167,7 @@ class InMemoryDB {
         (0, js_lib_1._assert)(this.cfg.persistenceEnabled, 'flushToDisk() called but persistenceEnabled=false');
         const { persistentStoragePath, persistZip } = this.cfg;
         const started = Date.now();
-        await fs.emptyDir(persistentStoragePath);
+        await (0, nodejs_lib_1._emptyDir)(persistentStoragePath);
         const transformZip = persistZip ? [(0, node_zlib_1.createGzip)()] : [];
         let tables = 0;
         // infinite concurrency for now
@@ -192,9 +193,9 @@ class InMemoryDB {
         (0, js_lib_1._assert)(this.cfg.persistenceEnabled, 'restoreFromDisk() called but persistenceEnabled=false');
         const { persistentStoragePath } = this.cfg;
         const started = Date.now();
-        await fs.ensureDir(persistentStoragePath);
+        await (0, nodejs_lib_1._ensureDir)(persistentStoragePath);
         this.data = {}; // empty it in the beginning!
-        const files = (await fs.readdir(persistentStoragePath)).filter(f => f.includes('.ndjson'));
+        const files = (await fsp.readdir(persistentStoragePath)).filter(f => f.includes('.ndjson'));
         // infinite concurrency for now
         await (0, js_lib_1.pMap)(files, async (file) => {
             const fname = `${persistentStoragePath}/${file}`;

package/dist/pipeline/dbPipelineBackup.js

@@ -1,11 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dbPipelineBackup = void 0;
+const fs = require("node:fs");
+const fsp = require("node:fs/promises");
 const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
-const fs = require("fs-extra");
 const index_1 = require("../index");
 /**
  * Pipeline from input stream(s) to a NDJSON file (optionally gzipped).
@@ -23,7 +24,7 @@ async function dbPipelineBackup(opt) {
     let { tables } = opt;
     const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, colors_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
     console.log(`>> ${(0, colors_1.dimWhite)('dbPipelineBackup')} started in ${(0, colors_1.grey)(outputDirPath)}...${sinceUpdatedStr}`);
-    fs.ensureDirSync(outputDirPath);
+    (0, nodejs_lib_1._ensureDirSync)(outputDirPath);
     tables ||= await db.getTables();
     console.log(`${(0, colors_1.yellow)(tables.length)} ${(0, colors_1.boldWhite)('table(s)')}:\n` + tables.join('\n'));
     const statsPerTable = {};
@@ -34,16 +35,16 @@ async function dbPipelineBackup(opt) {
         }
         const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '');
         const schemaFilePath = `${outputDirPath}/${table}.schema.json`;
-        if (protectFromOverwrite && (await fs.pathExists(filePath))) {
+        if (protectFromOverwrite && (0, nodejs_lib_1._pathExistsSync)(filePath)) {
             throw new js_lib_1.AppError(`dbPipelineBackup: output file exists: ${filePath}`);
         }
         const started = Date.now();
         let rows = 0;
-        await fs.ensureFile(filePath);
+        (0, nodejs_lib_1._ensureFileSync)(filePath);
         console.log(`>> ${(0, colors_1.grey)(filePath)} started...`);
         if (emitSchemaFromDB) {
             const schema = await db.getTableSchema(table);
-            await fs.writeJson(schemaFilePath, schema, { spaces: 2 });
+            await (0, nodejs_lib_1._writeJsonFile)(schemaFilePath, schema, { spaces: 2 });
             console.log(`>> ${(0, colors_1.grey)(schemaFilePath)} saved (generated from DB)`);
         }
         await (0, nodejs_lib_1._pipeline)([
@@ -66,7 +67,7 @@ async function dbPipelineBackup(opt) {
             ...(gzip ? [(0, node_zlib_1.createGzip)(zlibOptions)] : []),
             fs.createWriteStream(filePath),
         ]);
-        const { size: sizeBytes } = await fs.stat(filePath);
+        const { size: sizeBytes } = await fsp.stat(filePath);
         const stats = nodejs_lib_1.NDJsonStats.create({
             tookMillis: Date.now() - started,
             rows,

package/dist/pipeline/dbPipelineRestore.js

@@ -1,11 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dbPipelineRestore = void 0;
+const fs = require("node:fs");
 const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
-const fs = require("fs-extra");
 /**
  * Pipeline from NDJSON files in a folder (optionally gzipped) to CommonDB.
  * Allows to define a mapper and a predicate to map/filter objects between input and output.
@@ -19,7 +19,7 @@ async function dbPipelineRestore(opt) {
     const onlyTables = opt.tables && new Set(opt.tables);
     const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, colors_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
     console.log(`>> ${(0, colors_1.dimWhite)('dbPipelineRestore')} started in ${(0, colors_1.grey)(inputDirPath)}...${sinceUpdatedStr}`);
-    fs.ensureDirSync(inputDirPath);
+    (0, nodejs_lib_1._ensureDirSync)(inputDirPath);
     const tablesToGzip = new Set();
     const sizeByTable = {};
     const statsPerTable = {};
@@ -54,7 +54,7 @@ async function dbPipelineRestore(opt) {
                 console.warn(`${schemaFilePath} does not exist!`);
                 return;
             }
-            const schema = await fs.readJson(schemaFilePath);
+            const schema = await (0, nodejs_lib_1._readJsonFile)(schemaFilePath);
             await db.createTable(table, schema, { dropIfExists: true });
         });
     }

package/package.json

@@ -5,13 +5,12 @@
   },
   "dependencies": {
     "@naturalcycles/js-lib": "^14.116.0",
-    "@naturalcycles/nodejs-lib": "^12.0.0",
-    "fs-extra": "^11.1.0"
+    "@naturalcycles/nodejs-lib": "^12.0.0"
   },
   "devDependencies": {
     "@naturalcycles/bench-lib": "^1.0.0",
     "@naturalcycles/dev-lib": "^13.0.0",
-    "@types/node": "^18.0.3",
+    "@types/node": "^20.2.1",
     "jest": "^29.0.0"
   },
   "files": [
@@ -41,7 +40,7 @@
   "engines": {
     "node": ">=18.12"
   },
-  "version": "8.54.0",
+  "version": "8.54.1",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/adapter/file/localFile.persistence.plugin.ts

@@ -1,4 +1,6 @@
+import * as fs from 'node:fs'
 import { Readable } from 'node:stream'
+import * as fsp from 'node:fs/promises'
 import { createGzip, createUnzip } from 'node:zlib'
 import { pMap, ObjectWithId } from '@naturalcycles/js-lib'
 import {
@@ -7,8 +9,9 @@ import {
   transformToNDJson,
   writablePushToArray,
   _pipeline,
+  _ensureDir,
+  _pathExists,
 } from '@naturalcycles/nodejs-lib'
-import * as fs from 'fs-extra'
 import { DBSaveBatchOperation } from '../../db.model'
 import { FileDBPersistencePlugin } from './file.db.model'
 
@@ -41,17 +44,17 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
   async ping(): Promise<void> {}
 
   async getTables(): Promise<string[]> {
-    return (await fs.readdir(this.cfg.storagePath))
+    return (await fsp.readdir(this.cfg.storagePath))
       .filter(f => f.includes('.ndjson'))
       .map(f => f.split('.ndjson')[0]!)
   }
 
   async loadFile<ROW extends ObjectWithId>(table: string): Promise<ROW[]> {
-    await fs.ensureDir(this.cfg.storagePath)
+    await _ensureDir(this.cfg.storagePath)
     const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
     const filePath = `${this.cfg.storagePath}/${table}.${ext}`
 
-    if (!(await fs.pathExists(filePath))) return []
+    if (!(await _pathExists(filePath))) return []
 
     const transformUnzip = this.cfg.gzip ? [createUnzip()] : []
 
@@ -73,7 +76,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
   }
 
   async saveFile<ROW extends ObjectWithId>(table: string, rows: ROW[]): Promise<void> {
-    await fs.ensureDir(this.cfg.storagePath)
+    await _ensureDir(this.cfg.storagePath)
     const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
     const filePath = `${this.cfg.storagePath}/${table}.${ext}`
     const transformZip = this.cfg.gzip ? [createGzip()] : []

package/src/adapter/inmemory/inMemory.db.ts

@@ -1,3 +1,5 @@
+import * as fs from 'node:fs'
+import * as fsp from 'node:fs/promises'
 import { Readable } from 'node:stream'
 import { createGzip, createUnzip } from 'node:zlib'
 import {
@@ -23,9 +25,10 @@ import {
   transformToNDJson,
   writablePushToArray,
   _pipeline,
+  _emptyDir,
+  _ensureDir,
 } from '@naturalcycles/nodejs-lib'
 import { dimGrey, yellow } from '@naturalcycles/nodejs-lib/dist/colors'
-import * as fs from 'fs-extra'
 import { CommonDB, DBIncrement, DBPatch, DBTransaction, queryInMemory } from '../..'
 import {
   CommonDBCreateOptions,
@@ -279,7 +282,7 @@ export class InMemoryDB implements CommonDB {
 
     const started = Date.now()
 
-    await fs.emptyDir(persistentStoragePath)
+    await _emptyDir(persistentStoragePath)
 
     const transformZip = persistZip ? [createGzip()] : []
     let tables = 0
@@ -314,11 +317,11 @@ export class InMemoryDB implements CommonDB {
 
     const started = Date.now()
 
-    await fs.ensureDir(persistentStoragePath)
+    await _ensureDir(persistentStoragePath)
 
     this.data = {} // empty it in the beginning!
 
-    const files = (await fs.readdir(persistentStoragePath)).filter(f => f.includes('.ndjson'))
+    const files = (await fsp.readdir(persistentStoragePath)).filter(f => f.includes('.ndjson'))
 
     // infinite concurrency for now
     await pMap(files, async file => {

package/src/pipeline/dbPipelineBackup.ts

@@ -1,3 +1,5 @@
+import * as fs from 'node:fs'
+import * as fsp from 'node:fs/promises'
 import { createGzip, ZlibOptions } from 'node:zlib'
 import {
   AppError,
@@ -16,9 +18,12 @@ import {
   transformTap,
   transformToNDJson,
   _pipeline,
+  _ensureDirSync,
+  _pathExistsSync,
+  _ensureFileSync,
+  _writeJsonFile,
 } from '@naturalcycles/nodejs-lib'
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/dist/colors'
-import * as fs from 'fs-extra'
 import { CommonDB } from '../common.db'
 import { DBQuery } from '../index'
 
@@ -156,7 +161,7 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
     `>> ${dimWhite('dbPipelineBackup')} started in ${grey(outputDirPath)}...${sinceUpdatedStr}`,
   )
 
-  fs.ensureDirSync(outputDirPath)
+  _ensureDirSync(outputDirPath)
 
   tables ||= await db.getTables()
 
@@ -176,20 +181,20 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
       const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '')
       const schemaFilePath = `${outputDirPath}/${table}.schema.json`
 
-      if (protectFromOverwrite && (await fs.pathExists(filePath))) {
+      if (protectFromOverwrite && _pathExistsSync(filePath)) {
         throw new AppError(`dbPipelineBackup: output file exists: ${filePath}`)
       }
 
       const started = Date.now()
       let rows = 0
 
-      await fs.ensureFile(filePath)
+      _ensureFileSync(filePath)
 
       console.log(`>> ${grey(filePath)} started...`)
 
       if (emitSchemaFromDB) {
         const schema = await db.getTableSchema(table)
-        await fs.writeJson(schemaFilePath, schema, { spaces: 2 })
+        await _writeJsonFile(schemaFilePath, schema, { spaces: 2 })
         console.log(`>> ${grey(schemaFilePath)} saved (generated from DB)`)
       }
 
@@ -214,7 +219,7 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
         fs.createWriteStream(filePath),
       ])
 
-      const { size: sizeBytes } = await fs.stat(filePath)
+      const { size: sizeBytes } = await fsp.stat(filePath)
 
       const stats = NDJsonStats.create({
         tookMillis: Date.now() - started,

package/src/pipeline/dbPipelineRestore.ts

@@ -1,3 +1,4 @@
+import * as fs from 'node:fs'
 import { createUnzip } from 'node:zlib'
 import {
   AsyncMapper,
@@ -8,6 +9,7 @@ import {
   _passthroughMapper,
   SavedDBEntity,
   localTime,
+  JsonSchemaObject,
 } from '@naturalcycles/js-lib'
 import {
   NDJsonStats,
@@ -23,9 +25,10 @@ import {
   transformTap,
   writableForEach,
   _pipeline,
+  _ensureDirSync,
+  _readJsonFile,
 } from '@naturalcycles/nodejs-lib'
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/dist/colors'
-import * as fs from 'fs-extra'
 import { CommonDB } from '../common.db'
 import { CommonDBSaveOptions } from '../index'
 
@@ -139,7 +142,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
     `>> ${dimWhite('dbPipelineRestore')} started in ${grey(inputDirPath)}...${sinceUpdatedStr}`,
   )
 
-  fs.ensureDirSync(inputDirPath)
+  _ensureDirSync(inputDirPath)
 
   const tablesToGzip = new Set<string>()
   const sizeByTable: Record<string, number> = {}
@@ -179,7 +182,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
         return
       }
 
-      const schema = await fs.readJson(schemaFilePath)
+      const schema = await _readJsonFile<JsonSchemaObject<any>>(schemaFilePath)
       await db.createTable(table, schema, { dropIfExists: true })
     })
   }