@naturalcycles/db-lib 10.7.0 → 10.9.0

package/dist/adapter/file/file.db.js

@@ -1,6 +1,6 @@
-import { _by, _sortBy } from '@naturalcycles/js-lib';
+import { _by, _sortBy } from '@naturalcycles/js-lib/array';
 import { _since, localTime } from '@naturalcycles/js-lib/datetime';
-import { _assert } from '@naturalcycles/js-lib/error';
+import { _assert } from '@naturalcycles/js-lib/error/assert.js';
 import { generateJsonSchemaFromData } from '@naturalcycles/js-lib/json-schema';
 import { _deepEquals, _filterUndefinedValues, _sortObjectDeep } from '@naturalcycles/js-lib/object';
 import { _stringMapValues, } from '@naturalcycles/js-lib/types';

package/dist/adapter/file/inMemory.persistence.plugin.js

@@ -1,4 +1,4 @@
-import { _by } from '@naturalcycles/js-lib';
+import { _by } from '@naturalcycles/js-lib/array/array.util.js';
 /**
  * Mostly useful for testing.
  */

package/dist/adapter/file/localFile.persistence.plugin.js

@@ -1,5 +1,5 @@
 import { Readable } from 'node:stream';
-import { pMap } from '@naturalcycles/js-lib/promise';
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
 import { _pipeline, createReadStreamAsNDJSON, createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
 /**

package/dist/commondao/common.dao.js

@@ -1,9 +1,12 @@
-import { _isTruthy, _uniqBy } from '@naturalcycles/js-lib';
-import { _since, localTime } from '@naturalcycles/js-lib/datetime';
+import { _isTruthy } from '@naturalcycles/js-lib';
+import { _uniqBy } from '@naturalcycles/js-lib/array/array.util.js';
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
+import { _since } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { _assert, AppError, ErrorMode } from '@naturalcycles/js-lib/error';
-import { _deepCopy, _deepJsonEquals, _filterUndefinedValues, _objectAssignExact, } from '@naturalcycles/js-lib/object';
-import { pMap } from '@naturalcycles/js-lib/promise';
-import { _truncate } from '@naturalcycles/js-lib/string';
+import { _deepJsonEquals } from '@naturalcycles/js-lib/object/deepEquals.js';
+import { _deepCopy, _filterUndefinedValues, _objectAssignExact, } from '@naturalcycles/js-lib/object/object.util.js';
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
+import { _truncate } from '@naturalcycles/js-lib/string/string.util.js';
 import { _passthroughPredicate, _typeCast, SKIP } from '@naturalcycles/js-lib/types';
 import { ZodType, zSafeValidate } from '@naturalcycles/js-lib/zod';
 import { stringId } from '@naturalcycles/nodejs-lib';
@@ -962,7 +965,14 @@ export class CommonDao {
         const objectName = table;
         let error;
         let convertedValue;
-        if (schema instanceof ZodType) {
+        if (this.cfg.validateBM) {
+            const [err, value] = this.cfg.validateBM(obj, {
+                itemName: table,
+            });
+            error = err;
+            convertedValue = value;
+        }
+        else if (schema instanceof ZodType) {
             // Zod schema
             const vr = zSafeValidate(obj, schema);
             error = vr.error;
@@ -977,14 +987,7 @@ export class CommonDao {
         }
         else {
             // Joi
-            const start = localTime.nowUnixMillis();
             const vr = getValidationResult(obj, schema, objectName);
-            const tookMillis = localTime.nowUnixMillis() - start;
-            this.cfg.onValidationTime?.({
-                tookMillis,
-                table,
-                obj,
-            });
             error = vr.error;
             convertedValue = vr.value;
         }

package/dist/commondao/common.dao.model.d.ts

@@ -1,3 +1,4 @@
+import type { ValidationFunction } from '@naturalcycles/js-lib';
 import type { ErrorMode } from '@naturalcycles/js-lib/error';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import type { BaseDBEntity, NumberOfMilliseconds, Promisable, UnixTimestamp } from '@naturalcycles/js-lib/types';
@@ -105,6 +106,15 @@ export interface CommonDaoCfg<BM extends BaseDBEntity, DBM extends BaseDBEntity
      * Joi, AjvSchema or ZodSchema is supported.
      */
     bmSchema?: ObjectSchema<BM> | AjvSchema<BM> | ZodType<BM>;
+    /**
+     * Experimental alternative to bmSchema.
+     * "Bring your own validation function".
+     * It removes the knowledge from CommonDao about the validation library used
+     * and abstracts it away.
+     *
+     * @experimental
+     */
+    validateBM?: ValidationFunction<BM, any>;
     excludeFromIndexes?: (keyof DBM)[];
     /**
      * Defaults to true.
@@ -116,12 +126,6 @@ export interface CommonDaoCfg<BM extends BaseDBEntity, DBM extends BaseDBEntity
      * If set to false - save (write) operations will skip validation (and conversion).
      */
     validateOnSave?: boolean;
-    /**
-     * The hook allows to get a callback, to instrument "validation time".
-     *
-     * @experimental
-     */
-    onValidationTime?: (data: OnValidationTimeData) => void;
     /**
      * Defaults to false.
      * Setting it to true will set saveMethod to `insert` for save/saveBatch, which will

package/dist/inmemory/inMemory.db.d.ts

@@ -1,7 +1,7 @@
 import { type JsonSchemaObject, type JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import { type AnyObjectWithId, type ObjectWithId, type StringMap } from '@naturalcycles/js-lib/types';
-import { type ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream/stream.model.js';
 import type { CommonDB, CommonDBSupport } from '../commondb/common.db.js';
 import { CommonDBType } from '../commondb/common.db.js';
 import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBTransactionOptions, DBOperation, DBTransaction, DBTransactionFn, RunQueryResult } from '../db.model.js';
@@ -25,24 +25,6 @@ export interface InMemoryDBCfg {
      * Defaults to true.
      */
     forbidTransactionReadAfterWrite?: boolean;
-    /**
-     * @default false
-     *
-     * Set to true to enable disk persistence (!).
-     */
-    persistenceEnabled: boolean;
-    /**
-     * @default ./tmp/inmemorydb.ndjson.gz
-     *
-     * Will store one ndjson file per table.
-     * Will only flush on demand (see .flushToDisk() and .restoreFromDisk() methods).
-     * Even if persistence is enabled - nothing is flushed or restored automatically.
-     */
-    persistentStoragePath: string;
-    /**
-     * @default true
-     */
-    persistZip: boolean;
     /**
      * Defaults to `console`.
      */
@@ -78,14 +60,6 @@ export declare class InMemoryDB implements CommonDB {
     runInTransaction(fn: DBTransactionFn, opt?: CommonDBTransactionOptions): Promise<void>;
     createTransaction(opt?: CommonDBTransactionOptions): Promise<DBTransaction>;
     incrementBatch(table: string, prop: string, incrementMap: StringMap<number>, _opt?: CommonDBOptions): Promise<StringMap<number>>;
-    /**
-     * Flushes all tables (all namespaces) at once.
-     */
-    flushToDisk(): Promise<void>;
-    /**
-     * Restores all tables (all namespaces) at once.
-     */
-    restoreFromDisk(): Promise<void>;
 }
 export declare class InMemoryDBTransaction implements DBTransaction {
     private db;

package/dist/inmemory/inMemory.db.js

@@ -1,14 +1,10 @@
 import { Readable } from 'node:stream';
-import { _by, _isEmptyObject } from '@naturalcycles/js-lib';
-import { _since, localTime } from '@naturalcycles/js-lib/datetime';
-import { _assert } from '@naturalcycles/js-lib/error';
+import { _isEmptyObject } from '@naturalcycles/js-lib';
+import { _assert } from '@naturalcycles/js-lib/error/assert.js';
 import { generateJsonSchemaFromData, } from '@naturalcycles/js-lib/json-schema';
 import { _deepCopy, _sortObjectDeep } from '@naturalcycles/js-lib/object';
-import { pMap } from '@naturalcycles/js-lib/promise';
 import { _stringMapEntries, _stringMapValues, } from '@naturalcycles/js-lib/types';
-import { dimGrey, yellow } from '@naturalcycles/nodejs-lib/colors';
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { _pipeline, bufferReviver, createReadStreamAsNDJSON, createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
+import { bufferReviver } from '@naturalcycles/nodejs-lib/stream/ndjson/transformJsonParse.js';
 import { commonDBFullSupport, CommonDBType } from '../commondb/common.db.js';
 import { queryInMemory } from './queryInMemory.js';
 export class InMemoryDB {
@@ -22,9 +18,6 @@ export class InMemoryDB {
             // defaults
             tablesPrefix: '',
             forbidTransactionReadAfterWrite: true,
-            persistenceEnabled: false,
-            persistZip: true,
-            persistentStoragePath: './tmp/inmemorydb',
             logger: console,
             ...cfg,
         };
@@ -173,45 +166,6 @@ export class InMemoryDB {
         }
         return result;
     }
-    /**
-     * Flushes all tables (all namespaces) at once.
-     */
-    async flushToDisk() {
-        _assert(this.cfg.persistenceEnabled, 'flushToDisk() called but persistenceEnabled=false');
-        const { persistentStoragePath, persistZip } = this.cfg;
-        const started = localTime.nowUnixMillis();
-        await fs2.emptyDirAsync(persistentStoragePath);
-        let tables = 0;
-        // infinite concurrency for now
-        await pMap(Object.keys(this.data), async (table) => {
-            const rows = Object.values(this.data[table]);
-            if (rows.length === 0)
-                return; // 0 rows
-            tables++;
-            const fname = `${persistentStoragePath}/${table}.ndjson${persistZip ? '.gz' : ''}`;
-            await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(fname)]);
-        });
-        this.cfg.logger.log(`flushToDisk took ${dimGrey(_since(started))} to save ${yellow(tables)} tables`);
-    }
-    /**
-     * Restores all tables (all namespaces) at once.
-     */
-    async restoreFromDisk() {
-        _assert(this.cfg.persistenceEnabled, 'restoreFromDisk() called but persistenceEnabled=false');
-        const { persistentStoragePath } = this.cfg;
-        const started = localTime.nowUnixMillis();
-        await fs2.ensureDirAsync(persistentStoragePath);
-        this.data = {}; // empty it in the beginning!
-        const files = (await fs2.readdirAsync(persistentStoragePath)).filter(f => f.includes('.ndjson'));
-        // infinite concurrency for now
-        await pMap(files, async (file) => {
-            const fname = `${persistentStoragePath}/${file}`;
-            const table = file.split('.ndjson')[0];
-            const rows = await createReadStreamAsNDJSON(fname).toArray();
-            this.data[table] = _by(rows, r => r.id);
-        });
-        this.cfg.logger.log(`restoreFromDisk took ${dimGrey(_since(started))} to read ${yellow(files.length)} tables`);
-    }
 }
 export class InMemoryDBTransaction {
     db;

package/dist/inmemory/queryInMemory.js

@@ -1,4 +1,4 @@
-import { _get, _pick } from '@naturalcycles/js-lib/object';
+import { _get, _pick } from '@naturalcycles/js-lib/object/object.util.js';
 const FILTER_FNS = {
     '==': (v, val) => v === val,
     '!=': (v, val) => v !== val,

package/dist/kv/commonKeyValueDao.js

@@ -1,5 +1,5 @@
-import { AppError } from '@naturalcycles/js-lib/error';
-import { pMap } from '@naturalcycles/js-lib/promise';
+import { AppError } from '@naturalcycles/js-lib/error/error.util.js';
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib/zip';
 export const commonKeyValueDaoDeflatedJsonTransformer = {
     valueToBuffer: async (v) => await deflateString(JSON.stringify(v)),

package/dist/kv/commonKeyValueDaoMemoCache.js

@@ -1,4 +1,4 @@
-import { localTime } from '@naturalcycles/js-lib/datetime';
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { MISS } from '@naturalcycles/js-lib/types';
 /**
  * AsyncMemoCache implementation, backed by CommonKeyValueDao.

package/dist/model.util.js

@@ -1,4 +1,4 @@
-import { localTime } from '@naturalcycles/js-lib/datetime';
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { stringId } from '@naturalcycles/nodejs-lib';
 export function createdUpdatedFields(existingObject) {
     const now = localTime.nowUnix();

package/dist/pipeline/dbPipelineBackup.js

@@ -1,6 +1,6 @@
-import { localTime } from '@naturalcycles/js-lib/datetime';
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { AppError, ErrorMode } from '@naturalcycles/js-lib/error';
-import { pMap } from '@naturalcycles/js-lib/promise';
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2';

package/dist/pipeline/dbPipelineCopy.d.ts

@@ -1,4 +1,4 @@
-import { ErrorMode } from '@naturalcycles/js-lib/error';
+import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
 import type { AsyncMapper, UnixTimestamp } from '@naturalcycles/js-lib/types';
 import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';

package/dist/pipeline/dbPipelineCopy.js

@@ -1,6 +1,6 @@
-import { localTime } from '@naturalcycles/js-lib/datetime';
-import { ErrorMode } from '@naturalcycles/js-lib/error';
-import { pMap } from '@naturalcycles/js-lib/promise';
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
+import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
 import { _pipeline, NDJsonStats, transformChunk, transformLogProgress, transformMap, transformTap, writableForEach, } from '@naturalcycles/nodejs-lib/stream';

package/dist/pipeline/dbPipelineRestore.d.ts

@@ -1,4 +1,4 @@
-import { ErrorMode } from '@naturalcycles/js-lib/error';
+import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
 import type { AsyncMapper, UnixTimestamp } from '@naturalcycles/js-lib/types';
 import { type TransformLogProgressOptions, type TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';

package/dist/pipeline/dbPipelineRestore.js

@@ -1,8 +1,8 @@
 import { _hb } from '@naturalcycles/js-lib';
-import { localTime } from '@naturalcycles/js-lib/datetime';
-import { ErrorMode } from '@naturalcycles/js-lib/error';
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
+import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
 import { _mapValues } from '@naturalcycles/js-lib/object';
-import { pMap } from '@naturalcycles/js-lib/promise';
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2';

package/dist/query/dbQuery.js

@@ -1,4 +1,4 @@
-import { _truncate } from '@naturalcycles/js-lib/string';
+import { _truncate } from '@naturalcycles/js-lib/string/string.util.js';
 import { _objectAssign } from '@naturalcycles/js-lib/types';
 export const dbQueryFilterOperatorValues = [
     '<',

package/dist/testing/commonDBTest.js

@@ -1,7 +1,7 @@
-import { _sortBy } from '@naturalcycles/js-lib';
-import { localTime } from '@naturalcycles/js-lib/datetime';
+import { _sortBy } from '@naturalcycles/js-lib/array/array.util.js';
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { _deepFreeze, _filterObject, _pick } from '@naturalcycles/js-lib/object';
-import { pMap } from '@naturalcycles/js-lib/promise';
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { CommonDBType } from '../commondb/common.db.js';
 import { DBQuery } from '../query/dbQuery.js';
 import { createTestItemDBM, createTestItemsDBM, TEST_TABLE, testItemBMJsonSchema, } from './test.model.js';

package/dist/testing/commonDaoTest.js

@@ -1,6 +1,6 @@
 import { Readable } from 'node:stream';
-import { _sortBy } from '@naturalcycles/js-lib';
-import { localTime } from '@naturalcycles/js-lib/datetime';
+import { _sortBy } from '@naturalcycles/js-lib/array/array.util.js';
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { _deepCopy, _filterObject, _omit, _pick } from '@naturalcycles/js-lib/object';
 import { _pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { CommonDao } from '../commondao/common.dao.js';

package/dist/testing/keyValueDBTest.js

@@ -1,4 +1,4 @@
-import { _range, _sortBy } from '@naturalcycles/js-lib';
+import { _range, _sortBy } from '@naturalcycles/js-lib/array';
 import { TEST_TABLE } from './test.model.js';
 const testIds = _range(1, 4).map(n => `id${n}`);
 const testEntries = testIds.map(id => [

package/dist/testing/keyValueDaoTest.js

@@ -1,4 +1,4 @@
-import { _sortBy } from '@naturalcycles/js-lib';
+import { _sortBy } from '@naturalcycles/js-lib/array/array.util.js';
 import { CommonKeyValueDao } from '../kv/commonKeyValueDao.js';
 import { createTestItemsBM, TEST_TABLE } from './test.model.js';
 const testItems = createTestItemsBM(4);

package/dist/testing/test.model.js

@@ -1,4 +1,4 @@
-import { _range } from '@naturalcycles/js-lib';
+import { _range } from '@naturalcycles/js-lib/array/range.js';
 import { jsonSchema } from '@naturalcycles/js-lib/json-schema';
 import { baseDBEntitySchema, binarySchema, booleanSchema, numberSchema, objectSchema, stringSchema, } from '@naturalcycles/nodejs-lib/joi';
 const MOCK_TS_2018_06_21 = 1529539200;

package/dist/testing/timeSeriesTest.util.js

@@ -1,4 +1,5 @@
-import { _randomInt, _range } from '@naturalcycles/js-lib';
+import { _randomInt } from '@naturalcycles/js-lib';
+import { _range } from '@naturalcycles/js-lib/array/range.js';
 export function createTestTimeSeries(count = 10) {
     const ts = Date.now();
     return _range(1, count + 1).map(i => [ts - i * 60_000, _randomInt(10, 20)]);

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@naturalcycles/db-lib",
   "type": "module",
-  "version": "10.7.0",
+  "version": "10.9.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@naturalcycles/nodejs-lib": "^15"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "19.8.0"
+    "@naturalcycles/dev-lib": "19.11.0"
   },
   "files": [
     "dist",
@@ -28,7 +28,8 @@
     "./dao": "./dist/commondao/index.js",
     "./cachedb": "./dist/adapter/cachedb/index.js",
     "./file": "./dist/adapter/file/index.js",
-    "./inmemory": "./dist/inmemory/index.js"
+    "./inmemory": "./dist/inmemory/index.js",
+    "./inmemory/*": "./dist/inmemory/*.js"
   },
   "publishConfig": {
     "access": "public"

package/src/adapter/file/file.db.ts

@@ -1,6 +1,6 @@
-import { _by, _sortBy } from '@naturalcycles/js-lib'
+import { _by, _sortBy } from '@naturalcycles/js-lib/array'
 import { _since, localTime } from '@naturalcycles/js-lib/datetime'
-import { _assert } from '@naturalcycles/js-lib/error'
+import { _assert } from '@naturalcycles/js-lib/error/assert.js'
 import type { JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema'
 import { generateJsonSchemaFromData } from '@naturalcycles/js-lib/json-schema'
 import { _deepEquals, _filterUndefinedValues, _sortObjectDeep } from '@naturalcycles/js-lib/object'

package/src/adapter/file/inMemory.persistence.plugin.ts

@@ -1,4 +1,4 @@
-import { _by } from '@naturalcycles/js-lib'
+import { _by } from '@naturalcycles/js-lib/array/array.util.js'
 import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
 import type { DBSaveBatchOperation } from '../../db.model.js'
 import type { FileDBPersistencePlugin } from './file.db.model.js'

package/src/adapter/file/localFile.persistence.plugin.ts

@@ -1,5 +1,5 @@
 import { Readable } from 'node:stream'
-import { pMap } from '@naturalcycles/js-lib/promise'
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { ObjectWithId } from '@naturalcycles/js-lib/types'
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
 import {

package/src/commondao/common.dao.model.ts

@@ -1,3 +1,4 @@
+import type { ValidationFunction } from '@naturalcycles/js-lib'
 import type { ErrorMode } from '@naturalcycles/js-lib/error'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import type {
@@ -132,6 +133,16 @@ export interface CommonDaoCfg<
    */
   bmSchema?: ObjectSchema<BM> | AjvSchema<BM> | ZodType<BM>
 
+  /**
+   * Experimental alternative to bmSchema.
+   * "Bring your own validation function".
+   * It removes the knowledge from CommonDao about the validation library used
+   * and abstracts it away.
+   *
+   * @experimental
+   */
+  validateBM?: ValidationFunction<BM, any>
+
   excludeFromIndexes?: (keyof DBM)[]
 
   /**
@@ -146,13 +157,6 @@ export interface CommonDaoCfg<
    */
   validateOnSave?: boolean
 
-  /**
-   * The hook allows to get a callback, to instrument "validation time".
-   *
-   * @experimental
-   */
-  onValidationTime?: (data: OnValidationTimeData) => void
-
   /**
    * Defaults to false.
    * Setting it to true will set saveMethod to `insert` for save/saveBatch, which will

package/src/commondao/common.dao.ts

@@ -1,17 +1,19 @@
 import type { Transform } from 'node:stream'
-import { _isTruthy, _uniqBy } from '@naturalcycles/js-lib'
-import { _since, localTime } from '@naturalcycles/js-lib/datetime'
+import { _isTruthy } from '@naturalcycles/js-lib'
+import { _uniqBy } from '@naturalcycles/js-lib/array/array.util.js'
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
+import { _since } from '@naturalcycles/js-lib/datetime/time.util.js'
 import { _assert, AppError, ErrorMode } from '@naturalcycles/js-lib/error'
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
+import { _deepJsonEquals } from '@naturalcycles/js-lib/object/deepEquals.js'
 import {
   _deepCopy,
-  _deepJsonEquals,
   _filterUndefinedValues,
   _objectAssignExact,
-} from '@naturalcycles/js-lib/object'
-import { pMap } from '@naturalcycles/js-lib/promise'
-import { _truncate } from '@naturalcycles/js-lib/string'
+} from '@naturalcycles/js-lib/object/object.util.js'
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
+import { _truncate } from '@naturalcycles/js-lib/string/string.util.js'
 import type {
   AsyncIndexedMapper,
   BaseDBEntity,
@@ -1240,7 +1242,13 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM, I
     let error: JoiValidationError | AjvValidationError | ZodValidationError | undefined
     let convertedValue: any
 
-    if (schema instanceof ZodType) {
+    if (this.cfg.validateBM) {
+      const [err, value] = this.cfg.validateBM(obj as any as BM, {
+        itemName: table,
+      })
+      error = err
+      convertedValue = value
+    } else if (schema instanceof ZodType) {
       // Zod schema
       const vr = zSafeValidate(obj as T, schema)
       error = vr.error
@@ -1254,16 +1262,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM, I
       })
     } else {
       // Joi
-      const start = localTime.nowUnixMillis()
       const vr = getValidationResult(obj, schema, objectName)
-      const tookMillis = localTime.nowUnixMillis() - start
-
-      this.cfg.onValidationTime?.({
-        tookMillis,
-        table,
-        obj,
-      })
-
       error = vr.error
       convertedValue = vr.value
     }

package/src/inmemory/inMemory.db.ts

@@ -1,7 +1,6 @@
 import { Readable } from 'node:stream'
-import { _by, _isEmptyObject } from '@naturalcycles/js-lib'
-import { _since, localTime } from '@naturalcycles/js-lib/datetime'
-import { _assert } from '@naturalcycles/js-lib/error'
+import { _isEmptyObject } from '@naturalcycles/js-lib'
+import { _assert } from '@naturalcycles/js-lib/error/assert.js'
 import {
   generateJsonSchemaFromData,
   type JsonSchemaObject,
@@ -9,7 +8,6 @@ import {
 } from '@naturalcycles/js-lib/json-schema'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { _deepCopy, _sortObjectDeep } from '@naturalcycles/js-lib/object'
-import { pMap } from '@naturalcycles/js-lib/promise'
 import {
   _stringMapEntries,
   _stringMapValues,
@@ -17,15 +15,8 @@ import {
   type ObjectWithId,
   type StringMap,
 } from '@naturalcycles/js-lib/types'
-import { dimGrey, yellow } from '@naturalcycles/nodejs-lib/colors'
-import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import {
-  _pipeline,
-  bufferReviver,
-  createReadStreamAsNDJSON,
-  createWriteStreamAsNDJSON,
-  type ReadableTyped,
-} from '@naturalcycles/nodejs-lib/stream'
+import { bufferReviver } from '@naturalcycles/nodejs-lib/stream/ndjson/transformJsonParse.js'
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream/stream.model.js'
 import type { CommonDB, CommonDBSupport } from '../commondb/common.db.js'
 import { commonDBFullSupport, CommonDBType } from '../commondb/common.db.js'
 import type {
@@ -62,27 +53,6 @@ export interface InMemoryDBCfg {
    */
   forbidTransactionReadAfterWrite?: boolean
 
-  /**
-   * @default false
-   *
-   * Set to true to enable disk persistence (!).
-   */
-  persistenceEnabled: boolean
-
-  /**
-   * @default ./tmp/inmemorydb.ndjson.gz
-   *
-   * Will store one ndjson file per table.
-   * Will only flush on demand (see .flushToDisk() and .restoreFromDisk() methods).
-   * Even if persistence is enabled - nothing is flushed or restored automatically.
-   */
-  persistentStoragePath: string
-
-  /**
-   * @default true
-   */
-  persistZip: boolean
-
   /**
    * Defaults to `console`.
    */
@@ -102,9 +72,6 @@ export class InMemoryDB implements CommonDB {
       // defaults
       tablesPrefix: '',
       forbidTransactionReadAfterWrite: true,
-      persistenceEnabled: false,
-      persistZip: true,
-      persistentStoragePath: './tmp/inmemorydb',
       logger: console,
       ...cfg,
     }
@@ -310,65 +277,6 @@ export class InMemoryDB implements CommonDB {
 
     return result
   }
-
-  /**
-   * Flushes all tables (all namespaces) at once.
-   */
-  async flushToDisk(): Promise<void> {
-    _assert(this.cfg.persistenceEnabled, 'flushToDisk() called but persistenceEnabled=false')
-    const { persistentStoragePath, persistZip } = this.cfg
-
-    const started = localTime.nowUnixMillis()
-
-    await fs2.emptyDirAsync(persistentStoragePath)
-
-    let tables = 0
-
-    // infinite concurrency for now
-    await pMap(Object.keys(this.data), async table => {
-      const rows = Object.values(this.data[table]!)
-      if (rows.length === 0) return // 0 rows
-
-      tables++
-      const fname = `${persistentStoragePath}/${table}.ndjson${persistZip ? '.gz' : ''}`
-
-      await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(fname)])
-    })
-
-    this.cfg.logger!.log(
-      `flushToDisk took ${dimGrey(_since(started))} to save ${yellow(tables)} tables`,
-    )
-  }
-
-  /**
-   * Restores all tables (all namespaces) at once.
-   */
-  async restoreFromDisk(): Promise<void> {
-    _assert(this.cfg.persistenceEnabled, 'restoreFromDisk() called but persistenceEnabled=false')
-    const { persistentStoragePath } = this.cfg
-
-    const started = localTime.nowUnixMillis()
-
-    await fs2.ensureDirAsync(persistentStoragePath)
-
-    this.data = {} // empty it in the beginning!
-
-    const files = (await fs2.readdirAsync(persistentStoragePath)).filter(f => f.includes('.ndjson'))
-
-    // infinite concurrency for now
-    await pMap(files, async file => {
-      const fname = `${persistentStoragePath}/${file}`
-      const table = file.split('.ndjson')[0]!
-
-      const rows = await createReadStreamAsNDJSON(fname).toArray()
-
-      this.data[table] = _by(rows, r => r.id)
-    })
-
-    this.cfg.logger!.log(
-      `restoreFromDisk took ${dimGrey(_since(started))} to read ${yellow(files.length)} tables`,
-    )
-  }
 }
 
 export class InMemoryDBTransaction implements DBTransaction {

package/src/inmemory/queryInMemory.ts

@@ -1,4 +1,4 @@
-import { _get, _pick } from '@naturalcycles/js-lib/object'
+import { _get, _pick } from '@naturalcycles/js-lib/object/object.util.js'
 import type { ObjectWithId } from '@naturalcycles/js-lib/types'
 import type { DBQuery, DBQueryFilterOperator } from '../query/dbQuery.js'
 

package/src/kv/commonKeyValueDao.ts

@@ -1,6 +1,6 @@
-import { AppError } from '@naturalcycles/js-lib/error'
+import { AppError } from '@naturalcycles/js-lib/error/error.util.js'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import { pMap } from '@naturalcycles/js-lib/promise'
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { KeyValueTuple } from '@naturalcycles/js-lib/types'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
 import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib/zip'

package/src/kv/commonKeyValueDaoMemoCache.ts

@@ -1,4 +1,4 @@
-import { localTime } from '@naturalcycles/js-lib/datetime'
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import type { AsyncMemoCache } from '@naturalcycles/js-lib/decorators'
 import type { NumberOfSeconds, UnixTimestamp } from '@naturalcycles/js-lib/types'
 import { MISS } from '@naturalcycles/js-lib/types'

package/src/model.util.ts

@@ -1,4 +1,4 @@
-import { localTime } from '@naturalcycles/js-lib/datetime'
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import type { CreatedUpdated, CreatedUpdatedId } from '@naturalcycles/js-lib/types'
 import { stringId } from '@naturalcycles/nodejs-lib'
 

package/src/pipeline/dbPipelineBackup.ts

@@ -1,6 +1,6 @@
-import { localTime } from '@naturalcycles/js-lib/datetime'
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { AppError, ErrorMode } from '@naturalcycles/js-lib/error'
-import { pMap } from '@naturalcycles/js-lib/promise'
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { AsyncMapper, StringMap, UnixTimestamp } from '@naturalcycles/js-lib/types'
 import { _passthroughMapper } from '@naturalcycles/js-lib/types'
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'

package/src/pipeline/dbPipelineCopy.ts

@@ -1,6 +1,6 @@
-import { localTime } from '@naturalcycles/js-lib/datetime'
-import { ErrorMode } from '@naturalcycles/js-lib/error'
-import { pMap } from '@naturalcycles/js-lib/promise'
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
+import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js'
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { AsyncMapper, BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types'
 import { _passthroughMapper } from '@naturalcycles/js-lib/types'
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'

package/src/pipeline/dbPipelineRestore.ts

@@ -1,9 +1,9 @@
 import { _hb } from '@naturalcycles/js-lib'
-import { localTime } from '@naturalcycles/js-lib/datetime'
-import { ErrorMode } from '@naturalcycles/js-lib/error'
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
+import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js'
 import type { JsonSchemaObject } from '@naturalcycles/js-lib/json-schema'
 import { _mapValues } from '@naturalcycles/js-lib/object'
-import { pMap } from '@naturalcycles/js-lib/promise'
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { AsyncMapper, BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types'
 import { _passthroughMapper } from '@naturalcycles/js-lib/types'
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'

package/src/query/dbQuery.ts

@@ -1,4 +1,4 @@
-import { _truncate } from '@naturalcycles/js-lib/string'
+import { _truncate } from '@naturalcycles/js-lib/string/string.util.js'
 import type { AsyncIndexedMapper, BaseDBEntity, ObjectWithId } from '@naturalcycles/js-lib/types'
 import { _objectAssign } from '@naturalcycles/js-lib/types'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'

package/src/testing/commonDBTest.ts

@@ -1,7 +1,7 @@
-import { _sortBy } from '@naturalcycles/js-lib'
-import { localTime } from '@naturalcycles/js-lib/datetime'
+import { _sortBy } from '@naturalcycles/js-lib/array/array.util.js'
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { _deepFreeze, _filterObject, _pick } from '@naturalcycles/js-lib/object'
-import { pMap } from '@naturalcycles/js-lib/promise'
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { CommonDB } from '../commondb/common.db.js'
 import { CommonDBType } from '../commondb/common.db.js'
 import { DBQuery } from '../query/dbQuery.js'

package/src/testing/commonDaoTest.ts

@@ -1,6 +1,6 @@
 import { Readable } from 'node:stream'
-import { _sortBy } from '@naturalcycles/js-lib'
-import { localTime } from '@naturalcycles/js-lib/datetime'
+import { _sortBy } from '@naturalcycles/js-lib/array/array.util.js'
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { _deepCopy, _filterObject, _omit, _pick } from '@naturalcycles/js-lib/object'
 import { _pipeline } from '@naturalcycles/nodejs-lib/stream'
 import { CommonDao } from '../commondao/common.dao.js'

package/src/testing/keyValueDBTest.ts

@@ -1,4 +1,4 @@
-import { _range, _sortBy } from '@naturalcycles/js-lib'
+import { _range, _sortBy } from '@naturalcycles/js-lib/array'
 import type { KeyValueTuple } from '@naturalcycles/js-lib/types'
 import type { CommonKeyValueDB } from '../kv/commonKeyValueDB.js'
 import { TEST_TABLE } from './test.model.js'

package/src/testing/keyValueDaoTest.ts

@@ -1,4 +1,4 @@
-import { _sortBy } from '@naturalcycles/js-lib'
+import { _sortBy } from '@naturalcycles/js-lib/array/array.util.js'
 import { CommonKeyValueDao } from '../kv/commonKeyValueDao.js'
 import type { CommonKeyValueDB, KeyValueDBTuple } from '../kv/commonKeyValueDB.js'
 import { createTestItemsBM, TEST_TABLE } from './test.model.js'

package/src/testing/test.model.ts

@@ -1,4 +1,4 @@
-import { _range } from '@naturalcycles/js-lib'
+import { _range } from '@naturalcycles/js-lib/array/range.js'
 import type { JsonSchemaObject } from '@naturalcycles/js-lib/json-schema'
 import { jsonSchema } from '@naturalcycles/js-lib/json-schema'
 import type { BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types'

package/src/testing/timeSeriesTest.util.ts

@@ -1,4 +1,5 @@
-import { _randomInt, _range } from '@naturalcycles/js-lib'
+import { _randomInt } from '@naturalcycles/js-lib'
+import { _range } from '@naturalcycles/js-lib/array/range.js'
 import type { TimeSeriesDataPoint } from '../timeseries/timeSeries.model.js'
 
 export function createTestTimeSeries(count = 10): TimeSeriesDataPoint[] {