@naturalcycles/db-lib 8.19.0 → 8.21.1

package/CHANGELOG.md

@@ -1,3 +1,31 @@
+## [8.21.1](https://github.com/NaturalCycles/db-lib/compare/v8.21.0...v8.21.1) (2021-10-18)
+
+
+### Bug Fixes
+
+* test for saving/loading undefined values ([e7a8dcf](https://github.com/NaturalCycles/db-lib/commit/e7a8dcff296ba7c8c23c4157510dd79fe7ef1727))
+
+# [8.21.0](https://github.com/NaturalCycles/db-lib/compare/v8.20.1...v8.21.0) (2021-10-18)
+
+
+### Features
+
+* preserve `null` values in CommonDao load and save ([6f9b0d6](https://github.com/NaturalCycles/db-lib/commit/6f9b0d6e8419d2d1a0c025b7375c2504c3b267d5))
+
+## [8.20.1](https://github.com/NaturalCycles/db-lib/compare/v8.20.0...v8.20.1) (2021-10-17)
+
+
+### Bug Fixes
+
+* types, test ([fb2f741](https://github.com/NaturalCycles/db-lib/commit/fb2f741d8c5c63a04fb70f48fcafbb529669caf2))
+
+# [8.20.0](https://github.com/NaturalCycles/db-lib/compare/v8.19.0...v8.20.0) (2021-10-17)
+
+
+### Features
+
+* CommonDao.runUnionQueries ([ed6c986](https://github.com/NaturalCycles/db-lib/commit/ed6c986e8ba25b660903e235c062237b5100e63c))
+
 # [8.19.0](https://github.com/NaturalCycles/db-lib/compare/v8.18.0...v8.19.0) (2021-10-17)
 
 

package/dist/adapter/file/file.db.js

@@ -41,7 +41,7 @@ class FileDB extends __1.BaseCommonDB {
     async saveBatch(table, rows, _opt) {
         if (!rows.length)
             return; // save some api calls
-        // 1. Load the whole file from gh
+        // 1. Load the whole file
         const byId = (0, js_lib_1._by)(await this.loadFile(table), r => r.id);
         // 2. Merge with new data (using ids)
         let saved = 0;
@@ -173,6 +173,7 @@ class FileDB extends __1.BaseCommonDB {
      * Mutates
      */
     sortRows(rows) {
+        rows.forEach(r => (0, js_lib_1._filterUndefinedValues)(r, true));
         if (this.cfg.sortOnSave) {
             (0, js_lib_1._sortBy)(rows, r => r[this.cfg.sortOnSave.name], true);
             if (this.cfg.sortOnSave.descending)

package/dist/base.common.db.d.ts

@@ -1,7 +1,7 @@
 import { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { CommonDB } from './common.db';
-import { CommonDBOptions, ObjectWithId, RunQueryResult } from './db.model';
+import { CommonDBOptions, CommonDBSaveOptions, ObjectWithId, RunQueryResult } from './db.model';
 import { DBQuery } from './query/dbQuery';
 import { DBTransaction } from './transaction/dbTransaction';
 /**
@@ -18,7 +18,7 @@ export declare class BaseCommonDB implements CommonDB {
     getByIds<ROW extends ObjectWithId>(_table: string, _ids: string[]): Promise<ROW[]>;
     runQuery<ROW extends ObjectWithId>(_q: DBQuery<ROW>): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(_q: DBQuery<ROW>): Promise<number>;
-    saveBatch<ROW extends ObjectWithId>(_table: string, _rows: ROW[]): Promise<void>;
+    saveBatch<ROW extends ObjectWithId>(_table: string, _rows: ROW[], _opt?: CommonDBSaveOptions<ROW>): Promise<void>;
     streamQuery<ROW extends ObjectWithId>(_q: DBQuery<ROW>): ReadableTyped<ROW>;
     /**
      * Naive implementation.

package/dist/base.common.db.js

@@ -37,7 +37,7 @@ class BaseCommonDB {
     async runQueryCount(_q) {
         return 0;
     }
-    async saveBatch(_table, _rows) { }
+    async saveBatch(_table, _rows, _opt) { }
     streamQuery(_q) {
         return stream_1.Readable.from([]);
     }

package/dist/commondao/common.dao.d.ts

@@ -40,6 +40,12 @@ export declare class CommonDao<BM extends Partial<ObjectWithId>, DBM extends Obj
      */
     query(table?: string): RunnableDBQuery<BM, DBM, TM>;
     runQuery(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<Saved<BM>[]>;
+    /**
+     * Convenience method that runs multiple queries in parallel and then merges their results together.
+     * Does deduplication by id.
+     * Order is not guaranteed, as queries run in parallel.
+     */
+    runUnionQueries(queries: DBQuery<DBM>[], opt?: CommonDaoOptions): Promise<Saved<BM>[]>;
     runQueryExtended(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<RunQueryResult<Saved<BM>>>;
     runQueryAsDBM(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<DBM[]>;
     runQueryExtendedAsDBM(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<RunQueryResult<DBM>>;

package/dist/commondao/common.dao.js

@@ -175,6 +175,15 @@ class CommonDao {
         const { rows } = await this.runQueryExtended(q, opt);
         return rows;
     }
+    /**
+     * Convenience method that runs multiple queries in parallel and then merges their results together.
+     * Does deduplication by id.
+     * Order is not guaranteed, as queries run in parallel.
+     */
+    async runUnionQueries(queries, opt) {
+        const results = (await (0, js_lib_1.pMap)(queries, async (q) => (await this.runQueryExtended(q, opt)).rows)).flat();
+        return (0, js_lib_1._uniqBy)(results, r => r.id);
+    }
     async runQueryExtended(q, opt = {}) {
         q.table = opt.table || q.table;
         const op = `runQuery(${q.pretty()})`;
@@ -630,8 +639,14 @@ class CommonDao {
         // `raw` option completely bypasses any processing
         if (opt.raw)
             return obj;
+        // Kirill 2021-10-18: I realized that there's little reason to keep removing null values
+        // So, from now on we'll preserve them
+        // "undefined" values, I believe, are/were not saved to/from DB anyway (due to e.g JSON.stringify removing them)
+        // But let's keep watching it!
+        //
         // Filter null and undefined values
-        obj = (0, js_lib_1._filterNullishValues)(obj);
+        // obj = _filterNullishValues(obj as any)
+        obj = { ...obj }; // prevent mutation
         // Pre-validation hooks
         if (modelType === db_model_1.DBModelType.DBM) {
             obj = this.cfg.hooks.beforeDBMValidate(obj);

package/dist/testing/daoTest.js

@@ -22,7 +22,7 @@ function runCommonDaoTest(db, features = {}, quirks = {}) {
     // tableSchemas = true,
     createTable = true, dbQueryFilter = true, 
     // dbQueryFilterIn = true,
-    dbQueryOrder = true, dbQuerySelectFields = true, streaming = true, strongConsistency = true, } = features;
+    dbQueryOrder = true, dbQuerySelectFields = true, streaming = true, strongConsistency = true, nullValues = true, } = features;
     // const {
     // allowExtraPropertiesInResponse,
     // allowBooleansAsUndefined,
@@ -72,6 +72,29 @@ function runCommonDaoTest(db, features = {}, quirks = {}) {
         expect(await dao.getByIds(['abc', 'abcd'])).toEqual([]);
     });
     // SAVE
+    if (nullValues) {
+        test('should allow to save and load null values', async () => {
+            const item3 = {
+                ...(0, test_model_1.createTestItemBM)(3),
+                k2: null,
+            };
+            await dao.save(item3);
+            const item3Loaded = await dao.requireById(item3.id);
+            (0, dbTest_1.expectMatch)([item3], [item3Loaded], quirks);
+            expect(item3Loaded.k2).toBe(null);
+        });
+    }
+    test('undefined values should not be saved/loaded', async () => {
+        const item3 = {
+            ...(0, test_model_1.createTestItemBM)(3),
+            k2: undefined,
+        };
+        await dao.save(item3);
+        const item3Loaded = await dao.requireById(item3.id);
+        (0, dbTest_1.expectMatch)([item3], [item3Loaded], quirks);
+        expect(item3Loaded.k2).toBe(undefined);
+        expect(Object.keys(item3Loaded)).not.toContain('k2');
+    });
     test('saveBatch test items', async () => {
         const itemsSaved = await dao.saveBatch(items);
         expect(itemsSaved[0]).toBe(items[0]); // expect "same object" returned

package/dist/testing/dbTest.d.ts

@@ -17,6 +17,7 @@ export interface CommonDBImplementationFeatures {
     strongConsistency?: boolean;
     streaming?: boolean;
     bufferSupport?: boolean;
+    nullValues?: boolean;
 }
 /**
  * All options default to `false`.

package/dist/testing/dbTest.js

@@ -12,7 +12,7 @@ const test_util_1 = require("./test.util");
 function runCommonDBTest(db, features = {}, quirks = {}) {
     const { querying = true, tableSchemas = true, createTable = true, dbQueryFilter = true, 
     // dbQueryFilterIn = true,
-    dbQueryOrder = true, dbQuerySelectFields = true, streaming = true, strongConsistency = true, bufferSupport = true, } = features;
+    dbQueryOrder = true, dbQuerySelectFields = true, streaming = true, strongConsistency = true, bufferSupport = true, nullValues = true, } = features;
     // const {
     // allowExtraPropertiesInResponse,
     // allowBooleansAsUndefined,
@@ -58,6 +58,29 @@ function runCommonDBTest(db, features = {}, quirks = {}) {
         expect(await db.getByIds(test_model_1.TEST_TABLE, ['abc', 'abcd'])).toEqual([]);
     });
     // SAVE
+    if (nullValues) {
+        test('should allow to save and load null values', async () => {
+            const item3 = {
+                ...(0, test_model_1.createTestItemDBM)(3),
+                k2: null,
+            };
+            await db.saveBatch(test_model_1.TEST_TABLE, [item3]);
+            const item3Loaded = (await db.getByIds(test_model_1.TEST_TABLE, [item3.id]))[0];
+            expectMatch([item3], [item3Loaded], quirks);
+            expect(item3Loaded.k2).toBe(null);
+        });
+    }
+    test('undefined values should not be saved/loaded', async () => {
+        const item3 = {
+            ...(0, test_model_1.createTestItemDBM)(3),
+            k2: undefined,
+        };
+        await db.saveBatch(test_model_1.TEST_TABLE, [item3]);
+        const item3Loaded = (await db.getByIds(test_model_1.TEST_TABLE, [item3.id]))[0];
+        expectMatch([item3], [item3Loaded], quirks);
+        expect(item3Loaded.k2).toBe(undefined);
+        expect(Object.keys(item3Loaded)).not.toContain('k2');
+    });
     test('saveBatch test items', async () => {
         await db.saveBatch(test_model_1.TEST_TABLE, items);
     });

package/dist/testing/test.model.d.ts

@@ -3,7 +3,7 @@ import { JsonSchemaObject, BaseDBEntity, Saved } from '@naturalcycles/js-lib';
 export declare const TEST_TABLE = "TEST_TABLE";
 export interface TestItemBM extends BaseDBEntity {
     k1: string;
-    k2?: string;
+    k2?: string | null;
     k3?: number;
     even?: boolean;
     b1?: Buffer;

package/dist/testing/test.model.js

@@ -7,14 +7,14 @@ const MOCK_TS_2018_06_21 = 1529539200;
 exports.TEST_TABLE = 'TEST_TABLE';
 exports.testItemBMSchema = (0, nodejs_lib_1.objectSchema)({
     k1: nodejs_lib_1.stringSchema,
-    k2: nodejs_lib_1.stringSchema.optional(),
+    k2: nodejs_lib_1.stringSchema.allow(null).optional(),
     k3: nodejs_lib_1.numberSchema.optional(),
     even: nodejs_lib_1.booleanSchema.optional(),
     b1: nodejs_lib_1.binarySchema.optional(),
 }).concat(nodejs_lib_1.baseDBEntitySchema);
 exports.testItemDBMSchema = (0, nodejs_lib_1.objectSchema)({
     k1: nodejs_lib_1.stringSchema,
-    k2: nodejs_lib_1.stringSchema.optional(),
+    k2: nodejs_lib_1.stringSchema.allow(null).optional(),
     k3: nodejs_lib_1.numberSchema.optional(),
     even: nodejs_lib_1.booleanSchema.optional(),
     b1: nodejs_lib_1.binarySchema.optional(),
@@ -26,7 +26,7 @@ exports.testItemTMSchema = (0, nodejs_lib_1.objectSchema)({
 exports.testItemBMJsonSchema = js_lib_1.jsonSchema
     .rootObject({
     k1: js_lib_1.jsonSchema.string(),
-    k2: js_lib_1.jsonSchema.string().optional(),
+    k2: js_lib_1.jsonSchema.oneOf([js_lib_1.jsonSchema.string(), js_lib_1.jsonSchema.null()]).optional(),
     k3: js_lib_1.jsonSchema.number().optional(),
     even: js_lib_1.jsonSchema.boolean().optional(),
     b1: js_lib_1.jsonSchema.buffer().optional(),

package/package.json

@@ -42,7 +42,7 @@
   "engines": {
     "node": ">=12.13"
   },
-  "version": "8.19.0",
+  "version": "8.21.1",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/adapter/file/file.db.ts

@@ -10,6 +10,7 @@ import {
   _stringMapValues,
   _uniq,
   JsonSchemaRootObject,
+  _filterUndefinedValues,
 } from '@naturalcycles/js-lib'
 import { Debug, readableCreate, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { dimGrey } from '@naturalcycles/nodejs-lib/dist/colors'
@@ -82,7 +83,7 @@ export class FileDB extends BaseCommonDB implements CommonDB {
   ): Promise<void> {
     if (!rows.length) return // save some api calls
 
-    // 1. Load the whole file from gh
+    // 1. Load the whole file
     const byId = _by(await this.loadFile<ROW>(table), r => r.id)
 
     // 2. Merge with new data (using ids)
@@ -261,6 +262,8 @@ export class FileDB extends BaseCommonDB implements CommonDB {
    * Mutates
    */
   sortRows<ROW>(rows: ROW[]): ROW[] {
+    rows.forEach(r => _filterUndefinedValues(r, true))
+
     if (this.cfg.sortOnSave) {
       _sortBy(rows, r => r[this.cfg.sortOnSave!.name], true)
       if (this.cfg.sortOnSave.descending) rows.reverse() // mutates

package/src/base.common.db.ts

@@ -2,7 +2,7 @@ import { Readable } from 'stream'
 import { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { CommonDB } from './common.db'
-import { CommonDBOptions, ObjectWithId, RunQueryResult } from './db.model'
+import { CommonDBOptions, CommonDBSaveOptions, ObjectWithId, RunQueryResult } from './db.model'
 import { DBQuery } from './query/dbQuery'
 import { DBTransaction } from './transaction/dbTransaction'
 import { commitDBTransactionSimple } from './transaction/dbTransaction.util'
@@ -55,7 +55,11 @@ export class BaseCommonDB implements CommonDB {
     return 0
   }
 
-  async saveBatch<ROW extends ObjectWithId>(_table: string, _rows: ROW[]): Promise<void> {}
+  async saveBatch<ROW extends ObjectWithId>(
+    _table: string,
+    _rows: ROW[],
+    _opt?: CommonDBSaveOptions<ROW>,
+  ): Promise<void> {}
 
   streamQuery<ROW extends ObjectWithId>(_q: DBQuery<ROW>): ReadableTyped<ROW> {
     return Readable.from([])

package/src/commondao/common.dao.ts

@@ -3,13 +3,13 @@ import {
   AsyncMapper,
   ErrorMode,
   JsonSchemaObject,
-  _filterNullishValues,
   _passthroughPredicate,
   _since,
   _truncate,
   pMap,
   JsonSchemaRootObject,
   Saved,
+  _uniqBy,
 } from '@naturalcycles/js-lib'
 import {
   AjvSchema,
@@ -242,6 +242,18 @@ export class CommonDao<
     return rows
   }
 
+  /**
+   * Convenience method that runs multiple queries in parallel and then merges their results together.
+   * Does deduplication by id.
+   * Order is not guaranteed, as queries run in parallel.
+   */
+  async runUnionQueries(queries: DBQuery<DBM>[], opt?: CommonDaoOptions): Promise<Saved<BM>[]> {
+    const results = (
+      await pMap(queries, async q => (await this.runQueryExtended(q, opt)).rows)
+    ).flat()
+    return _uniqBy(results, r => r.id)
+  }
+
   async runQueryExtended(
     q: DBQuery<DBM>,
     opt: CommonDaoOptions = {},
@@ -852,8 +864,14 @@ export class CommonDao<
     // `raw` option completely bypasses any processing
     if (opt.raw) return obj as any as OUT
 
+    // Kirill 2021-10-18: I realized that there's little reason to keep removing null values
+    // So, from now on we'll preserve them
+    // "undefined" values, I believe, are/were not saved to/from DB anyway (due to e.g JSON.stringify removing them)
+    // But let's keep watching it!
+    //
     // Filter null and undefined values
-    obj = _filterNullishValues(obj as any)
+    // obj = _filterNullishValues(obj as any)
+    obj = { ...obj } // prevent mutation
 
     // Pre-validation hooks
     if (modelType === DBModelType.DBM) {

package/src/testing/daoTest.ts

@@ -10,6 +10,7 @@ import {
   testItemDBMSchema,
   testItemTMSchema,
   TEST_TABLE,
+  createTestItemBM,
 } from './test.model'
 import { getTestItemSchema, TestItemBM } from '.'
 
@@ -38,6 +39,7 @@ export function runCommonDaoTest(
     dbQuerySelectFields = true,
     streaming = true,
     strongConsistency = true,
+    nullValues = true,
   } = features
 
   // const {
@@ -101,6 +103,31 @@ export function runCommonDaoTest(
   })
 
   // SAVE
+  if (nullValues) {
+    test('should allow to save and load null values', async () => {
+      const item3 = {
+        ...createTestItemBM(3),
+        k2: null,
+      }
+      await dao.save(item3)
+      const item3Loaded = await dao.requireById(item3.id)
+      expectMatch([item3], [item3Loaded], quirks)
+      expect(item3Loaded.k2).toBe(null)
+    })
+  }
+
+  test('undefined values should not be saved/loaded', async () => {
+    const item3 = {
+      ...createTestItemBM(3),
+      k2: undefined,
+    }
+    await dao.save(item3)
+    const item3Loaded = await dao.requireById(item3.id)
+    expectMatch([item3], [item3Loaded], quirks)
+    expect(item3Loaded.k2).toBe(undefined)
+    expect(Object.keys(item3Loaded)).not.toContain('k2')
+  })
+
   test('saveBatch test items', async () => {
     const itemsSaved = await dao.saveBatch(items)
     expect(itemsSaved[0]).toBe(items[0]) // expect "same object" returned

package/src/testing/dbTest.ts

@@ -34,6 +34,7 @@ export interface CommonDBImplementationFeatures {
   streaming?: boolean
 
   bufferSupport?: boolean
+  nullValues?: boolean
 }
 
 /**
@@ -76,6 +77,7 @@ export function runCommonDBTest(
     streaming = true,
     strongConsistency = true,
     bufferSupport = true,
+    nullValues = true,
   } = features
 
   // const {
@@ -135,6 +137,31 @@ export function runCommonDBTest(
   })
 
   // SAVE
+  if (nullValues) {
+    test('should allow to save and load null values', async () => {
+      const item3 = {
+        ...createTestItemDBM(3),
+        k2: null,
+      }
+      await db.saveBatch(TEST_TABLE, [item3])
+      const item3Loaded = (await db.getByIds<TestItemDBM>(TEST_TABLE, [item3.id]))[0]!
+      expectMatch([item3], [item3Loaded], quirks)
+      expect(item3Loaded.k2).toBe(null)
+    })
+  }
+
+  test('undefined values should not be saved/loaded', async () => {
+    const item3 = {
+      ...createTestItemDBM(3),
+      k2: undefined,
+    }
+    await db.saveBatch(TEST_TABLE, [item3])
+    const item3Loaded = (await db.getByIds<TestItemDBM>(TEST_TABLE, [item3.id]))[0]!
+    expectMatch([item3], [item3Loaded], quirks)
+    expect(item3Loaded.k2).toBe(undefined)
+    expect(Object.keys(item3Loaded)).not.toContain('k2')
+  })
+
   test('saveBatch test items', async () => {
     await db.saveBatch(TEST_TABLE, items)
   })

package/src/testing/test.model.ts

@@ -15,7 +15,7 @@ export const TEST_TABLE = 'TEST_TABLE'
 
 export interface TestItemBM extends BaseDBEntity {
   k1: string
-  k2?: string
+  k2?: string | null
   k3?: number
   even?: boolean
   b1?: Buffer
@@ -30,7 +30,7 @@ export interface TestItemTM {
 
 export const testItemBMSchema = objectSchema<TestItemBM>({
   k1: stringSchema,
-  k2: stringSchema.optional(),
+  k2: stringSchema.allow(null).optional(),
   k3: numberSchema.optional(),
   even: booleanSchema.optional(),
   b1: binarySchema.optional(),
@@ -38,7 +38,7 @@ export const testItemBMSchema = objectSchema<TestItemBM>({
 
 export const testItemDBMSchema = objectSchema<TestItemDBM>({
   k1: stringSchema,
-  k2: stringSchema.optional(),
+  k2: stringSchema.allow(null).optional(),
   k3: numberSchema.optional(),
   even: booleanSchema.optional(),
   b1: binarySchema.optional(),
@@ -52,7 +52,7 @@ export const testItemTMSchema = objectSchema<TestItemTM>({
 export const testItemBMJsonSchema = jsonSchema
   .rootObject<TestItemBM>({
     k1: jsonSchema.string(),
-    k2: jsonSchema.string().optional(),
+    k2: jsonSchema.oneOf<string | null>([jsonSchema.string(), jsonSchema.null()]).optional(),
     k3: jsonSchema.number().optional(),
     even: jsonSchema.boolean().optional(),
     b1: jsonSchema.buffer().optional(),