@naturalcycles/db-lib 8.24.2 → 8.26.0

package/dist/adapter/inmemory/inMemoryKeyValueDB.d.ts

@@ -17,4 +17,5 @@ export declare class InMemoryKeyValueDB implements CommonKeyValueDB {
     streamIds(table: string, limit?: number): ReadableTyped<string>;
     streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
     streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    count(table: string): Promise<number>;
 }

package/dist/adapter/inmemory/inMemoryKeyValueDB.js

@@ -34,5 +34,10 @@ class InMemoryKeyValueDB {
     streamEntries(table, limit) {
         return stream_1.Readable.from(Object.entries(this.data[table] || {}).slice(0, limit));
     }
+    async count(table) {
+        var _a;
+        (_a = this.data)[table] || (_a[table] = {});
+        return Object.keys(this.data[table]).length;
+    }
 }
 exports.InMemoryKeyValueDB = InMemoryKeyValueDB;

package/dist/commondao/common.dao.d.ts

@@ -137,7 +137,7 @@ export declare class CommonDao<BM extends Partial<ObjectWithId>, DBM extends Obj
      *
      * Does NOT mutate the object.
      */
-    validateAndConvert<IN, OUT = IN>(obj: IN, schema?: ObjectSchemaTyped<IN> | AjvSchema<IN>, modelType?: DBModelType, opt?: CommonDaoOptions): OUT;
+    validateAndConvert<IN, OUT = IN>(obj: Partial<IN>, schema: ObjectSchemaTyped<IN> | AjvSchema<IN> | undefined, modelType: DBModelType, opt?: CommonDaoOptions): OUT;
     getTableSchema(): Promise<JsonSchemaRootObject<DBM>>;
     createTable(schema: JsonSchemaObject<DBM>, opt?: CommonDaoCreateOptions): Promise<void>;
     /**

package/dist/commondao/common.dao.js

@@ -56,7 +56,17 @@ class CommonDao {
         const op = `getById(${id})`;
         const table = opt.table || this.cfg.table;
         const started = this.logStarted(op, table);
-        const [dbm] = await this.cfg.db.getByIds(table, [id]);
+        let dbm;
+        if (opt.timeout) {
+            // todo: possibly remove it after debugging is done
+            dbm = (await (0, js_lib_1.pTimeout)(this.cfg.db.getByIds(table, [id]), {
+                timeout: opt.timeout,
+                name: `getById(${table})`,
+            }))[0];
+        }
+        else {
+            dbm = (await this.cfg.db.getByIds(table, [id]))[0];
+        }
         const bm = opt.raw ? dbm : await this.dbmToBM(dbm, opt);
         this.logResult(started, op, bm, table);
         return bm || null;

package/dist/commondao/common.dao.model.d.ts

@@ -1,4 +1,4 @@
-import { CommonLogger, ErrorMode, ObjectWithId } from '@naturalcycles/js-lib';
+import { CommonLogger, ErrorMode, ObjectWithId, Saved } from '@naturalcycles/js-lib';
 import { AjvSchema, AjvValidationError, JoiValidationError, ObjectSchemaTyped, TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib';
 import { CommonDB } from '../common.db';
 import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model';
@@ -47,7 +47,7 @@ export declare enum CommonDaoLogLevel {
      */
     DATA_FULL = 30
 }
-export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId, TM> {
+export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId = Saved<BM>, TM = BM> {
     db: CommonDB;
     table: string;
     /**
@@ -135,6 +135,13 @@ export interface CommonDaoOptions extends CommonDBOptions {
      * Useful e.g in AirtableDB where you can have one Dao to control multiple tables.
      */
     table?: string;
+    /**
+     * If set - wraps the method in `pTimeout` with a timeout of given number of milliseconds.
+     * Currently, it is only used to debug an ongoing GCP infra issue.
+     *
+     * @experimental
+     */
+    timeout?: number;
 }
 /**
  * All properties default to undefined.

package/dist/kv/commonKeyValueDB.d.ts

@@ -28,4 +28,5 @@ export interface CommonKeyValueDB {
     streamIds(table: string, limit?: number): ReadableTyped<string>;
     streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
     streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    count(table: string): Promise<number>;
 }

package/dist/pipeline/dbPipelineRestore.js

@@ -78,7 +78,7 @@ async function dbPipelineRestore(opt) {
                 ...opt,
                 metric: table,
             }),
-            (0, nodejs_lib_1.transformLimit)(limit),
+            (0, nodejs_lib_1.transformLimit)({ limit }),
             ...(sinceUpdated
                 ? [(0, nodejs_lib_1.transformFilterSync)(r => r.updated >= sinceUpdated)]
                 : []),

package/dist/testing/dbTest.d.ts

@@ -18,6 +18,11 @@ export interface CommonDBImplementationFeatures {
     streaming?: boolean;
     bufferSupport?: boolean;
     nullValues?: boolean;
+    /**
+     * Set false for SQL (relational) databases,
+     * they will return `null` for all missing properties.
+     */
+    documentDB?: boolean;
 }
 /**
  * All options default to `false`.

package/dist/testing/dbTest.js

@@ -12,7 +12,7 @@ const test_util_1 = require("./test.util");
 function runCommonDBTest(db, features = {}, quirks = {}) {
     const { querying = true, tableSchemas = true, createTable = true, dbQueryFilter = true, 
     // dbQueryFilterIn = true,
-    dbQueryOrder = true, dbQuerySelectFields = true, streaming = true, strongConsistency = true, bufferSupport = true, nullValues = true, } = features;
+    dbQueryOrder = true, dbQuerySelectFields = true, streaming = true, strongConsistency = true, bufferSupport = true, nullValues = true, documentDB = true, } = features;
     // const {
     // allowExtraPropertiesInResponse,
     // allowBooleansAsUndefined,
@@ -71,20 +71,22 @@ function runCommonDBTest(db, features = {}, quirks = {}) {
             expect(item3Loaded.k2).toBe(null);
         });
     }
-    test('undefined values should not be saved/loaded', async () => {
-        const item3 = {
-            ...(0, test_model_1.createTestItemDBM)(3),
-            k2: undefined,
-        };
-        (0, test_util_1.deepFreeze)(item3);
-        const expected = { ...item3 };
-        delete expected.k2;
-        await db.saveBatch(test_model_1.TEST_TABLE, [item3]);
-        const item3Loaded = (await db.getByIds(test_model_1.TEST_TABLE, [item3.id]))[0];
-        expectMatch([expected], [item3Loaded], quirks);
-        expect(item3Loaded.k2).toBe(undefined);
-        expect(Object.keys(item3Loaded)).not.toContain('k2');
-    });
+    if (documentDB) {
+        test('undefined values should not be saved/loaded', async () => {
+            const item3 = {
+                ...(0, test_model_1.createTestItemDBM)(3),
+                k2: undefined,
+            };
+            (0, test_util_1.deepFreeze)(item3);
+            const expected = { ...item3 };
+            delete expected.k2;
+            await db.saveBatch(test_model_1.TEST_TABLE, [item3]);
+            const item3Loaded = (await db.getByIds(test_model_1.TEST_TABLE, [item3.id]))[0];
+            expectMatch([expected], [item3Loaded], quirks);
+            expect(item3Loaded.k2).toBe(undefined);
+            expect(Object.keys(item3Loaded)).not.toContain('k2');
+        });
+    }
     test('saveBatch test items', async () => {
         await db.saveBatch(test_model_1.TEST_TABLE, items);
     });

package/dist/testing/keyValueDBTest.js

@@ -20,12 +20,18 @@ function runCommonKeyValueDBTest(db) {
         const results = await db.getByIds(test_model_1.TEST_TABLE, testIds);
         expect(results).toEqual([]);
     });
+    test('count should be 0', async () => {
+        expect(await db.count(test_model_1.TEST_TABLE)).toBe(0);
+    });
     test('saveBatch, then getByIds', async () => {
         await db.saveBatch(test_model_1.TEST_TABLE, testEntries);
         const entries = await db.getByIds(test_model_1.TEST_TABLE, testIds);
         (0, js_lib_1._sortBy)(entries, e => e[0], true);
         expect(entries).toEqual(testEntries);
     });
+    test('count should be 3', async () => {
+        expect(await db.count(test_model_1.TEST_TABLE)).toBe(3);
+    });
     test('streamIds', async () => {
         const ids = await (0, nodejs_lib_1.readableToArray)(db.streamIds(test_model_1.TEST_TABLE));
         ids.sort();

package/package.json

@@ -12,8 +12,9 @@
   "devDependencies": {
     "@naturalcycles/bench-lib": "^1.0.0",
     "@naturalcycles/dev-lib": "^12.0.1",
-    "@types/node": "^16.0.0",
-    "jest": "^27.0.3"
+    "@types/node": "^17.0.0",
+    "jest": "^27.0.3",
+    "weak-napi": "^2.0.2"
   },
   "files": [
     "dist",
@@ -42,7 +43,7 @@
   "engines": {
     "node": ">=14.15"
   },
-  "version": "8.24.2",
+  "version": "8.26.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/adapter/inmemory/inMemoryKeyValueDB.ts

@@ -42,4 +42,9 @@ export class InMemoryKeyValueDB implements CommonKeyValueDB {
   streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
     return Readable.from(Object.entries(this.data[table] || {}).slice(0, limit))
   }
+
+  async count(table: string): Promise<number> {
+    this.data[table] ||= {}
+    return Object.keys(this.data[table]!).length
+  }
 }

package/src/commondao/common.dao.model.ts

@@ -1,4 +1,4 @@
-import { CommonLogger, ErrorMode, ObjectWithId } from '@naturalcycles/js-lib'
+import { CommonLogger, ErrorMode, ObjectWithId, Saved } from '@naturalcycles/js-lib'
 import {
   AjvSchema,
   AjvValidationError,
@@ -60,7 +60,11 @@ export enum CommonDaoLogLevel {
   DATA_FULL = 30,
 }
 
-export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId, TM> {
+export interface CommonDaoCfg<
+  BM extends Partial<ObjectWithId>,
+  DBM extends ObjectWithId = Saved<BM>,
+  TM = BM,
+> {
   db: CommonDB
   table: string
 
@@ -164,6 +168,14 @@ export interface CommonDaoOptions extends CommonDBOptions {
    * Useful e.g in AirtableDB where you can have one Dao to control multiple tables.
    */
   table?: string
+
+  /**
+   * If set - wraps the method in `pTimeout` with a timeout of given number of milliseconds.
+   * Currently, it is only used to debug an ongoing GCP infra issue.
+   *
+   * @experimental
+   */
+  timeout?: number
 }
 
 /**

package/src/commondao/common.dao.ts

@@ -12,6 +12,7 @@ import {
   JsonSchemaRootObject,
   ObjectWithId,
   pMap,
+  pTimeout,
   Saved,
 } from '@naturalcycles/js-lib'
 import {
@@ -103,7 +104,21 @@ export class CommonDao<
     const op = `getById(${id})`
     const table = opt.table || this.cfg.table
     const started = this.logStarted(op, table)
-    const [dbm] = await this.cfg.db.getByIds<DBM>(table, [id])
+
+    let dbm: DBM | undefined
+
+    if (opt.timeout) {
+      // todo: possibly remove it after debugging is done
+      dbm = (
+        await pTimeout(this.cfg.db.getByIds<DBM>(table, [id]), {
+          timeout: opt.timeout,
+          name: `getById(${table})`,
+        })
+      )[0]
+    } else {
+      dbm = (await this.cfg.db.getByIds<DBM>(table, [id]))[0]
+    }
+
     const bm = opt.raw ? (dbm as any) : await this.dbmToBM(dbm, opt)
     this.logResult(started, op, bm, table)
     return bm || null
@@ -903,9 +918,9 @@ export class CommonDao<
    * Does NOT mutate the object.
    */
   validateAndConvert<IN, OUT = IN>(
-    obj: IN,
-    schema?: ObjectSchemaTyped<IN> | AjvSchema<IN>,
-    modelType?: DBModelType,
+    obj: Partial<IN>,
+    schema: ObjectSchemaTyped<IN> | AjvSchema<IN> | undefined,
+    modelType: DBModelType,
     opt: CommonDaoOptions = {},
   ): OUT {
     // `raw` option completely bypasses any processing
@@ -928,12 +943,12 @@ export class CommonDao<
 
     // Pre-validation hooks
     if (modelType === DBModelType.DBM) {
-      obj = this.cfg.hooks!.beforeDBMValidate!(obj as any) as any
+      obj = this.cfg.hooks!.beforeDBMValidate!(obj as any) as IN
     }
 
     // Return as is if no schema is passed or if `skipConversion` is set
     if (!schema || opt.skipConversion) {
-      return obj as any
+      return obj as OUT
     }
 
     // This will Convert and Validate
@@ -947,12 +962,12 @@ export class CommonDao<
       // Ajv schema
       convertedValue = obj // because Ajv mutates original object
 
-      error = schema.getValidationError(obj, {
+      error = schema.getValidationError(obj as IN, {
         objectName,
       })
     } else {
       // Joi
-      const vr = getValidationResult<IN, OUT>(obj, schema, objectName)
+      const vr = getValidationResult<IN, OUT>(obj as IN, schema, objectName)
       error = vr.error
       convertedValue = vr.value
     }

package/src/kv/commonKeyValueDB.ts

@@ -34,4 +34,6 @@ export interface CommonKeyValueDB {
   streamIds(table: string, limit?: number): ReadableTyped<string>
   streamValues(table: string, limit?: number): ReadableTyped<Buffer>
   streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>
+
+  count(table: string): Promise<number>
 }

package/src/pipeline/dbPipelineRestore.ts

@@ -209,7 +209,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
           ...opt,
           metric: table,
         }),
-        transformLimit(limit),
+        transformLimit({ limit }),
         ...(sinceUpdated
           ? [transformFilterSync<SavedDBEntity>(r => r.updated >= sinceUpdated)]
           : []),

package/src/testing/dbTest.ts

@@ -35,6 +35,12 @@ export interface CommonDBImplementationFeatures {
 
   bufferSupport?: boolean
   nullValues?: boolean
+
+  /**
+   * Set false for SQL (relational) databases,
+   * they will return `null` for all missing properties.
+   */
+  documentDB?: boolean
 }
 
 /**
@@ -78,6 +84,7 @@ export function runCommonDBTest(
     strongConsistency = true,
     bufferSupport = true,
     nullValues = true,
+    documentDB = true,
   } = features
 
   // const {
@@ -151,21 +158,23 @@ export function runCommonDBTest(
     })
   }
 
-  test('undefined values should not be saved/loaded', async () => {
-    const item3 = {
-      ...createTestItemDBM(3),
-      k2: undefined,
-    }
-    deepFreeze(item3)
-    const expected = { ...item3 }
-    delete expected.k2
-
-    await db.saveBatch(TEST_TABLE, [item3])
-    const item3Loaded = (await db.getByIds<TestItemDBM>(TEST_TABLE, [item3.id]))[0]!
-    expectMatch([expected], [item3Loaded], quirks)
-    expect(item3Loaded.k2).toBe(undefined)
-    expect(Object.keys(item3Loaded)).not.toContain('k2')
-  })
+  if (documentDB) {
+    test('undefined values should not be saved/loaded', async () => {
+      const item3 = {
+        ...createTestItemDBM(3),
+        k2: undefined,
+      }
+      deepFreeze(item3)
+      const expected = { ...item3 }
+      delete expected.k2
+
+      await db.saveBatch(TEST_TABLE, [item3])
+      const item3Loaded = (await db.getByIds<TestItemDBM>(TEST_TABLE, [item3.id]))[0]!
+      expectMatch([expected], [item3Loaded], quirks)
+      expect(item3Loaded.k2).toBe(undefined)
+      expect(Object.keys(item3Loaded)).not.toContain('k2')
+    })
+  }
 
   test('saveBatch test items', async () => {
     await db.saveBatch(TEST_TABLE, items)

package/src/testing/keyValueDBTest.ts

@@ -25,6 +25,10 @@ export function runCommonKeyValueDBTest(db: CommonKeyValueDB): void {
     expect(results).toEqual([])
   })
 
+  test('count should be 0', async () => {
+    expect(await db.count(TEST_TABLE)).toBe(0)
+  })
+
   test('saveBatch, then getByIds', async () => {
     await db.saveBatch(TEST_TABLE, testEntries)
 
@@ -33,6 +37,10 @@ export function runCommonKeyValueDBTest(db: CommonKeyValueDB): void {
     expect(entries).toEqual(testEntries)
   })
 
+  test('count should be 3', async () => {
+    expect(await db.count(TEST_TABLE)).toBe(3)
+  })
+
   test('streamIds', async () => {
     const ids = await readableToArray(db.streamIds(TEST_TABLE))
     ids.sort()