@naturalcycles/db-lib 8.58.0 → 8.60.0

package/dist/adapter/file/localFile.persistence.plugin.js

@@ -36,7 +36,7 @@ class LocalFilePersistencePlugin {
         await (0, nodejs_lib_1._pipeline)([
             node_fs_1.default.createReadStream(filePath),
             ...transformUnzip,
-            (0, nodejs_lib_1.transformSplit)(),
+            (0, nodejs_lib_1.transformSplit)(), // splits by \n
             (0, nodejs_lib_1.transformJsonParse)(),
             (0, nodejs_lib_1.writablePushToArray)(rows),
         ]);

package/dist/adapter/inmemory/inMemory.db.js

@@ -205,7 +205,7 @@ class InMemoryDB {
             await (0, nodejs_lib_1._pipeline)([
                 node_fs_1.default.createReadStream(fname),
                 ...transformUnzip,
-                (0, nodejs_lib_1.transformSplit)(),
+                (0, nodejs_lib_1.transformSplit)(), // splits by \n
                 (0, nodejs_lib_1.transformJsonParse)(),
                 (0, nodejs_lib_1.writablePushToArray)(rows),
             ]);

package/dist/commondao/common.dao.d.ts

@@ -12,22 +12,22 @@ import { CommonDaoCfg, CommonDaoCreateOptions, CommonDaoOptions, CommonDaoSaveBa
  * BM = Backend model (optimized for API access)
  * TM = Transport model (optimized to be sent over the wire)
  */
-export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends ObjectWithId<ID> = Saved<BM>, TM extends AnyObject = BM, ID extends string | number = NonNullable<BM['id']>> {
-    cfg: CommonDaoCfg<BM, DBM, TM, ID>;
-    constructor(cfg: CommonDaoCfg<BM, DBM, TM, ID>);
+export declare class CommonDao<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId = Saved<BM>, TM extends AnyObject = BM> {
+    cfg: CommonDaoCfg<BM, DBM, TM>;
+    constructor(cfg: CommonDaoCfg<BM, DBM, TM>);
     create(part?: Partial<BM>, opt?: CommonDaoOptions): Saved<BM>;
     getById(id: undefined | null, opt?: CommonDaoOptions): Promise<null>;
-    getById(id?: ID | null, opt?: CommonDaoOptions): Promise<Saved<BM> | null>;
-    getByIdOrEmpty(id: ID, part?: Partial<BM>, opt?: CommonDaoOptions): Promise<Saved<BM>>;
-    getByIdAsDBMOrEmpty(id: ID, part?: Partial<BM>, opt?: CommonDaoOptions): Promise<DBM>;
+    getById(id?: string | null, opt?: CommonDaoOptions): Promise<Saved<BM> | null>;
+    getByIdOrEmpty(id: string, part?: Partial<BM>, opt?: CommonDaoOptions): Promise<Saved<BM>>;
+    getByIdAsDBMOrEmpty(id: string, part?: Partial<BM>, opt?: CommonDaoOptions): Promise<DBM>;
     getByIdAsDBM(id: undefined | null, opt?: CommonDaoOptions): Promise<null>;
-    getByIdAsDBM(id?: ID | null, opt?: CommonDaoOptions): Promise<DBM | null>;
+    getByIdAsDBM(id?: string | null, opt?: CommonDaoOptions): Promise<DBM | null>;
     getByIdAsTM(id: undefined | null, opt?: CommonDaoOptions): Promise<null>;
-    getByIdAsTM(id?: ID | null, opt?: CommonDaoOptions): Promise<TM | null>;
-    getByIds(ids: ID[], opt?: CommonDaoOptions): Promise<Saved<BM>[]>;
-    getByIdsAsDBM(ids: ID[], opt?: CommonDaoOptions): Promise<DBM[]>;
-    requireById(id: ID, opt?: CommonDaoOptions): Promise<Saved<BM>>;
-    requireByIdAsDBM(id: ID, opt?: CommonDaoOptions): Promise<DBM>;
+    getByIdAsTM(id?: string | null, opt?: CommonDaoOptions): Promise<TM | null>;
+    getByIds(ids: string[], opt?: CommonDaoOptions): Promise<Saved<BM>[]>;
+    getByIdsAsDBM(ids: string[], opt?: CommonDaoOptions): Promise<DBM[]>;
+    requireById(id: string, opt?: CommonDaoOptions): Promise<Saved<BM>>;
+    requireByIdAsDBM(id: string, opt?: CommonDaoOptions): Promise<DBM>;
     private throwRequiredError;
     /**
      * Throws if readOnly is true
@@ -44,7 +44,7 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
     /**
      * Pass `table` to override table
      */
-    query(table?: string): RunnableDBQuery<BM, DBM, TM, ID>;
+    query(table?: string): RunnableDBQuery<BM, DBM, TM>;
     runQuery(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<Saved<BM>[]>;
     runQuerySingleColumn<T = any>(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<T[]>;
     /**
@@ -75,9 +75,9 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
      * You can do `.pipe(transformNoOp)` to make it "valid again".
      */
     streamQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<Saved<BM>>): ReadableTyped<Saved<BM>>;
-    queryIds(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<ID[]>;
-    streamQueryIds(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<ID>): ReadableTyped<ID>;
-    streamQueryIdsForEach(q: DBQuery<DBM>, mapper: AsyncMapper<ID, void>, opt?: CommonDaoStreamForEachOptions<ID>): Promise<void>;
+    queryIds(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<string[]>;
+    streamQueryIds(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<string>): ReadableTyped<string>;
+    streamQueryIdsForEach(q: DBQuery<DBM>, mapper: AsyncMapper<string, void>, opt?: CommonDaoStreamForEachOptions<string>): Promise<void>;
     /**
      * Mutates!
      * "Returns", just to have a type of "Saved"
@@ -88,8 +88,8 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
     tx: {
         save: (bm: Unsaved<BM>, opt?: CommonDaoSaveBatchOptions<DBM>) => Promise<DBSaveBatchOperation | undefined>;
         saveBatch: (bms: Unsaved<BM>[], opt?: CommonDaoSaveBatchOptions<DBM>) => Promise<DBSaveBatchOperation | undefined>;
-        deleteByIds: (ids: ID[], opt?: CommonDaoOptions) => Promise<DBDeleteByIdsOperation | undefined>;
-        deleteById: (id: ID | null | undefined, opt?: CommonDaoOptions) => Promise<DBDeleteByIdsOperation | undefined>;
+        deleteByIds: (ids: string[], opt?: CommonDaoOptions) => Promise<DBDeleteByIdsOperation | undefined>;
+        deleteById: (id: string | null | undefined, opt?: CommonDaoOptions) => Promise<DBDeleteByIdsOperation | undefined>;
     };
     /**
      * Mutates with id, created, updated
@@ -113,7 +113,7 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
      * 2. Applies the patch on top of loaded data.
      * 3. Saves (as fast as possible since the read) with the Patch applied, but only if the data has changed.
      */
-    patchById(id: ID, patch: Partial<BM>, opt?: CommonDaoSaveBatchOptions<DBM>): Promise<Saved<BM>>;
+    patchById(id: string, patch: Partial<BM>, opt?: CommonDaoSaveBatchOptions<DBM>): Promise<Saved<BM>>;
     /**
      * Same as patchById, but takes the whole object as input.
      * This "whole object" is mutated with the patch and returned.
@@ -135,16 +135,16 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
      * @returns number of deleted items
      */
     deleteById(id: undefined | null, opt?: CommonDaoOptions): Promise<0>;
-    deleteById(id?: ID | null, opt?: CommonDaoOptions): Promise<number>;
-    deleteByIds(ids: ID[], opt?: CommonDaoOptions): Promise<number>;
+    deleteById(id?: string | null, opt?: CommonDaoOptions): Promise<number>;
+    deleteByIds(ids: string[], opt?: CommonDaoOptions): Promise<number>;
     /**
      * Pass `stream: true` option to use Streaming: it will Stream the query, batch by 500, and execute
      * `deleteByIds` for each batch concurrently (infinite concurrency).
      * This is expected to be more memory-efficient way of deleting big numbers of rows.
      */
     deleteByQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamDeleteOptions<DBM>): Promise<number>;
-    updateById(id: ID, patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;
-    updateByIds(ids: ID[], patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;
+    updateById(id: string, patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;
+    updateByIds(ids: string[], patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;
     updateByQuery(q: DBQuery<DBM>, patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;
     dbmToBM(_dbm: undefined, opt?: CommonDaoOptions): Promise<undefined>;
     dbmToBM(_dbm?: DBM, opt?: CommonDaoOptions): Promise<Saved<BM>>;

package/dist/commondao/common.dao.js

@@ -56,7 +56,7 @@ class CommonDao {
                 return {
                     type: 'deleteByIds',
                     table: this.cfg.table,
-                    ids: ids,
+                    ids,
                     opt,
                 };
             },
@@ -76,7 +76,6 @@ class CommonDao {
             // otherwise to log Operations
             // e.g in Dev (local machine), Test - it will log operations (useful for debugging)
             logLevel: isGAE || isCI ? common_dao_model_1.CommonDaoLogLevel.NONE : common_dao_model_1.CommonDaoLogLevel.OPERATIONS,
-            idType: 'string',
             createId: true,
             assignGeneratedIds: false,
             created: true,
@@ -96,7 +95,6 @@ class CommonDao {
             },
         };
         if (this.cfg.createId) {
-            (0, js_lib_1._assert)(this.cfg.idType === 'string', 'db-lib: automatic generation of non-string ids is not supported');
             this.cfg.hooks.createRandomId ||= () => (0, nodejs_lib_1.stringId)();
         }
         else {
@@ -875,7 +873,7 @@ class CommonDao {
                 // LogProgress should be AFTER the mapper, to be able to report correct stats
                 (0, nodejs_lib_1.transformLogProgress)({
                     metric: q.table,
-                    logEvery: 2,
+                    logEvery: 2, // 500 * 2 === 1000
                     batchSize,
                     ...opt,
                 }),

package/dist/commondao/common.dao.model.d.ts

@@ -2,24 +2,24 @@ import { CommonLogger, ErrorMode, ObjectWithId, Promisable, Saved, ZodError, Zod
 import { AjvSchema, AjvValidationError, JoiValidationError, ObjectSchema, TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib';
 import { CommonDB } from '../common.db';
 import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model';
-export interface CommonDaoHooks<BM extends Partial<ObjectWithId<ID>>, DBM extends ObjectWithId<ID>, TM, ID extends string | number> {
+export interface CommonDaoHooks<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId, TM> {
     /**
      * Allows to override the id generation function.
      * By default it uses `stringId` from nodejs-lib
      * (which uses lowercase alphanumberic alphabet and the size of 16).
      */
-    createRandomId: () => ID;
+    createRandomId: () => string;
     /**
      * createNaturalId hook is called (tried) first.
      * If it doesn't exist - createRandomId is called.
      */
-    createNaturalId: (obj: DBM | BM) => ID;
+    createNaturalId: (obj: DBM | BM) => string;
     /**
      * It's a counter-part of `createNaturalId`.
      * Allows to provide a parser function to parse "natural id" into
      * DBM components (e.g accountId and some other property that is part of the id).
      */
-    parseNaturalId: (id: ID) => Partial<DBM>;
+    parseNaturalId: (id: string) => Partial<DBM>;
     /**
      * It is called only on `dao.create` method.
      * Dao.create method is called in:
@@ -104,7 +104,7 @@ export declare enum CommonDaoLogLevel {
      */
     DATA_FULL = 30
 }
-export interface CommonDaoCfg<BM extends Partial<ObjectWithId<ID>>, DBM extends ObjectWithId<ID> = Saved<BM>, TM = BM, ID extends string | number = string> {
+export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId = Saved<BM>, TM = BM> {
     db: CommonDB;
     table: string;
     /**
@@ -141,11 +141,7 @@ export interface CommonDaoCfg<BM extends Partial<ObjectWithId<ID>>, DBM extends
      * @default false
      */
     logStarted?: boolean;
-    hooks?: Partial<CommonDaoHooks<BM, DBM, TM, ID>>;
-    /**
-     * Defaults to 'string'
-     */
-    idType?: 'string' | 'number';
+    hooks?: Partial<CommonDaoHooks<BM, DBM, TM>>;
     /**
      * Defaults to true.
      * Set to false to disable auto-generation of `id`.

package/dist/model.util.d.ts

@@ -1,5 +1,5 @@
 import { CreatedUpdated, CreatedUpdatedId } from '@naturalcycles/js-lib';
 export declare function createdUpdatedFields(existingObject?: Partial<CreatedUpdated> | null): CreatedUpdated;
-export declare function createdUpdatedIdFields(existingObject?: Partial<CreatedUpdatedId<string>> | null): CreatedUpdatedId<string>;
+export declare function createdUpdatedIdFields(existingObject?: Partial<CreatedUpdatedId> | null): CreatedUpdatedId;
 export declare function deserializeJsonField<T = any>(f?: string): T;
 export declare function serializeJsonField(f: any): string | undefined;

package/dist/pipeline/dbPipelineBackup.d.ts

@@ -3,6 +3,7 @@ import { ZlibOptions } from 'node:zlib';
 import { AsyncMapper, ErrorMode, UnixTimestampNumber, StringMap } from '@naturalcycles/js-lib';
 import { NDJsonStats, TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib';
 import { CommonDB } from '../common.db';
+import { DBQuery } from '../index';
 export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
     /**
      * DB to dump data from.
@@ -40,6 +41,12 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
      * If set - will do "incremental backup" (not full), only for entities that updated >= `sinceUpdated` (on a per table basis)
      */
     sinceUpdatedPerTable?: StringMap<UnixTimestampNumber>;
+    /**
+     * By default, dbPipelineBackup creates a Query based on sinceUpdated.
+     * But if queryPerTable is set for a table - it will override the Query that is ran for that table
+     * (and ignore sinceUpdated, sinceUpdatedPerTable, limit, and any other properties that modify the query).
+     */
+    queryPerTable?: StringMap<DBQuery>;
     /**
      * Directory path to store dumped files. Will create `${tableName}.ndjson` (or .ndjson.gz if gzip=true) files.
      * All parent directories will be created.
@@ -67,6 +74,11 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
      * Default mappers will be "passthroughMapper" (pass all data as-is).
      */
     mapperPerTable?: StringMap<AsyncMapper>;
+    /**
+     * If defined - it'll use that `logEvery` for that table.
+     * Default logEvery is 1000.
+     */
+    logEveryPerTable?: StringMap<number>;
     /**
      * You can alter default `transformMapOptions` here.
      *

package/dist/pipeline/dbPipelineBackup.js

@@ -18,7 +18,7 @@ const index_1 = require("../index");
  * Optionally you can provide mapperPerTable and @param transformMapOptions (one for all mappers) - it will run for each table.
  */
 async function dbPipelineBackup(opt) {
-    const { db, concurrency = 16, limit = 0, outputDirPath, protectFromOverwrite = false, zlibOptions, mapperPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, emitSchemaFromDB = false, sortObjects = false, } = opt;
+    const { db, concurrency = 16, limit = 0, outputDirPath, protectFromOverwrite = false, zlibOptions, mapperPerTable = {}, queryPerTable = {}, logEveryPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, emitSchemaFromDB = false, sortObjects = false, } = opt;
     const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
     const gzip = opt.gzip !== false; // default to true
     let { tables } = opt;
@@ -28,15 +28,22 @@ async function dbPipelineBackup(opt) {
     console.log(`${(0, nodejs_lib_1.yellow)(tables.length)} ${(0, nodejs_lib_1.boldWhite)('table(s)')}:\n` + tables.join('\n'));
     const statsPerTable = {};
     await (0, js_lib_1.pMap)(tables, async (table) => {
-        const sinceUpdated = opt.sinceUpdatedPerTable?.[table] || opt.sinceUpdated;
-        const sinceUpdatedStr = sinceUpdated
-            ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty())
-            : '';
-        console.log(`>> ${(0, nodejs_lib_1.grey)(table)}${sinceUpdatedStr}`);
         let q = index_1.DBQuery.create(table).limit(limit);
+        const sinceUpdated = opt.sinceUpdatedPerTable?.[table] ?? opt.sinceUpdated;
         if (sinceUpdated) {
             q = q.filter('updated', '>=', sinceUpdated);
         }
+        if (queryPerTable[table]) {
+            // Override the Query with this Query, completely ingoring any of the other query-related options
+            q = queryPerTable[table];
+            console.log(`>> ${(0, nodejs_lib_1.grey)(table)} ${q.pretty()}`);
+        }
+        else {
+            const sinceUpdatedStr = sinceUpdated
+                ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty())
+                : '';
+            console.log(`>> ${(0, nodejs_lib_1.grey)(table)}${sinceUpdatedStr}`);
+        }
         const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '');
         const schemaFilePath = `${outputDirPath}/${table}.schema.json`;
         if (protectFromOverwrite && (0, nodejs_lib_1._pathExistsSync)(filePath)) {
@@ -45,7 +52,7 @@ async function dbPipelineBackup(opt) {
         const started = Date.now();
         let rows = 0;
         (0, nodejs_lib_1._ensureFileSync)(filePath);
-        console.log(`>> ${(0, nodejs_lib_1.grey)(filePath)} started...`);
+        // console.log(`>> ${grey(filePath)} started...`)
         if (emitSchemaFromDB) {
             const schema = await db.getTableSchema(table);
             await (0, nodejs_lib_1._writeJson)(schemaFilePath, schema, { spaces: 2 });
@@ -54,8 +61,8 @@ async function dbPipelineBackup(opt) {
         await (0, nodejs_lib_1._pipeline)([
             db.streamQuery(q),
             (0, nodejs_lib_1.transformLogProgress)({
-                logEvery: 1000,
                 ...opt,
+                logEvery: logEveryPerTable[table] ?? opt.logEvery ?? 1000,
                 metric: table,
             }),
             (0, nodejs_lib_1.transformMap)(mapperPerTable[table] || js_lib_1._passthroughMapper, {
@@ -68,7 +75,7 @@ async function dbPipelineBackup(opt) {
                 rows++;
             }),
             (0, nodejs_lib_1.transformToNDJson)({ strict, sortObjects }),
-            ...(gzip ? [(0, node_zlib_1.createGzip)(zlibOptions)] : []),
+            ...(gzip ? [(0, node_zlib_1.createGzip)(zlibOptions)] : []), // optional gzip
             node_fs_1.default.createWriteStream(filePath),
         ]);
         const { size: sizeBytes } = await promises_1.default.stat(filePath);

package/dist/pipeline/dbPipelineRestore.js

@@ -69,7 +69,7 @@ async function dbPipelineRestore(opt) {
         await (0, nodejs_lib_1._pipeline)([
             node_fs_1.default.createReadStream(filePath),
             ...(gzip ? [(0, node_zlib_1.createUnzip)()] : []),
-            (0, nodejs_lib_1.transformSplit)(),
+            (0, nodejs_lib_1.transformSplit)(), // splits by \n
             (0, nodejs_lib_1.transformJsonParse)({ strict }),
             (0, nodejs_lib_1.transformTap)(() => rows++),
             (0, nodejs_lib_1.transformLogProgress)({

package/dist/query/dbQuery.d.ts

@@ -84,12 +84,12 @@ export declare class DBQuery<ROW extends ObjectWithId = AnyObjectWithId> {
 /**
  * DBQuery that has additional method to support Fluent API style.
  */
-export declare class RunnableDBQuery<BM extends Partial<ObjectWithId<ID>>, DBM extends ObjectWithId<ID> = Saved<BM>, TM extends AnyObject = BM, ID extends string | number = string> extends DBQuery<DBM> {
-    dao: CommonDao<BM, DBM, TM, ID>;
+export declare class RunnableDBQuery<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId = Saved<BM>, TM extends AnyObject = BM> extends DBQuery<DBM> {
+    dao: CommonDao<BM, DBM, TM>;
     /**
      * Pass `table` to override table.
      */
-    constructor(dao: CommonDao<BM, DBM, TM, ID>, table?: string);
+    constructor(dao: CommonDao<BM, DBM, TM>, table?: string);
     runQuery(opt?: CommonDaoOptions): Promise<Saved<BM>[]>;
     runQuerySingleColumn<T = any>(opt?: CommonDaoOptions): Promise<T[]>;
     runQueryAsDBM(opt?: CommonDaoOptions): Promise<DBM[]>;
@@ -103,8 +103,8 @@ export declare class RunnableDBQuery<BM extends Partial<ObjectWithId<ID>>, DBM e
     streamQueryAsDBMForEach(mapper: AsyncMapper<DBM, void>, opt?: CommonDaoStreamForEachOptions<DBM>): Promise<void>;
     streamQuery(opt?: CommonDaoStreamOptions<Saved<BM>>): ReadableTyped<Saved<BM>>;
     streamQueryAsDBM(opt?: CommonDaoStreamOptions<DBM>): ReadableTyped<DBM>;
-    queryIds(opt?: CommonDaoOptions): Promise<ID[]>;
-    streamQueryIds(opt?: CommonDaoStreamOptions<ID>): ReadableTyped<ID>;
-    streamQueryIdsForEach(mapper: AsyncMapper<ID, void>, opt?: CommonDaoStreamForEachOptions<ID>): Promise<void>;
+    queryIds(opt?: CommonDaoOptions): Promise<string[]>;
+    streamQueryIds(opt?: CommonDaoStreamOptions<string>): ReadableTyped<string>;
+    streamQueryIdsForEach(mapper: AsyncMapper<string, void>, opt?: CommonDaoStreamForEachOptions<string>): Promise<void>;
     deleteByQuery(opt?: CommonDaoStreamDeleteOptions<DBM>): Promise<number>;
 }

package/dist/testing/test.model.js

@@ -25,7 +25,7 @@ exports.testItemTMSchema = (0, nodejs_lib_1.objectSchema)({
 });
 exports.testItemBMJsonSchema = js_lib_1.jsonSchema
     .rootObject({
-    id: js_lib_1.jsonSchema.string(),
+    id: js_lib_1.jsonSchema.string(), // todo: not strictly needed here
     k1: js_lib_1.jsonSchema.string(),
     k2: js_lib_1.jsonSchema.oneOf([js_lib_1.jsonSchema.string(), js_lib_1.jsonSchema.null()]).optional(),
     k3: js_lib_1.jsonSchema.number().optional(),

package/dist/timeseries/commonTimeSeriesDao.js

@@ -31,8 +31,8 @@ class CommonTimeSeriesDao {
         if (!dataPoints.length)
             return;
         const rows = dataPoints.map(([ts, v]) => ({
-            id: String(ts),
-            ts,
+            id: String(ts), // Convert Number id into String id, as per CommonDB
+            ts, // to allow querying by ts, since querying by id is not always available (Datastore is one example)
             v,
         }));
         await this.cfg.db.saveBatch(`${series}${_TIMESERIES_RAW}`, rows);
@@ -46,8 +46,8 @@ class CommonTimeSeriesDao {
         const tx = __1.DBTransaction.create();
         ops.forEach(op => {
             const rows = op.dataPoints.map(([ts, v]) => ({
-                id: String(ts),
-                ts,
+                id: String(ts), // Convert Number id into String id, as per CommonDB
+                ts, // to allow querying by ts, since querying by id is not always available (Datastore is one example)
                 v,
             }));
             tx.saveBatch(`${op.series}${_TIMESERIES_RAW}`, rows);

package/dist/validation/index.d.ts

@@ -3,6 +3,6 @@ import { DBQuery, DBQueryFilter, DBQueryFilterOperator, DBQueryOrder } from '../
 export declare const commonDBOptionsSchema: import("joi").ObjectSchema<CommonDBOptions>;
 export declare const commonDBSaveOptionsSchema: import("joi").ObjectSchema<CommonDBSaveOptions<any>>;
 export declare const dbQueryFilterOperatorSchema: import("@naturalcycles/nodejs-lib/dist/validation/joi/string.extensions").StringSchema<DBQueryFilterOperator>;
-export declare const dbQueryFilterSchema: import("joi").ObjectSchema<DBQueryFilter<import("@naturalcycles/js-lib").AnyObjectWithId<string | number>>>;
-export declare const dbQueryOrderSchema: import("joi").ObjectSchema<DBQueryOrder<import("@naturalcycles/js-lib").AnyObjectWithId<string | number>>>;
+export declare const dbQueryFilterSchema: import("joi").ObjectSchema<DBQueryFilter<import("@naturalcycles/js-lib").AnyObjectWithId>>;
+export declare const dbQueryOrderSchema: import("joi").ObjectSchema<DBQueryOrder<import("@naturalcycles/js-lib").AnyObjectWithId>>;
 export declare const dbQuerySchema: import("joi").ObjectSchema<DBQuery<any>>;

package/package.json

@@ -40,7 +40,7 @@
   "engines": {
     "node": ">=18.12"
   },
-  "version": "8.58.0",
+  "version": "8.60.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/commondao/common.dao.model.ts

@@ -18,31 +18,26 @@ import {
 import { CommonDB } from '../common.db'
 import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model'
 
-export interface CommonDaoHooks<
-  BM extends Partial<ObjectWithId<ID>>,
-  DBM extends ObjectWithId<ID>,
-  TM,
-  ID extends string | number,
-> {
+export interface CommonDaoHooks<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId, TM> {
   /**
    * Allows to override the id generation function.
    * By default it uses `stringId` from nodejs-lib
    * (which uses lowercase alphanumberic alphabet and the size of 16).
    */
-  createRandomId: () => ID
+  createRandomId: () => string
 
   /**
    * createNaturalId hook is called (tried) first.
    * If it doesn't exist - createRandomId is called.
    */
-  createNaturalId: (obj: DBM | BM) => ID
+  createNaturalId: (obj: DBM | BM) => string
 
   /**
    * It's a counter-part of `createNaturalId`.
    * Allows to provide a parser function to parse "natural id" into
    * DBM components (e.g accountId and some other property that is part of the id).
    */
-  parseNaturalId: (id: ID) => Partial<DBM>
+  parseNaturalId: (id: string) => Partial<DBM>
 
   /**
    * It is called only on `dao.create` method.
@@ -137,10 +132,9 @@ export enum CommonDaoLogLevel {
 }
 
 export interface CommonDaoCfg<
-  BM extends Partial<ObjectWithId<ID>>,
-  DBM extends ObjectWithId<ID> = Saved<BM>,
+  BM extends Partial<ObjectWithId>,
+  DBM extends ObjectWithId = Saved<BM>,
   TM = BM,
-  ID extends string | number = string,
 > {
   db: CommonDB
   table: string
@@ -187,12 +181,7 @@ export interface CommonDaoCfg<
   logStarted?: boolean
 
   // Hooks are designed with inspiration from got/ky interface
-  hooks?: Partial<CommonDaoHooks<BM, DBM, TM, ID>>
-
-  /**
-   * Defaults to 'string'
-   */
-  idType?: 'string' | 'number'
+  hooks?: Partial<CommonDaoHooks<BM, DBM, TM>>
 
   /**
    * Defaults to true.

package/src/commondao/common.dao.ts

@@ -78,18 +78,16 @@ const isCI = !!process.env['CI']
  * TM = Transport model (optimized to be sent over the wire)
  */
 export class CommonDao<
-  BM extends Partial<ObjectWithId<ID>>,
-  DBM extends ObjectWithId<ID> = Saved<BM>,
+  BM extends Partial<ObjectWithId>,
+  DBM extends ObjectWithId = Saved<BM>,
   TM extends AnyObject = BM,
-  ID extends string | number = NonNullable<BM['id']>,
 > {
-  constructor(public cfg: CommonDaoCfg<BM, DBM, TM, ID>) {
+  constructor(public cfg: CommonDaoCfg<BM, DBM, TM>) {
     this.cfg = {
       // Default is to NOT log in AppEngine and in CI,
       // otherwise to log Operations
       // e.g in Dev (local machine), Test - it will log operations (useful for debugging)
       logLevel: isGAE || isCI ? CommonDaoLogLevel.NONE : CommonDaoLogLevel.OPERATIONS,
-      idType: 'string',
       createId: true,
       assignGeneratedIds: false,
       created: true,
@@ -106,16 +104,11 @@ export class CommonDao<
         anonymize: dbm => dbm,
         onValidationError: err => err,
         ...cfg.hooks,
-      } satisfies Partial<CommonDaoHooks<BM, DBM, TM, ID>>,
+      } satisfies Partial<CommonDaoHooks<BM, DBM, TM>>,
     }
 
     if (this.cfg.createId) {
-      _assert(
-        this.cfg.idType === 'string',
-        'db-lib: automatic generation of non-string ids is not supported',
-      )
-
-      this.cfg.hooks!.createRandomId ||= () => stringId() as ID
+      this.cfg.hooks!.createRandomId ||= () => stringId()
     } else {
       delete this.cfg.hooks!.createRandomId
     }
@@ -131,8 +124,8 @@ export class CommonDao<
 
   // GET
   async getById(id: undefined | null, opt?: CommonDaoOptions): Promise<null>
-  async getById(id?: ID | null, opt?: CommonDaoOptions): Promise<Saved<BM> | null>
-  async getById(id?: ID | null, opt: CommonDaoOptions = {}): Promise<Saved<BM> | null> {
+  async getById(id?: string | null, opt?: CommonDaoOptions): Promise<Saved<BM> | null>
+  async getById(id?: string | null, opt: CommonDaoOptions = {}): Promise<Saved<BM> | null> {
     if (!id) return null
     const op = `getById(${id})`
     const table = opt.table || this.cfg.table
@@ -148,14 +141,22 @@ export class CommonDao<
     return bm || null
   }
 
-  async getByIdOrEmpty(id: ID, part: Partial<BM> = {}, opt?: CommonDaoOptions): Promise<Saved<BM>> {
+  async getByIdOrEmpty(
+    id: string,
+    part: Partial<BM> = {},
+    opt?: CommonDaoOptions,
+  ): Promise<Saved<BM>> {
     const bm = await this.getById(id, opt)
     if (bm) return bm
 
     return this.create({ ...part, id }, opt)
   }
 
-  async getByIdAsDBMOrEmpty(id: ID, part: Partial<BM> = {}, opt?: CommonDaoOptions): Promise<DBM> {
+  async getByIdAsDBMOrEmpty(
+    id: string,
+    part: Partial<BM> = {},
+    opt?: CommonDaoOptions,
+  ): Promise<DBM> {
     const dbm = await this.getByIdAsDBM(id, opt)
     if (dbm) return dbm
 
@@ -164,8 +165,8 @@ export class CommonDao<
   }
 
   async getByIdAsDBM(id: undefined | null, opt?: CommonDaoOptions): Promise<null>
-  async getByIdAsDBM(id?: ID | null, opt?: CommonDaoOptions): Promise<DBM | null>
-  async getByIdAsDBM(id?: ID | null, opt: CommonDaoOptions = {}): Promise<DBM | null> {
+  async getByIdAsDBM(id?: string | null, opt?: CommonDaoOptions): Promise<DBM | null>
+  async getByIdAsDBM(id?: string | null, opt: CommonDaoOptions = {}): Promise<DBM | null> {
     if (!id) return null
     const op = `getByIdAsDBM(${id})`
     const table = opt.table || this.cfg.table
@@ -183,8 +184,8 @@ export class CommonDao<
   }
 
   async getByIdAsTM(id: undefined | null, opt?: CommonDaoOptions): Promise<null>
-  async getByIdAsTM(id?: ID | null, opt?: CommonDaoOptions): Promise<TM | null>
-  async getByIdAsTM(id?: ID | null, opt: CommonDaoOptions = {}): Promise<TM | null> {
+  async getByIdAsTM(id?: string | null, opt?: CommonDaoOptions): Promise<TM | null>
+  async getByIdAsTM(id?: string | null, opt: CommonDaoOptions = {}): Promise<TM | null> {
     if (!id) return null
     const op = `getByIdAsTM(${id})`
     const table = opt.table || this.cfg.table
@@ -204,7 +205,7 @@ export class CommonDao<
     return tm || null
   }
 
-  async getByIds(ids: ID[], opt: CommonDaoOptions = {}): Promise<Saved<BM>[]> {
+  async getByIds(ids: string[], opt: CommonDaoOptions = {}): Promise<Saved<BM>[]> {
     if (!ids.length) return []
     const op = `getByIds ${ids.length} id(s) (${_truncate(ids.slice(0, 10).join(', '), 50)})`
     const table = opt.table || this.cfg.table
@@ -221,7 +222,7 @@ export class CommonDao<
     return bms
   }
 
-  async getByIdsAsDBM(ids: ID[], opt: CommonDaoOptions = {}): Promise<DBM[]> {
+  async getByIdsAsDBM(ids: string[], opt: CommonDaoOptions = {}): Promise<DBM[]> {
     if (!ids.length) return []
     const op = `getByIdsAsDBM ${ids.length} id(s) (${_truncate(ids.slice(0, 10).join(', '), 50)})`
     const table = opt.table || this.cfg.table
@@ -237,7 +238,7 @@ export class CommonDao<
     return dbms
   }
 
-  async requireById(id: ID, opt: CommonDaoOptions = {}): Promise<Saved<BM>> {
+  async requireById(id: string, opt: CommonDaoOptions = {}): Promise<Saved<BM>> {
     const r = await this.getById(id, opt)
     if (!r) {
       this.throwRequiredError(id, opt)
@@ -245,7 +246,7 @@ export class CommonDao<
     return r
   }
 
-  async requireByIdAsDBM(id: ID, opt: CommonDaoOptions = {}): Promise<DBM> {
+  async requireByIdAsDBM(id: string, opt: CommonDaoOptions = {}): Promise<DBM> {
     const r = await this.getByIdAsDBM(id, opt)
     if (!r) {
       this.throwRequiredError(id, opt)
@@ -253,7 +254,7 @@ export class CommonDao<
     return r
   }
 
-  private throwRequiredError(id: ID, opt: CommonDaoOptions): never {
+  private throwRequiredError(id: string, opt: CommonDaoOptions): never {
     const table = opt.table || this.cfg.table
     throw new AppError(`DB row required, but not found in ${table}`, {
       table,
@@ -311,8 +312,8 @@ export class CommonDao<
   /**
    * Pass `table` to override table
    */
-  query(table?: string): RunnableDBQuery<BM, DBM, TM, ID> {
-    return new RunnableDBQuery<BM, DBM, TM, ID>(this, table)
+  query(table?: string): RunnableDBQuery<BM, DBM, TM> {
+    return new RunnableDBQuery<BM, DBM, TM>(this, table)
   }
 
   async runQuery(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<Saved<BM>[]> {
@@ -614,21 +615,21 @@ export class CommonDao<
     )
   }
 
-  async queryIds(q: DBQuery<DBM>, opt: CommonDaoOptions = {}): Promise<ID[]> {
+  async queryIds(q: DBQuery<DBM>, opt: CommonDaoOptions = {}): Promise<string[]> {
     q.table = opt.table || q.table
     const { rows } = await this.cfg.db.runQuery(q.select(['id']), opt)
     return rows.map(r => r.id)
   }
 
-  streamQueryIds(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<ID> = {}): ReadableTyped<ID> {
+  streamQueryIds(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<string> = {}): ReadableTyped<string> {
     q.table = opt.table || q.table
     opt.errorMode ||= ErrorMode.SUPPRESS
 
-    const stream: ReadableTyped<ID> = this.cfg.db
+    const stream: ReadableTyped<string> = this.cfg.db
       .streamQuery<DBM>(q.select(['id']), opt)
       .on('error', err => stream.emit('error', err))
       .pipe(
-        transformMapSimple<DBM, ID>(objectWithId => objectWithId.id, {
+        transformMapSimple<DBM, string>(objectWithId => objectWithId.id, {
           errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }),
       )
@@ -638,8 +639,8 @@ export class CommonDao<
 
   async streamQueryIdsForEach(
     q: DBQuery<DBM>,
-    mapper: AsyncMapper<ID, void>,
-    opt: CommonDaoStreamForEachOptions<ID> = {},
+    mapper: AsyncMapper<string, void>,
+    opt: CommonDaoStreamForEachOptions<string> = {},
   ): Promise<void> {
     q.table = opt.table || q.table
     opt.errorMode ||= ErrorMode.SUPPRESS
@@ -650,11 +651,11 @@ export class CommonDao<
 
     await _pipeline([
       this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
-      transformMapSimple<DBM, ID>(objectWithId => {
+      transformMapSimple<DBM, string>(objectWithId => {
         count++
         return objectWithId.id
       }),
-      transformMap<ID, void>(mapper, {
+      transformMap<string, void>(mapper, {
         ...opt,
         predicate: _passthroughPredicate,
       }),
@@ -734,26 +735,26 @@ export class CommonDao<
       }
     },
     deleteByIds: async (
-      ids: ID[],
+      ids: string[],
       opt: CommonDaoOptions = {},
     ): Promise<DBDeleteByIdsOperation | undefined> => {
       if (!ids.length) return
       return {
         type: 'deleteByIds',
         table: this.cfg.table,
-        ids: ids as string[],
+        ids,
         opt,
       }
     },
     deleteById: async (
-      id: ID | null | undefined,
+      id: string | null | undefined,
       opt: CommonDaoOptions = {},
     ): Promise<DBDeleteByIdsOperation | undefined> => {
       if (!id) return
       return {
         type: 'deleteByIds',
         table: this.cfg.table,
-        ids: [id as string],
+        ids: [id],
         opt,
       }
     },
@@ -848,7 +849,7 @@ export class CommonDao<
    * 3. Saves (as fast as possible since the read) with the Patch applied, but only if the data has changed.
    */
   async patchById(
-    id: ID,
+    id: string,
     patch: Partial<BM>,
     opt: CommonDaoSaveBatchOptions<DBM> = {},
   ): Promise<Saved<BM>> {
@@ -1109,8 +1110,8 @@ export class CommonDao<
    * @returns number of deleted items
    */
   async deleteById(id: undefined | null, opt?: CommonDaoOptions): Promise<0>
-  async deleteById(id?: ID | null, opt?: CommonDaoOptions): Promise<number>
-  async deleteById(id?: ID | null, opt: CommonDaoOptions = {}): Promise<number> {
+  async deleteById(id?: string | null, opt?: CommonDaoOptions): Promise<number>
+  async deleteById(id?: string | null, opt: CommonDaoOptions = {}): Promise<number> {
     if (!id) return 0
     this.requireWriteAccess()
     this.requireObjectMutability(opt)
@@ -1122,7 +1123,7 @@ export class CommonDao<
     return count
   }
 
-  async deleteByIds(ids: ID[], opt: CommonDaoOptions = {}): Promise<number> {
+  async deleteByIds(ids: string[], opt: CommonDaoOptions = {}): Promise<number> {
     if (!ids.length) return 0
     this.requireWriteAccess()
     this.requireObjectMutability(opt)
@@ -1155,7 +1156,7 @@ export class CommonDao<
 
       await _pipeline([
         this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
-        transformMapSimple<DBM, ID>(objectWithId => objectWithId.id, {
+        transformMapSimple<DBM, string>(objectWithId => objectWithId.id, {
           errorMode: ErrorMode.SUPPRESS,
         }),
         transformBuffer<string>({ batchSize }),
@@ -1188,11 +1189,15 @@ export class CommonDao<
     return deleted
   }
 
-  async updateById(id: ID, patch: DBPatch<DBM>, opt: CommonDaoOptions = {}): Promise<number> {
+  async updateById(id: string, patch: DBPatch<DBM>, opt: CommonDaoOptions = {}): Promise<number> {
     return await this.updateByQuery(this.query().filterEq('id', id), patch, opt)
   }
 
-  async updateByIds(ids: ID[], patch: DBPatch<DBM>, opt: CommonDaoOptions = {}): Promise<number> {
+  async updateByIds(
+    ids: string[],
+    patch: DBPatch<DBM>,
+    opt: CommonDaoOptions = {},
+  ): Promise<number> {
     if (!ids.length) return 0
     return await this.updateByQuery(this.query().filterIn('id', ids), patch, opt)
   }

package/src/model.util.ts

@@ -12,8 +12,8 @@ export function createdUpdatedFields(
 }
 
 export function createdUpdatedIdFields(
-  existingObject?: Partial<CreatedUpdatedId<string>> | null,
-): CreatedUpdatedId<string> {
+  existingObject?: Partial<CreatedUpdatedId> | null,
+): CreatedUpdatedId {
   const now = Math.floor(Date.now() / 1000)
   return {
     created: existingObject?.created || now,

package/src/pipeline/dbPipelineBackup.ts

@@ -76,6 +76,13 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
    */
   sinceUpdatedPerTable?: StringMap<UnixTimestampNumber>
 
+  /**
+   * By default, dbPipelineBackup creates a Query based on sinceUpdated.
+   * But if queryPerTable is set for a table - it will override the Query that is ran for that table
+   * (and ignore sinceUpdated, sinceUpdatedPerTable, limit, and any other properties that modify the query).
+   */
+  queryPerTable?: StringMap<DBQuery>
+
   /**
    * Directory path to store dumped files. Will create `${tableName}.ndjson` (or .ndjson.gz if gzip=true) files.
    * All parent directories will be created.
@@ -108,6 +115,12 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
    */
   mapperPerTable?: StringMap<AsyncMapper>
 
+  /**
+   * If defined - it'll use that `logEvery` for that table.
+   * Default logEvery is 1000.
+   */
+  logEveryPerTable?: StringMap<number>
+
   /**
    * You can alter default `transformMapOptions` here.
    *
@@ -153,6 +166,8 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
     protectFromOverwrite = false,
     zlibOptions,
     mapperPerTable = {},
+    queryPerTable = {},
+    logEveryPerTable = {},
     transformMapOptions,
     errorMode = ErrorMode.SUPPRESS,
     emitSchemaFromDB = false,
@@ -176,20 +191,25 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
   await pMap(
     tables,
     async table => {
-      const sinceUpdated = opt.sinceUpdatedPerTable?.[table] || opt.sinceUpdated
-
-      const sinceUpdatedStr = sinceUpdated
-        ? ' since ' + grey(localTime(sinceUpdated).toPretty())
-        : ''
-
-      console.log(`>> ${grey(table)}${sinceUpdatedStr}`)
-
-      let q = DBQuery.create(table).limit(limit)
+      let q = DBQuery.create<any>(table).limit(limit)
 
+      const sinceUpdated = opt.sinceUpdatedPerTable?.[table] ?? opt.sinceUpdated
       if (sinceUpdated) {
         q = q.filter('updated', '>=', sinceUpdated)
       }
 
+      if (queryPerTable[table]) {
+        // Override the Query with this Query, completely ingoring any of the other query-related options
+        q = queryPerTable[table]!
+
+        console.log(`>> ${grey(table)} ${q.pretty()}`)
+      } else {
+        const sinceUpdatedStr = sinceUpdated
+          ? ' since ' + grey(localTime(sinceUpdated).toPretty())
+          : ''
+        console.log(`>> ${grey(table)}${sinceUpdatedStr}`)
+      }
+
       const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '')
       const schemaFilePath = `${outputDirPath}/${table}.schema.json`
 
@@ -202,7 +222,7 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
 
       _ensureFileSync(filePath)
 
-      console.log(`>> ${grey(filePath)} started...`)
+      // console.log(`>> ${grey(filePath)} started...`)
 
       if (emitSchemaFromDB) {
         const schema = await db.getTableSchema(table)
@@ -213,8 +233,8 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
       await _pipeline([
         db.streamQuery(q),
         transformLogProgress({
-          logEvery: 1000,
           ...opt,
+          logEvery: logEveryPerTable[table] ?? opt.logEvery ?? 1000,
           metric: table,
         }),
         transformMap(mapperPerTable[table] || _passthroughMapper, {

package/src/query/dbQuery.ts

@@ -237,16 +237,15 @@ export class DBQuery<ROW extends ObjectWithId = AnyObjectWithId> {
  * DBQuery that has additional method to support Fluent API style.
  */
 export class RunnableDBQuery<
-  BM extends Partial<ObjectWithId<ID>>,
-  DBM extends ObjectWithId<ID> = Saved<BM>,
+  BM extends Partial<ObjectWithId>,
+  DBM extends ObjectWithId = Saved<BM>,
   TM extends AnyObject = BM,
-  ID extends string | number = string,
 > extends DBQuery<DBM> {
   /**
    * Pass `table` to override table.
    */
   constructor(
-    public dao: CommonDao<BM, DBM, TM, ID>,
+    public dao: CommonDao<BM, DBM, TM>,
     table?: string,
   ) {
     super(table || dao.cfg.table)
@@ -310,17 +309,17 @@ export class RunnableDBQuery<
     return this.dao.streamQueryAsDBM(this, opt)
   }
 
-  async queryIds(opt?: CommonDaoOptions): Promise<ID[]> {
+  async queryIds(opt?: CommonDaoOptions): Promise<string[]> {
     return await this.dao.queryIds(this, opt)
   }
 
-  streamQueryIds(opt?: CommonDaoStreamOptions<ID>): ReadableTyped<ID> {
+  streamQueryIds(opt?: CommonDaoStreamOptions<string>): ReadableTyped<string> {
     return this.dao.streamQueryIds(this, opt)
   }
 
   async streamQueryIdsForEach(
-    mapper: AsyncMapper<ID, void>,
-    opt?: CommonDaoStreamForEachOptions<ID>,
+    mapper: AsyncMapper<string, void>,
+    opt?: CommonDaoStreamForEachOptions<string>,
   ): Promise<void> {
     await this.dao.streamQueryIdsForEach(this, mapper, opt)
   }