@naturalcycles/db-lib 8.58.0 → 8.59.0

package/dist/pipeline/dbPipelineBackup.d.ts

@@ -3,6 +3,7 @@ import { ZlibOptions } from 'node:zlib';
 import { AsyncMapper, ErrorMode, UnixTimestampNumber, StringMap } from '@naturalcycles/js-lib';
 import { NDJsonStats, TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib';
 import { CommonDB } from '../common.db';
+import { DBQuery } from '../index';
 export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
     /**
      * DB to dump data from.
@@ -40,6 +41,12 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
      * If set - will do "incremental backup" (not full), only for entities that updated >= `sinceUpdated` (on a per table basis)
      */
     sinceUpdatedPerTable?: StringMap<UnixTimestampNumber>;
+    /**
+     * By default, dbPipelineBackup creates a Query based on sinceUpdated.
+     * But if queryPerTable is set for a table - it will override the Query that is ran for that table
+     * (and ignore sinceUpdated, sinceUpdatedPerTable, limit, and any other properties that modify the query).
+     */
+    queryPerTable?: StringMap<DBQuery>;
     /**
      * Directory path to store dumped files. Will create `${tableName}.ndjson` (or .ndjson.gz if gzip=true) files.
      * All parent directories will be created.
@@ -67,6 +74,11 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
      * Default mappers will be "passthroughMapper" (pass all data as-is).
      */
     mapperPerTable?: StringMap<AsyncMapper>;
+    /**
+     * If defined - it'll use that `logEvery` for that table.
+     * Default logEvery is 1000.
+     */
+    logEveryPerTable?: StringMap<number>;
     /**
      * You can alter default `transformMapOptions` here.
      *

package/dist/pipeline/dbPipelineBackup.js

@@ -18,7 +18,7 @@ const index_1 = require("../index");
  * Optionally you can provide mapperPerTable and @param transformMapOptions (one for all mappers) - it will run for each table.
  */
 async function dbPipelineBackup(opt) {
-    const { db, concurrency = 16, limit = 0, outputDirPath, protectFromOverwrite = false, zlibOptions, mapperPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, emitSchemaFromDB = false, sortObjects = false, } = opt;
+    const { db, concurrency = 16, limit = 0, outputDirPath, protectFromOverwrite = false, zlibOptions, mapperPerTable = {}, queryPerTable = {}, logEveryPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, emitSchemaFromDB = false, sortObjects = false, } = opt;
     const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
     const gzip = opt.gzip !== false; // default to true
     let { tables } = opt;
@@ -28,15 +28,22 @@ async function dbPipelineBackup(opt) {
     console.log(`${(0, nodejs_lib_1.yellow)(tables.length)} ${(0, nodejs_lib_1.boldWhite)('table(s)')}:\n` + tables.join('\n'));
     const statsPerTable = {};
     await (0, js_lib_1.pMap)(tables, async (table) => {
-        const sinceUpdated = opt.sinceUpdatedPerTable?.[table] || opt.sinceUpdated;
-        const sinceUpdatedStr = sinceUpdated
-            ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty())
-            : '';
-        console.log(`>> ${(0, nodejs_lib_1.grey)(table)}${sinceUpdatedStr}`);
         let q = index_1.DBQuery.create(table).limit(limit);
+        const sinceUpdated = opt.sinceUpdatedPerTable?.[table] ?? opt.sinceUpdated;
         if (sinceUpdated) {
             q = q.filter('updated', '>=', sinceUpdated);
         }
+        if (queryPerTable[table]) {
+            // Override the Query with this Query, completely ingoring any of the other query-related options
+            q = queryPerTable[table];
+            console.log(`>> ${(0, nodejs_lib_1.grey)(table)} ${q.pretty()}`);
+        }
+        else {
+            const sinceUpdatedStr = sinceUpdated
+                ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty())
+                : '';
+            console.log(`>> ${(0, nodejs_lib_1.grey)(table)}${sinceUpdatedStr}`);
+        }
         const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '');
         const schemaFilePath = `${outputDirPath}/${table}.schema.json`;
         if (protectFromOverwrite && (0, nodejs_lib_1._pathExistsSync)(filePath)) {
@@ -45,7 +52,7 @@ async function dbPipelineBackup(opt) {
         const started = Date.now();
         let rows = 0;
         (0, nodejs_lib_1._ensureFileSync)(filePath);
-        console.log(`>> ${(0, nodejs_lib_1.grey)(filePath)} started...`);
+        // console.log(`>> ${grey(filePath)} started...`)
         if (emitSchemaFromDB) {
             const schema = await db.getTableSchema(table);
             await (0, nodejs_lib_1._writeJson)(schemaFilePath, schema, { spaces: 2 });
@@ -54,8 +61,8 @@ async function dbPipelineBackup(opt) {
         await (0, nodejs_lib_1._pipeline)([
             db.streamQuery(q),
             (0, nodejs_lib_1.transformLogProgress)({
-                logEvery: 1000,
                 ...opt,
+                logEvery: logEveryPerTable[table] ?? opt.logEvery ?? 1000,
                 metric: table,
             }),
             (0, nodejs_lib_1.transformMap)(mapperPerTable[table] || js_lib_1._passthroughMapper, {

package/package.json

@@ -40,7 +40,7 @@
   "engines": {
     "node": ">=18.12"
   },
-  "version": "8.58.0",
+  "version": "8.59.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/pipeline/dbPipelineBackup.ts

@@ -76,6 +76,13 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
    */
   sinceUpdatedPerTable?: StringMap<UnixTimestampNumber>
 
+  /**
+   * By default, dbPipelineBackup creates a Query based on sinceUpdated.
+   * But if queryPerTable is set for a table - it will override the Query that is ran for that table
+   * (and ignore sinceUpdated, sinceUpdatedPerTable, limit, and any other properties that modify the query).
+   */
+  queryPerTable?: StringMap<DBQuery>
+
   /**
    * Directory path to store dumped files. Will create `${tableName}.ndjson` (or .ndjson.gz if gzip=true) files.
    * All parent directories will be created.
@@ -108,6 +115,12 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
    */
   mapperPerTable?: StringMap<AsyncMapper>
 
+  /**
+   * If defined - it'll use that `logEvery` for that table.
+   * Default logEvery is 1000.
+   */
+  logEveryPerTable?: StringMap<number>
+
   /**
    * You can alter default `transformMapOptions` here.
    *
@@ -153,6 +166,8 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
     protectFromOverwrite = false,
     zlibOptions,
     mapperPerTable = {},
+    queryPerTable = {},
+    logEveryPerTable = {},
     transformMapOptions,
     errorMode = ErrorMode.SUPPRESS,
     emitSchemaFromDB = false,
@@ -176,20 +191,25 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
   await pMap(
     tables,
     async table => {
-      const sinceUpdated = opt.sinceUpdatedPerTable?.[table] || opt.sinceUpdated
-
-      const sinceUpdatedStr = sinceUpdated
-        ? ' since ' + grey(localTime(sinceUpdated).toPretty())
-        : ''
-
-      console.log(`>> ${grey(table)}${sinceUpdatedStr}`)
-
-      let q = DBQuery.create(table).limit(limit)
+      let q = DBQuery.create<any>(table).limit(limit)
 
+      const sinceUpdated = opt.sinceUpdatedPerTable?.[table] ?? opt.sinceUpdated
       if (sinceUpdated) {
         q = q.filter('updated', '>=', sinceUpdated)
       }
 
+      if (queryPerTable[table]) {
+        // Override the Query with this Query, completely ingoring any of the other query-related options
+        q = queryPerTable[table]!
+
+        console.log(`>> ${grey(table)} ${q.pretty()}`)
+      } else {
+        const sinceUpdatedStr = sinceUpdated
+          ? ' since ' + grey(localTime(sinceUpdated).toPretty())
+          : ''
+        console.log(`>> ${grey(table)}${sinceUpdatedStr}`)
+      }
+
       const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '')
       const schemaFilePath = `${outputDirPath}/${table}.schema.json`
 
@@ -202,7 +222,7 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
 
       _ensureFileSync(filePath)
 
-      console.log(`>> ${grey(filePath)} started...`)
+      // console.log(`>> ${grey(filePath)} started...`)
 
       if (emitSchemaFromDB) {
         const schema = await db.getTableSchema(table)
@@ -213,8 +233,8 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
       await _pipeline([
         db.streamQuery(q),
         transformLogProgress({
-          logEvery: 1000,
           ...opt,
+          logEvery: logEveryPerTable[table] ?? opt.logEvery ?? 1000,
           metric: table,
         }),
         transformMap(mapperPerTable[table] || _passthroughMapper, {