@naturalcycles/db-lib 10.15.1 → 10.17.0

package/dist/commondao/common.dao.d.ts

@@ -2,7 +2,7 @@ import type { Transform } from 'node:stream';
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import type { AsyncIndexedMapper, BaseDBEntity, StringMap, UnixTimestampMillis, Unsaved } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { type ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDBTransactionOptions, DBTransaction, RunQueryResult } from '../db.model.js';
 import type { DBQuery } from '../query/dbQuery.js';
 import { RunnableDBQuery } from '../query/dbQuery.js';

package/dist/commondao/common.dao.js

@@ -9,6 +9,7 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _truncate } from '@naturalcycles/js-lib/string/string.util.js';
 import { _passthroughPredicate, _typeCast, SKIP } from '@naturalcycles/js-lib/types';
 import { stringId } from '@naturalcycles/nodejs-lib';
+import { transformFlatten } from '@naturalcycles/nodejs-lib/stream';
 import { _pipeline, transformChunk, transformLogProgress, transformMap, transformNoOp, writableVoid, } from '@naturalcycles/nodejs-lib/stream';
 import { DBLibError } from '../cnst.js';
 import { RunnableDBQuery } from '../query/dbQuery.js';
@@ -473,7 +474,7 @@ export class CommonDao {
             obj.created ||= obj.updated || now;
         }
         if (this.cfg.useUpdatedProperty) {
-            obj.updated = opt.preserveUpdatedCreated && obj.updated ? obj.updated : now;
+            obj.updated = opt.preserveUpdated && obj.updated ? obj.updated : now;
         }
         if (this.cfg.generateId) {
             obj.id ||= (this.cfg.hooks.createNaturalId?.(obj) ||
@@ -766,8 +767,8 @@ export class CommonDao {
             }, {
                 concurrency: chunkConcurrency,
                 errorMode,
-                flattenArrayOutput: true,
             }),
+            transformFlatten(),
             transformLogProgress({
                 metric: 'saved',
                 ...opt,

package/dist/commondao/common.dao.model.d.ts

@@ -106,7 +106,15 @@ export interface CommonDaoCfg<BM extends BaseDBEntity, DBM extends BaseDBEntity
      * and abstracts it away.
      */
     validateBM?: ValidationFunction<BM, any>;
+    /**
+     * Used by e.g Datastore.
+     */
     excludeFromIndexes?: (keyof DBM)[];
+    /**
+     * Used by e.g Firestore.
+     */
+    indexes?: CommonDaoIndex<DBM>[];
+    ttl?: CommonDaoTTL<DBM>;
     /**
      * Defaults to true.
      * If set to false - load (read) operations will skip validation (and conversion).
@@ -174,6 +182,28 @@ export interface CommonDaoCfg<BM extends BaseDBEntity, DBM extends BaseDBEntity
      */
     patchInTransaction?: boolean;
 }
+export interface CommonDaoIndex<DBM extends BaseDBEntity> {
+    /**
+     * Name of the property to index.
+     */
+    name: keyof DBM;
+    /**
+     * Defaults to ['asc']
+     */
+    order?: CommonDaoIndexOrder[];
+}
+export type CommonDaoIndexOrder = 'asc' | 'desc' | 'array-contains';
+/**
+ * TTL definition - a map from Table name to an array of properties that should be used for TTL.
+ * Example:
+ *
+ * {
+ *   myTable: ['deleteAt']
+ * }
+ */
+export interface CommonDaoTTL<DBM extends BaseDBEntity> {
+    [table: string]: (keyof DBM)[];
+}
 /**
  * All properties default to undefined.
  */
@@ -202,9 +232,12 @@ export interface CommonDaoOptions extends CommonDBOptions {
      */
     mutateInput?: boolean;
     /**
-     * @default false
+     * Defaults to false.
+     *
+     * If false (default) - will set `updated` property to the current timestamp.
+     * If true - will NOT set it (preserve the original value).
      */
-    preserveUpdatedCreated?: boolean;
+    preserveUpdated?: boolean;
     /**
      * @default false (for streams). Setting to true enables deletion of immutable objects
      */

package/dist/pipeline/dbPipelineBackup.js

@@ -4,7 +4,7 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
+import { createWriteStreamAsNDJSON, transformFlatten, } from '@naturalcycles/nodejs-lib/stream';
 import { _pipeline, NDJsonStats, transformLogProgress, transformMap, transformTap, } from '@naturalcycles/nodejs-lib/stream';
 import { DBQuery } from '../query/dbQuery.js';
 /**
@@ -65,10 +65,10 @@ export async function dbPipelineBackup(opt) {
             }),
             transformMap(mapperPerTable[table] || _passthroughMapper, {
                 errorMode,
-                flattenArrayOutput: true,
                 ...transformMapOptions,
                 metric: table,
             }),
+            transformFlatten(),
             transformTap(() => {
                 rows++;
             }),

package/dist/pipeline/dbPipelineCopy.d.ts

@@ -1,6 +1,6 @@
 import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
 import type { AsyncMapper, UnixTimestamp } from '@naturalcycles/js-lib/types';
-import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
+import { type TransformLogProgressOptions, type TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDB } from '../commondb/common.db.js';
 import type { CommonDBSaveOptions } from '../db.model.js';

package/dist/pipeline/dbPipelineCopy.js

@@ -3,6 +3,7 @@ import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
+import { transformFlatten, } from '@naturalcycles/nodejs-lib/stream';
 import { _pipeline, NDJsonStats, transformChunk, transformLogProgress, transformMap, transformTap, writableForEach, } from '@naturalcycles/nodejs-lib/stream';
 import { DBQuery } from '../query/dbQuery.js';
 /**
@@ -38,10 +39,10 @@ export async function dbPipelineCopy(opt) {
             }),
             transformMap(mapper, {
                 errorMode,
-                flattenArrayOutput: true,
                 ...transformMapOptions,
                 metric: table,
             }),
+            transformFlatten(),
             transformTap(() => rows++),
             transformChunk({ chunkSize }),
             writableForEach(async (dbms) => {

package/dist/pipeline/dbPipelineRestore.js

@@ -6,7 +6,7 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { createReadStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
+import { createReadStreamAsNDJSON, transformFlatten, } from '@naturalcycles/nodejs-lib/stream';
 import { _pipeline, NDJsonStats, transformChunk, transformFilterSync, transformLogProgress, transformMap, transformTap, writableForEach, } from '@naturalcycles/nodejs-lib/stream';
 /**
  * Pipeline from NDJSON files in a folder (optionally gzipped) to CommonDB.
@@ -80,10 +80,10 @@ export async function dbPipelineRestore(opt) {
                 : []),
             transformMap(mapperPerTable[table] || _passthroughMapper, {
                 errorMode,
-                flattenArrayOutput: true,
                 ...transformMapOptions,
                 metric: table,
             }),
+            transformFlatten(),
             transformChunk({ chunkSize }),
             writableForEach(async (dbms) => {
                 await db.saveBatch(table, dbms, saveOptions);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/db-lib",
   "type": "module",
-  "version": "10.15.1",
+  "version": "10.17.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@naturalcycles/nodejs-lib": "^15"

package/src/commondao/common.dao.model.ts

@@ -131,8 +131,18 @@ export interface CommonDaoCfg<
    */
   validateBM?: ValidationFunction<BM, any>
 
+  /**
+   * Used by e.g Datastore.
+   */
   excludeFromIndexes?: (keyof DBM)[]
 
+  /**
+   * Used by e.g Firestore.
+   */
+  indexes?: CommonDaoIndex<DBM>[]
+
+  ttl?: CommonDaoTTL<DBM>
+
   /**
    * Defaults to true.
    * If set to false - load (read) operations will skip validation (and conversion).
@@ -214,6 +224,31 @@ export interface CommonDaoCfg<
   patchInTransaction?: boolean
 }
 
+export interface CommonDaoIndex<DBM extends BaseDBEntity> {
+  /**
+   * Name of the property to index.
+   */
+  name: keyof DBM
+  /**
+   * Defaults to ['asc']
+   */
+  order?: CommonDaoIndexOrder[]
+}
+
+export type CommonDaoIndexOrder = 'asc' | 'desc' | 'array-contains'
+
+/**
+ * TTL definition - a map from Table name to an array of properties that should be used for TTL.
+ * Example:
+ *
+ * {
+ *   myTable: ['deleteAt']
+ * }
+ */
+export interface CommonDaoTTL<DBM extends BaseDBEntity> {
+  [table: string]: (keyof DBM)[]
+}
+
 /**
  * All properties default to undefined.
  */
@@ -244,9 +279,12 @@ export interface CommonDaoOptions extends CommonDBOptions {
   mutateInput?: boolean
 
   /**
-   * @default false
+   * Defaults to false.
+   *
+   * If false (default) - will set `updated` property to the current timestamp.
+   * If true - will NOT set it (preserve the original value).
    */
-  preserveUpdatedCreated?: boolean
+  preserveUpdated?: boolean
 
   /**
    * @default false (for streams). Setting to true enables deletion of immutable objects

package/src/commondao/common.dao.ts

@@ -24,7 +24,7 @@ import type {
 } from '@naturalcycles/js-lib/types'
 import { _passthroughPredicate, _typeCast, SKIP } from '@naturalcycles/js-lib/types'
 import { stringId } from '@naturalcycles/nodejs-lib'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { type ReadableTyped, transformFlatten } from '@naturalcycles/nodejs-lib/stream'
 import {
   _pipeline,
   transformChunk,
@@ -617,7 +617,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM, I
     }
 
     if (this.cfg.useUpdatedProperty) {
-      obj.updated = opt.preserveUpdatedCreated && obj.updated ? obj.updated : now
+      obj.updated = opt.preserveUpdated && obj.updated ? obj.updated : now
     }
 
     if (this.cfg.generateId) {
@@ -983,9 +983,9 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM, I
         {
           concurrency: chunkConcurrency,
           errorMode,
-          flattenArrayOutput: true,
         },
       ),
+      transformFlatten(),
       transformLogProgress({
         metric: 'saved',
         ...opt,

package/src/pipeline/dbPipelineBackup.ts

@@ -7,6 +7,7 @@ import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/col
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
 import {
   createWriteStreamAsNDJSON,
+  transformFlatten,
   type TransformLogProgressOptions,
   type TransformMapOptions,
 } from '@naturalcycles/nodejs-lib/stream'
@@ -220,10 +221,10 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
         }),
         transformMap(mapperPerTable[table] || _passthroughMapper, {
           errorMode,
-          flattenArrayOutput: true,
           ...transformMapOptions,
           metric: table,
         }),
+        transformFlatten(),
         transformTap(() => {
           rows++
         }),

package/src/pipeline/dbPipelineCopy.ts

@@ -4,9 +4,10 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { AsyncMapper, BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types'
 import { _passthroughMapper } from '@naturalcycles/js-lib/types'
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
-import type {
-  TransformLogProgressOptions,
-  TransformMapOptions,
+import {
+  transformFlatten,
+  type TransformLogProgressOptions,
+  type TransformMapOptions,
 } from '@naturalcycles/nodejs-lib/stream'
 import {
   _pipeline,
@@ -144,10 +145,10 @@ export async function dbPipelineCopy(opt: DBPipelineCopyOptions): Promise<NDJson
         }),
         transformMap(mapper, {
           errorMode,
-          flattenArrayOutput: true,
           ...transformMapOptions,
           metric: table,
         }),
+        transformFlatten(),
         transformTap(() => rows++),
         transformChunk({ chunkSize }),
         writableForEach(async dbms => {

package/src/pipeline/dbPipelineRestore.ts

@@ -10,6 +10,7 @@ import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/col
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
 import {
   createReadStreamAsNDJSON,
+  transformFlatten,
   type TransformLogProgressOptions,
   type TransformMapOptions,
 } from '@naturalcycles/nodejs-lib/stream'
@@ -207,10 +208,10 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
           : []),
         transformMap(mapperPerTable[table] || _passthroughMapper, {
           errorMode,
-          flattenArrayOutput: true,
           ...transformMapOptions,
           metric: table,
         }),
+        transformFlatten(),
         transformChunk({ chunkSize }),
         writableForEach(async dbms => {
           await db.saveBatch(table, dbms, saveOptions)