@naturalcycles/db-lib 10.46.1 → 10.47.1

package/dist/commondao/common.dao.js

@@ -178,7 +178,10 @@ export class CommonDao {
         q.table = opt.table || q.table;
         let pipeline = this.cfg.db.streamQuery(q, opt);
         if (this.cfg.compress?.keys.length) {
-            pipeline = pipeline.map(async row => await this.storageRowToDBM(row));
+            pipeline = pipeline.map(async row => await this.storageRowToDBM(row), {
+                // lowered, to reduce the total buffer size a bit for uncompressed objects
+                highWaterMark: 16,
+            });
         }
         const isPartialQuery = !!q._selectedFieldNames;
         if (isPartialQuery)
@@ -192,7 +195,10 @@ export class CommonDao {
         q.table = opt.table || q.table;
         let pipeline = this.cfg.db.streamQuery(q, opt);
         if (this.cfg.compress?.keys.length) {
-            pipeline = pipeline.map(async row => await this.storageRowToDBM(row));
+            pipeline = pipeline.map(async row => await this.storageRowToDBM(row), {
+                // lowered, to reduce the total buffer size a bit for uncompressed objects
+                highWaterMark: 16,
+            });
         }
         const isPartialQuery = !!q._selectedFieldNames;
         if (isPartialQuery)
@@ -645,10 +651,10 @@ export class CommonDao {
     async compress(dbm) {
         if (!this.cfg.compress?.keys.length)
             return; // No compression requested
-        const { keys } = this.cfg.compress;
+        const { keys, level = 1 } = this.cfg.compress;
         const properties = _pick(dbm, keys);
         const bufferString = JSON.stringify(properties);
-        const __compressed = await zstdCompress(bufferString);
+        const __compressed = await zstdCompress(bufferString, level);
         _omitWithUndefined(dbm, _objectKeys(properties), { mutate: true });
         Object.assign(dbm, { __compressed });
     }

package/dist/commondao/common.dao.model.d.ts

@@ -1,7 +1,7 @@
 import type { ValidationFunction } from '@naturalcycles/js-lib';
 import type { AppError, ErrorMode } from '@naturalcycles/js-lib/error';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import type { BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types';
+import type { BaseDBEntity, Integer, UnixTimestamp } from '@naturalcycles/js-lib/types';
 import type { TransformLogProgressOptions } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDB } from '../commondb/common.db.js';
 import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model.js';
@@ -180,6 +180,11 @@ export interface CommonDaoCfg<BM extends BaseDBEntity, DBM extends BaseDBEntity
      */
     compress?: {
         keys: (keyof DBM)[];
+        /**
+         * zstd compression level.
+         * Undefined will default to level 1 (not the 3, which is the zstd default)
+         */
+        level?: Integer;
     };
 }
 /**

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/db-lib",
   "type": "module",
-  "version": "10.46.1",
+  "version": "10.47.1",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@naturalcycles/nodejs-lib": "^15"

package/src/commondao/common.dao.model.ts

@@ -1,7 +1,7 @@
 import type { ValidationFunction } from '@naturalcycles/js-lib'
 import type { AppError, ErrorMode } from '@naturalcycles/js-lib/error'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import type { BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types'
+import type { BaseDBEntity, Integer, UnixTimestamp } from '@naturalcycles/js-lib/types'
 import type { TransformLogProgressOptions } from '@naturalcycles/nodejs-lib/stream'
 import type { CommonDB } from '../commondb/common.db.js'
 import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model.js'
@@ -219,6 +219,11 @@ export interface CommonDaoCfg<
    */
   compress?: {
     keys: (keyof DBM)[]
+    /**
+     * zstd compression level.
+     * Undefined will default to level 1 (not the 3, which is the zstd default)
+     */
+    level?: Integer
   }
 }
 

package/src/commondao/common.dao.ts

@@ -254,7 +254,10 @@ export class CommonDao<
     let pipeline = this.cfg.db.streamQuery<DBM>(q, opt)
 
     if (this.cfg.compress?.keys.length) {
-      pipeline = pipeline.map(async row => await this.storageRowToDBM(row))
+      pipeline = pipeline.map(async row => await this.storageRowToDBM(row), {
+        // lowered, to reduce the total buffer size a bit for uncompressed objects
+        highWaterMark: 16,
+      })
     }
 
     const isPartialQuery = !!q._selectedFieldNames
@@ -272,7 +275,10 @@ export class CommonDao<
     let pipeline = this.cfg.db.streamQuery<DBM>(q, opt)
 
     if (this.cfg.compress?.keys.length) {
-      pipeline = pipeline.map(async row => await this.storageRowToDBM(row))
+      pipeline = pipeline.map(async row => await this.storageRowToDBM(row), {
+        // lowered, to reduce the total buffer size a bit for uncompressed objects
+        highWaterMark: 16,
+      })
     }
 
     const isPartialQuery = !!q._selectedFieldNames
@@ -841,10 +847,10 @@ export class CommonDao<
   private async compress(dbm: DBM): Promise<void> {
     if (!this.cfg.compress?.keys.length) return // No compression requested
 
-    const { keys } = this.cfg.compress
+    const { keys, level = 1 } = this.cfg.compress
     const properties = _pick(dbm, keys)
     const bufferString = JSON.stringify(properties)
-    const __compressed = await zstdCompress(bufferString)
+    const __compressed = await zstdCompress(bufferString, level)
     _omitWithUndefined(dbm as any, _objectKeys(properties), { mutate: true })
     Object.assign(dbm, { __compressed })
   }