@naturalcycles/db-lib 10.35.1 → 10.37.0

package/dist/adapter/file/localFile.persistence.plugin.d.ts

@@ -9,7 +9,7 @@ export interface LocalFilePersistencePluginCfg {
     /**
      * @default true
      */
-    gzip: boolean;
+    zst: boolean;
 }
 /**
  * Persists in local filesystem as ndjson.

package/dist/adapter/file/localFile.persistence.plugin.js

@@ -8,7 +8,7 @@ export class LocalFilePersistencePlugin {
     constructor(cfg = {}) {
         this.cfg = {
             storagePath: './tmp/localdb',
-            gzip: true,
+            zst: true,
             ...cfg,
         };
     }
@@ -21,7 +21,7 @@ export class LocalFilePersistencePlugin {
     }
     async loadFile(table) {
         await fs2.ensureDirAsync(this.cfg.storagePath);
-        const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
+        const ext = `ndjson${this.cfg.zst ? '.zst' : ''}`;
         const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
         if (!(await fs2.pathExistsAsync(filePath)))
             return [];
@@ -32,7 +32,7 @@ export class LocalFilePersistencePlugin {
     }
     async saveFile(table, rows) {
         await fs2.ensureDirAsync(this.cfg.storagePath);
-        const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
+        const ext = `ndjson${this.cfg.zst ? '.zst' : ''}`;
         const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
         await Pipeline.fromArray(rows).toNDJsonFile(filePath);
     }

package/dist/kv/commonKeyValueDao.d.ts

@@ -32,6 +32,12 @@ export interface CommonKeyValueDaoTransformer<V> {
     bufferToValue: (buf: Buffer) => Promise<V>;
 }
 export declare const commonKeyValueDaoDeflatedJsonTransformer: CommonKeyValueDaoTransformer<any>;
+export declare const commonKeyValueDaoZstdJsonTransformer: CommonKeyValueDaoTransformer<any>;
+/**
+ * Saves: zstd
+ * Reads: zstd or deflate (backwards compatible)
+ */
+export declare const commonKeyValueDaoCompressedTransformer: CommonKeyValueDaoTransformer<any>;
 export declare class CommonKeyValueDao<K extends string = string, V = Buffer> {
     constructor(cfg: CommonKeyValueDaoCfg<V>);
     cfg: CommonKeyValueDaoCfg<V> & {

package/dist/kv/commonKeyValueDao.js

@@ -1,11 +1,23 @@
 import { AppError } from '@naturalcycles/js-lib/error/error.util.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { SKIP } from '@naturalcycles/js-lib/types';
-import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib/zip';
+import { decompressZstdOrInflateToString, deflateString, inflateToString, zstdCompress, zstdDecompressToString, } from '@naturalcycles/nodejs-lib/zip';
 export const commonKeyValueDaoDeflatedJsonTransformer = {
     valueToBuffer: async (v) => await deflateString(JSON.stringify(v)),
     bufferToValue: async (buf) => JSON.parse(await inflateToString(buf)),
 };
+export const commonKeyValueDaoZstdJsonTransformer = {
+    valueToBuffer: async (v) => await zstdCompress(JSON.stringify(v)),
+    bufferToValue: async (buf) => JSON.parse(await zstdDecompressToString(buf)),
+};
+/**
+ * Saves: zstd
+ * Reads: zstd or deflate (backwards compatible)
+ */
+export const commonKeyValueDaoCompressedTransformer = {
+    valueToBuffer: async (v) => await zstdCompress(JSON.stringify(v)),
+    bufferToValue: async (buf) => JSON.parse(await decompressZstdOrInflateToString(buf)),
+};
 // todo: logging
 // todo: readonly
 export class CommonKeyValueDao {

package/dist/pipeline/dbPipelineBackup.d.ts

@@ -60,9 +60,10 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
      */
     protectFromOverwrite?: boolean;
     /**
+     * Compress as .zst
      * @default true
      */
-    gzip?: boolean;
+    zst?: boolean;
     /**
      * Only applicable if `gzip` is enabled
      * Currently not available.

package/dist/pipeline/dbPipelineBackup.js

@@ -16,8 +16,7 @@ import { DBQuery } from '../query/dbQuery.js';
  * Optionally you can provide mapperPerTable and @param transformMapOptions (one for all mappers) - it will run for each table.
  */
 export async function dbPipelineBackup(opt) {
-    const { db, concurrency = 16, limit = 0, outputDirPath, protectFromOverwrite = false, mapperPerTable = {}, queryPerTable = {}, logEveryPerTable = {}, transformMapOptions, errorMode = ErrorMode.SUPPRESS, emitSchemaFromDB = false, } = opt;
-    const gzip = opt.gzip !== false; // default to true
+    const { db, concurrency = 16, limit = 0, outputDirPath, protectFromOverwrite = false, mapperPerTable = {}, queryPerTable = {}, logEveryPerTable = {}, transformMapOptions, errorMode = ErrorMode.SUPPRESS, emitSchemaFromDB = false, zst = true, } = opt;
     let { tables } = opt;
     console.log(`>> ${dimWhite('dbPipelineBackup')} started in ${grey(outputDirPath)}...`);
     fs2.ensureDir(outputDirPath);
@@ -41,7 +40,7 @@ export async function dbPipelineBackup(opt) {
                 : '';
             console.log(`>> ${grey(table)}${sinceUpdatedStr}`);
         }
-        const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '');
+        const filePath = `${outputDirPath}/${table}.ndjson` + (zst ? '.zst' : '');
         const schemaFilePath = `${outputDirPath}/${table}.schema.json`;
         if (protectFromOverwrite && fs2.pathExists(filePath)) {
             throw new AppError(`dbPipelineBackup: output file exists: ${filePath}`);

package/dist/pipeline/dbPipelineRestore.d.ts

@@ -9,7 +9,7 @@ export interface DBPipelineRestoreOptions extends TransformLogProgressOptions {
      */
     db: CommonDB;
     /**
-     * Directory path to store dumped files. Will create `${tableName}.ndjson` (or .ndjson.gz if gzip=true) files.
+     * Directory path to store dumped files. Will create `${tableName}.ndjson` (or .ndjson.zst if zst=true) files.
      * All parent directories will be created.
      */
     inputDirPath: string;

package/dist/pipeline/dbPipelineRestore.js

@@ -20,19 +20,19 @@ export async function dbPipelineRestore(opt) {
     const sinceUpdatedStr = sinceUpdated ? ' since ' + grey(localTime(sinceUpdated).toPretty()) : '';
     console.log(`>> ${dimWhite('dbPipelineRestore')} started in ${grey(inputDirPath)}...${sinceUpdatedStr}`);
     fs2.ensureDir(inputDirPath);
-    const tablesToGzip = new Set();
+    const tablesToCompress = new Set();
     const sizeByTable = {};
     const statsPerTable = {};
     const tables = [];
     fs2.readdir(inputDirPath).forEach(f => {
         let table;
-        let gzip = false;
+        let zst = false;
         if (f.endsWith('.ndjson')) {
             table = f.slice(0, f.length - '.ndjson'.length);
         }
-        else if (f.endsWith('.ndjson.gz')) {
-            table = f.slice(0, f.length - '.ndjson.gz'.length);
-            gzip = true;
+        else if (f.endsWith('.ndjson.zst')) {
+            table = f.slice(0, f.length - '.ndjson.zst'.length);
+            zst = true;
         }
         else {
             return;
@@ -40,8 +40,8 @@ export async function dbPipelineRestore(opt) {
         if (onlyTables && !onlyTables.has(table))
             return; // skip table
         tables.push(table);
-        if (gzip)
-            tablesToGzip.add(table);
+        if (zst)
+            tablesToCompress.add(table);
         sizeByTable[table] = fs2.stat(`${inputDirPath}/${f}`).size;
     });
     const sizeStrByTable = _mapValues(sizeByTable, (_k, b) => _hb(b));
@@ -59,8 +59,8 @@ export async function dbPipelineRestore(opt) {
         });
     }
     await pMap(tables, async (table) => {
-        const gzip = tablesToGzip.has(table);
-        const filePath = `${inputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '');
+        const zst = tablesToCompress.has(table);
+        const filePath = `${inputDirPath}/${table}.ndjson` + (zst ? '.zst' : '');
         const saveOptions = saveOptionsPerTable[table] || {};
         const started = Date.now();
         let rows = 0;

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@naturalcycles/db-lib",
   "type": "module",
-  "version": "10.35.1",
+  "version": "10.37.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@naturalcycles/nodejs-lib": "^15"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "20.12.10"
+    "@naturalcycles/dev-lib": "20.15.0"
   },
   "files": [
     "dist",

package/src/adapter/file/localFile.persistence.plugin.ts

@@ -14,7 +14,7 @@ export interface LocalFilePersistencePluginCfg {
   /**
    * @default true
    */
-  gzip: boolean
+  zst: boolean
 }
 
 /**
@@ -24,7 +24,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
   constructor(cfg: Partial<LocalFilePersistencePluginCfg> = {}) {
     this.cfg = {
       storagePath: './tmp/localdb',
-      gzip: true,
+      zst: true,
       ...cfg,
     }
   }
@@ -41,7 +41,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
 
   async loadFile<ROW extends ObjectWithId>(table: string): Promise<ROW[]> {
     await fs2.ensureDirAsync(this.cfg.storagePath)
-    const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
+    const ext = `ndjson${this.cfg.zst ? '.zst' : ''}`
     const filePath = `${this.cfg.storagePath}/${table}.${ext}`
 
     if (!(await fs2.pathExistsAsync(filePath))) return []
@@ -55,7 +55,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
 
   async saveFile<ROW extends ObjectWithId>(table: string, rows: ROW[]): Promise<void> {
     await fs2.ensureDirAsync(this.cfg.storagePath)
-    const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
+    const ext = `ndjson${this.cfg.zst ? '.zst' : ''}`
     const filePath = `${this.cfg.storagePath}/${table}.${ext}`
 
     await Pipeline.fromArray(rows).toNDJsonFile(filePath)

package/src/kv/commonKeyValueDao.ts

@@ -3,7 +3,13 @@ import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import { type KeyValueTuple, SKIP } from '@naturalcycles/js-lib/types'
 import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
-import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib/zip'
+import {
+  decompressZstdOrInflateToString,
+  deflateString,
+  inflateToString,
+  zstdCompress,
+  zstdDecompressToString,
+} from '@naturalcycles/nodejs-lib/zip'
 import type { CommonDaoLogLevel } from '../commondao/common.dao.model.js'
 import type { CommonDBCreateOptions } from '../db.model.js'
 import type {
@@ -54,6 +60,20 @@ export const commonKeyValueDaoDeflatedJsonTransformer: CommonKeyValueDaoTransfor
   bufferToValue: async buf => JSON.parse(await inflateToString(buf)),
 }
 
+export const commonKeyValueDaoZstdJsonTransformer: CommonKeyValueDaoTransformer<any> = {
+  valueToBuffer: async v => await zstdCompress(JSON.stringify(v)),
+  bufferToValue: async buf => JSON.parse(await zstdDecompressToString(buf)),
+}
+
+/**
+ * Saves: zstd
+ * Reads: zstd or deflate (backwards compatible)
+ */
+export const commonKeyValueDaoCompressedTransformer: CommonKeyValueDaoTransformer<any> = {
+  valueToBuffer: async v => await zstdCompress(JSON.stringify(v)),
+  bufferToValue: async buf => JSON.parse(await decompressZstdOrInflateToString(buf)),
+}
+
 // todo: logging
 // todo: readonly
 

package/src/pipeline/dbPipelineBackup.ts

@@ -79,9 +79,10 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
   protectFromOverwrite?: boolean
 
   /**
+   * Compress as .zst
    * @default true
    */
-  gzip?: boolean
+  zst?: boolean
 
   /**
    * Only applicable if `gzip` is enabled
@@ -147,8 +148,8 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
     transformMapOptions,
     errorMode = ErrorMode.SUPPRESS,
     emitSchemaFromDB = false,
+    zst = true,
   } = opt
-  const gzip = opt.gzip !== false // default to true
 
   let { tables } = opt
 
@@ -184,7 +185,7 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
         console.log(`>> ${grey(table)}${sinceUpdatedStr}`)
       }
 
-      const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '')
+      const filePath = `${outputDirPath}/${table}.ndjson` + (zst ? '.zst' : '')
       const schemaFilePath = `${outputDirPath}/${table}.schema.json`
 
       if (protectFromOverwrite && fs2.pathExists(filePath)) {

package/src/pipeline/dbPipelineRestore.ts

@@ -24,7 +24,7 @@ export interface DBPipelineRestoreOptions extends TransformLogProgressOptions {
   db: CommonDB
 
   /**
-   * Directory path to store dumped files. Will create `${tableName}.ndjson` (or .ndjson.gz if gzip=true) files.
+   * Directory path to store dumped files. Will create `${tableName}.ndjson` (or .ndjson.zst if zst=true) files.
    * All parent directories will be created.
    */
   inputDirPath: string
@@ -128,19 +128,19 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
 
   fs2.ensureDir(inputDirPath)
 
-  const tablesToGzip = new Set<string>()
+  const tablesToCompress = new Set<string>()
   const sizeByTable: Record<string, number> = {}
   const statsPerTable: Record<string, NDJsonStats> = {}
   const tables: string[] = []
   fs2.readdir(inputDirPath).forEach(f => {
     let table: string
-    let gzip = false
+    let zst = false
 
     if (f.endsWith('.ndjson')) {
       table = f.slice(0, f.length - '.ndjson'.length)
-    } else if (f.endsWith('.ndjson.gz')) {
-      table = f.slice(0, f.length - '.ndjson.gz'.length)
-      gzip = true
+    } else if (f.endsWith('.ndjson.zst')) {
+      table = f.slice(0, f.length - '.ndjson.zst'.length)
+      zst = true
     } else {
       return
     }
@@ -148,7 +148,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
     if (onlyTables && !onlyTables.has(table)) return // skip table
 
     tables.push(table)
-    if (gzip) tablesToGzip.add(table)
+    if (zst) tablesToCompress.add(table)
     sizeByTable[table] = fs2.stat(`${inputDirPath}/${f}`).size
   })
 
@@ -174,8 +174,8 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
   await pMap(
     tables,
     async table => {
-      const gzip = tablesToGzip.has(table)
-      const filePath = `${inputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '')
+      const zst = tablesToCompress.has(table)
+      const filePath = `${inputDirPath}/${table}.ndjson` + (zst ? '.zst' : '')
       const saveOptions: CommonDBSaveOptions<any> = saveOptionsPerTable[table] || {}
 
       const started = Date.now()