@naturalcycles/nodejs-lib 15.71.0 → 15.72.0

package/dist/stream/pipeline.d.ts

@@ -1,7 +1,7 @@
 import { type Transform } from 'node:stream';
 import type { ReadableStream as WebReadableStream } from 'node:stream/web';
 import { type ZlibOptions, type ZstdOptions } from 'node:zlib';
-import { type AbortableAsyncMapper, type AsyncIndexedMapper, type AsyncPredicate, type END, type IndexedMapper, type NonNegativeInteger, type PositiveInteger, type Predicate, type SKIP } from '@naturalcycles/js-lib/types';
+import { type AbortableAsyncMapper, type AsyncIndexedMapper, type AsyncPredicate, type END, type IndexedMapper, type Integer, type NonNegativeInteger, type PositiveInteger, type Predicate, type SKIP } from '@naturalcycles/js-lib/types';
 import type { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream.model.js';
 import { type TransformLogProgressOptions } from './transform/transformLogProgress.js';
 import { type TransformMapOptions } from './transform/transformMap.js';
@@ -97,11 +97,19 @@ export declare class Pipeline<T = unknown> {
     parseJson<TO = unknown>(this: Pipeline<Buffer> | Pipeline<Uint8Array> | Pipeline<string>): Pipeline<TO>;
     gzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
     gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
-    zstdCompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array>;
+    zstdCompress(this: Pipeline<Uint8Array>, level?: Integer, // defaults to 3
+    opt?: ZstdOptions): Pipeline<Uint8Array>;
     zstdDecompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array>;
     toArray(opt?: TransformOptions): Promise<T[]>;
     toFile(outputFilePath: string): Promise<void>;
-    toNDJsonFile(outputFilePath: string): Promise<void>;
+    /**
+     * level corresponds to zstd compression level (if filename ends with .zst),
+     * or gzip compression level (if filename ends with .gz).
+     * Default levels are:
+     * gzip: 6
+     * zlib: 3 (optimized for throughput, not size, may be larger than gzip at its default level)
+     */
+    toNDJsonFile(outputFilePath: string, level?: Integer): Promise<void>;
     to(destination: WritableTyped<T>): Promise<void>;
     forEach(fn: AsyncIndexedMapper<T, void>, opt?: TransformMapOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
     forEachSync(fn: IndexedMapper<T, void>, opt?: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;

package/dist/stream/pipeline.js

@@ -4,6 +4,7 @@ import { createGzip, createUnzip, createZstdCompress, createZstdDecompress, } fr
 import { createAbortableSignal } from '@naturalcycles/js-lib';
 import { _passthroughPredicate, } from '@naturalcycles/js-lib/types';
 import { fs2 } from '../fs/fs2.js';
+import { zstdLevelToOptions } from '../zip/zip.util.js';
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js';
 import { transformJsonParse } from './ndjson/transformJsonParse.js';
 import { transformToNDJson } from './ndjson/transformToNDJson.js';
@@ -253,11 +254,9 @@ export class Pipeline {
         this.objectMode = false;
         return this;
     }
-    zstdCompress(opt) {
-        this.transforms.push(createZstdCompress({
-            // chunkSize: 64 * 1024, // no observed speedup
-            ...opt,
-        }));
+    zstdCompress(level, // defaults to 3
+    opt) {
+        this.transforms.push(createZstdCompress(zstdLevelToOptions(level, opt)));
         this.objectMode = false;
         return this;
     }
@@ -280,18 +279,24 @@ export class Pipeline {
         this.destination = fs2.createWriteStream(outputFilePath);
         await this.run();
     }
-    async toNDJsonFile(outputFilePath) {
+    /**
+     * level corresponds to zstd compression level (if filename ends with .zst),
+     * or gzip compression level (if filename ends with .gz).
+     * Default levels are:
+     * gzip: 6
+     * zlib: 3 (optimized for throughput, not size, may be larger than gzip at its default level)
+     */
+    async toNDJsonFile(outputFilePath, level) {
         fs2.ensureFile(outputFilePath);
         this.transforms.push(transformToNDJson());
         if (outputFilePath.endsWith('.gz')) {
             this.transforms.push(createGzip({
-            // chunkSize: 64 * 1024, // no observed speedup
+                level,
+                // chunkSize: 64 * 1024, // no observed speedup
             }));
         }
         else if (outputFilePath.endsWith('.zst')) {
-            this.transforms.push(createZstdCompress({
-            // chunkSize: 64 * 1024, // no observed speedup
-            }));
+            this.transforms.push(createZstdCompress(zstdLevelToOptions(level)));
         }
         this.destination = fs2.createWriteStream(outputFilePath, {
         // highWaterMark: 64 * 1024, // no observed speedup

package/dist/zip/zip.util.d.ts

@@ -1,4 +1,5 @@
 import type { ZlibOptions, ZstdOptions } from 'node:zlib';
+import type { Integer } from '@naturalcycles/js-lib/types';
 export declare function decompressZstdOrInflateToString(buf: Buffer): Promise<string>;
 /**
  * Detects if Buffer is zstd-compressed.
@@ -29,7 +30,9 @@ export declare function gunzipBuffer(buf: Buffer, options?: ZlibOptions): Promis
  */
 export declare function gzipString(s: string, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
 export declare function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
-export declare function zstdCompress(input: Buffer | string, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
+export declare function zstdCompress(input: Buffer | string, level?: Integer, // defaults to 3
+options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
+export declare function zstdLevelToOptions(level: Integer | undefined, opt?: ZstdOptions): ZstdOptions;
 export declare function zstdDecompressToString(input: Buffer, options?: ZstdOptions): Promise<string>;
 export declare function zstdDecompress(input: Buffer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
 export declare function isZstdBuffer(input: Buffer): boolean;

package/dist/zip/zip.util.js

@@ -59,8 +59,20 @@ export async function gzipString(s, options) {
 export async function gunzipToString(buf, options) {
     return (await gunzipBuffer(buf, options)).toString();
 }
-export async function zstdCompress(input, options = {}) {
-    return await zstdCompressAsync(input, options);
+export async function zstdCompress(input, level, // defaults to 3
+options = {}) {
+    return await zstdCompressAsync(input, zstdLevelToOptions(level, options));
+}
+export function zstdLevelToOptions(level, opt = {}) {
+    if (!level)
+        return opt;
+    return {
+        ...opt,
+        params: {
+            ...opt.params,
+            [zlib.constants.ZSTD_c_compressionLevel]: level,
+        },
+    };
 }
 export async function zstdDecompressToString(input, options = {}) {
     return (await zstdDecompressAsync(input, options)).toString();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.71.0",
+  "version": "15.72.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/pipeline.ts

@@ -24,6 +24,7 @@ import {
   type SKIP,
 } from '@naturalcycles/js-lib/types'
 import { fs2 } from '../fs/fs2.js'
+import { zstdLevelToOptions } from '../zip/zip.util.js'
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js'
 import { transformJsonParse } from './ndjson/transformJsonParse.js'
 import { transformToNDJson } from './ndjson/transformToNDJson.js'
@@ -349,13 +350,12 @@ export class Pipeline<T = unknown> {
     return this as any
   }
 
-  zstdCompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array> {
-    this.transforms.push(
-      createZstdCompress({
-        // chunkSize: 64 * 1024, // no observed speedup
-        ...opt,
-      }),
-    )
+  zstdCompress(
+    this: Pipeline<Uint8Array>,
+    level?: Integer, // defaults to 3
+    opt?: ZstdOptions,
+  ): Pipeline<Uint8Array> {
+    this.transforms.push(createZstdCompress(zstdLevelToOptions(level, opt)))
     this.objectMode = false
     return this as any
   }
@@ -384,21 +384,25 @@ export class Pipeline<T = unknown> {
     await this.run()
   }
 
-  async toNDJsonFile(outputFilePath: string): Promise<void> {
+  /**
+   * level corresponds to zstd compression level (if filename ends with .zst),
+   * or gzip compression level (if filename ends with .gz).
+   * Default levels are:
+   * gzip: 6
+   * zlib: 3 (optimized for throughput, not size, may be larger than gzip at its default level)
+   */
+  async toNDJsonFile(outputFilePath: string, level?: Integer): Promise<void> {
     fs2.ensureFile(outputFilePath)
     this.transforms.push(transformToNDJson())
     if (outputFilePath.endsWith('.gz')) {
       this.transforms.push(
         createGzip({
+          level,
           // chunkSize: 64 * 1024, // no observed speedup
         }),
       )
     } else if (outputFilePath.endsWith('.zst')) {
-      this.transforms.push(
-        createZstdCompress({
-          // chunkSize: 64 * 1024, // no observed speedup
-        }),
-      )
+      this.transforms.push(createZstdCompress(zstdLevelToOptions(level)))
     }
     this.destination = fs2.createWriteStream(outputFilePath, {
       // highWaterMark: 64 * 1024, // no observed speedup

package/src/zip/zip.util.ts

@@ -1,6 +1,7 @@
 import { promisify } from 'node:util'
 import type { ZlibOptions, ZstdOptions } from 'node:zlib'
 import zlib from 'node:zlib'
+import type { Integer } from '@naturalcycles/js-lib/types'
 
 const deflate = promisify(zlib.deflate.bind(zlib))
 const inflate = promisify(zlib.inflate.bind(zlib))
@@ -89,9 +90,22 @@ export async function gunzipToString(buf: Buffer, options?: ZlibOptions): Promis
 
 export async function zstdCompress(
   input: Buffer | string,
+  level?: Integer, // defaults to 3
   options: ZstdOptions = {},
 ): Promise<Buffer<ArrayBuffer>> {
-  return await zstdCompressAsync(input, options)
+  return await zstdCompressAsync(input, zstdLevelToOptions(level, options))
+}
+
+export function zstdLevelToOptions(level: Integer | undefined, opt: ZstdOptions = {}): ZstdOptions {
+  if (!level) return opt
+
+  return {
+    ...opt,
+    params: {
+      ...opt.params,
+      [zlib.constants.ZSTD_c_compressionLevel]: level,
+    },
+  }
 }
 
 export async function zstdDecompressToString(