@naturalcycles/nodejs-lib 15.71.0 → 15.72.1

package/dist/exec2/exec2.js

@@ -129,7 +129,7 @@ class Exec2 {
      * log: true
      */
     async spawnAsync(cmd, opt = {}) {
-        const { shell = true, cwd, env, passProcessEnv = true, forceColor = hasColors } = opt;
+        const { shell = true, cwd, env, passProcessEnv = true, forceColor = hasColors, stdio = 'inherit', } = opt;
         opt.log ??= true; // by default log should be true, as we are printing the output
         opt.logStart ??= opt.log;
         opt.logFinish ??= opt.log;
@@ -139,7 +139,7 @@ class Exec2 {
             const p = spawn(cmd, opt.args || [], {
                 shell,
                 cwd,
-                stdio: 'inherit',
+                stdio,
                 env: {
                     ...(passProcessEnv ? process.env : {}),
                     ...(forceColor ? { FORCE_COLOR: '1' } : {}),

package/dist/infra/process.util.js

@@ -43,11 +43,12 @@ class ProcessUtil {
         };
     }
     cpuInfo() {
-        const c = os.cpus()[0];
+        const cpus = os.cpus();
+        const c = cpus[0];
         return {
-            count: os.cpus().length,
-            model: c.model,
-            speed: c.speed,
+            count: cpus.length,
+            model: c?.model || '',
+            speed: c?.speed || 0,
         };
     }
     async cpuPercent(ms) {
@@ -61,6 +62,10 @@ class ProcessUtil {
                 const endTotal = stats2.total;
                 const idle = endIdle - startIdle;
                 const total = endTotal - startTotal;
+                if (!total) {
+                    resolve(0);
+                    return;
+                }
                 const perc = idle / total;
                 resolve(Math.round((1 - perc) * 100));
             }, ms);

package/dist/stream/pipeline.d.ts

@@ -1,7 +1,7 @@
 import { type Transform } from 'node:stream';
 import type { ReadableStream as WebReadableStream } from 'node:stream/web';
 import { type ZlibOptions, type ZstdOptions } from 'node:zlib';
-import { type AbortableAsyncMapper, type AsyncIndexedMapper, type AsyncPredicate, type END, type IndexedMapper, type NonNegativeInteger, type PositiveInteger, type Predicate, type SKIP } from '@naturalcycles/js-lib/types';
+import { type AbortableAsyncMapper, type AsyncIndexedMapper, type AsyncPredicate, type END, type IndexedMapper, type Integer, type NonNegativeInteger, type PositiveInteger, type Predicate, type SKIP } from '@naturalcycles/js-lib/types';
 import type { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream.model.js';
 import { type TransformLogProgressOptions } from './transform/transformLogProgress.js';
 import { type TransformMapOptions } from './transform/transformMap.js';
@@ -97,11 +97,19 @@ export declare class Pipeline<T = unknown> {
     parseJson<TO = unknown>(this: Pipeline<Buffer> | Pipeline<Uint8Array> | Pipeline<string>): Pipeline<TO>;
     gzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
     gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
-    zstdCompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array>;
+    zstdCompress(this: Pipeline<Uint8Array>, level?: Integer, // defaults to 3
+    opt?: ZstdOptions): Pipeline<Uint8Array>;
     zstdDecompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array>;
     toArray(opt?: TransformOptions): Promise<T[]>;
     toFile(outputFilePath: string): Promise<void>;
-    toNDJsonFile(outputFilePath: string): Promise<void>;
+    /**
+     * level corresponds to zstd compression level (if filename ends with .zst),
+     * or gzip compression level (if filename ends with .gz).
+     * Default levels are:
+     * gzip: 6
+     * zlib: 3 (optimized for throughput, not size, may be larger than gzip at its default level)
+     */
+    toNDJsonFile(outputFilePath: string, level?: Integer): Promise<void>;
     to(destination: WritableTyped<T>): Promise<void>;
     forEach(fn: AsyncIndexedMapper<T, void>, opt?: TransformMapOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
     forEachSync(fn: IndexedMapper<T, void>, opt?: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;

package/dist/stream/pipeline.js

@@ -4,6 +4,7 @@ import { createGzip, createUnzip, createZstdCompress, createZstdDecompress, } fr
 import { createAbortableSignal } from '@naturalcycles/js-lib';
 import { _passthroughPredicate, } from '@naturalcycles/js-lib/types';
 import { fs2 } from '../fs/fs2.js';
+import { zstdLevelToOptions } from '../zip/zip.util.js';
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js';
 import { transformJsonParse } from './ndjson/transformJsonParse.js';
 import { transformToNDJson } from './ndjson/transformToNDJson.js';
@@ -253,11 +254,9 @@ export class Pipeline {
         this.objectMode = false;
         return this;
     }
-    zstdCompress(opt) {
-        this.transforms.push(createZstdCompress({
-            // chunkSize: 64 * 1024, // no observed speedup
-            ...opt,
-        }));
+    zstdCompress(level, // defaults to 3
+    opt) {
+        this.transforms.push(createZstdCompress(zstdLevelToOptions(level, opt)));
         this.objectMode = false;
         return this;
     }
@@ -280,18 +279,24 @@ export class Pipeline {
         this.destination = fs2.createWriteStream(outputFilePath);
         await this.run();
     }
-    async toNDJsonFile(outputFilePath) {
+    /**
+     * level corresponds to zstd compression level (if filename ends with .zst),
+     * or gzip compression level (if filename ends with .gz).
+     * Default levels are:
+     * gzip: 6
+     * zlib: 3 (optimized for throughput, not size, may be larger than gzip at its default level)
+     */
+    async toNDJsonFile(outputFilePath, level) {
         fs2.ensureFile(outputFilePath);
         this.transforms.push(transformToNDJson());
         if (outputFilePath.endsWith('.gz')) {
             this.transforms.push(createGzip({
-            // chunkSize: 64 * 1024, // no observed speedup
+                level,
+                // chunkSize: 64 * 1024, // no observed speedup
             }));
         }
         else if (outputFilePath.endsWith('.zst')) {
-            this.transforms.push(createZstdCompress({
-            // chunkSize: 64 * 1024, // no observed speedup
-            }));
+            this.transforms.push(createZstdCompress(zstdLevelToOptions(level)));
         }
         this.destination = fs2.createWriteStream(outputFilePath, {
         // highWaterMark: 64 * 1024, // no observed speedup

package/dist/stream/transform/worker/transformMultiThreaded.model.d.ts

@@ -34,4 +34,5 @@ export interface BaseWorkerData {
      * @default 1000
      */
     logEvery: number;
+    silent?: boolean;
 }

package/dist/zip/zip.util.d.ts

@@ -1,4 +1,5 @@
 import type { ZlibOptions, ZstdOptions } from 'node:zlib';
+import type { Integer } from '@naturalcycles/js-lib/types';
 export declare function decompressZstdOrInflateToString(buf: Buffer): Promise<string>;
 /**
  * Detects if Buffer is zstd-compressed.
@@ -29,7 +30,9 @@ export declare function gunzipBuffer(buf: Buffer, options?: ZlibOptions): Promis
  */
 export declare function gzipString(s: string, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
 export declare function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
-export declare function zstdCompress(input: Buffer | string, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
+export declare function zstdCompress(input: Buffer | string, level?: Integer, // defaults to 3
+options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
+export declare function zstdLevelToOptions(level: Integer | undefined, opt?: ZstdOptions): ZstdOptions;
 export declare function zstdDecompressToString(input: Buffer, options?: ZstdOptions): Promise<string>;
 export declare function zstdDecompress(input: Buffer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
 export declare function isZstdBuffer(input: Buffer): boolean;

package/dist/zip/zip.util.js

@@ -59,8 +59,20 @@ export async function gzipString(s, options) {
 export async function gunzipToString(buf, options) {
     return (await gunzipBuffer(buf, options)).toString();
 }
-export async function zstdCompress(input, options = {}) {
-    return await zstdCompressAsync(input, options);
+export async function zstdCompress(input, level, // defaults to 3
+options = {}) {
+    return await zstdCompressAsync(input, zstdLevelToOptions(level, options));
+}
+export function zstdLevelToOptions(level, opt = {}) {
+    if (!level)
+        return opt;
+    return {
+        ...opt,
+        params: {
+            ...opt.params,
+            [zlib.constants.ZSTD_c_compressionLevel]: level,
+        },
+    };
 }
 export async function zstdDecompressToString(input, options = {}) {
     return (await zstdDecompressAsync(input, options)).toString();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.71.0",
+  "version": "15.72.1",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",
@@ -19,7 +19,7 @@
     "yargs": "^18"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "18.4.2"
+    "@naturalcycles/dev-lib": "20.20.0"
   },
   "exports": {
     ".": "./dist/index.js",

package/src/exec2/exec2.ts

@@ -148,7 +148,14 @@ class Exec2 {
    * log: true
    */
   async spawnAsync(cmd: string, opt: SpawnOptions = {}): Promise<void> {
-    const { shell = true, cwd, env, passProcessEnv = true, forceColor = hasColors } = opt
+    const {
+      shell = true,
+      cwd,
+      env,
+      passProcessEnv = true,
+      forceColor = hasColors,
+      stdio = 'inherit',
+    } = opt
     opt.log ??= true // by default log should be true, as we are printing the output
     opt.logStart ??= opt.log
     opt.logFinish ??= opt.log
@@ -159,7 +166,7 @@ class Exec2 {
       const p = spawn(cmd, opt.args || [], {
         shell,
         cwd,
-        stdio: 'inherit',
+        stdio,
         env: {
           ...(passProcessEnv ? process.env : {}),
           ...(forceColor ? { FORCE_COLOR: '1' } : {}),

package/src/infra/process.util.ts

@@ -64,11 +64,12 @@ class ProcessUtil {
   }
 
   cpuInfo(): { count: number; model: string; speed: number } {
-    const c = os.cpus()[0]!
+    const cpus = os.cpus()
+    const c = cpus[0]
     return {
-      count: os.cpus().length,
-      model: c.model,
-      speed: c.speed,
+      count: cpus.length,
+      model: c?.model || '',
+      speed: c?.speed || 0,
     }
   }
 
@@ -85,8 +86,12 @@ class ProcessUtil {
 
         const idle = endIdle - startIdle
         const total = endTotal - startTotal
-        const perc = idle / total
+        if (!total) {
+          resolve(0)
+          return
+        }
 
+        const perc = idle / total
         resolve(Math.round((1 - perc) * 100))
       }, ms)
     })

package/src/stream/pipeline.ts

@@ -24,6 +24,7 @@ import {
   type SKIP,
 } from '@naturalcycles/js-lib/types'
 import { fs2 } from '../fs/fs2.js'
+import { zstdLevelToOptions } from '../zip/zip.util.js'
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js'
 import { transformJsonParse } from './ndjson/transformJsonParse.js'
 import { transformToNDJson } from './ndjson/transformToNDJson.js'
@@ -349,13 +350,12 @@ export class Pipeline<T = unknown> {
     return this as any
   }
 
-  zstdCompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array> {
-    this.transforms.push(
-      createZstdCompress({
-        // chunkSize: 64 * 1024, // no observed speedup
-        ...opt,
-      }),
-    )
+  zstdCompress(
+    this: Pipeline<Uint8Array>,
+    level?: Integer, // defaults to 3
+    opt?: ZstdOptions,
+  ): Pipeline<Uint8Array> {
+    this.transforms.push(createZstdCompress(zstdLevelToOptions(level, opt)))
     this.objectMode = false
     return this as any
   }
@@ -384,21 +384,25 @@ export class Pipeline<T = unknown> {
     await this.run()
   }
 
-  async toNDJsonFile(outputFilePath: string): Promise<void> {
+  /**
+   * level corresponds to zstd compression level (if filename ends with .zst),
+   * or gzip compression level (if filename ends with .gz).
+   * Default levels are:
+   * gzip: 6
+   * zlib: 3 (optimized for throughput, not size, may be larger than gzip at its default level)
+   */
+  async toNDJsonFile(outputFilePath: string, level?: Integer): Promise<void> {
     fs2.ensureFile(outputFilePath)
     this.transforms.push(transformToNDJson())
     if (outputFilePath.endsWith('.gz')) {
       this.transforms.push(
         createGzip({
+          level,
           // chunkSize: 64 * 1024, // no observed speedup
         }),
       )
     } else if (outputFilePath.endsWith('.zst')) {
-      this.transforms.push(
-        createZstdCompress({
-          // chunkSize: 64 * 1024, // no observed speedup
-        }),
-      )
+      this.transforms.push(createZstdCompress(zstdLevelToOptions(level)))
     }
     this.destination = fs2.createWriteStream(outputFilePath, {
       // highWaterMark: 64 * 1024, // no observed speedup

package/src/stream/transform/worker/transformMultiThreaded.model.ts

@@ -42,4 +42,6 @@ export interface BaseWorkerData {
    * @default 1000
    */
   logEvery: number
+
+  silent?: boolean
 }

package/src/stream/transform/worker/workerClassProxy.js

@@ -1,7 +1,7 @@
 const started = Date.now()
 import { workerData, parentPort } from 'node:worker_threads'
 import { inspect } from 'node:util'
-const { workerFile, workerIndex, logEvery = 1000, metric = 'worker' } = workerData || {}
+const { workerFile, workerIndex, logEvery = 1000, metric = 'worker', silent } = workerData || {}
 
 if (!workerFile) {
   throw new Error('workerData.workerFile is required!')
@@ -17,7 +17,7 @@ try {
 const { WorkerClass } = await import(workerFile)
 const worker = new WorkerClass(workerData)
 
-console.log(`${metric}#${workerIndex} loaded in ${Date.now() - started} ms`)
+log(`${metric}#${workerIndex} loaded in ${Date.now() - started} ms`)
 
 let errors = 0
 let processed = 0
@@ -52,7 +52,7 @@ parentPort.on('message', async msg => {
     })
 
     errors++
-    console.log(`${metric}#${workerIndex} errors: ${errors}`)
+    log(`${metric}#${workerIndex} errors: ${errors}`)
   }
 })
 
@@ -64,7 +64,7 @@ const inspectOpt = {
 function logStats(final) {
   const { rss, heapUsed, heapTotal, external } = process.memoryUsage()
 
-  console.log(
+  log(
     inspect(
       {
         [`${metric}${workerIndex}`]: processed,
@@ -83,3 +83,8 @@ function logStats(final) {
 function mb(b) {
   return Math.round(b / (1024 * 1024))
 }
+
+function log(...args) {
+  if (silent) return
+  console.log(...args)
+}

package/src/zip/zip.util.ts

@@ -1,6 +1,7 @@
 import { promisify } from 'node:util'
 import type { ZlibOptions, ZstdOptions } from 'node:zlib'
 import zlib from 'node:zlib'
+import type { Integer } from '@naturalcycles/js-lib/types'
 
 const deflate = promisify(zlib.deflate.bind(zlib))
 const inflate = promisify(zlib.inflate.bind(zlib))
@@ -89,9 +90,22 @@ export async function gunzipToString(buf: Buffer, options?: ZlibOptions): Promis
 
 export async function zstdCompress(
   input: Buffer | string,
+  level?: Integer, // defaults to 3
   options: ZstdOptions = {},
 ): Promise<Buffer<ArrayBuffer>> {
-  return await zstdCompressAsync(input, options)
+  return await zstdCompressAsync(input, zstdLevelToOptions(level, options))
+}
+
+export function zstdLevelToOptions(level: Integer | undefined, opt: ZstdOptions = {}): ZstdOptions {
+  if (!level) return opt
+
+  return {
+    ...opt,
+    params: {
+      ...opt.params,
+      [zlib.constants.ZSTD_c_compressionLevel]: level,
+    },
+  }
 }
 
 export async function zstdDecompressToString(