@naturalcycles/nodejs-lib 15.70.1 → 15.72.0

package/dist/stream/transform/worker/transformMultiThreaded.js

@@ -1,7 +1,7 @@
+import { Transform } from 'node:stream';
 import { Worker } from 'node:worker_threads';
 import { _range } from '@naturalcycles/js-lib/array/range.js';
 import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
-import through2Concurrent from 'through2-concurrent';
 const workerProxyFilePath = `${import.meta.dirname}/workerClassProxy.js`;
 /**
  * Spawns a pool of Workers (threads).
@@ -21,6 +21,10 @@ export function transformMultiThreaded(opt) {
     const workerDonePromises = [];
     const messageDonePromises = {};
     let index = -1; // input chunk index, will start from 0
+    // Concurrency control
+    let inFlight = 0;
+    let blockedCallback = null;
+    let flushBlocked = null;
     const workers = _range(0, poolSize).map(workerIndex => {
         workerDonePromises.push(pDefer());
         const worker = new Worker(workerProxyFilePath, {
@@ -30,20 +34,14 @@ export function transformMultiThreaded(opt) {
                 ...workerData,
             },
         });
-        // const {threadId} = worker
-        // console.log({threadId})
         worker.on('error', err => {
             console.error(`Worker ${workerIndex} error`, err);
             workerDonePromises[workerIndex].reject(err);
         });
         worker.on('exit', _exitCode => {
-            // console.log(`Worker ${index} exit: ${exitCode}`)
             workerDonePromises[workerIndex].resolve(undefined);
         });
         worker.on('message', (out) => {
-            // console.log(`Message from Worker ${workerIndex}:`, out)
-            // console.log(Object.keys(messageDonePromises))
-            // tr.push(out.payload)
             if (out.error) {
                 messageDonePromises[out.index].reject(out.error);
             }
@@ -53,48 +51,70 @@ export function transformMultiThreaded(opt) {
         });
         return worker;
     });
-    return through2Concurrent.obj({
-        maxConcurrency,
-        highWaterMark,
-        async final(cb) {
+    return new Transform({
+        objectMode: true,
+        readableHighWaterMark: highWaterMark,
+        writableHighWaterMark: highWaterMark,
+        async transform(chunk, _, cb) {
+            const currentIndex = ++index;
+            inFlight++;
+            // Apply backpressure if at capacity, otherwise request more input
+            if (inFlight < maxConcurrency) {
+                cb();
+            }
+            else {
+                blockedCallback = cb;
+            }
+            // Create the unresolved promise (to await)
+            messageDonePromises[currentIndex] = pDefer();
+            const worker = workers[currentIndex % poolSize]; // round-robin
+            worker.postMessage({
+                index: currentIndex,
+                payload: chunk,
+            });
+            try {
+                const out = await messageDonePromises[currentIndex];
+                this.push(out);
+            }
+            catch (err) {
+                // Currently we only support ErrorMode.SUPPRESS
+                // Error is logged and output continues
+                console.error(err);
+            }
+            finally {
+                delete messageDonePromises[currentIndex];
+                inFlight--;
+                // Release blocked callback if we now have capacity
+                if (blockedCallback && inFlight < maxConcurrency) {
+                    const pendingCb = blockedCallback;
+                    blockedCallback = null;
+                    pendingCb();
+                }
+                // Trigger flush completion if all done
+                if (inFlight === 0 && flushBlocked) {
+                    flushBlocked.resolve();
+                }
+            }
+        },
+        async flush(cb) {
+            // Wait for all in-flight operations to complete
+            if (inFlight > 0) {
+                flushBlocked = pDefer();
+                await flushBlocked;
+            }
             try {
-                // Push null (complete) to all sub-streams
+                // Push null (complete) to all workers
                 for (const worker of workers) {
                     worker.postMessage(null);
                 }
-                console.log(`transformMultiThreaded.final is waiting for all chains to be done`);
+                console.log(`transformMultiThreaded.flush is waiting for all workers to be done`);
                 await Promise.all(workerDonePromises);
-                console.log(`transformMultiThreaded.final all chains done`);
+                console.log(`transformMultiThreaded.flush all workers done`);
                 cb();
             }
             catch (err) {
                 cb(err);
             }
         },
-    }, async function transformMapFn(chunk, _, cb) {
-        // Freezing the index, because it may change due to concurrency
-        const currentIndex = ++index;
-        // Create the unresolved promise (to avait)
-        messageDonePromises[currentIndex] = pDefer();
-        const worker = workers[currentIndex % poolSize]; // round-robin
-        worker.postMessage({
-            index: currentIndex,
-            payload: chunk,
-        });
-        try {
-            // awaiting for result
-            const out = await messageDonePromises[currentIndex];
-            // console.log('awaited!')
-            // return the result
-            cb(null, out);
-        }
-        catch (err) {
-            // Currently we only support ErrorMode.SUPPRESS
-            // Error is logged and output continues
-            console.error(err);
-            cb(); // emit nothing in case of an error
-        }
-        // clean up
-        delete messageDonePromises[currentIndex];
     });
 }

package/dist/zip/zip.util.d.ts

@@ -1,4 +1,5 @@
 import type { ZlibOptions, ZstdOptions } from 'node:zlib';
+import type { Integer } from '@naturalcycles/js-lib/types';
 export declare function decompressZstdOrInflateToString(buf: Buffer): Promise<string>;
 /**
  * Detects if Buffer is zstd-compressed.
@@ -29,7 +30,9 @@ export declare function gunzipBuffer(buf: Buffer, options?: ZlibOptions): Promis
  */
 export declare function gzipString(s: string, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
 export declare function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
-export declare function zstdCompress(input: Buffer | string, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
+export declare function zstdCompress(input: Buffer | string, level?: Integer, // defaults to 3
+options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
+export declare function zstdLevelToOptions(level: Integer | undefined, opt?: ZstdOptions): ZstdOptions;
 export declare function zstdDecompressToString(input: Buffer, options?: ZstdOptions): Promise<string>;
 export declare function zstdDecompress(input: Buffer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
 export declare function isZstdBuffer(input: Buffer): boolean;

package/dist/zip/zip.util.js

@@ -59,8 +59,20 @@ export async function gzipString(s, options) {
 export async function gunzipToString(buf, options) {
     return (await gunzipBuffer(buf, options)).toString();
 }
-export async function zstdCompress(input, options = {}) {
-    return await zstdCompressAsync(input, options);
+export async function zstdCompress(input, level, // defaults to 3
+options = {}) {
+    return await zstdCompressAsync(input, zstdLevelToOptions(level, options));
+}
+export function zstdLevelToOptions(level, opt = {}) {
+    if (!level)
+        return opt;
+    return {
+        ...opt,
+        params: {
+            ...opt.params,
+            [zlib.constants.ZSTD_c_compressionLevel]: level,
+        },
+    };
 }
 export async function zstdDecompressToString(input, options = {}) {
     return (await zstdDecompressAsync(input, options)).toString();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.70.1",
+  "version": "15.72.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",
@@ -14,13 +14,11 @@
     "js-yaml": "^4",
     "jsonwebtoken": "^9",
     "lru-cache": "^11",
-    "through2-concurrent": "^2",
     "tinyglobby": "^0.2",
     "tslib": "^2",
     "yargs": "^18"
   },
   "devDependencies": {
-    "@types/through2-concurrent": "^2",
     "@naturalcycles/dev-lib": "18.4.2"
   },
   "exports": {

package/src/stream/index.ts

@@ -16,7 +16,6 @@ export * from './transform/transformFork.js'
 export * from './transform/transformLimit.js'
 export * from './transform/transformLogProgress.js'
 export * from './transform/transformMap.js'
-export * from './transform/transformMap2.js'
 export * from './transform/transformMapSimple.js'
 export * from './transform/transformMapSync.js'
 export * from './transform/transformNoOp.js'

package/src/stream/pipeline.ts

@@ -24,6 +24,7 @@ import {
   type SKIP,
 } from '@naturalcycles/js-lib/types'
 import { fs2 } from '../fs/fs2.js'
+import { zstdLevelToOptions } from '../zip/zip.util.js'
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js'
 import { transformJsonParse } from './ndjson/transformJsonParse.js'
 import { transformToNDJson } from './ndjson/transformToNDJson.js'
@@ -45,7 +46,6 @@ import {
   type TransformLogProgressOptions,
 } from './transform/transformLogProgress.js'
 import { transformMap, type TransformMapOptions } from './transform/transformMap.js'
-import { transformMap2, type TransformMap2Options } from './transform/transformMap2.js'
 import {
   transformMapSimple,
   type TransformMapSimpleOptions,
@@ -185,7 +185,7 @@ export class Pipeline<T = unknown> {
     return this
   }
 
-  mapLegacy<TO>(
+  map<TO>(
     mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>,
     opt?: TransformMapOptions<T, TO>,
   ): Pipeline<TO> {
@@ -198,19 +198,6 @@ export class Pipeline<T = unknown> {
     return this as any
   }
 
-  map<TO>(
-    mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>,
-    opt?: TransformMap2Options<T, TO>,
-  ): Pipeline<TO> {
-    this.transforms.push(
-      transformMap2(mapper, {
-        ...opt,
-        signal: this.abortableSignal,
-      }),
-    )
-    return this as any
-  }
-
   mapSync<TO>(
     mapper: IndexedMapper<T, TO | typeof SKIP | typeof END>,
     opt?: TransformMapSyncOptions,
@@ -231,7 +218,7 @@ export class Pipeline<T = unknown> {
 
   filter(asyncPredicate: AsyncPredicate<T>, opt?: TransformMapOptions): this {
     this.transforms.push(
-      transformMap2(v => v, {
+      transformMap(v => v, {
         asyncPredicate,
         ...opt,
         signal: this.abortableSignal,
@@ -363,13 +350,12 @@ export class Pipeline<T = unknown> {
     return this as any
   }
 
-  zstdCompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array> {
-    this.transforms.push(
-      createZstdCompress({
-        // chunkSize: 64 * 1024, // no observed speedup
-        ...opt,
-      }),
-    )
+  zstdCompress(
+    this: Pipeline<Uint8Array>,
+    level?: Integer, // defaults to 3
+    opt?: ZstdOptions,
+  ): Pipeline<Uint8Array> {
+    this.transforms.push(createZstdCompress(zstdLevelToOptions(level, opt)))
     this.objectMode = false
     return this as any
   }
@@ -398,21 +384,25 @@ export class Pipeline<T = unknown> {
     await this.run()
   }
 
-  async toNDJsonFile(outputFilePath: string): Promise<void> {
+  /**
+   * level corresponds to zstd compression level (if filename ends with .zst),
+   * or gzip compression level (if filename ends with .gz).
+   * Default levels are:
+   * gzip: 6
+   * zlib: 3 (optimized for throughput, not size, may be larger than gzip at its default level)
+   */
+  async toNDJsonFile(outputFilePath: string, level?: Integer): Promise<void> {
     fs2.ensureFile(outputFilePath)
     this.transforms.push(transformToNDJson())
     if (outputFilePath.endsWith('.gz')) {
       this.transforms.push(
         createGzip({
+          level,
           // chunkSize: 64 * 1024, // no observed speedup
         }),
       )
     } else if (outputFilePath.endsWith('.zst')) {
-      this.transforms.push(
-        createZstdCompress({
-          // chunkSize: 64 * 1024, // no observed speedup
-        }),
-      )
+      this.transforms.push(createZstdCompress(zstdLevelToOptions(level)))
     }
     this.destination = fs2.createWriteStream(outputFilePath, {
       // highWaterMark: 64 * 1024, // no observed speedup
@@ -425,29 +415,12 @@ export class Pipeline<T = unknown> {
     await this.run()
   }
 
-  async forEachLegacy(
-    fn: AsyncIndexedMapper<T, void>,
-    opt: TransformMapOptions<T, void> & TransformLogProgressOptions<T> = {},
-  ): Promise<void> {
-    this.transforms.push(
-      transformMap2(fn, {
-        predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
-        ...opt,
-        signal: this.abortableSignal,
-      }),
-    )
-    if (opt.logEvery) {
-      this.transforms.push(transformLogProgress(opt))
-    }
-    await this.run()
-  }
-
   async forEach(
     fn: AsyncIndexedMapper<T, void>,
-    opt: TransformMap2Options<T, void> & TransformLogProgressOptions<T> = {},
+    opt: TransformMapOptions<T, void> & TransformLogProgressOptions<T> = {},
   ): Promise<void> {
     this.transforms.push(
-      transformMap2(fn, {
+      transformMap(fn, {
         predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
         ...opt,
         signal: this.abortableSignal,

package/src/stream/transform/transformFilter.ts

@@ -2,7 +2,7 @@ import { Transform } from 'node:stream'
 import type { AsyncPredicate, Predicate } from '@naturalcycles/js-lib/types'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
 import type { TransformMapOptions } from './transformMap.js'
-import { transformMap2 } from './transformMap2.js'
+import { transformMap } from './transformMap.js'
 
 /**
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
@@ -11,7 +11,7 @@ export function transformFilter<IN = any>(
   asyncPredicate: AsyncPredicate<IN>,
   opt: TransformMapOptions = {},
 ): TransformTyped<IN, IN> {
-  return transformMap2(v => v, {
+  return transformMap(v => v, {
     asyncPredicate,
     ...opt,
   })