@naturalcycles/nodejs-lib 15.70.1 → 15.72.0

package/dist/stream/index.d.ts

@@ -16,7 +16,6 @@ export * from './transform/transformFork.js';
 export * from './transform/transformLimit.js';
 export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
-export * from './transform/transformMap2.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
 export * from './transform/transformNoOp.js';

package/dist/stream/index.js

@@ -16,7 +16,6 @@ export * from './transform/transformFork.js';
 export * from './transform/transformLimit.js';
 export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
-export * from './transform/transformMap2.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
 export * from './transform/transformNoOp.js';

package/dist/stream/pipeline.d.ts

@@ -1,11 +1,10 @@
 import { type Transform } from 'node:stream';
 import type { ReadableStream as WebReadableStream } from 'node:stream/web';
 import { type ZlibOptions, type ZstdOptions } from 'node:zlib';
-import { type AbortableAsyncMapper, type AsyncIndexedMapper, type AsyncPredicate, type END, type IndexedMapper, type NonNegativeInteger, type PositiveInteger, type Predicate, type SKIP } from '@naturalcycles/js-lib/types';
+import { type AbortableAsyncMapper, type AsyncIndexedMapper, type AsyncPredicate, type END, type IndexedMapper, type Integer, type NonNegativeInteger, type PositiveInteger, type Predicate, type SKIP } from '@naturalcycles/js-lib/types';
 import type { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream.model.js';
 import { type TransformLogProgressOptions } from './transform/transformLogProgress.js';
 import { type TransformMapOptions } from './transform/transformMap.js';
-import { type TransformMap2Options } from './transform/transformMap2.js';
 import { type TransformMapSimpleOptions } from './transform/transformMapSimple.js';
 import { type TransformMapSyncOptions } from './transform/transformMapSync.js';
 import { type TransformOffsetOptions } from './transform/transformOffset.js';
@@ -62,8 +61,7 @@ export declare class Pipeline<T = unknown> {
     flatten<TO>(this: Pipeline<readonly TO[]>): Pipeline<TO>;
     flattenIfNeeded(): Pipeline<T extends readonly (infer TO)[] ? TO : T>;
     logProgress(opt?: TransformLogProgressOptions): this;
-    mapLegacy<TO>(mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapOptions<T, TO>): Pipeline<TO>;
-    map<TO>(mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMap2Options<T, TO>): Pipeline<TO>;
+    map<TO>(mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapOptions<T, TO>): Pipeline<TO>;
     mapSync<TO>(mapper: IndexedMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapSyncOptions): Pipeline<TO>;
     mapSimple<TO>(mapper: IndexedMapper<T, TO>, opt?: TransformMapSimpleOptions): Pipeline<TO>;
     filter(asyncPredicate: AsyncPredicate<T>, opt?: TransformMapOptions): this;
@@ -99,14 +97,21 @@ export declare class Pipeline<T = unknown> {
     parseJson<TO = unknown>(this: Pipeline<Buffer> | Pipeline<Uint8Array> | Pipeline<string>): Pipeline<TO>;
     gzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
     gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
-    zstdCompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array>;
+    zstdCompress(this: Pipeline<Uint8Array>, level?: Integer, // defaults to 3
+    opt?: ZstdOptions): Pipeline<Uint8Array>;
     zstdDecompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array>;
     toArray(opt?: TransformOptions): Promise<T[]>;
     toFile(outputFilePath: string): Promise<void>;
-    toNDJsonFile(outputFilePath: string): Promise<void>;
+    /**
+     * level corresponds to zstd compression level (if filename ends with .zst),
+     * or gzip compression level (if filename ends with .gz).
+     * Default levels are:
+     * gzip: 6
+     * zlib: 3 (optimized for throughput, not size, may be larger than gzip at its default level)
+     */
+    toNDJsonFile(outputFilePath: string, level?: Integer): Promise<void>;
     to(destination: WritableTyped<T>): Promise<void>;
-    forEachLegacy(fn: AsyncIndexedMapper<T, void>, opt?: TransformMapOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
-    forEach(fn: AsyncIndexedMapper<T, void>, opt?: TransformMap2Options<T, void> & TransformLogProgressOptions<T>): Promise<void>;
+    forEach(fn: AsyncIndexedMapper<T, void>, opt?: TransformMapOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
     forEachSync(fn: IndexedMapper<T, void>, opt?: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
     run(): Promise<void>;
 }

package/dist/stream/pipeline.js

@@ -4,6 +4,7 @@ import { createGzip, createUnzip, createZstdCompress, createZstdDecompress, } fr
 import { createAbortableSignal } from '@naturalcycles/js-lib';
 import { _passthroughPredicate, } from '@naturalcycles/js-lib/types';
 import { fs2 } from '../fs/fs2.js';
+import { zstdLevelToOptions } from '../zip/zip.util.js';
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js';
 import { transformJsonParse } from './ndjson/transformJsonParse.js';
 import { transformToNDJson } from './ndjson/transformToNDJson.js';
@@ -16,7 +17,6 @@ import { transformFork } from './transform/transformFork.js';
 import { transformLimit } from './transform/transformLimit.js';
 import { transformLogProgress, } from './transform/transformLogProgress.js';
 import { transformMap } from './transform/transformMap.js';
-import { transformMap2 } from './transform/transformMap2.js';
 import { transformMapSimple, } from './transform/transformMapSimple.js';
 import { transformMapSync } from './transform/transformMapSync.js';
 import { transformOffset } from './transform/transformOffset.js';
@@ -128,15 +128,8 @@ export class Pipeline {
         this.transforms.push(transformLogProgress(opt));
         return this;
     }
-    mapLegacy(mapper, opt) {
-        this.transforms.push(transformMap(mapper, {
-            ...opt,
-            signal: this.abortableSignal,
-        }));
-        return this;
-    }
     map(mapper, opt) {
-        this.transforms.push(transformMap2(mapper, {
+        this.transforms.push(transformMap(mapper, {
             ...opt,
             signal: this.abortableSignal,
         }));
@@ -154,7 +147,7 @@ export class Pipeline {
         return this;
     }
     filter(asyncPredicate, opt) {
-        this.transforms.push(transformMap2(v => v, {
+        this.transforms.push(transformMap(v => v, {
             asyncPredicate,
             ...opt,
             signal: this.abortableSignal,
@@ -261,11 +254,9 @@ export class Pipeline {
         this.objectMode = false;
         return this;
     }
-    zstdCompress(opt) {
-        this.transforms.push(createZstdCompress({
-            // chunkSize: 64 * 1024, // no observed speedup
-            ...opt,
-        }));
+    zstdCompress(level, // defaults to 3
+    opt) {
+        this.transforms.push(createZstdCompress(zstdLevelToOptions(level, opt)));
         this.objectMode = false;
         return this;
     }
@@ -288,18 +279,24 @@ export class Pipeline {
         this.destination = fs2.createWriteStream(outputFilePath);
         await this.run();
     }
-    async toNDJsonFile(outputFilePath) {
+    /**
+     * level corresponds to zstd compression level (if filename ends with .zst),
+     * or gzip compression level (if filename ends with .gz).
+     * Default levels are:
+     * gzip: 6
+     * zlib: 3 (optimized for throughput, not size, may be larger than gzip at its default level)
+     */
+    async toNDJsonFile(outputFilePath, level) {
         fs2.ensureFile(outputFilePath);
         this.transforms.push(transformToNDJson());
         if (outputFilePath.endsWith('.gz')) {
             this.transforms.push(createGzip({
-            // chunkSize: 64 * 1024, // no observed speedup
+                level,
+                // chunkSize: 64 * 1024, // no observed speedup
             }));
         }
         else if (outputFilePath.endsWith('.zst')) {
-            this.transforms.push(createZstdCompress({
-            // chunkSize: 64 * 1024, // no observed speedup
-            }));
+            this.transforms.push(createZstdCompress(zstdLevelToOptions(level)));
         }
         this.destination = fs2.createWriteStream(outputFilePath, {
         // highWaterMark: 64 * 1024, // no observed speedup
@@ -310,19 +307,8 @@ export class Pipeline {
         this.destination = destination;
         await this.run();
     }
-    async forEachLegacy(fn, opt = {}) {
-        this.transforms.push(transformMap2(fn, {
-            predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
-            ...opt,
-            signal: this.abortableSignal,
-        }));
-        if (opt.logEvery) {
-            this.transforms.push(transformLogProgress(opt));
-        }
-        await this.run();
-    }
     async forEach(fn, opt = {}) {
-        this.transforms.push(transformMap2(fn, {
+        this.transforms.push(transformMap(fn, {
             predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
             ...opt,
             signal: this.abortableSignal,

package/dist/stream/transform/transformFilter.js

@@ -1,10 +1,10 @@
 import { Transform } from 'node:stream';
-import { transformMap2 } from './transformMap2.js';
+import { transformMap } from './transformMap.js';
 /**
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
 export function transformFilter(asyncPredicate, opt = {}) {
-    return transformMap2(v => v, {
+    return transformMap(v => v, {
         asyncPredicate,
         ...opt,
     });

package/dist/stream/transform/transformMap.d.ts

@@ -1,6 +1,6 @@
 import { type AbortableSignal } from '@naturalcycles/js-lib';
 import { ErrorMode } from '@naturalcycles/js-lib/error';
-import { type AbortableAsyncMapper, type AsyncPredicate, END, type PositiveInteger, type Predicate, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
+import { type AbortableAsyncMapper, type AsyncPredicate, END, type NumberOfSeconds, type PositiveInteger, type Predicate, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
 import type { TransformOptions, TransformTyped } from '../stream.model.js';
 export interface TransformMapOptions<IN = any, OUT = IN> extends TransformOptions {
     /**
@@ -15,22 +15,16 @@ export interface TransformMapOptions<IN = any, OUT = IN> extends TransformOption
     /**
      * Number of concurrently pending promises returned by `mapper`.
      *
-     * Default is 16.
-     * It was recently changed up from 16, after some testing that shown that
-     * for simple low-cpu mapper functions 32 produces almost 2x throughput.
-     * For example, in scenarios like streaming a query from Datastore.
-     * UPD: changed back from 32 to 16, "to be on a safe side", as 32 sometimes
-     * causes "Datastore timeout errors".
+     * @default 16
      */
     concurrency?: PositiveInteger;
     /**
-     * Defaults to 64 items.
-     * (objectMode default is 16, but we increased it)
+     * Time in seconds to gradually increase concurrency from 1 to `concurrency`.
+     * Useful for warming up connections to databases, APIs, etc.
      *
-     * Affects both readable and writable highWaterMark (buffer).
-     * So, 64 means a total buffer of 128 (64 input and 64 output buffer).
+     * Set to 0 to disable warmup (default).
      */
-    highWaterMark?: PositiveInteger;
+    warmupSeconds?: NumberOfSeconds;
     /**
      * @default THROW_IMMEDIATELY
      */
@@ -62,6 +56,13 @@ export interface TransformMapOptions<IN = any, OUT = IN> extends TransformOption
      */
     signal?: AbortableSignal;
 }
+/**
+ * Like transformMap, but with native concurrency control (no through2-concurrent dependency)
+ * and support for gradual warmup.
+ *
+ * @experimental
+ */
+export declare function transformMap<IN = any, OUT = IN>(mapper: AbortableAsyncMapper<IN, OUT | typeof SKIP | typeof END>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
 export interface TransformMapStats {
     /**
      * True if transform was successful (didn't throw Immediate or Aggregated error).
@@ -88,19 +89,6 @@ export interface TransformMapStatsSummary extends TransformMapStats {
      */
     extra?: StringMap<any>;
 }
-/**
- * Like pMap, but for streams.
- * Inspired by `through2`.
- * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
- * Using this allows native stream .pipe() to work and use backpressure.
- *
- * Only works in objectMode (due to through2Concurrent).
- *
- * Concurrency defaults to 16.
- *
- * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
- */
-export declare function transformMap<IN = any, OUT = IN>(mapper: AbortableAsyncMapper<IN, OUT | typeof SKIP | typeof END>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
 /**
  * Renders TransformMapStatsSummary into a friendly string,
  * to be used e.g in Github Actions summary or Slack.

package/dist/stream/transform/transformMap.js

@@ -1,159 +1,175 @@
+import { Transform } from 'node:stream';
 import { _hc } from '@naturalcycles/js-lib';
-import { _since } from '@naturalcycles/js-lib/datetime/time.util.js';
+import { _since } from '@naturalcycles/js-lib/datetime';
 import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error';
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
-import { _stringify } from '@naturalcycles/js-lib/string/stringify.js';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+import { _stringify } from '@naturalcycles/js-lib/string';
 import { END, SKIP, } from '@naturalcycles/js-lib/types';
-import through2Concurrent from 'through2-concurrent';
 import { yellow } from '../../colors/colors.js';
 import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
-// doesn't work, cause here we don't construct our Transform instance ourselves
-// export class TransformMap extends AbortableTransform {}
+const WARMUP_CHECK_INTERVAL_MS = 1000;
 /**
- * Like pMap, but for streams.
- * Inspired by `through2`.
- * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
- * Using this allows native stream .pipe() to work and use backpressure.
+ * Like transformMap, but with native concurrency control (no through2-concurrent dependency)
+ * and support for gradual warmup.
  *
- * Only works in objectMode (due to through2Concurrent).
- *
- * Concurrency defaults to 16.
- *
- * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
+ * @experimental
  */
 export function transformMap(mapper, opt = {}) {
-    const { concurrency = 16, highWaterMark = 64, predicate, // we now default to "no predicate" (meaning pass-everything)
-    asyncPredicate, errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', signal, } = opt;
-    const started = Date.now();
+    const { concurrency: maxConcurrency = 16, warmupSeconds = 0, predicate, asyncPredicate, errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', signal, objectMode = true, highWaterMark = 64, } = opt;
+    const warmupMs = warmupSeconds * 1000;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+    // Stats
+    let started = 0;
     let index = -1;
     let countOut = 0;
     let isSettled = false;
     let ok = true;
     let errors = 0;
-    const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
-    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
-    return through2Concurrent.obj({
-        maxConcurrency: concurrency,
+    const collectedErrors = [];
+    // Concurrency control - single counter, single callback for backpressure
+    let inFlight = 0;
+    let blockedCallback = null;
+    let flushBlocked = null;
+    // Warmup - cached concurrency to reduce Date.now() syscalls
+    let warmupComplete = warmupSeconds <= 0 || maxConcurrency <= 1;
+    let concurrency = warmupComplete ? maxConcurrency : 1;
+    let lastWarmupCheck = 0;
+    return new Transform({
+        objectMode,
         readableHighWaterMark: highWaterMark,
         writableHighWaterMark: highWaterMark,
-        async final(cb) {
-            logErrorStats(true);
-            if (collectedErrors.length) {
-                try {
-                    await onDone?.({
-                        ok: false,
-                        collectedErrors,
-                        countErrors: errors,
-                        countIn: index + 1,
-                        countOut,
-                        started,
-                    });
-                }
-                catch (err) {
-                    logger.error(err);
-                }
-                // emit Aggregated error
-                cb(new AggregateError(collectedErrors, `transformMap resulted in ${collectedErrors.length} error(s)`));
-            }
-            else {
-                // emit no error
-                try {
-                    await onDone?.({
-                        ok,
-                        collectedErrors,
-                        countErrors: errors,
-                        countIn: index + 1,
-                        countOut,
-                        started,
-                    });
-                }
-                catch (err) {
-                    logger.error(err);
-                }
-                cb();
+        async transform(chunk, _, cb) {
+            // Initialize start time on first item
+            if (started === 0) {
+                started = Date.now();
+                lastWarmupCheck = started;
             }
-        },
-    }, async function transformMapFn(chunk, _, cb) {
-        // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
-        if (isSettled)
-            return cb();
-        const currentIndex = ++index;
-        try {
-            const res = await mapper(chunk, currentIndex);
-            // Check for isSettled again, as it may happen while mapper was running
             if (isSettled)
                 return cb();
-            if (res === END) {
-                isSettled = true;
-                logger.log(`transformMap END received at index ${currentIndex}`);
-                _assert(signal, 'signal is required when using END');
-                signal.abort(new Error(PIPELINE_GRACEFUL_ABORT));
-                return cb();
+            const currentIndex = ++index;
+            inFlight++;
+            if (!warmupComplete) {
+                updateConcurrency();
             }
-            if (res === SKIP) {
-                // do nothing, don't push
-                return cb();
+            // Apply backpressure if at capacity, otherwise request more input
+            if (inFlight < concurrency) {
+                cb();
             }
-            if (predicate) {
-                if (predicate(res, currentIndex)) {
-                    countOut++;
-                    this.push(res);
-                }
+            else {
+                blockedCallback = cb;
             }
-            else if (asyncPredicate) {
-                if ((await asyncPredicate(res, currentIndex)) && !isSettled) {
-                    // isSettled could have happened in parallel, hence the extra check
+            try {
+                const res = await mapper(chunk, currentIndex);
+                if (isSettled)
+                    return;
+                if (res === END) {
+                    isSettled = true;
+                    logger.log(`transformMap2 END received at index ${currentIndex}`);
+                    _assert(signal, 'signal is required when using END');
+                    signal.abort(new Error(PIPELINE_GRACEFUL_ABORT));
+                    return;
+                }
+                if (res === SKIP)
+                    return;
+                let shouldPush = true;
+                if (predicate) {
+                    shouldPush = predicate(res, currentIndex);
+                }
+                else if (asyncPredicate) {
+                    shouldPush = (await asyncPredicate(res, currentIndex)) && !isSettled;
+                }
+                if (shouldPush) {
                     countOut++;
                     this.push(res);
                 }
             }
-            else {
-                countOut++;
-                this.push(res);
+            catch (err) {
+                logger.error(err);
+                errors++;
+                logErrorStats();
+                if (onError) {
+                    try {
+                        onError(_anyToError(err), chunk);
+                    }
+                    catch { }
+                }
+                if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
+                    isSettled = true;
+                    ok = false;
+                    await callOnDone();
+                    this.destroy(_anyToError(err));
+                    return;
+                }
+                if (errorMode === ErrorMode.THROW_AGGREGATED) {
+                    collectedErrors.push(_anyToError(err));
+                }
             }
-            cb(); // done processing
-        }
-        catch (err) {
-            logger.error(err);
-            errors++;
-            logErrorStats();
-            if (onError) {
-                try {
-                    onError(_anyToError(err), chunk);
+            finally {
+                inFlight--;
+                // Release blocked callback if we now have capacity
+                if (blockedCallback && inFlight < concurrency) {
+                    const pendingCb = blockedCallback;
+                    blockedCallback = null;
+                    pendingCb();
+                }
+                // Trigger flush completion if all done
+                if (inFlight === 0 && flushBlocked) {
+                    flushBlocked.resolve();
                 }
-                catch { }
             }
-            if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
-                isSettled = true;
-                ok = false;
-                // Tests show that onDone is still called at `final` (second time),
-                // so, we no longer call it here
-                // try {
-                //   await onDone?.({
-                //     ok: false,
-                //     collectedErrors,
-                //     countErrors: errors,
-                //     countIn: index + 1,
-                //     countOut,
-                //     started,
-                //   })
-                // } catch (err) {
-                //   logger.error(err)
-                // }
-                return cb(err); // Emit error immediately
+        },
+        async flush(cb) {
+            // Wait for all in-flight operations to complete
+            if (inFlight > 0) {
+                flushBlocked = pDefer();
+                await flushBlocked;
             }
-            if (errorMode === ErrorMode.THROW_AGGREGATED) {
-                collectedErrors.push(err);
+            logErrorStats(true);
+            await callOnDone();
+            if (collectedErrors.length) {
+                cb(new AggregateError(collectedErrors, `transformMap2 resulted in ${collectedErrors.length} error(s)`));
             }
-            // Tell input stream that we're done processing, but emit nothing to output - not error nor result
-            cb();
-        }
+            else {
+                cb();
+            }
+        },
     });
+    function updateConcurrency() {
+        const now = Date.now();
+        if (now - lastWarmupCheck < WARMUP_CHECK_INTERVAL_MS)
+            return;
+        lastWarmupCheck = now;
+        const elapsed = now - started;
+        if (elapsed >= warmupMs) {
+            warmupComplete = true;
+            concurrency = maxConcurrency;
+            logger.log(`transformMap2: warmup complete in ${_since(started)}`);
+            return;
+        }
+        const progress = elapsed / warmupMs;
+        concurrency = Math.max(1, Math.floor(1 + (maxConcurrency - 1) * progress));
+    }
     function logErrorStats(final = false) {
         if (!errors)
             return;
         logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`);
     }
+    async function callOnDone() {
+        try {
+            await onDone?.({
+                ok: collectedErrors.length === 0 && ok,
+                collectedErrors,
+                countErrors: errors,
+                countIn: index + 1,
+                countOut,
+                started,
+            });
+        }
+        catch (err) {
+            logger.error(err);
+        }
+    }
 }
 /**
  * Renders TransformMapStatsSummary into a friendly string,