@naturalcycles/nodejs-lib 15.67.0 → 15.68.0

package/dist/stream/index.d.ts

@@ -16,6 +16,7 @@ export * from './transform/transformFork.js';
 export * from './transform/transformLimit.js';
 export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
+export * from './transform/transformMap2.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
 export * from './transform/transformNoOp.js';
@@ -23,6 +24,7 @@ export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';
 export * from './transform/transformTap.js';
 export * from './transform/transformThrottle.js';
+export * from './transform/transformWarmup.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';

package/dist/stream/index.js

@@ -16,6 +16,7 @@ export * from './transform/transformFork.js';
 export * from './transform/transformLimit.js';
 export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
+export * from './transform/transformMap2.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
 export * from './transform/transformNoOp.js';
@@ -23,6 +24,7 @@ export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';
 export * from './transform/transformTap.js';
 export * from './transform/transformThrottle.js';
+export * from './transform/transformWarmup.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';

package/dist/stream/pipeline.d.ts

@@ -5,10 +5,12 @@ import { type AbortableAsyncMapper, type AsyncIndexedMapper, type AsyncPredicate
 import type { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream.model.js';
 import { type TransformLogProgressOptions } from './transform/transformLogProgress.js';
 import { type TransformMapOptions } from './transform/transformMap.js';
+import { type TransformMap2Options } from './transform/transformMap2.js';
 import { type TransformMapSimpleOptions } from './transform/transformMapSimple.js';
 import { type TransformMapSyncOptions } from './transform/transformMapSync.js';
 import { type TransformOffsetOptions } from './transform/transformOffset.js';
 import { type TransformThrottleOptions } from './transform/transformThrottle.js';
+import { type TransformWarmupOptions } from './transform/transformWarmup.js';
 export declare class Pipeline<T = unknown> {
     private readonly source;
     private transforms;
@@ -61,6 +63,10 @@ export declare class Pipeline<T = unknown> {
     flattenIfNeeded(): Pipeline<T extends readonly (infer TO)[] ? TO : T>;
     logProgress(opt?: TransformLogProgressOptions): this;
     map<TO>(mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapOptions<T, TO>): Pipeline<TO>;
+    /**
+     * @experimental if proven to be stable - will replace transformMap
+     */
+    map2<TO>(mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMap2Options<T, TO>): Pipeline<TO>;
     mapSync<TO>(mapper: IndexedMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapSyncOptions): Pipeline<TO>;
     mapSimple<TO>(mapper: IndexedMapper<T, TO>, opt?: TransformMapSimpleOptions): Pipeline<TO>;
     filter(asyncPredicate: AsyncPredicate<T>, opt?: TransformMapOptions): this;
@@ -69,6 +75,10 @@ export declare class Pipeline<T = unknown> {
     tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformOptions): this;
     tapSync(fn: IndexedMapper<T, any>, opt?: TransformOptions): this;
     throttle(opt: TransformThrottleOptions): this;
+    /**
+     * @experimental to be removed after transformMap2 is stable
+     */
+    warmup(opt: TransformWarmupOptions): this;
     transform<TO>(transform: TransformTyped<T, TO>): Pipeline<TO>;
     /**
      * Helper method to add multiple transforms at once.

package/dist/stream/pipeline.js

@@ -16,12 +16,14 @@ import { transformFork } from './transform/transformFork.js';
 import { transformLimit } from './transform/transformLimit.js';
 import { transformLogProgress, } from './transform/transformLogProgress.js';
 import { transformMap } from './transform/transformMap.js';
+import { transformMap2 } from './transform/transformMap2.js';
 import { transformMapSimple, } from './transform/transformMapSimple.js';
 import { transformMapSync } from './transform/transformMapSync.js';
 import { transformOffset } from './transform/transformOffset.js';
 import { transformSplitOnNewline } from './transform/transformSplit.js';
 import { transformTap, transformTapSync } from './transform/transformTap.js';
 import { transformThrottle } from './transform/transformThrottle.js';
+import { transformWarmup } from './transform/transformWarmup.js';
 import { writablePushToArray } from './writable/writablePushToArray.js';
 import { writableVoid } from './writable/writableVoid.js';
 export class Pipeline {
@@ -133,6 +135,16 @@ export class Pipeline {
         }));
         return this;
     }
+    /**
+     * @experimental if proven to be stable - will replace transformMap
+     */
+    map2(mapper, opt) {
+        this.transforms.push(transformMap2(mapper, {
+            ...opt,
+            signal: this.abortableSignal,
+        }));
+        return this;
+    }
     mapSync(mapper, opt) {
         this.transforms.push(transformMapSync(mapper, {
             ...opt,
@@ -172,6 +184,13 @@ export class Pipeline {
         this.transforms.push(transformThrottle(opt));
         return this;
     }
+    /**
+     * @experimental to be removed after transformMap2 is stable
+     */
+    warmup(opt) {
+        this.transforms.push(transformWarmup(opt));
+        return this;
+    }
     transform(transform) {
         this.transforms.push(transform);
         return this;

package/dist/stream/transform/transformMap2.d.ts

@@ -0,0 +1,66 @@
+import type { AbortableSignal } from '@naturalcycles/js-lib';
+import { ErrorMode } from '@naturalcycles/js-lib/error';
+import { type AbortableAsyncMapper, type AsyncPredicate, END, type NumberOfSeconds, type PositiveInteger, type Predicate, type Promisable, SKIP } from '@naturalcycles/js-lib/types';
+import type { TransformOptions, TransformTyped } from '../stream.model.js';
+import type { TransformMapStats } from './transformMap.js';
+export interface TransformMap2Options<IN = any, OUT = IN> extends TransformOptions {
+    /**
+     * Predicate to filter outgoing results (after mapper).
+     * Allows to not emit all results.
+     *
+     * Defaults to "pass everything" (including null, undefined, etc).
+     * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
+     */
+    predicate?: Predicate<OUT>;
+    asyncPredicate?: AsyncPredicate<OUT>;
+    /**
+     * Number of concurrently pending promises returned by `mapper`.
+     *
+     * @default 16
+     */
+    concurrency?: PositiveInteger;
+    /**
+     * Time in seconds to gradually increase concurrency from 1 to `concurrency`.
+     * Useful for warming up connections to databases, APIs, etc.
+     *
+     * Set to 0 to disable warmup (default).
+     */
+    warmupSeconds?: NumberOfSeconds;
+    /**
+     * @default THROW_IMMEDIATELY
+     */
+    errorMode?: ErrorMode;
+    /**
+     * If defined - will be called on every error happening in the stream.
+     * Called BEFORE observable will emit error (unless skipErrors is set to true).
+     */
+    onError?: (err: Error, input: IN) => any;
+    /**
+     * A hook that is called when the last item is finished processing.
+     * stats object is passed, containing countIn and countOut -
+     * number of items that entered the transform and number of items that left it.
+     *
+     * Callback is called **before** [possible] Aggregated error is thrown,
+     * and before [possible] THROW_IMMEDIATELY error.
+     *
+     * onDone callback will be awaited before Error is thrown.
+     */
+    onDone?: (stats: TransformMapStats) => Promisable<any>;
+    /**
+     * Progress metric
+     *
+     * @default `stream`
+     */
+    metric?: string;
+    /**
+     * Allows to abort (gracefully stop) the stream from inside the Transform.
+     */
+    signal?: AbortableSignal;
+}
+/**
+ * Like transformMap, but with native concurrency control (no through2-concurrent dependency)
+ * and support for gradual warmup.
+ *
+ * @experimental
+ */
+export declare function transformMap2<IN = any, OUT = IN>(mapper: AbortableAsyncMapper<IN, OUT | typeof SKIP | typeof END>, opt?: TransformMap2Options<IN, OUT>): TransformTyped<IN, OUT>;

package/dist/stream/transform/transformMap2.js

@@ -0,0 +1,183 @@
+import { Transform } from 'node:stream';
+import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+import { END, SKIP, } from '@naturalcycles/js-lib/types';
+import { yellow } from '../../colors/colors.js';
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
+/**
+ * Like transformMap, but with native concurrency control (no through2-concurrent dependency)
+ * and support for gradual warmup.
+ *
+ * @experimental
+ */
+export function transformMap2(mapper, opt = {}) {
+    const { concurrency = 16, warmupSeconds = 0, predicate, asyncPredicate, errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', signal, objectMode = true, highWaterMark = 64, } = opt;
+    const warmupMs = warmupSeconds * 1000;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+    // Stats
+    const started = Date.now();
+    let index = -1;
+    let countOut = 0;
+    let isSettled = false;
+    let ok = true;
+    let errors = 0;
+    const collectedErrors = [];
+    // Concurrency control
+    let startTime = 0;
+    let warmupComplete = warmupSeconds <= 0 || concurrency <= 1;
+    let inFlight = 0;
+    const waiters = [];
+    // Track pending operations for proper flush
+    let pendingOperations = 0;
+    return new Transform({
+        objectMode,
+        readableHighWaterMark: highWaterMark,
+        writableHighWaterMark: highWaterMark,
+        async transform(chunk, _, cb) {
+            // Initialize start time on first item
+            if (startTime === 0) {
+                startTime = Date.now();
+            }
+            // Stop processing if isSettled
+            if (isSettled)
+                return cb();
+            const currentIndex = ++index;
+            const currentConcurrency = getCurrentConcurrency();
+            // Wait for a slot if at capacity
+            if (inFlight >= currentConcurrency) {
+                const waiter = pDefer();
+                waiters.push(waiter);
+                await waiter;
+            }
+            else {
+                inFlight++;
+            }
+            // Signal that we're ready for more input
+            cb();
+            // Track this operation
+            pendingOperations++;
+            // Process the item asynchronously
+            try {
+                const res = await mapper(chunk, currentIndex);
+                if (isSettled) {
+                    release();
+                    pendingOperations--;
+                    return;
+                }
+                if (res === END) {
+                    isSettled = true;
+                    logger.log(`transformMap2 END received at index ${currentIndex}`);
+                    _assert(signal, 'signal is required when using END');
+                    signal.abort(new Error(PIPELINE_GRACEFUL_ABORT));
+                    release();
+                    pendingOperations--;
+                    return;
+                }
+                if (res === SKIP) {
+                    release();
+                    pendingOperations--;
+                    return;
+                }
+                let shouldPush = true;
+                if (predicate) {
+                    shouldPush = predicate(res, currentIndex);
+                }
+                else if (asyncPredicate) {
+                    shouldPush = (await asyncPredicate(res, currentIndex)) && !isSettled;
+                }
+                if (shouldPush) {
+                    countOut++;
+                    this.push(res);
+                }
+            }
+            catch (err) {
+                logger.error(err);
+                errors++;
+                logErrorStats();
+                if (onError) {
+                    try {
+                        onError(_anyToError(err), chunk);
+                    }
+                    catch { }
+                }
+                if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
+                    isSettled = true;
+                    ok = false;
+                    // Call onDone before destroying, since flush won't be called
+                    await callOnDone();
+                    this.destroy(_anyToError(err));
+                }
+                else if (errorMode === ErrorMode.THROW_AGGREGATED) {
+                    collectedErrors.push(_anyToError(err));
+                }
+            }
+            finally {
+                release();
+                pendingOperations--;
+            }
+        },
+        async flush(cb) {
+            // Wait for all pending operations to complete
+            // Polling is simple and race-condition-free
+            // Timeout prevents infinite loop if something goes wrong
+            const flushStart = Date.now();
+            const flushTimeoutMs = 60_000;
+            while (pendingOperations > 0) {
+                await new Promise(resolve => setImmediate(resolve));
+                if (Date.now() - flushStart > flushTimeoutMs) {
+                    logger.error(`transformMap2 flush timeout: ${pendingOperations} operations still pending after ${flushTimeoutMs}ms`);
+                    break;
+                }
+            }
+            logErrorStats(true);
+            await callOnDone();
+            if (collectedErrors.length) {
+                cb(new AggregateError(collectedErrors, `transformMap2 resulted in ${collectedErrors.length} error(s)`));
+            }
+            else {
+                cb();
+            }
+        },
+    });
+    function getCurrentConcurrency() {
+        if (warmupComplete)
+            return concurrency;
+        const elapsed = Date.now() - startTime;
+        if (elapsed >= warmupMs) {
+            warmupComplete = true;
+            logger.debug('warmup complete');
+            return concurrency;
+        }
+        const progress = elapsed / warmupMs;
+        return Math.max(1, Math.floor(1 + (concurrency - 1) * progress));
+    }
+    function release() {
+        inFlight--;
+        const currentConcurrency = getCurrentConcurrency();
+        while (waiters.length && inFlight < currentConcurrency) {
+            inFlight++;
+            waiters.shift().resolve();
+        }
+    }
+    function logErrorStats(final = false) {
+        if (!errors)
+            return;
+        logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`);
+    }
+    async function callOnDone() {
+        try {
+            await onDone?.({
+                ok: collectedErrors.length === 0 && ok,
+                collectedErrors,
+                countErrors: errors,
+                countIn: index + 1,
+                countOut,
+                started,
+            });
+        }
+        catch (err) {
+            logger.error(err);
+        }
+    }
+}

package/dist/stream/transform/transformWarmup.d.ts

@@ -0,0 +1,24 @@
+import type { NumberOfSeconds, PositiveInteger } from '@naturalcycles/js-lib/types';
+import type { TransformOptions, TransformTyped } from '../stream.model.js';
+export interface TransformWarmupOptions extends TransformOptions {
+    /**
+     * Target concurrency after warmup completes.
+     */
+    concurrency: PositiveInteger;
+    /**
+     * Time in seconds to gradually increase concurrency from 1 to `concurrency`.
+     * Set to 0 to disable warmup (pass-through mode from the start).
+     */
+    warmupSeconds: NumberOfSeconds;
+}
+/**
+ * Transform that gradually increases concurrency from 1 to the configured maximum
+ * over a warmup period. Useful for scenarios where you want to avoid overwhelming
+ * a system at startup (e.g., database connections, API rate limits).
+ *
+ * During warmup: limits concurrent items based on elapsed time.
+ * After warmup: passes items through immediately with zero overhead.
+ *
+ * @experimental
+ */
+export declare function transformWarmup<T>(opt: TransformWarmupOptions): TransformTyped<T, T>;

package/dist/stream/transform/transformWarmup.js

@@ -0,0 +1,79 @@
+import { Transform } from 'node:stream';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+/**
+ * Transform that gradually increases concurrency from 1 to the configured maximum
+ * over a warmup period. Useful for scenarios where you want to avoid overwhelming
+ * a system at startup (e.g., database connections, API rate limits).
+ *
+ * During warmup: limits concurrent items based on elapsed time.
+ * After warmup: passes items through immediately with zero overhead.
+ *
+ * @experimental
+ */
+export function transformWarmup(opt) {
+    const { concurrency, warmupSeconds, objectMode = true, highWaterMark } = opt;
+    const warmupMs = warmupSeconds * 1000;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+    let startTime = 0;
+    let warmupComplete = warmupSeconds <= 0 || concurrency <= 1;
+    let inFlight = 0;
+    const waiters = [];
+    return new Transform({
+        objectMode,
+        highWaterMark,
+        async transform(item, _, cb) {
+            // Initialize start time on first item
+            if (startTime === 0) {
+                startTime = Date.now();
+            }
+            // Fast-path: after warmup, just pass through with zero overhead
+            if (warmupComplete) {
+                cb(null, item);
+                return;
+            }
+            const currentConcurrency = getCurrentConcurrency();
+            if (inFlight < currentConcurrency) {
+                // Have room, proceed immediately
+                inFlight++;
+                logger.debug(`inFlight++ ${inFlight}/${currentConcurrency}, waiters ${waiters.length}`);
+            }
+            else {
+                // Wait for a slot
+                const waiter = pDefer();
+                waiters.push(waiter);
+                logger.debug(`inFlight ${inFlight}/${currentConcurrency}, waiters++ ${waiters.length}`);
+                await waiter;
+                logger.debug(`waiter resolved, inFlight ${inFlight}/${getCurrentConcurrency()}`);
+            }
+            // Push the item
+            cb(null, item);
+            // Release slot on next microtask - essential for concurrency control.
+            // Without this, the slot would be freed immediately and items would
+            // flow through without any limiting effect.
+            queueMicrotask(release);
+        },
+    });
+    function getCurrentConcurrency() {
+        if (warmupComplete)
+            return concurrency;
+        const elapsed = Date.now() - startTime;
+        if (elapsed >= warmupMs) {
+            warmupComplete = true;
+            logger.debug('warmup complete');
+            return concurrency;
+        }
+        // Linear interpolation from 1 to concurrency
+        const progress = elapsed / warmupMs;
+        return Math.max(1, Math.floor(1 + (concurrency - 1) * progress));
+    }
+    function release() {
+        inFlight--;
+        // Wake up waiters based on current concurrency (may have increased)
+        const currentConcurrency = getCurrentConcurrency();
+        while (waiters.length && inFlight < currentConcurrency) {
+            inFlight++;
+            waiters.shift().resolve();
+        }
+    }
+}

package/dist/zip/zip.util.js

@@ -34,7 +34,7 @@ export async function inflateBuffer(buf, options = {}) {
  * It's 9 bytes shorter than `gzip`.
  */
 export async function deflateString(s, options) {
-    return await deflateBuffer(Buffer.from(s), options);
+    return await deflate(s, options);
 }
 export async function inflateToString(buf, options) {
     return (await inflateBuffer(buf, options)).toString();
@@ -54,7 +54,7 @@ export async function gunzipBuffer(buf, options = {}) {
  * It's 9 bytes longer than `deflate`.
  */
 export async function gzipString(s, options) {
-    return await gzipBuffer(Buffer.from(s), options);
+    return await gzip(s, options);
 }
 export async function gunzipToString(buf, options) {
     return (await gunzipBuffer(buf, options)).toString();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.67.0",
+  "version": "15.68.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/index.ts

@@ -16,6 +16,7 @@ export * from './transform/transformFork.js'
 export * from './transform/transformLimit.js'
 export * from './transform/transformLogProgress.js'
 export * from './transform/transformMap.js'
+export * from './transform/transformMap2.js'
 export * from './transform/transformMapSimple.js'
 export * from './transform/transformMapSync.js'
 export * from './transform/transformNoOp.js'
@@ -23,6 +24,7 @@ export * from './transform/transformOffset.js'
 export * from './transform/transformSplit.js'
 export * from './transform/transformTap.js'
 export * from './transform/transformThrottle.js'
+export * from './transform/transformWarmup.js'
 export * from './transform/worker/baseWorkerClass.js'
 export * from './transform/worker/transformMultiThreaded.js'
 export * from './transform/worker/transformMultiThreaded.model.js'

package/src/stream/pipeline.ts

@@ -45,6 +45,7 @@ import {
   type TransformLogProgressOptions,
 } from './transform/transformLogProgress.js'
 import { transformMap, type TransformMapOptions } from './transform/transformMap.js'
+import { transformMap2, type TransformMap2Options } from './transform/transformMap2.js'
 import {
   transformMapSimple,
   type TransformMapSimpleOptions,
@@ -54,6 +55,7 @@ import { transformOffset, type TransformOffsetOptions } from './transform/transf
 import { transformSplitOnNewline } from './transform/transformSplit.js'
 import { transformTap, transformTapSync } from './transform/transformTap.js'
 import { transformThrottle, type TransformThrottleOptions } from './transform/transformThrottle.js'
+import { transformWarmup, type TransformWarmupOptions } from './transform/transformWarmup.js'
 import { writablePushToArray } from './writable/writablePushToArray.js'
 import { writableVoid } from './writable/writableVoid.js'
 
@@ -196,6 +198,22 @@ export class Pipeline<T = unknown> {
     return this as any
   }
 
+  /**
+   * @experimental if proven to be stable - will replace transformMap
+   */
+  map2<TO>(
+    mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>,
+    opt?: TransformMap2Options<T, TO>,
+  ): Pipeline<TO> {
+    this.transforms.push(
+      transformMap2(mapper, {
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    return this as any
+  }
+
   mapSync<TO>(
     mapper: IndexedMapper<T, TO | typeof SKIP | typeof END>,
     opt?: TransformMapSyncOptions,
@@ -250,6 +268,14 @@ export class Pipeline<T = unknown> {
     return this
   }
 
+  /**
+   * @experimental to be removed after transformMap2 is stable
+   */
+  warmup(opt: TransformWarmupOptions): this {
+    this.transforms.push(transformWarmup(opt))
+    return this
+  }
+
   transform<TO>(transform: TransformTyped<T, TO>): Pipeline<TO> {
     this.transforms.push(transform)
     return this as any

package/src/stream/transform/transformMap2.ts

@@ -0,0 +1,298 @@
+import { Transform } from 'node:stream'
+import type { AbortableSignal } from '@naturalcycles/js-lib'
+import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
+import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import {
+  type AbortableAsyncMapper,
+  type AsyncPredicate,
+  END,
+  type NumberOfSeconds,
+  type PositiveInteger,
+  type Predicate,
+  type Promisable,
+  SKIP,
+  type UnixTimestampMillis,
+} from '@naturalcycles/js-lib/types'
+import { yellow } from '../../colors/colors.js'
+import type { TransformOptions, TransformTyped } from '../stream.model.js'
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
+import type { TransformMapStats } from './transformMap.js'
+
+export interface TransformMap2Options<IN = any, OUT = IN> extends TransformOptions {
+  /**
+   * Predicate to filter outgoing results (after mapper).
+   * Allows to not emit all results.
+   *
+   * Defaults to "pass everything" (including null, undefined, etc).
+   * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
+   */
+  predicate?: Predicate<OUT>
+
+  asyncPredicate?: AsyncPredicate<OUT>
+
+  /**
+   * Number of concurrently pending promises returned by `mapper`.
+   *
+   * @default 16
+   */
+  concurrency?: PositiveInteger
+
+  /**
+   * Time in seconds to gradually increase concurrency from 1 to `concurrency`.
+   * Useful for warming up connections to databases, APIs, etc.
+   *
+   * Set to 0 to disable warmup (default).
+   */
+  warmupSeconds?: NumberOfSeconds
+
+  /**
+   * @default THROW_IMMEDIATELY
+   */
+  errorMode?: ErrorMode
+
+  /**
+   * If defined - will be called on every error happening in the stream.
+   * Called BEFORE observable will emit error (unless skipErrors is set to true).
+   */
+  onError?: (err: Error, input: IN) => any
+
+  /**
+   * A hook that is called when the last item is finished processing.
+   * stats object is passed, containing countIn and countOut -
+   * number of items that entered the transform and number of items that left it.
+   *
+   * Callback is called **before** [possible] Aggregated error is thrown,
+   * and before [possible] THROW_IMMEDIATELY error.
+   *
+   * onDone callback will be awaited before Error is thrown.
+   */
+  onDone?: (stats: TransformMapStats) => Promisable<any>
+
+  /**
+   * Progress metric
+   *
+   * @default `stream`
+   */
+  metric?: string
+
+  /**
+   * Allows to abort (gracefully stop) the stream from inside the Transform.
+   */
+  signal?: AbortableSignal
+}
+
+/**
+ * Like transformMap, but with native concurrency control (no through2-concurrent dependency)
+ * and support for gradual warmup.
+ *
+ * @experimental
+ */
+export function transformMap2<IN = any, OUT = IN>(
+  mapper: AbortableAsyncMapper<IN, OUT | typeof SKIP | typeof END>,
+  opt: TransformMap2Options<IN, OUT> = {},
+): TransformTyped<IN, OUT> {
+  const {
+    concurrency = 16,
+    warmupSeconds = 0,
+    predicate,
+    asyncPredicate,
+    errorMode = ErrorMode.THROW_IMMEDIATELY,
+    onError,
+    onDone,
+    metric = 'stream',
+    signal,
+    objectMode = true,
+    highWaterMark = 64,
+  } = opt
+
+  const warmupMs = warmupSeconds * 1000
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+
+  // Stats
+  const started = Date.now() as UnixTimestampMillis
+  let index = -1
+  let countOut = 0
+  let isSettled = false
+  let ok = true
+  let errors = 0
+  const collectedErrors: Error[] = []
+
+  // Concurrency control
+  let startTime = 0
+  let warmupComplete = warmupSeconds <= 0 || concurrency <= 1
+  let inFlight = 0
+  const waiters: DeferredPromise[] = []
+
+  // Track pending operations for proper flush
+  let pendingOperations = 0
+
+  return new Transform({
+    objectMode,
+    readableHighWaterMark: highWaterMark,
+    writableHighWaterMark: highWaterMark,
+    async transform(this: Transform, chunk: IN, _, cb) {
+      // Initialize start time on first item
+      if (startTime === 0) {
+        startTime = Date.now()
+      }
+
+      // Stop processing if isSettled
+      if (isSettled) return cb()
+
+      const currentIndex = ++index
+      const currentConcurrency = getCurrentConcurrency()
+
+      // Wait for a slot if at capacity
+      if (inFlight >= currentConcurrency) {
+        const waiter = pDefer()
+        waiters.push(waiter)
+        await waiter
+      } else {
+        inFlight++
+      }
+
+      // Signal that we're ready for more input
+      cb()
+
+      // Track this operation
+      pendingOperations++
+
+      // Process the item asynchronously
+      try {
+        const res: OUT | typeof SKIP | typeof END = await mapper(chunk, currentIndex)
+
+        if (isSettled) {
+          release()
+          pendingOperations--
+          return
+        }
+
+        if (res === END) {
+          isSettled = true
+          logger.log(`transformMap2 END received at index ${currentIndex}`)
+          _assert(signal, 'signal is required when using END')
+          signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
+          release()
+          pendingOperations--
+          return
+        }
+
+        if (res === SKIP) {
+          release()
+          pendingOperations--
+          return
+        }
+
+        let shouldPush = true
+        if (predicate) {
+          shouldPush = predicate(res, currentIndex)
+        } else if (asyncPredicate) {
+          shouldPush = (await asyncPredicate(res, currentIndex)) && !isSettled
+        }
+
+        if (shouldPush) {
+          countOut++
+          this.push(res)
+        }
+      } catch (err) {
+        logger.error(err)
+        errors++
+        logErrorStats()
+
+        if (onError) {
+          try {
+            onError(_anyToError(err), chunk)
+          } catch {}
+        }
+
+        if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
+          isSettled = true
+          ok = false
+          // Call onDone before destroying, since flush won't be called
+          await callOnDone()
+          this.destroy(_anyToError(err))
+        } else if (errorMode === ErrorMode.THROW_AGGREGATED) {
+          collectedErrors.push(_anyToError(err))
+        }
+      } finally {
+        release()
+        pendingOperations--
+      }
+    },
+    async flush(cb) {
+      // Wait for all pending operations to complete
+      // Polling is simple and race-condition-free
+      // Timeout prevents infinite loop if something goes wrong
+      const flushStart = Date.now()
+      const flushTimeoutMs = 60_000
+      while (pendingOperations > 0) {
+        await new Promise(resolve => setImmediate(resolve))
+        if (Date.now() - flushStart > flushTimeoutMs) {
+          logger.error(
+            `transformMap2 flush timeout: ${pendingOperations} operations still pending after ${flushTimeoutMs}ms`,
+          )
+          break
+        }
+      }
+
+      logErrorStats(true)
+      await callOnDone()
+
+      if (collectedErrors.length) {
+        cb(
+          new AggregateError(
+            collectedErrors,
+            `transformMap2 resulted in ${collectedErrors.length} error(s)`,
+          ),
+        )
+      } else {
+        cb()
+      }
+    },
+  })
+
+  function getCurrentConcurrency(): number {
+    if (warmupComplete) return concurrency
+
+    const elapsed = Date.now() - startTime
+    if (elapsed >= warmupMs) {
+      warmupComplete = true
+      logger.debug('warmup complete')
+      return concurrency
+    }
+
+    const progress = elapsed / warmupMs
+    return Math.max(1, Math.floor(1 + (concurrency - 1) * progress))
+  }
+
+  function release(): void {
+    inFlight--
+    const currentConcurrency = getCurrentConcurrency()
+    while (waiters.length && inFlight < currentConcurrency) {
+      inFlight++
+      waiters.shift()!.resolve()
+    }
+  }
+
+  function logErrorStats(final = false): void {
+    if (!errors) return
+    logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
+  }
+
+  async function callOnDone(): Promise<void> {
+    try {
+      await onDone?.({
+        ok: collectedErrors.length === 0 && ok,
+        collectedErrors,
+        countErrors: errors,
+        countIn: index + 1,
+        countOut,
+        started,
+      })
+    } catch (err) {
+      logger.error(err)
+    }
+  }
+}

package/src/stream/transform/transformWarmup.ts

@@ -0,0 +1,105 @@
+import { Transform } from 'node:stream'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
+import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import type { NumberOfSeconds, PositiveInteger } from '@naturalcycles/js-lib/types'
+import type { TransformOptions, TransformTyped } from '../stream.model.js'
+
+export interface TransformWarmupOptions extends TransformOptions {
+  /**
+   * Target concurrency after warmup completes.
+   */
+  concurrency: PositiveInteger
+
+  /**
+   * Time in seconds to gradually increase concurrency from 1 to `concurrency`.
+   * Set to 0 to disable warmup (pass-through mode from the start).
+   */
+  warmupSeconds: NumberOfSeconds
+}
+
+/**
+ * Transform that gradually increases concurrency from 1 to the configured maximum
+ * over a warmup period. Useful for scenarios where you want to avoid overwhelming
+ * a system at startup (e.g., database connections, API rate limits).
+ *
+ * During warmup: limits concurrent items based on elapsed time.
+ * After warmup: passes items through immediately with zero overhead.
+ *
+ * @experimental
+ */
+export function transformWarmup<T>(opt: TransformWarmupOptions): TransformTyped<T, T> {
+  const { concurrency, warmupSeconds, objectMode = true, highWaterMark } = opt
+  const warmupMs = warmupSeconds * 1000
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+
+  let startTime = 0
+  let warmupComplete = warmupSeconds <= 0 || concurrency <= 1
+  let inFlight = 0
+  const waiters: DeferredPromise[] = []
+
+  return new Transform({
+    objectMode,
+    highWaterMark,
+    async transform(item: T, _, cb) {
+      // Initialize start time on first item
+      if (startTime === 0) {
+        startTime = Date.now()
+      }
+
+      // Fast-path: after warmup, just pass through with zero overhead
+      if (warmupComplete) {
+        cb(null, item)
+        return
+      }
+
+      const currentConcurrency = getCurrentConcurrency()
+
+      if (inFlight < currentConcurrency) {
+        // Have room, proceed immediately
+        inFlight++
+        logger.debug(`inFlight++ ${inFlight}/${currentConcurrency}, waiters ${waiters.length}`)
+      } else {
+        // Wait for a slot
+        const waiter = pDefer()
+        waiters.push(waiter)
+        logger.debug(`inFlight ${inFlight}/${currentConcurrency}, waiters++ ${waiters.length}`)
+        await waiter
+        logger.debug(`waiter resolved, inFlight ${inFlight}/${getCurrentConcurrency()}`)
+      }
+
+      // Push the item
+      cb(null, item)
+
+      // Release slot on next microtask - essential for concurrency control.
+      // Without this, the slot would be freed immediately and items would
+      // flow through without any limiting effect.
+      queueMicrotask(release)
+    },
+  })
+
+  function getCurrentConcurrency(): number {
+    if (warmupComplete) return concurrency
+
+    const elapsed = Date.now() - startTime
+    if (elapsed >= warmupMs) {
+      warmupComplete = true
+      logger.debug('warmup complete')
+      return concurrency
+    }
+
+    // Linear interpolation from 1 to concurrency
+    const progress = elapsed / warmupMs
+    return Math.max(1, Math.floor(1 + (concurrency - 1) * progress))
+  }
+
+  function release(): void {
+    inFlight--
+    // Wake up waiters based on current concurrency (may have increased)
+    const currentConcurrency = getCurrentConcurrency()
+    while (waiters.length && inFlight < currentConcurrency) {
+      inFlight++
+      waiters.shift()!.resolve()
+    }
+  }
+}

package/src/zip/zip.util.ts

@@ -50,7 +50,7 @@ export async function deflateString(
   s: string,
   options?: ZlibOptions,
 ): Promise<Buffer<ArrayBuffer>> {
-  return await deflateBuffer(Buffer.from(s), options)
+  return await deflate(s, options)
 }
 
 export async function inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
@@ -80,7 +80,7 @@ export async function gunzipBuffer(
  * It's 9 bytes longer than `deflate`.
  */
 export async function gzipString(s: string, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>> {
-  return await gzipBuffer(Buffer.from(s), options)
+  return await gzip(s, options)
 }
 
 export async function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string> {