@naturalcycles/nodejs-lib 15.70.1 → 15.72.0

package/src/stream/transform/transformMap.ts

@@ -1,12 +1,16 @@
+import { Transform } from 'node:stream'
 import { _hc, type AbortableSignal } from '@naturalcycles/js-lib'
-import { _since } from '@naturalcycles/js-lib/datetime/time.util.js'
+import { _since } from '@naturalcycles/js-lib/datetime'
 import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
-import { _stringify } from '@naturalcycles/js-lib/string/stringify.js'
+import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import { _stringify } from '@naturalcycles/js-lib/string'
 import {
   type AbortableAsyncMapper,
   type AsyncPredicate,
   END,
+  type NumberOfSeconds,
   type PositiveInteger,
   type Predicate,
   type Promisable,
@@ -14,7 +18,6 @@ import {
   type StringMap,
   type UnixTimestampMillis,
 } from '@naturalcycles/js-lib/types'
-import through2Concurrent from 'through2-concurrent'
 import { yellow } from '../../colors/colors.js'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
 import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
@@ -34,23 +37,17 @@ export interface TransformMapOptions<IN = any, OUT = IN> extends TransformOption
   /**
    * Number of concurrently pending promises returned by `mapper`.
    *
-   * Default is 16.
-   * It was recently changed up from 16, after some testing that shown that
-   * for simple low-cpu mapper functions 32 produces almost 2x throughput.
-   * For example, in scenarios like streaming a query from Datastore.
-   * UPD: changed back from 32 to 16, "to be on a safe side", as 32 sometimes
-   * causes "Datastore timeout errors".
+   * @default 16
    */
   concurrency?: PositiveInteger
 
   /**
-   * Defaults to 64 items.
-   * (objectMode default is 16, but we increased it)
+   * Time in seconds to gradually increase concurrency from 1 to `concurrency`.
+   * Useful for warming up connections to databases, APIs, etc.
    *
-   * Affects both readable and writable highWaterMark (buffer).
-   * So, 64 means a total buffer of 128 (64 input and 64 output buffer).
+   * Set to 0 to disable warmup (default).
    */
-  highWaterMark?: PositiveInteger
+  warmupSeconds?: NumberOfSeconds
 
   /**
    * @default THROW_IMMEDIATELY
@@ -88,165 +85,106 @@ export interface TransformMapOptions<IN = any, OUT = IN> extends TransformOption
   signal?: AbortableSignal
 }
 
-export interface TransformMapStats {
-  /**
-   * True if transform was successful (didn't throw Immediate or Aggregated error).
-   */
-  ok: boolean
-  /**
-   * Only used (and returned) for ErrorMode.Aggregated
-   */
-  collectedErrors: Error[]
-  countErrors: number
-  countIn: number
-  countOut: number
-  started: UnixTimestampMillis
-}
-
-export interface TransformMapStatsSummary extends TransformMapStats {
-  /**
-   * Name of the summary, defaults to `Transform`
-   */
-  name?: string
-
-  /**
-   * Allows to pass extra key-value object, which will be rendered as:
-   * key: value
-   * key2: value2
-   */
-  extra?: StringMap<any>
-}
-
-// doesn't work, cause here we don't construct our Transform instance ourselves
-// export class TransformMap extends AbortableTransform {}
+const WARMUP_CHECK_INTERVAL_MS = 1000
 
 /**
- * Like pMap, but for streams.
- * Inspired by `through2`.
- * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
- * Using this allows native stream .pipe() to work and use backpressure.
+ * Like transformMap, but with native concurrency control (no through2-concurrent dependency)
+ * and support for gradual warmup.
  *
- * Only works in objectMode (due to through2Concurrent).
- *
- * Concurrency defaults to 16.
- *
- * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
+ * @experimental
  */
 export function transformMap<IN = any, OUT = IN>(
   mapper: AbortableAsyncMapper<IN, OUT | typeof SKIP | typeof END>,
   opt: TransformMapOptions<IN, OUT> = {},
 ): TransformTyped<IN, OUT> {
   const {
-    concurrency = 16,
-    highWaterMark = 64,
-    predicate, // we now default to "no predicate" (meaning pass-everything)
+    concurrency: maxConcurrency = 16,
+    warmupSeconds = 0,
+    predicate,
     asyncPredicate,
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     onError,
     onDone,
     metric = 'stream',
     signal,
+    objectMode = true,
+    highWaterMark = 64,
   } = opt
 
-  const started = Date.now() as UnixTimestampMillis
+  const warmupMs = warmupSeconds * 1000
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+
+  // Stats
+  let started = 0 as UnixTimestampMillis
   let index = -1
   let countOut = 0
   let isSettled = false
   let ok = true
   let errors = 0
-  const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
-  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
-
-  return through2Concurrent.obj(
-    {
-      maxConcurrency: concurrency,
-      readableHighWaterMark: highWaterMark,
-      writableHighWaterMark: highWaterMark,
-      async final(cb) {
-        logErrorStats(true)
-
-        if (collectedErrors.length) {
-          try {
-            await onDone?.({
-              ok: false,
-              collectedErrors,
-              countErrors: errors,
-              countIn: index + 1,
-              countOut,
-              started,
-            })
-          } catch (err) {
-            logger.error(err)
-          }
-
-          // emit Aggregated error
-          cb(
-            new AggregateError(
-              collectedErrors,
-              `transformMap resulted in ${collectedErrors.length} error(s)`,
-            ),
-          )
-        } else {
-          // emit no error
+  const collectedErrors: Error[] = []
+
+  // Concurrency control - single counter, single callback for backpressure
+  let inFlight = 0
+  let blockedCallback: (() => void) | null = null
+  let flushBlocked: DeferredPromise | null = null
+
+  // Warmup - cached concurrency to reduce Date.now() syscalls
+  let warmupComplete = warmupSeconds <= 0 || maxConcurrency <= 1
+  let concurrency = warmupComplete ? maxConcurrency : 1
+  let lastWarmupCheck = 0
+
+  return new Transform({
+    objectMode,
+    readableHighWaterMark: highWaterMark,
+    writableHighWaterMark: highWaterMark,
+    async transform(this: Transform, chunk: IN, _, cb) {
+      // Initialize start time on first item
+      if (started === 0) {
+        started = Date.now() as UnixTimestampMillis
+        lastWarmupCheck = started
+      }
 
-          try {
-            await onDone?.({
-              ok,
-              collectedErrors,
-              countErrors: errors,
-              countIn: index + 1,
-              countOut,
-              started,
-            })
-          } catch (err) {
-            logger.error(err)
-          }
-
-          cb()
-        }
-      },
-    },
-    async function transformMapFn(chunk: IN, _, cb) {
-      // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
       if (isSettled) return cb()
 
       const currentIndex = ++index
+      inFlight++
+      if (!warmupComplete) {
+        updateConcurrency()
+      }
+
+      // Apply backpressure if at capacity, otherwise request more input
+      if (inFlight < concurrency) {
+        cb()
+      } else {
+        blockedCallback = cb
+      }
 
       try {
         const res: OUT | typeof SKIP | typeof END = await mapper(chunk, currentIndex)
-        // Check for isSettled again, as it may happen while mapper was running
-        if (isSettled) return cb()
+
+        if (isSettled) return
 
         if (res === END) {
           isSettled = true
-          logger.log(`transformMap END received at index ${currentIndex}`)
+          logger.log(`transformMap2 END received at index ${currentIndex}`)
           _assert(signal, 'signal is required when using END')
           signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
-          return cb()
+          return
         }
 
-        if (res === SKIP) {
-          // do nothing, don't push
-          return cb()
-        }
+        if (res === SKIP) return
 
+        let shouldPush = true
         if (predicate) {
-          if (predicate(res, currentIndex)) {
-            countOut++
-            this.push(res)
-          }
+          shouldPush = predicate(res, currentIndex)
         } else if (asyncPredicate) {
-          if ((await asyncPredicate(res, currentIndex)) && !isSettled) {
-            // isSettled could have happened in parallel, hence the extra check
-            countOut++
-            this.push(res)
-          }
-        } else {
+          shouldPush = (await asyncPredicate(res, currentIndex)) && !isSettled
+        }
+
+        if (shouldPush) {
           countOut++
           this.push(res)
         }
-
-        cb() // done processing
       } catch (err) {
         logger.error(err)
         errors++
@@ -261,40 +199,117 @@ export function transformMap<IN = any, OUT = IN>(
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
           ok = false
-
-          // Tests show that onDone is still called at `final` (second time),
-          // so, we no longer call it here
-
-          // try {
-          //   await onDone?.({
-          //     ok: false,
-          //     collectedErrors,
-          //     countErrors: errors,
-          //     countIn: index + 1,
-          //     countOut,
-          //     started,
-          //   })
-          // } catch (err) {
-          //   logger.error(err)
-          // }
-
-          return cb(err) // Emit error immediately
+          await callOnDone()
+          this.destroy(_anyToError(err))
+          return
         }
-
         if (errorMode === ErrorMode.THROW_AGGREGATED) {
-          collectedErrors.push(err as Error)
+          collectedErrors.push(_anyToError(err))
+        }
+      } finally {
+        inFlight--
+
+        // Release blocked callback if we now have capacity
+        if (blockedCallback && inFlight < concurrency) {
+          const pendingCb = blockedCallback
+          blockedCallback = null
+          pendingCb()
         }
 
-        // Tell input stream that we're done processing, but emit nothing to output - not error nor result
+        // Trigger flush completion if all done
+        if (inFlight === 0 && flushBlocked) {
+          flushBlocked.resolve()
+        }
+      }
+    },
+    async flush(cb) {
+      // Wait for all in-flight operations to complete
+      if (inFlight > 0) {
+        flushBlocked = pDefer()
+        await flushBlocked
+      }
+
+      logErrorStats(true)
+      await callOnDone()
+
+      if (collectedErrors.length) {
+        cb(
+          new AggregateError(
+            collectedErrors,
+            `transformMap2 resulted in ${collectedErrors.length} error(s)`,
+          ),
+        )
+      } else {
         cb()
       }
     },
-  )
+  })
+
+  function updateConcurrency(): void {
+    const now = Date.now()
+    if (now - lastWarmupCheck < WARMUP_CHECK_INTERVAL_MS) return
+    lastWarmupCheck = now
+
+    const elapsed = now - started
+    if (elapsed >= warmupMs) {
+      warmupComplete = true
+      concurrency = maxConcurrency
+      logger.log(`transformMap2: warmup complete in ${_since(started)}`)
+      return
+    }
+
+    const progress = elapsed / warmupMs
+    concurrency = Math.max(1, Math.floor(1 + (maxConcurrency - 1) * progress))
+  }
 
   function logErrorStats(final = false): void {
     if (!errors) return
     logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
   }
+
+  async function callOnDone(): Promise<void> {
+    try {
+      await onDone?.({
+        ok: collectedErrors.length === 0 && ok,
+        collectedErrors,
+        countErrors: errors,
+        countIn: index + 1,
+        countOut,
+        started,
+      })
+    } catch (err) {
+      logger.error(err)
+    }
+  }
+}
+
+export interface TransformMapStats {
+  /**
+   * True if transform was successful (didn't throw Immediate or Aggregated error).
+   */
+  ok: boolean
+  /**
+   * Only used (and returned) for ErrorMode.Aggregated
+   */
+  collectedErrors: Error[]
+  countErrors: number
+  countIn: number
+  countOut: number
+  started: UnixTimestampMillis
+}
+
+export interface TransformMapStatsSummary extends TransformMapStats {
+  /**
+   * Name of the summary, defaults to `Transform`
+   */
+  name?: string
+
+  /**
+   * Allows to pass extra key-value object, which will be rendered as:
+   * key: value
+   * key2: value2
+   */
+  extra?: StringMap<any>
 }
 
 /**

package/src/stream/transform/worker/transformMultiThreaded.ts

@@ -1,9 +1,9 @@
+import { Transform } from 'node:stream'
 import { Worker } from 'node:worker_threads'
 import { _range } from '@naturalcycles/js-lib/array/range.js'
 import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
 import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
 import type { AnyObject } from '@naturalcycles/js-lib/types'
-import through2Concurrent from 'through2-concurrent'
 import type { TransformTyped } from '../../stream.model.js'
 import type { WorkerInput, WorkerOutput } from './transformMultiThreaded.model.js'
 
@@ -59,6 +59,11 @@ export function transformMultiThreaded<IN, OUT>(
   const messageDonePromises: Record<number, DeferredPromise<OUT>> = {}
   let index = -1 // input chunk index, will start from 0
 
+  // Concurrency control
+  let inFlight = 0
+  let blockedCallback: (() => void) | null = null
+  let flushBlocked: DeferredPromise | null = null
+
   const workers = _range(0, poolSize).map(workerIndex => {
     workerDonePromises.push(pDefer())
 
@@ -70,23 +75,16 @@ export function transformMultiThreaded<IN, OUT>(
       },
     })
 
-    // const {threadId} = worker
-    // console.log({threadId})
-
     worker.on('error', err => {
       console.error(`Worker ${workerIndex} error`, err)
       workerDonePromises[workerIndex]!.reject(err as Error)
     })
 
     worker.on('exit', _exitCode => {
-      // console.log(`Worker ${index} exit: ${exitCode}`)
       workerDonePromises[workerIndex]!.resolve(undefined)
     })
 
     worker.on('message', (out: WorkerOutput<OUT>) => {
-      // console.log(`Message from Worker ${workerIndex}:`, out)
-      // console.log(Object.keys(messageDonePromises))
-      // tr.push(out.payload)
       if (out.error) {
         messageDonePromises[out.index]!.reject(out.error)
       } else {
@@ -97,32 +95,22 @@ export function transformMultiThreaded<IN, OUT>(
     return worker
   })
 
-  return through2Concurrent.obj(
-    {
-      maxConcurrency,
-      highWaterMark,
-      async final(cb) {
-        try {
-          // Push null (complete) to all sub-streams
-          for (const worker of workers) {
-            worker.postMessage(null)
-          }
-
-          console.log(`transformMultiThreaded.final is waiting for all chains to be done`)
-          await Promise.all(workerDonePromises)
-          console.log(`transformMultiThreaded.final all chains done`)
-
-          cb()
-        } catch (err) {
-          cb(err as Error)
-        }
-      },
-    },
-    async function transformMapFn(chunk: IN, _, cb) {
-      // Freezing the index, because it may change due to concurrency
+  return new Transform({
+    objectMode: true,
+    readableHighWaterMark: highWaterMark,
+    writableHighWaterMark: highWaterMark,
+    async transform(this: Transform, chunk: IN, _, cb) {
       const currentIndex = ++index
+      inFlight++
 
-      // Create the unresolved promise (to avait)
+      // Apply backpressure if at capacity, otherwise request more input
+      if (inFlight < maxConcurrency) {
+        cb()
+      } else {
+        blockedCallback = cb
+      }
+
+      // Create the unresolved promise (to await)
       messageDonePromises[currentIndex] = pDefer<OUT>()
 
       const worker = workers[currentIndex % poolSize]! // round-robin
@@ -132,21 +120,50 @@ export function transformMultiThreaded<IN, OUT>(
       } as WorkerInput)
 
       try {
-        // awaiting for result
         const out = await messageDonePromises[currentIndex]
-        // console.log('awaited!')
-        // return the result
-        cb(null, out)
+        this.push(out)
       } catch (err) {
         // Currently we only support ErrorMode.SUPPRESS
         // Error is logged and output continues
         console.error(err)
+      } finally {
+        delete messageDonePromises[currentIndex]
+        inFlight--
+
+        // Release blocked callback if we now have capacity
+        if (blockedCallback && inFlight < maxConcurrency) {
+          const pendingCb = blockedCallback
+          blockedCallback = null
+          pendingCb()
+        }
 
-        cb() // emit nothing in case of an error
+        // Trigger flush completion if all done
+        if (inFlight === 0 && flushBlocked) {
+          flushBlocked.resolve()
+        }
       }
+    },
+    async flush(cb) {
+      // Wait for all in-flight operations to complete
+      if (inFlight > 0) {
+        flushBlocked = pDefer()
+        await flushBlocked
+      }
+
+      try {
+        // Push null (complete) to all workers
+        for (const worker of workers) {
+          worker.postMessage(null)
+        }
 
-      // clean up
-      delete messageDonePromises[currentIndex]
+        console.log(`transformMultiThreaded.flush is waiting for all workers to be done`)
+        await Promise.all(workerDonePromises)
+        console.log(`transformMultiThreaded.flush all workers done`)
+
+        cb()
+      } catch (err) {
+        cb(err as Error)
+      }
     },
-  )
+  })
 }

package/src/stream/transform/worker/workerClassProxy.js

@@ -10,12 +10,8 @@ if (!workerFile) {
 // console.log(`worker#${workerIndex} created`)
 
 try {
-  // require('esbuild-register') // alternative
-  // require('ts-node/register/transpile-only')
-  // require('tsx/cjs/api').register() // https://tsx.is/dev-api/register-cjs
   const { register } = await import('tsx/esm/api')
   register() // https://tsx.is/dev-api/register-esm
-  // require('tsconfig-paths/register')
 } catch {} // require if exists
 
 const { WorkerClass } = await import(workerFile)

package/src/zip/zip.util.ts

@@ -1,6 +1,7 @@
 import { promisify } from 'node:util'
 import type { ZlibOptions, ZstdOptions } from 'node:zlib'
 import zlib from 'node:zlib'
+import type { Integer } from '@naturalcycles/js-lib/types'
 
 const deflate = promisify(zlib.deflate.bind(zlib))
 const inflate = promisify(zlib.inflate.bind(zlib))
@@ -89,9 +90,22 @@ export async function gunzipToString(buf: Buffer, options?: ZlibOptions): Promis
 
 export async function zstdCompress(
   input: Buffer | string,
+  level?: Integer, // defaults to 3
   options: ZstdOptions = {},
 ): Promise<Buffer<ArrayBuffer>> {
-  return await zstdCompressAsync(input, options)
+  return await zstdCompressAsync(input, zstdLevelToOptions(level, options))
+}
+
+export function zstdLevelToOptions(level: Integer | undefined, opt: ZstdOptions = {}): ZstdOptions {
+  if (!level) return opt
+
+  return {
+    ...opt,
+    params: {
+      ...opt.params,
+      [zlib.constants.ZSTD_c_compressionLevel]: level,
+    },
+  }
 }
 
 export async function zstdDecompressToString(

package/dist/stream/transform/transformMap2.d.ts

@@ -1,66 +0,0 @@
-import type { AbortableSignal } from '@naturalcycles/js-lib';
-import { ErrorMode } from '@naturalcycles/js-lib/error';
-import { type AbortableAsyncMapper, type AsyncPredicate, END, type NumberOfSeconds, type PositiveInteger, type Predicate, type Promisable, SKIP } from '@naturalcycles/js-lib/types';
-import type { TransformOptions, TransformTyped } from '../stream.model.js';
-import type { TransformMapStats } from './transformMap.js';
-export interface TransformMap2Options<IN = any, OUT = IN> extends TransformOptions {
-    /**
-     * Predicate to filter outgoing results (after mapper).
-     * Allows to not emit all results.
-     *
-     * Defaults to "pass everything" (including null, undefined, etc).
-     * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
-     */
-    predicate?: Predicate<OUT>;
-    asyncPredicate?: AsyncPredicate<OUT>;
-    /**
-     * Number of concurrently pending promises returned by `mapper`.
-     *
-     * @default 16
-     */
-    concurrency?: PositiveInteger;
-    /**
-     * Time in seconds to gradually increase concurrency from 1 to `concurrency`.
-     * Useful for warming up connections to databases, APIs, etc.
-     *
-     * Set to 0 to disable warmup (default).
-     */
-    warmupSeconds?: NumberOfSeconds;
-    /**
-     * @default THROW_IMMEDIATELY
-     */
-    errorMode?: ErrorMode;
-    /**
-     * If defined - will be called on every error happening in the stream.
-     * Called BEFORE observable will emit error (unless skipErrors is set to true).
-     */
-    onError?: (err: Error, input: IN) => any;
-    /**
-     * A hook that is called when the last item is finished processing.
-     * stats object is passed, containing countIn and countOut -
-     * number of items that entered the transform and number of items that left it.
-     *
-     * Callback is called **before** [possible] Aggregated error is thrown,
-     * and before [possible] THROW_IMMEDIATELY error.
-     *
-     * onDone callback will be awaited before Error is thrown.
-     */
-    onDone?: (stats: TransformMapStats) => Promisable<any>;
-    /**
-     * Progress metric
-     *
-     * @default `stream`
-     */
-    metric?: string;
-    /**
-     * Allows to abort (gracefully stop) the stream from inside the Transform.
-     */
-    signal?: AbortableSignal;
-}
-/**
- * Like transformMap, but with native concurrency control (no through2-concurrent dependency)
- * and support for gradual warmup.
- *
- * @experimental
- */
-export declare function transformMap2<IN = any, OUT = IN>(mapper: AbortableAsyncMapper<IN, OUT | typeof SKIP | typeof END>, opt?: TransformMap2Options<IN, OUT>): TransformTyped<IN, OUT>;