@naturalcycles/nodejs-lib 15.26.0 → 15.27.0

package/src/stream/transform/transformMapSimple.ts

@@ -1,18 +1,15 @@
 import { Transform } from 'node:stream'
 import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import type { IndexedMapper } from '@naturalcycles/js-lib/types'
-import type { TransformTyped } from '../stream.model.js'
+import type { TransformOptions, TransformTyped } from '../stream.model.js'
 
-export interface TransformMapSimpleOptions {
+export interface TransformMapSimpleOptions extends TransformOptions {
   /**
    * Only supports THROW_IMMEDIATELY (default) and SUPPRESS.
    *
    * @default ErrorMode.THROW_IMMEDIATELY
    */
   errorMode?: ErrorMode.THROW_IMMEDIATELY | ErrorMode.SUPPRESS
-
-  logger?: CommonLogger
 }
 
 /**
@@ -29,10 +26,16 @@ export function transformMapSimple<IN = any, OUT = IN>(
   opt: TransformMapSimpleOptions = {},
 ): TransformTyped<IN, OUT> {
   let index = -1
-  const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = console } = opt
+  const {
+    errorMode = ErrorMode.THROW_IMMEDIATELY,
+    logger = console,
+    objectMode = true,
+    highWaterMark,
+  } = opt
 
   return new Transform({
-    objectMode: true,
+    objectMode,
+    highWaterMark,
     transform(chunk: IN, _, cb) {
       try {
         cb(null, mapper(chunk, ++index))

package/src/stream/transform/transformMapSync.ts

@@ -1,15 +1,15 @@
 import { Transform } from 'node:stream'
 import type { AbortableSignal } from '@naturalcycles/js-lib'
 import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
 import type { IndexedMapper, Predicate, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
 import { END, SKIP } from '@naturalcycles/js-lib/types'
 import { yellow } from '../../colors/colors.js'
-import type { TransformTyped } from '../stream.model.js'
+import type { TransformOptions, TransformTyped } from '../stream.model.js'
 import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
 import type { TransformMapStats } from './transformMap.js'
 
-export interface TransformMapSyncOptions<IN = any, OUT = IN> {
+export interface TransformMapSyncOptions<IN = any, OUT = IN> extends TransformOptions {
   /**
    * @default true
    */
@@ -54,8 +54,6 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
    */
   metric?: string
 
-  logger?: CommonLogger
-
   /**
    * Allows to abort (gracefully stop) the stream from inside the Transform.
    */
@@ -77,7 +75,6 @@ export function transformMapSync<IN = any, OUT = IN>(
     onDone,
     metric = 'stream',
     objectMode = true,
-    logger = console,
     signal,
   } = opt
 
@@ -87,6 +84,7 @@ export function transformMapSync<IN = any, OUT = IN>(
   let isSettled = false
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
 
   return new Transform({
     objectMode,

package/src/stream/transform/transformSplit.ts

@@ -19,7 +19,7 @@ export function transformSplitOnNewline(): TransformTyped<Buffer, Buffer> {
     writableHighWaterMark: 64 * 1024,
     readableObjectMode: true,
 
-    transform(buf: Buffer, _enc, done) {
+    transform(buf: Buffer, _enc, cb) {
       let offset = 0
       let lastMatch = 0
       if (buffered) {
@@ -42,7 +42,7 @@ export function transformSplitOnNewline(): TransformTyped<Buffer, Buffer> {
         }
       }
 
-      done()
+      cb()
     },
 
     flush(done) {

package/src/stream/transform/transformThrottle.ts

@@ -1,5 +1,6 @@
 import { Transform } from 'node:stream'
 import { _ms, _since, localTime } from '@naturalcycles/js-lib/datetime'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
 import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
 import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
 import type {
@@ -7,9 +8,9 @@ import type {
   PositiveInteger,
   UnixTimestampMillis,
 } from '@naturalcycles/js-lib/types'
-import type { TransformTyped } from '../stream.model.js'
+import type { TransformOptions, TransformTyped } from '../stream.model.js'
 
-export interface TransformThrottleOptions {
+export interface TransformThrottleOptions extends TransformOptions {
   /**
    * How many items to allow per `interval` of seconds.
    */
@@ -19,8 +20,6 @@ export interface TransformThrottleOptions {
    * How long is the interval (in seconds) where number of items should not exceed `throughput`.
    */
   interval: NumberOfSeconds
-
-  debug?: boolean
 }
 
 /**
@@ -41,42 +40,40 @@ export interface TransformThrottleOptions {
  * @experimental
  */
 export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTyped<T, T> {
-  const { throughput, interval, debug } = opt
+  const { throughput, interval, objectMode = true, highWaterMark } = opt
 
   let count = 0
   let start: UnixTimestampMillis
-  let paused: DeferredPromise | undefined
+  let lock: DeferredPromise | undefined
   let timeout: NodeJS.Timeout | undefined
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
 
   return new Transform({
-    objectMode: true,
+    objectMode,
+    highWaterMark,
     async transform(item: T, _, cb) {
       // console.log('incoming', item, { paused: !!paused, count })
       if (!start) {
         start = Date.now() as UnixTimestampMillis
         timeout = setTimeout(() => onInterval(this), interval * 1000)
-        if (debug) {
-          console.log(`${localTime.now().toPretty()} transformThrottle started with`, {
-            throughput,
-            interval,
-            rps: Math.round(throughput / interval),
-          })
-        }
+        logger.log(`${localTime.now().toPretty()} transformThrottle started with`, {
+          throughput,
+          interval,
+          rps: Math.round(throughput / interval),
+        })
       }
 
-      if (paused) {
-        // console.log('awaiting pause', {item, count})
-        await paused
+      if (lock) {
+        // console.log('awaiting lock', {item, count})
+        await lock
       }
 
       if (++count >= throughput) {
         // console.log('pausing now after', {item, count})
-        paused = pDefer()
-        if (debug) {
-          console.log(
-            `${localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${_since(start)}, will pause for ${_ms(interval * 1000 - (Date.now() - start))}`,
-          )
-        }
+        lock = pDefer()
+        logger.log(
+          `${localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${_since(start)}, will pause for ${_ms(interval * 1000 - (Date.now() - start))}`,
+        )
       }
 
       cb(null, item) // pass the item through
@@ -88,23 +85,18 @@ export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTy
   })
 
   function onInterval(transform: Transform): void {
-    if (paused) {
-      if (debug) {
-        console.log(`${localTime.now().toPretty()} transformThrottle resumed`)
-      }
-
-      paused.resolve()
-      paused = undefined
+    if (lock) {
+      logger.log(`${localTime.now().toPretty()} transformThrottle resumed`)
+      lock.resolve()
+      lock = undefined
     } else {
-      if (debug) {
-        console.log(
-          `${localTime.now().toPretty()} transformThrottle passed ${count} (of max ${throughput}) items in ${_since(start)}`,
-        )
-      }
+      logger.log(
+        `${localTime.now().toPretty()} transformThrottle passed ${count} (of max ${throughput}) items in ${_since(start)}`,
+      )
     }
 
     count = 0
-    start = Date.now() as UnixTimestampMillis
+    start = localTime.nowUnixMillis()
     timeout = setTimeout(() => onInterval(transform), interval * 1000)
   }
 }

package/dist/stream/transform/transformTee.d.ts

@@ -1,13 +0,0 @@
-import type { TransformTyped } from '../stream.model.js';
-type AnyStream = NodeJS.WritableStream | NodeJS.ReadWriteStream;
-/**
- * Allows to "tee"/"fork" away from the "main pipeline" into the "secondary pipeline".
- *
- * Important, that the main pipeline works "as normal", keeps backpressure, etc.
- * Secondary pipeline DOES NOT keep backpressure.
- * Therefor, the "slowest" pipeline should be made Primary (to keep backpressure).
- *
- * @experimental
- */
-export declare function transformTee<T>(streams: AnyStream[]): TransformTyped<T, T>;
-export {};

package/dist/stream/transform/transformTee.js

@@ -1,37 +0,0 @@
-import { Transform } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
-import { readableCreate } from '../readable/readableCreate.js';
-/**
- * Allows to "tee"/"fork" away from the "main pipeline" into the "secondary pipeline".
- *
- * Important, that the main pipeline works "as normal", keeps backpressure, etc.
- * Secondary pipeline DOES NOT keep backpressure.
- * Therefor, the "slowest" pipeline should be made Primary (to keep backpressure).
- *
- * @experimental
- */
-export function transformTee(streams) {
-    const readable = readableCreate();
-    const secondPipelinePromise = pipeline([readable, ...streams]);
-    return new Transform({
-        objectMode: true,
-        transform(chunk, _, cb) {
-            // todo: it's possible to start respecting backpressure,
-            // if we start to listen to the boolean output of .push()
-            // pass to the "secondary" pipeline
-            readable.push(chunk);
-            // pass through to the "main" pipeline
-            cb(null, chunk);
-        },
-        async final(cb) {
-            console.log('transformTee final');
-            // Pushing null "closes"/ends the secondary pipeline correctly
-            readable.push(null);
-            // Second pipeline is expected to finish now, let's await it
-            await secondPipelinePromise;
-            console.log('transformTee final secondPipeline done');
-            // Because second pipeline is done - now we can signal main pipeline to be done as well
-            cb();
-        },
-    });
-}

package/dist/stream/transform/transformToArray.d.ts

@@ -1,5 +0,0 @@
-import type { TransformOptions, TransformTyped } from '../stream.model.js';
-/**
- * Will collect all stream results in the array (keeping it in memory) and emit in the end as one result.
- */
-export declare function transformToArray<IN>(opt?: TransformOptions): TransformTyped<IN, IN[]>;

package/dist/stream/transform/transformToArray.js

@@ -1,20 +0,0 @@
-import { Transform } from 'node:stream';
-/**
- * Will collect all stream results in the array (keeping it in memory) and emit in the end as one result.
- */
-export function transformToArray(opt = {}) {
-    const res = [];
-    return new Transform({
-        objectMode: true,
-        ...opt,
-        transform(chunk, _, cb) {
-            res.push(chunk);
-            // callback to signal that we processed input, but not emitting any output
-            cb();
-        },
-        final(cb) {
-            this.push(res);
-            cb();
-        },
-    });
-}

package/dist/stream/writable/writableFork.d.ts

@@ -1,10 +0,0 @@
-import type { TransformOptions, WritableTyped } from '../stream.model.js';
-/**
- * Allows "forking" a stream inside pipeline into a number of pipeline chains (2 or more).
- * Currently does NOT (!) maintain backpressure.
- * Error in the forked pipeline will propagate up to the main pipeline (and log error, to be sure).
- * Will wait until all forked pipelines are completed before completing the stream.
- *
- * @experimental
- */
-export declare function writableFork<T>(chains: NodeJS.WritableStream[][], opt?: TransformOptions): WritableTyped<T>;

package/dist/stream/writable/writableFork.js

@@ -1,45 +0,0 @@
-import { Writable } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
-import { readableCreate } from '../readable/readableCreate.js';
-/**
- * Allows "forking" a stream inside pipeline into a number of pipeline chains (2 or more).
- * Currently does NOT (!) maintain backpressure.
- * Error in the forked pipeline will propagate up to the main pipeline (and log error, to be sure).
- * Will wait until all forked pipelines are completed before completing the stream.
- *
- * @experimental
- */
-export function writableFork(chains, opt) {
-    const readables = [];
-    const allChainsDone = Promise.all(chains.map(async (chain) => {
-        const readable = readableCreate();
-        readables.push(readable);
-        return await pipeline([readable, ...chain]);
-    })).catch(err => {
-        console.error(err); // ensure the error is logged
-        throw err;
-    });
-    return new Writable({
-        objectMode: true,
-        ...opt,
-        write(chunk, _, cb) {
-            // Push/fork to all sub-streams
-            // No backpressure is ensured here, it'll push regardless of the
-            readables.forEach(readable => readable.push(chunk));
-            cb();
-        },
-        async final(cb) {
-            try {
-                // Push null (complete) to all sub-streams
-                readables.forEach(readable => readable.push(null));
-                console.log(`writableFork.final is waiting for all chains to be done`);
-                await allChainsDone;
-                console.log(`writableFork.final all chains done`);
-                cb();
-            }
-            catch (err) {
-                cb(err);
-            }
-        },
-    });
-}

package/src/stream/transform/transformTee.ts

@@ -1,48 +0,0 @@
-import { Transform } from 'node:stream'
-import { pipeline } from 'node:stream/promises'
-import { readableCreate } from '../readable/readableCreate.js'
-import type { TransformTyped } from '../stream.model.js'
-
-type AnyStream = NodeJS.WritableStream | NodeJS.ReadWriteStream
-
-/**
- * Allows to "tee"/"fork" away from the "main pipeline" into the "secondary pipeline".
- *
- * Important, that the main pipeline works "as normal", keeps backpressure, etc.
- * Secondary pipeline DOES NOT keep backpressure.
- * Therefor, the "slowest" pipeline should be made Primary (to keep backpressure).
- *
- * @experimental
- */
-export function transformTee<T>(streams: AnyStream[]): TransformTyped<T, T> {
-  const readable = readableCreate<T>()
-
-  const secondPipelinePromise = pipeline([readable, ...streams])
-
-  return new Transform({
-    objectMode: true,
-    transform(chunk: T, _, cb) {
-      // todo: it's possible to start respecting backpressure,
-      // if we start to listen to the boolean output of .push()
-
-      // pass to the "secondary" pipeline
-      readable.push(chunk)
-
-      // pass through to the "main" pipeline
-      cb(null, chunk)
-    },
-    async final(cb) {
-      console.log('transformTee final')
-
-      // Pushing null "closes"/ends the secondary pipeline correctly
-      readable.push(null)
-
-      // Second pipeline is expected to finish now, let's await it
-      await secondPipelinePromise
-      console.log('transformTee final secondPipeline done')
-
-      // Because second pipeline is done - now we can signal main pipeline to be done as well
-      cb()
-    },
-  })
-}

package/src/stream/transform/transformToArray.ts

@@ -1,23 +0,0 @@
-import { Transform } from 'node:stream'
-import type { TransformOptions, TransformTyped } from '../stream.model.js'
-
-/**
- * Will collect all stream results in the array (keeping it in memory) and emit in the end as one result.
- */
-export function transformToArray<IN>(opt: TransformOptions = {}): TransformTyped<IN, IN[]> {
-  const res: IN[] = []
-
-  return new Transform({
-    objectMode: true,
-    ...opt,
-    transform(chunk: IN, _, cb) {
-      res.push(chunk)
-      // callback to signal that we processed input, but not emitting any output
-      cb()
-    },
-    final(this: Transform, cb) {
-      this.push(res)
-      cb()
-    },
-  })
-}

package/src/stream/writable/writableFork.ts

@@ -1,56 +0,0 @@
-import { Writable } from 'node:stream'
-import { pipeline } from 'node:stream/promises'
-import { readableCreate } from '../readable/readableCreate.js'
-import type { ReadableTyped, TransformOptions, WritableTyped } from '../stream.model.js'
-
-/**
- * Allows "forking" a stream inside pipeline into a number of pipeline chains (2 or more).
- * Currently does NOT (!) maintain backpressure.
- * Error in the forked pipeline will propagate up to the main pipeline (and log error, to be sure).
- * Will wait until all forked pipelines are completed before completing the stream.
- *
- * @experimental
- */
-export function writableFork<T>(
-  chains: NodeJS.WritableStream[][],
-  opt?: TransformOptions,
-): WritableTyped<T> {
-  const readables: ReadableTyped<T>[] = []
-
-  const allChainsDone = Promise.all(
-    chains.map(async chain => {
-      const readable = readableCreate<T>()
-      readables.push(readable)
-
-      return await pipeline([readable, ...chain])
-    }),
-  ).catch(err => {
-    console.error(err) // ensure the error is logged
-    throw err
-  })
-
-  return new Writable({
-    objectMode: true,
-    ...opt,
-    write(chunk: T, _, cb) {
-      // Push/fork to all sub-streams
-      // No backpressure is ensured here, it'll push regardless of the
-      readables.forEach(readable => readable.push(chunk))
-
-      cb()
-    },
-    async final(cb) {
-      try {
-        // Push null (complete) to all sub-streams
-        readables.forEach(readable => readable.push(null))
-
-        console.log(`writableFork.final is waiting for all chains to be done`)
-        await allChainsDone
-        console.log(`writableFork.final all chains done`)
-        cb()
-      } catch (err) {
-        cb(err as Error)
-      }
-    },
-  })
-}