@naturalcycles/nodejs-lib 15.25.0 → 15.27.0

package/dist/stream/transform/transformMapSimple.js

@@ -11,9 +11,10 @@ import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
  */
 export function transformMapSimple(mapper, opt = {}) {
     let index = -1;
-    const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = console } = opt;
+    const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = console, objectMode = true, highWaterMark, } = opt;
     return new Transform({
-        objectMode: true,
+        objectMode,
+        highWaterMark,
         transform(chunk, _, cb) {
             try {
                 cb(null, mapper(chunk, ++index));

package/dist/stream/transform/transformMapSync.d.ts

@@ -1,11 +1,10 @@
 import type { AbortableSignal } from '@naturalcycles/js-lib';
 import { ErrorMode } from '@naturalcycles/js-lib/error';
-import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import type { IndexedMapper, Predicate } from '@naturalcycles/js-lib/types';
 import { END, SKIP } from '@naturalcycles/js-lib/types';
-import type { TransformTyped } from '../stream.model.js';
+import type { TransformOptions, TransformTyped } from '../stream.model.js';
 import type { TransformMapStats } from './transformMap.js';
-export interface TransformMapSyncOptions<IN = any, OUT = IN> {
+export interface TransformMapSyncOptions<IN = any, OUT = IN> extends TransformOptions {
     /**
      * @default true
      */
@@ -44,7 +43,6 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
      * @default `stream`
      */
     metric?: string;
-    logger?: CommonLogger;
     /**
      * Allows to abort (gracefully stop) the stream from inside the Transform.
      */

package/dist/stream/transform/transformMapSync.js

@@ -1,5 +1,6 @@
 import { Transform } from 'node:stream';
 import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
 import { END, SKIP } from '@naturalcycles/js-lib/types';
 import { yellow } from '../../colors/colors.js';
 import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
@@ -9,13 +10,14 @@ import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
  */
 export function transformMapSync(mapper, opt = {}) {
     const { predicate, // defaults to "no predicate" (pass everything)
-    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', objectMode = true, logger = console, signal, } = opt;
+    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', objectMode = true, signal, } = opt;
     const started = Date.now();
     let index = -1;
     let countOut = 0;
     let isSettled = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     return new Transform({
         objectMode,
         ...opt,

package/dist/stream/transform/transformSplit.js

@@ -14,7 +14,7 @@ export function transformSplitOnNewline() {
         writableObjectMode: false,
         writableHighWaterMark: 64 * 1024,
         readableObjectMode: true,
-        transform(buf, _enc, done) {
+        transform(buf, _enc, cb) {
             let offset = 0;
             let lastMatch = 0;
             if (buffered) {
@@ -36,7 +36,7 @@ export function transformSplitOnNewline() {
                     break;
                 }
             }
-            done();
+            cb();
         },
         flush(done) {
             if (buffered && buffered.length > 0)

package/dist/stream/transform/transformThrottle.d.ts

@@ -1,6 +1,6 @@
 import type { NumberOfSeconds, PositiveInteger } from '@naturalcycles/js-lib/types';
-import type { TransformTyped } from '../stream.model.js';
-export interface TransformThrottleOptions {
+import type { TransformOptions, TransformTyped } from '../stream.model.js';
+export interface TransformThrottleOptions extends TransformOptions {
     /**
      * How many items to allow per `interval` of seconds.
      */
@@ -9,7 +9,6 @@ export interface TransformThrottleOptions {
      * How long is the interval (in seconds) where number of items should not exceed `throughput`.
      */
     interval: NumberOfSeconds;
-    debug?: boolean;
 }
 /**
  * Allows to throttle the throughput of the stream.

package/dist/stream/transform/transformThrottle.js

@@ -1,5 +1,6 @@
 import { Transform } from 'node:stream';
 import { _ms, _since, localTime } from '@naturalcycles/js-lib/datetime';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
 import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
 /**
  * Allows to throttle the throughput of the stream.
@@ -19,36 +20,34 @@ import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
  * @experimental
  */
 export function transformThrottle(opt) {
-    const { throughput, interval, debug } = opt;
+    const { throughput, interval, objectMode = true, highWaterMark } = opt;
     let count = 0;
     let start;
-    let paused;
+    let lock;
     let timeout;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     return new Transform({
-        objectMode: true,
+        objectMode,
+        highWaterMark,
         async transform(item, _, cb) {
             // console.log('incoming', item, { paused: !!paused, count })
             if (!start) {
                 start = Date.now();
                 timeout = setTimeout(() => onInterval(this), interval * 1000);
-                if (debug) {
-                    console.log(`${localTime.now().toPretty()} transformThrottle started with`, {
-                        throughput,
-                        interval,
-                        rps: Math.round(throughput / interval),
-                    });
-                }
+                logger.log(`${localTime.now().toPretty()} transformThrottle started with`, {
+                    throughput,
+                    interval,
+                    rps: Math.round(throughput / interval),
+                });
             }
-            if (paused) {
-                // console.log('awaiting pause', {item, count})
-                await paused;
+            if (lock) {
+                // console.log('awaiting lock', {item, count})
+                await lock;
             }
             if (++count >= throughput) {
                 // console.log('pausing now after', {item, count})
-                paused = pDefer();
-                if (debug) {
-                    console.log(`${localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${_since(start)}, will pause for ${_ms(interval * 1000 - (Date.now() - start))}`);
-                }
+                lock = pDefer();
+                logger.log(`${localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${_since(start)}, will pause for ${_ms(interval * 1000 - (Date.now() - start))}`);
             }
             cb(null, item); // pass the item through
         },
@@ -58,20 +57,16 @@ export function transformThrottle(opt) {
         },
     });
     function onInterval(transform) {
-        if (paused) {
-            if (debug) {
-                console.log(`${localTime.now().toPretty()} transformThrottle resumed`);
-            }
-            paused.resolve();
-            paused = undefined;
+        if (lock) {
+            logger.log(`${localTime.now().toPretty()} transformThrottle resumed`);
+            lock.resolve();
+            lock = undefined;
         }
         else {
-            if (debug) {
-                console.log(`${localTime.now().toPretty()} transformThrottle passed ${count} (of max ${throughput}) items in ${_since(start)}`);
-            }
+            logger.log(`${localTime.now().toPretty()} transformThrottle passed ${count} (of max ${throughput}) items in ${_since(start)}`);
         }
         count = 0;
-        start = Date.now();
+        start = localTime.nowUnixMillis();
         timeout = setTimeout(() => onInterval(transform), interval * 1000);
     }
 }

package/dist/stream/writable/writableVoid.d.ts

@@ -1,15 +1,8 @@
 import { Writable } from 'node:stream';
-import type { DeferredPromise } from '@naturalcycles/js-lib/promise';
 import type { TransformOptions } from '../stream.model.js';
-export interface WritableVoidOptions extends TransformOptions {
-    /**
-     * If set - it will be Resolved when the Stream is done (after final.cb)
-     */
-    streamDone?: DeferredPromise;
-}
 /**
  * Use as a "null-terminator" of stream.pipeline.
  * It consumes the stream as quickly as possible without doing anything.
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
-export declare function writableVoid(opt?: WritableVoidOptions): Writable;
+export declare function writableVoid(opt?: TransformOptions): Writable;

package/dist/stream/writable/writableVoid.js

@@ -13,7 +13,6 @@ export function writableVoid(opt = {}) {
         },
         final(cb) {
             cb();
-            opt.streamDone?.resolve();
         },
     });
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.25.0",
+  "version": "15.27.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/slack/slack.service.ts

@@ -2,8 +2,8 @@ import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { type Fetcher, getFetcher } from '@naturalcycles/js-lib/http'
 import {
   type CommonLogger,
-  commonLoggerMinLevel,
   type CommonLogLevel,
+  createCommonLoggerAtLevel,
 } from '@naturalcycles/js-lib/log'
 import { _omit } from '@naturalcycles/js-lib/object/object.util.js'
 import { PQueue } from '@naturalcycles/js-lib/promise/pQueue.js'
@@ -152,19 +152,22 @@ export class SlackService<CTX = any> {
    */
   getCommonLogger(opt: {
     minLogLevel: CommonLogLevel
+    debugChannel?: string
     logChannel?: string
     warnChannel?: string
     errorChannel?: string
   }): CommonLogger {
-    const { minLogLevel = 'log', logChannel, warnChannel, errorChannel } = opt
+    const { minLogLevel = 'log', debugChannel, logChannel, warnChannel, errorChannel } = opt
     const defaultChannel = this.cfg.defaults?.channel || DEFAULTS.channel!
 
     const q = new PQueue({
       concurrency: 1,
     })
 
-    return commonLoggerMinLevel(
+    return createCommonLoggerAtLevel(
       {
+        debug: (...args) =>
+          q.push(() => this.send({ items: args, channel: debugChannel || defaultChannel })),
         log: (...args) =>
           q.push(() => this.send({ items: args, channel: logChannel || defaultChannel })),
         warn: (...args) =>

package/src/stream/index.ts

@@ -1,8 +1,6 @@
-export * from './ndjson/createReadStreamAsNDJSON.js'
-export * from './ndjson/createWriteStreamAsNDJSON.js'
+export * from './ndjson/createReadStreamAsNDJson.js'
 export * from './ndjson/ndjson.model.js'
 export * from './ndjson/ndjsonMap.js'
-export * from './ndjson/ndjsonStreamForEach.js'
 export * from './ndjson/transformJsonParse.js'
 export * from './ndjson/transformToNDJson.js'
 export * from './pipeline.js'
@@ -10,11 +8,11 @@ export * from './progressLogger.js'
 export * from './readable/readableCombined.js'
 export * from './readable/readableCreate.js'
 export * from './readable/readableFromArray.js'
-export * from './readable/readableToArray.js'
 export * from './stream.model.js'
 export * from './transform/transformChunk.js'
 export * from './transform/transformFilter.js'
 export * from './transform/transformFlatten.js'
+export * from './transform/transformFork.js'
 export * from './transform/transformLimit.js'
 export * from './transform/transformLogProgress.js'
 export * from './transform/transformMap.js'
@@ -24,13 +22,9 @@ export * from './transform/transformNoOp.js'
 export * from './transform/transformOffset.js'
 export * from './transform/transformSplit.js'
 export * from './transform/transformTap.js'
-export * from './transform/transformTee.js'
 export * from './transform/transformThrottle.js'
-export * from './transform/transformToArray.js'
 export * from './transform/worker/baseWorkerClass.js'
 export * from './transform/worker/transformMultiThreaded.js'
 export * from './transform/worker/transformMultiThreaded.model.js'
-export * from './writable/writableForEach.js'
-export * from './writable/writableFork.js'
 export * from './writable/writablePushToArray.js'
 export * from './writable/writableVoid.js'

package/src/stream/ndjson/{createReadStreamAsNDJSON.ts → createReadStreamAsNDJson.ts}

@@ -4,24 +4,21 @@ import type { ReadableTyped } from '../stream.model.js'
 import { transformSplitOnNewline } from '../transform/transformSplit.js'
 
 /**
-  Returns a Readable of [already parsed] NDJSON objects.
+ Returns a Readable of [already parsed] NDJSON objects.
  
-  Replaces a list of operations:
-  - requireFileToExist(inputPath)
-  - fs.createReadStream
-  - createUnzip (only if path ends with '.gz')
-  - transformSplitOnNewline
-  - transformJsonParse
+ Replaces a list of operations:
+ - requireFileToExist(inputPath)
+ - fs.createReadStream
+ - createUnzip (only if path ends with '.gz')
+ - transformSplitOnNewline
+ - transformJsonParse
  
-  To add a Limit or Offset: just add .take() or .drop(), example:
+ To add a Limit or Offset: just add .take() or .drop(), example:
  
-  _pipeline([
-    fs2.createReadStreamAsNDJSON().take(100),
-    transformX(),
-  ])
+ createReadStreamAsNDJson().take(100)
  */
 
-export function createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW> {
+export function createReadStreamAsNDJson<ROW = any>(inputPath: string): ReadableTyped<ROW> {
   fs2.requireFileToExist(inputPath)
 
   let stream: ReadableTyped<ROW> = fs2

package/src/stream/ndjson/ndjsonMap.ts

@@ -21,6 +21,8 @@ export interface NDJSONMapOptions<IN = any, OUT = IN>
 /**
  * Unzips input file automatically, if it ends with `.gz`.
  * Zips output file automatically, if it ends with `.gz`.
+ *
+ * @deprecated use Pipeline directly
  */
 export async function ndjsonMap<IN = any, OUT = any>(
   mapper: AbortableAsyncMapper<IN, OUT>,

package/src/stream/pipeline.ts

@@ -17,7 +17,7 @@ import type {
   SKIP,
 } from '@naturalcycles/js-lib/types'
 import { fs2 } from '../fs/fs2.js'
-import { createReadStreamAsNDJSON } from './ndjson/createReadStreamAsNDJSON.js'
+import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js'
 import { transformJsonParse } from './ndjson/transformJsonParse.js'
 import { transformToNDJson } from './ndjson/transformToNDJson.js'
 import type {
@@ -30,6 +30,7 @@ import { PIPELINE_GRACEFUL_ABORT } from './stream.util.js'
 import { transformChunk } from './transform/transformChunk.js'
 import { transformFilterSync } from './transform/transformFilter.js'
 import { transformFlatten, transformFlattenIfNeeded } from './transform/transformFlatten.js'
+import { transformFork } from './transform/transformFork.js'
 import { transformLimit } from './transform/transformLimit.js'
 import {
   transformLogProgress,
@@ -83,12 +84,22 @@ export class Pipeline<T> {
     return new Pipeline(Readable.from(input))
   }
 
-  static fromFile(sourceFilePath: string): Pipeline<Uint8Array> {
-    return new Pipeline(fs2.createReadStream(sourceFilePath), false)
+  static fromNDJsonFile<T>(sourceFilePath: string): Pipeline<T> {
+    // Important that createReadStreamAsNDJson function is used
+    // (and not Pipeline set of individual transforms),
+    // because createReadStreamAsNDJson returns a Readable,
+    // hence it allows to apply .take(limit) on it
+    // e.g like Pipeline.fromNDJsonFile().limitSource(limit)
+    return new Pipeline<T>(createReadStreamAsNDJson(sourceFilePath))
   }
 
-  static fromNDJsonFile<T>(sourceFilePath: string): Pipeline<T> {
-    return new Pipeline(createReadStreamAsNDJSON<T>(sourceFilePath))
+  static fromFile(sourceFilePath: string): Pipeline<Uint8Array> {
+    return new Pipeline(
+      fs2.createReadStream(sourceFilePath, {
+        highWaterMark: 64 * 1024, // no observed speedup
+      }),
+      false,
+    )
   }
 
   /**
@@ -210,8 +221,6 @@ export class Pipeline<T> {
     return this
   }
 
-  // todo: tee/fork
-
   transform<TO>(transform: TransformTyped<T, TO>): Pipeline<TO> {
     this.transforms.push(transform)
     return this as any
@@ -226,6 +235,11 @@ export class Pipeline<T> {
     return this as any
   }
 
+  fork(fn: (pipeline: Pipeline<T>) => Pipeline<T>, opt?: TransformOptions): this {
+    this.transforms.push(transformFork(fn, opt))
+    return this
+  }
+
   /**
    * Utility method just to conveniently type-cast the current Pipeline type.
    * No runtime effect.
@@ -276,13 +290,23 @@ export class Pipeline<T> {
   }
 
   gzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array> {
-    this.transforms.push(createGzip(opt))
+    this.transforms.push(
+      createGzip({
+        // chunkSize: 64 * 1024, // no observed speedup
+        ...opt,
+      }),
+    )
     this.objectMode = false
     return this as any
   }
 
   gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array> {
-    this.transforms.push(createUnzip(opt))
+    this.transforms.push(
+      createUnzip({
+        chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+        ...opt,
+      }),
+    )
     this.objectMode = false
     return this as any
   }

package/src/stream/progressLogger.ts

@@ -4,7 +4,7 @@ import { _hc, _mb } from '@naturalcycles/js-lib'
 import { _since, localTime } from '@naturalcycles/js-lib/datetime'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { SimpleMovingAverage } from '@naturalcycles/js-lib/math'
-import type { AnyObject, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
+import type { AnyObject, PositiveInteger, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
 import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js'
 import { SizeStack } from './sizeStack.js'
 import type { ReadableMapper } from './stream.model.js'
@@ -87,7 +87,7 @@ export interface ProgressLoggerCfg<T = any> {
    *
    * @default 1000
    */
-  logEvery?: number
+  logEvery?: PositiveInteger
 
   logger?: CommonLogger
 
@@ -111,7 +111,7 @@ export interface ProgressLoggerCfg<T = any> {
    *
    * Defaults to 1.
    */
-  chunkSize?: number
+  chunkSize?: PositiveInteger
 
   /**
    * Experimental logging of item (shunk) sizes, when json-stringified.

package/src/stream/readable/readableCombined.ts

@@ -1,7 +1,9 @@
 import { Readable } from 'node:stream'
+import { type CommonLogger, createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
 import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import type { TransformOptions } from '../stream.model.js'
 
 /**
  * Allows to combine multiple Readables into 1 Readable.
@@ -14,15 +16,22 @@ import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
  * @experimental
  */
 export class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
-  static create<T>(inputs: Readable[]): ReadableCombined<T> {
-    return new ReadableCombined<T>(inputs)
+  static create<T>(inputs: Readable[], opt: TransformOptions = {}): ReadableCombined<T> {
+    return new ReadableCombined<T>(inputs, opt)
   }
 
-  private constructor(public inputs: Readable[]) {
-    super({ objectMode: true })
-    void this.start()
+  private constructor(
+    public inputs: Readable[],
+    opt: TransformOptions,
+  ) {
+    const { objectMode = true, highWaterMark } = opt
+    super({ objectMode, highWaterMark })
+    this.logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+    void this.run()
   }
 
+  private logger: CommonLogger
+
   /**
    * If defined - we are in Paused mode
    * and should await the lock to be resolved before proceeding.
@@ -38,7 +47,9 @@ export class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
   // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
   private countReads = 0
 
-  private async start(): Promise<void> {
+  private async run(): Promise<void> {
+    const { logger } = this
+
     await pMap(this.inputs, async (input, i) => {
       for await (const item of input) {
         this.countIn++
@@ -52,14 +63,14 @@ export class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
         this.countOut++
         if (!shouldContinue && !this.lock) {
           this.lock = pDefer()
-          console.log(`ReadableCombined.push #${i} returned false, pausing the flow!`)
+          logger.log(`ReadableCombined.push #${i} returned false, pausing the flow!`)
         }
       }
 
-      console.log(`ReadableCombined: input #${i} done`)
+      logger.log(`ReadableCombined: input #${i} done`)
     })
 
-    console.log(`ReadableCombined: all inputs done!`)
+    logger.log(`ReadableCombined: all inputs done!`)
     this.push(null)
   }
 
@@ -67,7 +78,7 @@ export class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
     this.countReads++
 
     if (this.lock) {
-      console.log(`ReadableCombined._read: resuming the flow!`)
+      this.logger.log(`ReadableCombined._read: resuming the flow!`)
       // calling it in this order is important!
       // this.lock should be undefined BEFORE we call lock.resolve()
       const { lock } = this
@@ -78,7 +89,7 @@ export class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
 
   private logStats(): void {
     const { countIn, countOut, countReads } = this
-    console.log({
+    this.logger.debug({
       countIn,
       countOut,
       countReads,

package/src/stream/readable/readableCreate.ts

@@ -13,17 +13,18 @@ import type { ReadableTyped } from '../stream.model.js'
  * e.g the read() method doesn't return anything, so, it will hang the Node process (or cause it to process.exit(0))
  * if read() will be called AFTER everything was pushed and Readable is closed (by pushing `null`).
  * Beware of it when e.g doing unit testing! Jest prefers to hang (not exit-0).
- *
- * @deprecated because of the caution above
  */
 export function readableCreate<T>(
   items: Iterable<T> = [],
   opt?: ReadableOptions,
+  onRead?: () => void, // read callback
 ): ReadableTyped<T> {
   const readable = new Readable({
     objectMode: true,
     ...opt,
-    read() {}, // Caution, if this is called and Readable has not finished yet (null wasn't pushed) - it'll hang the process!
+    read() {
+      onRead?.()
+    },
   })
   for (const item of items) {
     readable.push(item)

package/src/stream/stream.model.ts

@@ -1,4 +1,5 @@
 import type { Readable, Transform, Writable } from 'node:stream'
+import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log'
 import type { Promisable } from '@naturalcycles/js-lib/types'
 
 export interface ReadableSignalOptions {
@@ -72,4 +73,21 @@ export interface TransformOptions {
    * @default 16
    */
   highWaterMark?: number
+
+  /**
+   * Defaults to `console`.
+   */
+  logger?: CommonLogger
+
+  /**
+   * Not every Transform implements it.
+   * Can be one of:
+   * debug - most verbose, when debugging is needed
+   * log - default level
+   * error - logs errors and warnings only
+   *
+   * Default is 'log'.
+   *
+   */
+  logLevel?: CommonLogLevel
 }

package/src/stream/transform/transformFork.ts

@@ -0,0 +1,74 @@
+import { Transform } from 'node:stream'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
+import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import { Pipeline } from '../pipeline.js'
+import { readableCreate } from '../readable/readableCreate.js'
+import type { TransformOptions, TransformTyped } from '../stream.model.js'
+
+/**
+ * Allows to "fork" away from the "main pipeline" into the "forked pipeline".
+ *
+ * Correctly keeps backpressure from both "downstreams" (main and forked).
+ *
+ * @experimental
+ */
+export function transformFork<T>(
+  fn: (pipeline: Pipeline<T>) => Pipeline<T>,
+  opt: TransformOptions = {},
+): TransformTyped<T, T> {
+  const { objectMode = true, highWaterMark } = opt
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+
+  let lock: DeferredPromise | undefined
+
+  const fork = readableCreate<T>([], {}, () => {
+    // `_read` is called
+    if (!lock) return
+    // We had a lock - let's Resume
+    logger.log(`TransformFork: resume`)
+    const lockCopy = lock
+    lock = undefined
+    lockCopy.resolve()
+  })
+
+  const p = fn(Pipeline.from<T>(fork))
+  void p.run().then(() => {
+    logger.log('TransformFork: done')
+  })
+
+  return new Transform({
+    objectMode,
+    highWaterMark,
+    async transform(chunk: T, _, cb) {
+      // pass through to the "main" pipeline
+      // Main pipeline should handle backpressure "automatically",
+      // so, we're not maintaining a Lock for it
+      this.push(chunk)
+
+      if (lock) {
+        // Forked pipeline is locked - let's wait for it to call _read
+        await lock
+        // lock is undefined at this point
+      }
+      // pass to the "forked" pipeline
+      const shouldContinue = fork.push(chunk)
+      if (!shouldContinue && !lock) {
+        // Forked pipeline indicates that we should Pause
+        lock = pDefer()
+        logger.log(`TransformFork: pause`)
+      }
+
+      // acknowledge that we've finished processing the input chunk
+      cb()
+    },
+    async final(cb) {
+      logger.log('TransformFork: final')
+
+      // Pushing null "closes"/ends the secondary pipeline correctly
+      fork.push(null)
+
+      // Acknowledge that we've received `null` and passed it through to the fork
+      cb()
+    },
+  })
+}