@naturalcycles/nodejs-lib 15.37.1 → 15.38.0

package/dist/security/nanoid.js

@@ -4,7 +4,7 @@ This file is "vendored" from Nanoid, all credit is to Nanoid authors:
 https://github.com/ai/nanoid/
 
  */
-/* eslint-disable */
+// oxlint-disable no-bitwise -- NanoID implementation relies on bitwise operations
 import { randomFillSync } from 'node:crypto';
 export const ALPHABET_NONAMBIGUOUS = '23456789ABCDEFGHJKLMNPQRSTUVWXYZ';
 export const ALPHABET_NUMBER = '0123456789';

package/dist/stream/index.d.ts

@@ -18,7 +18,6 @@ export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
-export * from './transform/transformMultiFork.js';
 export * from './transform/transformNoOp.js';
 export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';

package/dist/stream/index.js

@@ -18,7 +18,6 @@ export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
-export * from './transform/transformMultiFork.js';
 export * from './transform/transformNoOp.js';
 export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';

package/dist/stream/writable/writableChunk.d.ts

@@ -1,15 +1,13 @@
-import type { Predicate } from '@naturalcycles/js-lib/types';
-import { type TransformOptions, type TransformTyped, type WritableTyped } from '../index.js';
-export interface WritableChunkOptions<T> extends TransformOptions {
-    splitPredicate: Predicate<T>;
-    transformFactories?: (() => TransformTyped<T, T>)[];
-    writableFactory: (splitIndex: number) => WritableTyped<T>;
-}
+import type { NonNegativeInteger, Predicate } from '@naturalcycles/js-lib/types';
+import { Pipeline } from '../pipeline.js';
+import type { TransformOptions, WritableTyped } from '../stream.model.js';
 /**
- * Allows to split the output to multiple files by splitting into chunks
- * based on `shouldSplitFn`.
- * `transformFactories` are used to create a chain of transforms for each chunk.
- * It was meant to be used with createGzip, which needs a proper start and end for each chunk
- * for the output file to be a valid gzip file.
+ * Allows to "split the stream" into chunks, and attach a new Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
  */
-export declare function writableChunk<T>(opt: WritableChunkOptions<T>): WritableTyped<T>;
+export declare function writableChunk<T>(splitPredicate: Predicate<T>, fn: (pipeline: Pipeline<T>, splitIndex: NonNegativeInteger) => Promise<void>, opt?: TransformOptions): WritableTyped<T>;

package/dist/stream/writable/writableChunk.js

@@ -1,83 +1,74 @@
 import { Writable } from 'node:stream';
-import { _first, _last } from '@naturalcycles/js-lib/array';
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
-import { _deepCopy } from '@naturalcycles/js-lib/object';
-import { transformNoOp, } from '../index.js';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+import { Pipeline } from '../pipeline.js';
+import { createReadable } from '../readable/createReadable.js';
 /**
- * Allows to split the output to multiple files by splitting into chunks
- * based on `shouldSplitFn`.
- * `transformFactories` are used to create a chain of transforms for each chunk.
- * It was meant to be used with createGzip, which needs a proper start and end for each chunk
- * for the output file to be a valid gzip file.
+ * Allows to "split the stream" into chunks, and attach a new Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
  */
-export function writableChunk(opt) {
-    const { highWaterMark, splitPredicate, transformFactories = [], writableFactory } = opt;
+export function writableChunk(splitPredicate, fn, opt = {}) {
+    const { objectMode = true, highWaterMark } = opt;
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     let indexWritten = 0;
-    let currentSplitIndex = 0;
-    // We don't want to have an empty chain, so we add a no-op transform
-    if (transformFactories.length === 0) {
-        transformFactories.push((transformNoOp));
-    }
-    // Create the transforms as well as the Writable, and pipe them together
-    let currentWritable = writableFactory(currentSplitIndex);
-    let transforms = transformFactories.map(f => f());
-    generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2));
-    _last(transforms).pipe(currentWritable);
-    // We keep track of all the pending writables, so we can await them in the final method
-    const writablesFinish = [awaitFinish(currentWritable)];
+    let splitIndex = 0;
+    let lock;
+    let fork = createNewFork();
     return new Writable({
-        objectMode: true,
+        objectMode,
         highWaterMark,
-        write(chunk, _, cb) {
-            // pipe will take care of piping the data through the different streams correctly
-            transforms[0].write(chunk, cb);
+        async write(chunk, _, cb) {
+            if (lock) {
+                // Forked pipeline is locked - let's wait for it to call _read
+                await lock;
+                // lock is undefined at this point
+            }
+            // pass to the "forked" pipeline
+            const shouldContinue = fork.push(chunk);
+            if (!shouldContinue && !lock) {
+                // Forked pipeline indicates that we should Pause
+                lock = pDefer();
+                logger.debug(`WritableChunk(${splitIndex}): pause`);
+            }
             if (splitPredicate(chunk, ++indexWritten)) {
-                logger.log(`writableChunk: splitting at index ${currentSplitIndex}`);
-                currentSplitIndex++;
-                transforms[0].end();
-                currentWritable = writableFactory(currentSplitIndex);
-                transforms = transformFactories.map(f => f());
-                generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2));
-                _last(transforms).pipe(currentWritable);
-                writablesFinish.push(awaitFinish(currentWritable));
+                logger.log(`WritableChunk(${splitIndex}): splitting to ${splitIndex + 1}`);
+                splitIndex++;
+                fork.push(null);
+                lock?.resolve();
+                lock = undefined;
+                fork = createNewFork();
             }
+            // acknowledge that we've finished processing the input chunk
+            cb();
         },
         async final(cb) {
-            try {
-                transforms[0].end();
-                await Promise.all(writablesFinish);
-                logger.log('writableChunk: all writables are finished');
-                cb();
-            }
-            catch (err) {
-                cb(err);
-            }
+            logger.log(`WritableChunk: final`);
+            // Pushing null "closes"/ends the secondary pipeline correctly
+            fork.push(null);
+            // Acknowledge that we've received `null` and passed it through to the fork
+            cb();
         },
     });
-}
-/**
- * This is a helper function to create a promise which resolves when the stream emits a 'finish'
- * event.
- * This is used to await all the writables in the final method of the writableChunk
- */
-async function awaitFinish(stream) {
-    return await new Promise(resolve => {
-        stream.on('finish', resolve);
-    });
-}
-/**
- * Generates an array of [arr[i], arr[i+1]] tuples from the input array.
- * The resulting array will have a length of `arr.length - 1`.
- * ```ts
- * generateTuples([1, 2, 3, 4]) // [[1, 2], [2, 3], [3, 4]]
- * ```
- */
-function generateTuples(arr) {
-    const tuples = [];
-    const arrCopy = _deepCopy(arr);
-    for (let i = 1; i < arrCopy.length; i++) {
-        tuples.push([arrCopy[i - 1], arrCopy[i]]);
+    function createNewFork() {
+        const currentSplitIndex = splitIndex;
+        const readable = createReadable([], {}, () => {
+            // `_read` is called
+            if (!lock)
+                return;
+            // We had a lock - let's Resume
+            logger.debug(`WritableChunk(${currentSplitIndex}): resume`);
+            const lockCopy = lock;
+            lock = undefined;
+            lockCopy.resolve();
+        });
+        void fn(Pipeline.from(readable), currentSplitIndex).then(() => {
+            logger.log(`WritableChunk(${currentSplitIndex}): done`);
+        });
+        return readable;
     }
-    return tuples;
 }

package/dist/validation/ajv/getAjv.js

@@ -1,9 +1,8 @@
-/* eslint-disable @typescript-eslint/prefer-string-starts-ends-with */
-/* eslint-disable unicorn/prefer-code-point */
 import { _lazyValue } from '@naturalcycles/js-lib';
-import { Ajv } from 'ajv';
+import { _, Ajv } from 'ajv';
 import ajvFormats from 'ajv-formats';
-import ajvKeywords from 'ajv-keywords';
+/* eslint-disable @typescript-eslint/prefer-string-starts-ends-with */
+// oxlint-disable unicorn/prefer-code-point
 const AJV_OPTIONS = {
     removeAdditional: true,
     allErrors: true,
@@ -51,17 +50,56 @@ export function createAjv(opt) {
     // https://ajv.js.org/guide/formats.html
     // @ts-expect-error types are wrong
     ajvFormats(ajv);
-    // https://ajv.js.org/packages/ajv-keywords.html
-    // @ts-expect-error types are wrong
-    ajvKeywords(ajv, [
-        'transform', // trim, toLowerCase, etc.
-        'uniqueItemProperties',
-        'instanceof',
-    ]);
     // Adds $merge, $patch keywords
     // https://github.com/ajv-validator/ajv-merge-patch
     // Kirill: temporarily disabled, as it creates a noise of CVE warnings
     // require('ajv-merge-patch')(ajv)
+    ajv.addKeyword({
+        keyword: 'transform',
+        type: 'string',
+        modifying: true,
+        schemaType: 'object',
+        code(cxt) {
+            const { gen, data, schema, it } = cxt;
+            const { parentData, parentDataProperty } = it;
+            if (schema.trim) {
+                gen.assign(_ `${data}`, _ `${data}.trim()`);
+            }
+            if (schema.toLowerCase) {
+                gen.assign(_ `${data}`, _ `${data}.toLowerCase()`);
+            }
+            if (schema.toUpperCase) {
+                gen.assign(_ `${data}`, _ `${data}.toUpperCase()`);
+            }
+            if (typeof schema.truncate === 'number' && schema.truncate >= 0) {
+                gen.assign(_ `${data}`, _ `${data}.slice(0, ${schema.truncate})`);
+                if (schema.trim) {
+                    gen.assign(_ `${data}`, _ `${data}.trim()`);
+                }
+            }
+            gen.if(_ `${parentData} !== undefined`, () => {
+                gen.assign(_ `${parentData}[${parentDataProperty}]`, data);
+            });
+        },
+    });
+    ajv.addKeyword({
+        keyword: 'instanceof',
+        modifying: true,
+        schemaType: 'string',
+        validate(instanceOf, data, _schema, _ctx) {
+            if (typeof data !== 'object')
+                return false;
+            if (data === null)
+                return false;
+            let proto = Object.getPrototypeOf(data);
+            while (proto) {
+                if (proto.constructor?.name === instanceOf)
+                    return true;
+                proto = Object.getPrototypeOf(proto);
+            }
+            return false;
+        },
+    });
     return ajv;
 }
 const TS_2500 = 16725225600; // 2500-01-01

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.37.1",
+  "version": "15.38.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",
@@ -9,7 +9,6 @@
     "@types/yargs": "^16",
     "ajv": "^8",
     "ajv-formats": "^3",
-    "ajv-keywords": "^5",
     "chalk": "^5",
     "dotenv": "^17",
     "joi": "^18",

package/src/security/nanoid.ts

@@ -5,7 +5,7 @@ https://github.com/ai/nanoid/
 
  */
 
-/* eslint-disable */
+// oxlint-disable no-bitwise -- NanoID implementation relies on bitwise operations
 
 import { randomFillSync } from 'node:crypto'
 

package/src/stream/index.ts

@@ -18,7 +18,6 @@ export * from './transform/transformLogProgress.js'
 export * from './transform/transformMap.js'
 export * from './transform/transformMapSimple.js'
 export * from './transform/transformMapSync.js'
-export * from './transform/transformMultiFork.js'
 export * from './transform/transformNoOp.js'
 export * from './transform/transformOffset.js'
 export * from './transform/transformSplit.js'

package/src/stream/writable/writableChunk.ts

@@ -1,104 +1,91 @@
 import { Writable } from 'node:stream'
-import { _first, _last } from '@naturalcycles/js-lib/array'
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
-import { _deepCopy } from '@naturalcycles/js-lib/object'
-import type { Predicate } from '@naturalcycles/js-lib/types'
-import {
-  transformNoOp,
-  type TransformOptions,
-  type TransformTyped,
-  type WritableTyped,
-} from '../index.js'
-
-export interface WritableChunkOptions<T> extends TransformOptions {
-  splitPredicate: Predicate<T>
-  transformFactories?: (() => TransformTyped<T, T>)[]
-  writableFactory: (splitIndex: number) => WritableTyped<T>
-}
+import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import type { NonNegativeInteger, Predicate } from '@naturalcycles/js-lib/types'
+import { Pipeline } from '../pipeline.js'
+import { createReadable } from '../readable/createReadable.js'
+import type { ReadableTyped, TransformOptions, WritableTyped } from '../stream.model.js'
 
 /**
- * Allows to split the output to multiple files by splitting into chunks
- * based on `shouldSplitFn`.
- * `transformFactories` are used to create a chain of transforms for each chunk.
- * It was meant to be used with createGzip, which needs a proper start and end for each chunk
- * for the output file to be a valid gzip file.
+ * Allows to "split the stream" into chunks, and attach a new Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
  */
-export function writableChunk<T>(opt: WritableChunkOptions<T>): WritableTyped<T> {
-  const { highWaterMark, splitPredicate, transformFactories = [], writableFactory } = opt
+export function writableChunk<T>(
+  splitPredicate: Predicate<T>,
+  fn: (pipeline: Pipeline<T>, splitIndex: NonNegativeInteger) => Promise<void>,
+  opt: TransformOptions = {},
+): WritableTyped<T> {
+  const { objectMode = true, highWaterMark } = opt
   const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
-
   let indexWritten = 0
-  let currentSplitIndex = 0
-  // We don't want to have an empty chain, so we add a no-op transform
-  if (transformFactories.length === 0) {
-    transformFactories.push(transformNoOp<T>)
-  }
-
-  // Create the transforms as well as the Writable, and pipe them together
-  let currentWritable = writableFactory(currentSplitIndex)
-  let transforms = transformFactories.map(f => f())
-  generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2))
-  _last(transforms).pipe(currentWritable)
+  let splitIndex = 0
 
-  // We keep track of all the pending writables, so we can await them in the final method
-  const writablesFinish: Promise<void>[] = [awaitFinish(currentWritable)]
+  let lock: DeferredPromise | undefined
+  let fork = createNewFork()
 
   return new Writable({
-    objectMode: true,
+    objectMode,
     highWaterMark,
-    write(chunk: T, _, cb) {
-      // pipe will take care of piping the data through the different streams correctly
-      transforms[0]!.write(chunk, cb)
-
-      if (splitPredicate(chunk, ++indexWritten)) {
-        logger.log(`writableChunk: splitting at index ${currentSplitIndex}`)
-        currentSplitIndex++
-        transforms[0]!.end()
+    async write(chunk: T, _, cb) {
+      if (lock) {
+        // Forked pipeline is locked - let's wait for it to call _read
+        await lock
+        // lock is undefined at this point
+      }
 
-        currentWritable = writableFactory(currentSplitIndex)
-        transforms = transformFactories.map(f => f())
-        generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2))
-        _last(transforms).pipe(currentWritable)
+      // pass to the "forked" pipeline
+      const shouldContinue = fork.push(chunk)
+      if (!shouldContinue && !lock) {
+        // Forked pipeline indicates that we should Pause
+        lock = pDefer()
+        logger.debug(`WritableChunk(${splitIndex}): pause`)
+      }
 
-        writablesFinish.push(awaitFinish(currentWritable))
+      if (splitPredicate(chunk, ++indexWritten)) {
+        logger.log(`WritableChunk(${splitIndex}): splitting to ${splitIndex + 1}`)
+        splitIndex++
+        fork.push(null)
+        lock?.resolve()
+        lock = undefined
+        fork = createNewFork()
       }
+
+      // acknowledge that we've finished processing the input chunk
+      cb()
     },
     async final(cb) {
-      try {
-        transforms[0]!.end()
-        await Promise.all(writablesFinish)
-        logger.log('writableChunk: all writables are finished')
-        cb()
-      } catch (err) {
-        cb(err as Error)
-      }
+      logger.log(`WritableChunk: final`)
+
+      // Pushing null "closes"/ends the secondary pipeline correctly
+      fork.push(null)
+
+      // Acknowledge that we've received `null` and passed it through to the fork
+      cb()
     },
   })
-}
 
-/**
- * This is a helper function to create a promise which resolves when the stream emits a 'finish'
- * event.
- * This is used to await all the writables in the final method of the writableChunk
- */
-async function awaitFinish(stream: Writable): Promise<void> {
-  return await new Promise(resolve => {
-    stream.on('finish', resolve)
-  })
-}
+  function createNewFork(): ReadableTyped<T> {
+    const currentSplitIndex = splitIndex
 
-/**
- * Generates an array of [arr[i], arr[i+1]] tuples from the input array.
- * The resulting array will have a length of `arr.length - 1`.
- * ```ts
- * generateTuples([1, 2, 3, 4]) // [[1, 2], [2, 3], [3, 4]]
- * ```
- */
-function generateTuples<T>(arr: T[]): [T, T][] {
-  const tuples: [T, T][] = []
-  const arrCopy = _deepCopy(arr)
-  for (let i = 1; i < arrCopy.length; i++) {
-    tuples.push([arrCopy[i - 1]!, arrCopy[i]!])
+    const readable = createReadable<T>([], {}, () => {
+      // `_read` is called
+      if (!lock) return
+      // We had a lock - let's Resume
+      logger.debug(`WritableChunk(${currentSplitIndex}): resume`)
+      const lockCopy = lock
+      lock = undefined
+      lockCopy.resolve()
+    })
+
+    void fn(Pipeline.from<T>(readable), currentSplitIndex).then(() => {
+      logger.log(`WritableChunk(${currentSplitIndex}): done`)
+    })
+
+    return readable
   }
-  return tuples
 }

package/src/validation/ajv/getAjv.ts

@@ -1,10 +1,10 @@
-/* eslint-disable @typescript-eslint/prefer-string-starts-ends-with */
-/* eslint-disable unicorn/prefer-code-point */
 import { _lazyValue } from '@naturalcycles/js-lib'
 import type { Options } from 'ajv'
-import { Ajv } from 'ajv'
+import { _, Ajv } from 'ajv'
 import ajvFormats from 'ajv-formats'
-import ajvKeywords from 'ajv-keywords'
+
+/* eslint-disable @typescript-eslint/prefer-string-starts-ends-with */
+// oxlint-disable unicorn/prefer-code-point
 
 const AJV_OPTIONS: Options = {
   removeAdditional: true,
@@ -60,19 +60,64 @@ export function createAjv(opt?: Options): Ajv {
   // @ts-expect-error types are wrong
   ajvFormats(ajv)
 
-  // https://ajv.js.org/packages/ajv-keywords.html
-  // @ts-expect-error types are wrong
-  ajvKeywords(ajv, [
-    'transform', // trim, toLowerCase, etc.
-    'uniqueItemProperties',
-    'instanceof',
-  ])
-
   // Adds $merge, $patch keywords
   // https://github.com/ajv-validator/ajv-merge-patch
   // Kirill: temporarily disabled, as it creates a noise of CVE warnings
   // require('ajv-merge-patch')(ajv)
 
+  ajv.addKeyword({
+    keyword: 'transform',
+    type: 'string',
+    modifying: true,
+    schemaType: 'object',
+    code(cxt) {
+      const { gen, data, schema, it } = cxt
+      const { parentData, parentDataProperty } = it
+
+      if (schema.trim) {
+        gen.assign(_`${data}`, _`${data}.trim()`)
+      }
+
+      if (schema.toLowerCase) {
+        gen.assign(_`${data}`, _`${data}.toLowerCase()`)
+      }
+
+      if (schema.toUpperCase) {
+        gen.assign(_`${data}`, _`${data}.toUpperCase()`)
+      }
+
+      if (typeof schema.truncate === 'number' && schema.truncate >= 0) {
+        gen.assign(_`${data}`, _`${data}.slice(0, ${schema.truncate})`)
+
+        if (schema.trim) {
+          gen.assign(_`${data}`, _`${data}.trim()`)
+        }
+      }
+
+      gen.if(_`${parentData} !== undefined`, () => {
+        gen.assign(_`${parentData}[${parentDataProperty}]`, data)
+      })
+    },
+  })
+
+  ajv.addKeyword({
+    keyword: 'instanceof',
+    modifying: true,
+    schemaType: 'string',
+    validate(instanceOf: string, data: unknown, _schema, _ctx) {
+      if (typeof data !== 'object') return false
+      if (data === null) return false
+
+      let proto = Object.getPrototypeOf(data)
+      while (proto) {
+        if (proto.constructor?.name === instanceOf) return true
+        proto = Object.getPrototypeOf(proto)
+      }
+
+      return false
+    },
+  })
+
   return ajv
 }
 

package/src/validation/joi/joi.extensions.ts

@@ -5,9 +5,9 @@ import type { StringSchema } from './string.extensions.js'
 import { stringExtensions } from './string.extensions.js'
 
 export interface ExtendedJoi extends JoiLib.Root {
-  // eslint-disable-next-line id-blacklist
+  // eslint-disable-next-line id-denylist
   string: <TSchema = string>() => StringSchema<TSchema>
-  // eslint-disable-next-line id-blacklist
+  // eslint-disable-next-line id-denylist
   number: <TSchema = number>() => NumberSchema<TSchema>
 }
 

package/dist/stream/transform/transformMultiFork.d.ts

@@ -1,14 +0,0 @@
-import type { NonNegativeInteger, Predicate } from '@naturalcycles/js-lib/types';
-import { Pipeline } from '../pipeline.js';
-import type { TransformOptions, TransformTyped } from '../stream.model.js';
-/**
- * Like transformFork, but allows to fork multiple times,
- * aka "split the stream" into chunks, and attach a Pipeline to
- * each of the chunks.
- *
- * Example use case: you want to write to Cloud Storage, 1000 rows per file,
- * each file needs its own destination Pipeline.
- *
- * @experimental
- */
-export declare function transformMultiFork<T>(splitPredicate: Predicate<T>, fn: (pipeline: Pipeline<T>, splitIndex: NonNegativeInteger) => Promise<void>, opt?: TransformOptions): TransformTyped<T, T>;

package/dist/stream/transform/transformMultiFork.js

@@ -1,79 +0,0 @@
-import { Transform } from 'node:stream';
-import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
-import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
-import { Pipeline } from '../pipeline.js';
-import { createReadable } from '../readable/createReadable.js';
-/**
- * Like transformFork, but allows to fork multiple times,
- * aka "split the stream" into chunks, and attach a Pipeline to
- * each of the chunks.
- *
- * Example use case: you want to write to Cloud Storage, 1000 rows per file,
- * each file needs its own destination Pipeline.
- *
- * @experimental
- */
-export function transformMultiFork(splitPredicate, fn, opt = {}) {
-    const { objectMode = true, highWaterMark } = opt;
-    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
-    let indexWritten = 0;
-    let splitIndex = 0;
-    let lock;
-    let fork = createNewFork();
-    return new Transform({
-        objectMode,
-        highWaterMark,
-        async transform(chunk, _, cb) {
-            // pass through to the "main" pipeline
-            // Main pipeline should handle backpressure "automatically",
-            // so, we're not maintaining a Lock for it
-            this.push(chunk);
-            if (lock) {
-                // Forked pipeline is locked - let's wait for it to call _read
-                await lock;
-                // lock is undefined at this point
-            }
-            // pass to the "forked" pipeline
-            const shouldContinue = fork.push(chunk);
-            if (!shouldContinue && !lock) {
-                // Forked pipeline indicates that we should Pause
-                lock = pDefer();
-                logger.debug(`TransformMultiFork(${splitIndex}): pause`);
-            }
-            if (splitPredicate(chunk, ++indexWritten)) {
-                logger.log(`TransformMultiFork(${splitIndex}): splitting to ${splitIndex + 1}`);
-                splitIndex++;
-                fork.push(null);
-                lock?.resolve();
-                lock = undefined;
-                fork = createNewFork();
-            }
-            // acknowledge that we've finished processing the input chunk
-            cb();
-        },
-        async final(cb) {
-            logger.log(`TransformMultiFork: final`);
-            // Pushing null "closes"/ends the secondary pipeline correctly
-            fork.push(null);
-            // Acknowledge that we've received `null` and passed it through to the fork
-            cb();
-        },
-    });
-    function createNewFork() {
-        const currentSplitIndex = splitIndex;
-        const readable = createReadable([], {}, () => {
-            // `_read` is called
-            if (!lock)
-                return;
-            // We had a lock - let's Resume
-            logger.debug(`TransformMultiFork(${currentSplitIndex}): resume`);
-            const lockCopy = lock;
-            lock = undefined;
-            lockCopy.resolve();
-        });
-        void fn(Pipeline.from(readable), currentSplitIndex).then(() => {
-            logger.log(`TransformMultiFork(${currentSplitIndex}): done`);
-        });
-        return readable;
-    }
-}

package/src/stream/transform/transformMultiFork.ts

@@ -1,97 +0,0 @@
-import { Transform } from 'node:stream'
-import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
-import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
-import type { NonNegativeInteger, Predicate } from '@naturalcycles/js-lib/types'
-import { Pipeline } from '../pipeline.js'
-import { createReadable } from '../readable/createReadable.js'
-import type { ReadableTyped, TransformOptions, TransformTyped } from '../stream.model.js'
-
-/**
- * Like transformFork, but allows to fork multiple times,
- * aka "split the stream" into chunks, and attach a Pipeline to
- * each of the chunks.
- *
- * Example use case: you want to write to Cloud Storage, 1000 rows per file,
- * each file needs its own destination Pipeline.
- *
- * @experimental
- */
-export function transformMultiFork<T>(
-  splitPredicate: Predicate<T>,
-  fn: (pipeline: Pipeline<T>, splitIndex: NonNegativeInteger) => Promise<void>,
-  opt: TransformOptions = {},
-): TransformTyped<T, T> {
-  const { objectMode = true, highWaterMark } = opt
-  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
-  let indexWritten = 0
-  let splitIndex = 0
-
-  let lock: DeferredPromise | undefined
-  let fork = createNewFork()
-
-  return new Transform({
-    objectMode,
-    highWaterMark,
-    async transform(chunk: T, _, cb) {
-      // pass through to the "main" pipeline
-      // Main pipeline should handle backpressure "automatically",
-      // so, we're not maintaining a Lock for it
-      this.push(chunk)
-
-      if (lock) {
-        // Forked pipeline is locked - let's wait for it to call _read
-        await lock
-        // lock is undefined at this point
-      }
-
-      // pass to the "forked" pipeline
-      const shouldContinue = fork.push(chunk)
-      if (!shouldContinue && !lock) {
-        // Forked pipeline indicates that we should Pause
-        lock = pDefer()
-        logger.debug(`TransformMultiFork(${splitIndex}): pause`)
-      }
-
-      if (splitPredicate(chunk, ++indexWritten)) {
-        logger.log(`TransformMultiFork(${splitIndex}): splitting to ${splitIndex + 1}`)
-        splitIndex++
-        fork.push(null)
-        lock?.resolve()
-        lock = undefined
-        fork = createNewFork()
-      }
-
-      // acknowledge that we've finished processing the input chunk
-      cb()
-    },
-    async final(cb) {
-      logger.log(`TransformMultiFork: final`)
-
-      // Pushing null "closes"/ends the secondary pipeline correctly
-      fork.push(null)
-
-      // Acknowledge that we've received `null` and passed it through to the fork
-      cb()
-    },
-  })
-
-  function createNewFork(): ReadableTyped<T> {
-    const currentSplitIndex = splitIndex
-
-    const readable = createReadable<T>([], {}, () => {
-      // `_read` is called
-      if (!lock) return
-      // We had a lock - let's Resume
-      logger.debug(`TransformMultiFork(${currentSplitIndex}): resume`)
-      const lockCopy = lock
-      lock = undefined
-      lockCopy.resolve()
-    })
-
-    void fn(Pipeline.from<T>(readable), currentSplitIndex).then(() => {
-      logger.log(`TransformMultiFork(${currentSplitIndex}): done`)
-    })
-
-    return readable
-  }
-}