@naturalcycles/nodejs-lib 15.37.1 → 15.37.2

package/dist/stream/index.d.ts

@@ -18,7 +18,6 @@ export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
-export * from './transform/transformMultiFork.js';
 export * from './transform/transformNoOp.js';
 export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';

package/dist/stream/index.js

@@ -18,7 +18,6 @@ export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
-export * from './transform/transformMultiFork.js';
 export * from './transform/transformNoOp.js';
 export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';

package/dist/stream/writable/writableChunk.d.ts

@@ -1,15 +1,13 @@
-import type { Predicate } from '@naturalcycles/js-lib/types';
-import { type TransformOptions, type TransformTyped, type WritableTyped } from '../index.js';
-export interface WritableChunkOptions<T> extends TransformOptions {
-    splitPredicate: Predicate<T>;
-    transformFactories?: (() => TransformTyped<T, T>)[];
-    writableFactory: (splitIndex: number) => WritableTyped<T>;
-}
+import type { NonNegativeInteger, Predicate } from '@naturalcycles/js-lib/types';
+import { Pipeline } from '../pipeline.js';
+import type { TransformOptions, WritableTyped } from '../stream.model.js';
 /**
- * Allows to split the output to multiple files by splitting into chunks
- * based on `shouldSplitFn`.
- * `transformFactories` are used to create a chain of transforms for each chunk.
- * It was meant to be used with createGzip, which needs a proper start and end for each chunk
- * for the output file to be a valid gzip file.
+ * Allows to "split the stream" into chunks, and attach a new Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
  */
-export declare function writableChunk<T>(opt: WritableChunkOptions<T>): WritableTyped<T>;
+export declare function writableChunk<T>(splitPredicate: Predicate<T>, fn: (pipeline: Pipeline<T>, splitIndex: NonNegativeInteger) => Promise<void>, opt?: TransformOptions): WritableTyped<T>;

package/dist/stream/writable/writableChunk.js

@@ -1,83 +1,74 @@
 import { Writable } from 'node:stream';
-import { _first, _last } from '@naturalcycles/js-lib/array';
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
-import { _deepCopy } from '@naturalcycles/js-lib/object';
-import { transformNoOp, } from '../index.js';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+import { Pipeline } from '../pipeline.js';
+import { createReadable } from '../readable/createReadable.js';
 /**
- * Allows to split the output to multiple files by splitting into chunks
- * based on `shouldSplitFn`.
- * `transformFactories` are used to create a chain of transforms for each chunk.
- * It was meant to be used with createGzip, which needs a proper start and end for each chunk
- * for the output file to be a valid gzip file.
+ * Allows to "split the stream" into chunks, and attach a new Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
  */
-export function writableChunk(opt) {
-    const { highWaterMark, splitPredicate, transformFactories = [], writableFactory } = opt;
+export function writableChunk(splitPredicate, fn, opt = {}) {
+    const { objectMode = true, highWaterMark } = opt;
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     let indexWritten = 0;
-    let currentSplitIndex = 0;
-    // We don't want to have an empty chain, so we add a no-op transform
-    if (transformFactories.length === 0) {
-        transformFactories.push((transformNoOp));
-    }
-    // Create the transforms as well as the Writable, and pipe them together
-    let currentWritable = writableFactory(currentSplitIndex);
-    let transforms = transformFactories.map(f => f());
-    generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2));
-    _last(transforms).pipe(currentWritable);
-    // We keep track of all the pending writables, so we can await them in the final method
-    const writablesFinish = [awaitFinish(currentWritable)];
+    let splitIndex = 0;
+    let lock;
+    let fork = createNewFork();
     return new Writable({
-        objectMode: true,
+        objectMode,
         highWaterMark,
-        write(chunk, _, cb) {
-            // pipe will take care of piping the data through the different streams correctly
-            transforms[0].write(chunk, cb);
+        async write(chunk, _, cb) {
+            if (lock) {
+                // Forked pipeline is locked - let's wait for it to call _read
+                await lock;
+                // lock is undefined at this point
+            }
+            // pass to the "forked" pipeline
+            const shouldContinue = fork.push(chunk);
+            if (!shouldContinue && !lock) {
+                // Forked pipeline indicates that we should Pause
+                lock = pDefer();
+                logger.debug(`WritableChunk(${splitIndex}): pause`);
+            }
             if (splitPredicate(chunk, ++indexWritten)) {
-                logger.log(`writableChunk: splitting at index ${currentSplitIndex}`);
-                currentSplitIndex++;
-                transforms[0].end();
-                currentWritable = writableFactory(currentSplitIndex);
-                transforms = transformFactories.map(f => f());
-                generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2));
-                _last(transforms).pipe(currentWritable);
-                writablesFinish.push(awaitFinish(currentWritable));
+                logger.log(`WritableChunk(${splitIndex}): splitting to ${splitIndex + 1}`);
+                splitIndex++;
+                fork.push(null);
+                lock?.resolve();
+                lock = undefined;
+                fork = createNewFork();
             }
+            // acknowledge that we've finished processing the input chunk
+            cb();
         },
         async final(cb) {
-            try {
-                transforms[0].end();
-                await Promise.all(writablesFinish);
-                logger.log('writableChunk: all writables are finished');
-                cb();
-            }
-            catch (err) {
-                cb(err);
-            }
+            logger.log(`WritableChunk: final`);
+            // Pushing null "closes"/ends the secondary pipeline correctly
+            fork.push(null);
+            // Acknowledge that we've received `null` and passed it through to the fork
+            cb();
         },
     });
-}
-/**
- * This is a helper function to create a promise which resolves when the stream emits a 'finish'
- * event.
- * This is used to await all the writables in the final method of the writableChunk
- */
-async function awaitFinish(stream) {
-    return await new Promise(resolve => {
-        stream.on('finish', resolve);
-    });
-}
-/**
- * Generates an array of [arr[i], arr[i+1]] tuples from the input array.
- * The resulting array will have a length of `arr.length - 1`.
- * ```ts
- * generateTuples([1, 2, 3, 4]) // [[1, 2], [2, 3], [3, 4]]
- * ```
- */
-function generateTuples(arr) {
-    const tuples = [];
-    const arrCopy = _deepCopy(arr);
-    for (let i = 1; i < arrCopy.length; i++) {
-        tuples.push([arrCopy[i - 1], arrCopy[i]]);
+    function createNewFork() {
+        const currentSplitIndex = splitIndex;
+        const readable = createReadable([], {}, () => {
+            // `_read` is called
+            if (!lock)
+                return;
+            // We had a lock - let's Resume
+            logger.debug(`WritableChunk(${currentSplitIndex}): resume`);
+            const lockCopy = lock;
+            lock = undefined;
+            lockCopy.resolve();
+        });
+        void fn(Pipeline.from(readable), currentSplitIndex).then(() => {
+            logger.log(`WritableChunk(${currentSplitIndex}): done`);
+        });
+        return readable;
     }
-    return tuples;
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.37.1",
+  "version": "15.37.2",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/index.ts

@@ -18,7 +18,6 @@ export * from './transform/transformLogProgress.js'
 export * from './transform/transformMap.js'
 export * from './transform/transformMapSimple.js'
 export * from './transform/transformMapSync.js'
-export * from './transform/transformMultiFork.js'
 export * from './transform/transformNoOp.js'
 export * from './transform/transformOffset.js'
 export * from './transform/transformSplit.js'

package/src/stream/writable/writableChunk.ts

@@ -1,104 +1,91 @@
 import { Writable } from 'node:stream'
-import { _first, _last } from '@naturalcycles/js-lib/array'
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
-import { _deepCopy } from '@naturalcycles/js-lib/object'
-import type { Predicate } from '@naturalcycles/js-lib/types'
-import {
-  transformNoOp,
-  type TransformOptions,
-  type TransformTyped,
-  type WritableTyped,
-} from '../index.js'
-
-export interface WritableChunkOptions<T> extends TransformOptions {
-  splitPredicate: Predicate<T>
-  transformFactories?: (() => TransformTyped<T, T>)[]
-  writableFactory: (splitIndex: number) => WritableTyped<T>
-}
+import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import type { NonNegativeInteger, Predicate } from '@naturalcycles/js-lib/types'
+import { Pipeline } from '../pipeline.js'
+import { createReadable } from '../readable/createReadable.js'
+import type { ReadableTyped, TransformOptions, WritableTyped } from '../stream.model.js'
 
 /**
- * Allows to split the output to multiple files by splitting into chunks
- * based on `shouldSplitFn`.
- * `transformFactories` are used to create a chain of transforms for each chunk.
- * It was meant to be used with createGzip, which needs a proper start and end for each chunk
- * for the output file to be a valid gzip file.
+ * Allows to "split the stream" into chunks, and attach a new Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
  */
-export function writableChunk<T>(opt: WritableChunkOptions<T>): WritableTyped<T> {
-  const { highWaterMark, splitPredicate, transformFactories = [], writableFactory } = opt
+export function writableChunk<T>(
+  splitPredicate: Predicate<T>,
+  fn: (pipeline: Pipeline<T>, splitIndex: NonNegativeInteger) => Promise<void>,
+  opt: TransformOptions = {},
+): WritableTyped<T> {
+  const { objectMode = true, highWaterMark } = opt
   const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
-
   let indexWritten = 0
-  let currentSplitIndex = 0
-  // We don't want to have an empty chain, so we add a no-op transform
-  if (transformFactories.length === 0) {
-    transformFactories.push(transformNoOp<T>)
-  }
-
-  // Create the transforms as well as the Writable, and pipe them together
-  let currentWritable = writableFactory(currentSplitIndex)
-  let transforms = transformFactories.map(f => f())
-  generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2))
-  _last(transforms).pipe(currentWritable)
+  let splitIndex = 0
 
-  // We keep track of all the pending writables, so we can await them in the final method
-  const writablesFinish: Promise<void>[] = [awaitFinish(currentWritable)]
+  let lock: DeferredPromise | undefined
+  let fork = createNewFork()
 
   return new Writable({
-    objectMode: true,
+    objectMode,
     highWaterMark,
-    write(chunk: T, _, cb) {
-      // pipe will take care of piping the data through the different streams correctly
-      transforms[0]!.write(chunk, cb)
-
-      if (splitPredicate(chunk, ++indexWritten)) {
-        logger.log(`writableChunk: splitting at index ${currentSplitIndex}`)
-        currentSplitIndex++
-        transforms[0]!.end()
+    async write(chunk: T, _, cb) {
+      if (lock) {
+        // Forked pipeline is locked - let's wait for it to call _read
+        await lock
+        // lock is undefined at this point
+      }
 
-        currentWritable = writableFactory(currentSplitIndex)
-        transforms = transformFactories.map(f => f())
-        generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2))
-        _last(transforms).pipe(currentWritable)
+      // pass to the "forked" pipeline
+      const shouldContinue = fork.push(chunk)
+      if (!shouldContinue && !lock) {
+        // Forked pipeline indicates that we should Pause
+        lock = pDefer()
+        logger.debug(`WritableChunk(${splitIndex}): pause`)
+      }
 
-        writablesFinish.push(awaitFinish(currentWritable))
+      if (splitPredicate(chunk, ++indexWritten)) {
+        logger.log(`WritableChunk(${splitIndex}): splitting to ${splitIndex + 1}`)
+        splitIndex++
+        fork.push(null)
+        lock?.resolve()
+        lock = undefined
+        fork = createNewFork()
       }
+
+      // acknowledge that we've finished processing the input chunk
+      cb()
     },
     async final(cb) {
-      try {
-        transforms[0]!.end()
-        await Promise.all(writablesFinish)
-        logger.log('writableChunk: all writables are finished')
-        cb()
-      } catch (err) {
-        cb(err as Error)
-      }
+      logger.log(`WritableChunk: final`)
+
+      // Pushing null "closes"/ends the secondary pipeline correctly
+      fork.push(null)
+
+      // Acknowledge that we've received `null` and passed it through to the fork
+      cb()
     },
   })
-}
 
-/**
- * This is a helper function to create a promise which resolves when the stream emits a 'finish'
- * event.
- * This is used to await all the writables in the final method of the writableChunk
- */
-async function awaitFinish(stream: Writable): Promise<void> {
-  return await new Promise(resolve => {
-    stream.on('finish', resolve)
-  })
-}
+  function createNewFork(): ReadableTyped<T> {
+    const currentSplitIndex = splitIndex
 
-/**
- * Generates an array of [arr[i], arr[i+1]] tuples from the input array.
- * The resulting array will have a length of `arr.length - 1`.
- * ```ts
- * generateTuples([1, 2, 3, 4]) // [[1, 2], [2, 3], [3, 4]]
- * ```
- */
-function generateTuples<T>(arr: T[]): [T, T][] {
-  const tuples: [T, T][] = []
-  const arrCopy = _deepCopy(arr)
-  for (let i = 1; i < arrCopy.length; i++) {
-    tuples.push([arrCopy[i - 1]!, arrCopy[i]!])
+    const readable = createReadable<T>([], {}, () => {
+      // `_read` is called
+      if (!lock) return
+      // We had a lock - let's Resume
+      logger.debug(`WritableChunk(${currentSplitIndex}): resume`)
+      const lockCopy = lock
+      lock = undefined
+      lockCopy.resolve()
+    })
+
+    void fn(Pipeline.from<T>(readable), currentSplitIndex).then(() => {
+      logger.log(`WritableChunk(${currentSplitIndex}): done`)
+    })
+
+    return readable
   }
-  return tuples
 }

package/dist/stream/transform/transformMultiFork.d.ts

@@ -1,14 +0,0 @@
-import type { NonNegativeInteger, Predicate } from '@naturalcycles/js-lib/types';
-import { Pipeline } from '../pipeline.js';
-import type { TransformOptions, TransformTyped } from '../stream.model.js';
-/**
- * Like transformFork, but allows to fork multiple times,
- * aka "split the stream" into chunks, and attach a Pipeline to
- * each of the chunks.
- *
- * Example use case: you want to write to Cloud Storage, 1000 rows per file,
- * each file needs its own destination Pipeline.
- *
- * @experimental
- */
-export declare function transformMultiFork<T>(splitPredicate: Predicate<T>, fn: (pipeline: Pipeline<T>, splitIndex: NonNegativeInteger) => Promise<void>, opt?: TransformOptions): TransformTyped<T, T>;

package/dist/stream/transform/transformMultiFork.js

@@ -1,79 +0,0 @@
-import { Transform } from 'node:stream';
-import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
-import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
-import { Pipeline } from '../pipeline.js';
-import { createReadable } from '../readable/createReadable.js';
-/**
- * Like transformFork, but allows to fork multiple times,
- * aka "split the stream" into chunks, and attach a Pipeline to
- * each of the chunks.
- *
- * Example use case: you want to write to Cloud Storage, 1000 rows per file,
- * each file needs its own destination Pipeline.
- *
- * @experimental
- */
-export function transformMultiFork(splitPredicate, fn, opt = {}) {
-    const { objectMode = true, highWaterMark } = opt;
-    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
-    let indexWritten = 0;
-    let splitIndex = 0;
-    let lock;
-    let fork = createNewFork();
-    return new Transform({
-        objectMode,
-        highWaterMark,
-        async transform(chunk, _, cb) {
-            // pass through to the "main" pipeline
-            // Main pipeline should handle backpressure "automatically",
-            // so, we're not maintaining a Lock for it
-            this.push(chunk);
-            if (lock) {
-                // Forked pipeline is locked - let's wait for it to call _read
-                await lock;
-                // lock is undefined at this point
-            }
-            // pass to the "forked" pipeline
-            const shouldContinue = fork.push(chunk);
-            if (!shouldContinue && !lock) {
-                // Forked pipeline indicates that we should Pause
-                lock = pDefer();
-                logger.debug(`TransformMultiFork(${splitIndex}): pause`);
-            }
-            if (splitPredicate(chunk, ++indexWritten)) {
-                logger.log(`TransformMultiFork(${splitIndex}): splitting to ${splitIndex + 1}`);
-                splitIndex++;
-                fork.push(null);
-                lock?.resolve();
-                lock = undefined;
-                fork = createNewFork();
-            }
-            // acknowledge that we've finished processing the input chunk
-            cb();
-        },
-        async final(cb) {
-            logger.log(`TransformMultiFork: final`);
-            // Pushing null "closes"/ends the secondary pipeline correctly
-            fork.push(null);
-            // Acknowledge that we've received `null` and passed it through to the fork
-            cb();
-        },
-    });
-    function createNewFork() {
-        const currentSplitIndex = splitIndex;
-        const readable = createReadable([], {}, () => {
-            // `_read` is called
-            if (!lock)
-                return;
-            // We had a lock - let's Resume
-            logger.debug(`TransformMultiFork(${currentSplitIndex}): resume`);
-            const lockCopy = lock;
-            lock = undefined;
-            lockCopy.resolve();
-        });
-        void fn(Pipeline.from(readable), currentSplitIndex).then(() => {
-            logger.log(`TransformMultiFork(${currentSplitIndex}): done`);
-        });
-        return readable;
-    }
-}

package/src/stream/transform/transformMultiFork.ts

@@ -1,97 +0,0 @@
-import { Transform } from 'node:stream'
-import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
-import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
-import type { NonNegativeInteger, Predicate } from '@naturalcycles/js-lib/types'
-import { Pipeline } from '../pipeline.js'
-import { createReadable } from '../readable/createReadable.js'
-import type { ReadableTyped, TransformOptions, TransformTyped } from '../stream.model.js'
-
-/**
- * Like transformFork, but allows to fork multiple times,
- * aka "split the stream" into chunks, and attach a Pipeline to
- * each of the chunks.
- *
- * Example use case: you want to write to Cloud Storage, 1000 rows per file,
- * each file needs its own destination Pipeline.
- *
- * @experimental
- */
-export function transformMultiFork<T>(
-  splitPredicate: Predicate<T>,
-  fn: (pipeline: Pipeline<T>, splitIndex: NonNegativeInteger) => Promise<void>,
-  opt: TransformOptions = {},
-): TransformTyped<T, T> {
-  const { objectMode = true, highWaterMark } = opt
-  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
-  let indexWritten = 0
-  let splitIndex = 0
-
-  let lock: DeferredPromise | undefined
-  let fork = createNewFork()
-
-  return new Transform({
-    objectMode,
-    highWaterMark,
-    async transform(chunk: T, _, cb) {
-      // pass through to the "main" pipeline
-      // Main pipeline should handle backpressure "automatically",
-      // so, we're not maintaining a Lock for it
-      this.push(chunk)
-
-      if (lock) {
-        // Forked pipeline is locked - let's wait for it to call _read
-        await lock
-        // lock is undefined at this point
-      }
-
-      // pass to the "forked" pipeline
-      const shouldContinue = fork.push(chunk)
-      if (!shouldContinue && !lock) {
-        // Forked pipeline indicates that we should Pause
-        lock = pDefer()
-        logger.debug(`TransformMultiFork(${splitIndex}): pause`)
-      }
-
-      if (splitPredicate(chunk, ++indexWritten)) {
-        logger.log(`TransformMultiFork(${splitIndex}): splitting to ${splitIndex + 1}`)
-        splitIndex++
-        fork.push(null)
-        lock?.resolve()
-        lock = undefined
-        fork = createNewFork()
-      }
-
-      // acknowledge that we've finished processing the input chunk
-      cb()
-    },
-    async final(cb) {
-      logger.log(`TransformMultiFork: final`)
-
-      // Pushing null "closes"/ends the secondary pipeline correctly
-      fork.push(null)
-
-      // Acknowledge that we've received `null` and passed it through to the fork
-      cb()
-    },
-  })
-
-  function createNewFork(): ReadableTyped<T> {
-    const currentSplitIndex = splitIndex
-
-    const readable = createReadable<T>([], {}, () => {
-      // `_read` is called
-      if (!lock) return
-      // We had a lock - let's Resume
-      logger.debug(`TransformMultiFork(${currentSplitIndex}): resume`)
-      const lockCopy = lock
-      lock = undefined
-      lockCopy.resolve()
-    })
-
-    void fn(Pipeline.from<T>(readable), currentSplitIndex).then(() => {
-      logger.log(`TransformMultiFork(${currentSplitIndex}): done`)
-    })
-
-    return readable
-  }
-}