@naturalcycles/nodejs-lib 15.35.0 → 15.37.0

package/dist/stream/index.d.ts

@@ -18,6 +18,7 @@ export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
+export * from './transform/transformMultiFork.js';
 export * from './transform/transformNoOp.js';
 export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';
@@ -26,5 +27,6 @@ export * from './transform/transformThrottle.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';
+export * from './writable/writableChunk.js';
 export * from './writable/writablePushToArray.js';
 export * from './writable/writableVoid.js';

package/dist/stream/index.js

@@ -18,6 +18,7 @@ export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
+export * from './transform/transformMultiFork.js';
 export * from './transform/transformNoOp.js';
 export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';
@@ -26,5 +27,6 @@ export * from './transform/transformThrottle.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';
+export * from './writable/writableChunk.js';
 export * from './writable/writablePushToArray.js';
 export * from './writable/writableVoid.js';

package/dist/stream/transform/transformMultiFork.d.ts

@@ -0,0 +1,14 @@
+import type { Predicate } from '@naturalcycles/js-lib/types';
+import { Pipeline } from '../pipeline.js';
+import type { TransformOptions, TransformTyped } from '../stream.model.js';
+/**
+ * Like transformFork, but allows to fork multiple times,
+ * aka "split the stream" into chunks, and attach a Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
+ */
+export declare function transformMultiFork<T>(splitPredicate: Predicate<T>, fn: (pipeline: Pipeline<T>) => Promise<void>, opt?: TransformOptions): TransformTyped<T, T>;

package/dist/stream/transform/transformMultiFork.js

@@ -0,0 +1,79 @@
+import { Transform } from 'node:stream';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+import { Pipeline } from '../pipeline.js';
+import { createReadable } from '../readable/createReadable.js';
+/**
+ * Like transformFork, but allows to fork multiple times,
+ * aka "split the stream" into chunks, and attach a Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
+ */
+export function transformMultiFork(splitPredicate, fn, opt = {}) {
+    const { objectMode = true, highWaterMark } = opt;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+    let indexWritten = 0;
+    let splitIndex = 0;
+    let lock;
+    let fork = createNewFork();
+    return new Transform({
+        objectMode,
+        highWaterMark,
+        async transform(chunk, _, cb) {
+            // pass through to the "main" pipeline
+            // Main pipeline should handle backpressure "automatically",
+            // so, we're not maintaining a Lock for it
+            this.push(chunk);
+            if (lock) {
+                // Forked pipeline is locked - let's wait for it to call _read
+                await lock;
+                // lock is undefined at this point
+            }
+            // pass to the "forked" pipeline
+            const shouldContinue = fork.push(chunk);
+            if (!shouldContinue && !lock) {
+                // Forked pipeline indicates that we should Pause
+                lock = pDefer();
+                logger.debug(`TransformMultiFork(${splitIndex}): pause`);
+            }
+            if (splitPredicate(chunk, ++indexWritten)) {
+                logger.log(`TransformMultiFork(${splitIndex}): splitting to ${splitIndex + 1}`);
+                splitIndex++;
+                fork.push(null);
+                lock?.resolve();
+                lock = undefined;
+                fork = createNewFork();
+            }
+            // acknowledge that we've finished processing the input chunk
+            cb();
+        },
+        async final(cb) {
+            logger.log(`TransformMultiFork: final`);
+            // Pushing null "closes"/ends the secondary pipeline correctly
+            fork.push(null);
+            // Acknowledge that we've received `null` and passed it through to the fork
+            cb();
+        },
+    });
+    function createNewFork() {
+        const mySplitIndex = splitIndex;
+        const readable = createReadable([], {}, () => {
+            // `_read` is called
+            if (!lock)
+                return;
+            // We had a lock - let's Resume
+            logger.debug(`TransformMultiFork(${mySplitIndex}): resume`);
+            const lockCopy = lock;
+            lock = undefined;
+            lockCopy.resolve();
+        });
+        void fn(Pipeline.from(readable)).then(() => {
+            logger.log(`TransformMultiFork(${mySplitIndex}): done`);
+        });
+        return readable;
+    }
+}

package/dist/stream/writable/writableChunk.d.ts

@@ -0,0 +1,15 @@
+import type { Predicate } from '@naturalcycles/js-lib/types';
+import { type TransformOptions, type TransformTyped, type WritableTyped } from '../index.js';
+export interface WritableChunkOptions<T> extends TransformOptions {
+    splitPredicate: Predicate<T>;
+    transformFactories?: (() => TransformTyped<T, T>)[];
+    writableFactory: (splitIndex: number) => WritableTyped<T>;
+}
+/**
+ * Allows to split the output to multiple files by splitting into chunks
+ * based on `shouldSplitFn`.
+ * `transformFactories` are used to create a chain of transforms for each chunk.
+ * It was meant to be used with createGzip, which needs a proper start and end for each chunk
+ * for the output file to be a valid gzip file.
+ */
+export declare function writableChunk<T>(opt: WritableChunkOptions<T>): WritableTyped<T>;

package/dist/stream/writable/writableChunk.js

@@ -0,0 +1,83 @@
+import { Writable } from 'node:stream';
+import { _first, _last } from '@naturalcycles/js-lib/array';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
+import { _deepCopy } from '@naturalcycles/js-lib/object';
+import { transformNoOp, } from '../index.js';
+/**
+ * Allows to split the output to multiple files by splitting into chunks
+ * based on `shouldSplitFn`.
+ * `transformFactories` are used to create a chain of transforms for each chunk.
+ * It was meant to be used with createGzip, which needs a proper start and end for each chunk
+ * for the output file to be a valid gzip file.
+ */
+export function writableChunk(opt) {
+    const { highWaterMark, splitPredicate, transformFactories = [], writableFactory } = opt;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+    let indexWritten = 0;
+    let currentSplitIndex = 0;
+    // We don't want to have an empty chain, so we add a no-op transform
+    if (transformFactories.length === 0) {
+        transformFactories.push((transformNoOp));
+    }
+    // Create the transforms as well as the Writable, and pipe them together
+    let currentWritable = writableFactory(currentSplitIndex);
+    let transforms = transformFactories.map(f => f());
+    generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2));
+    _last(transforms).pipe(currentWritable);
+    // We keep track of all the pending writables, so we can await them in the final method
+    const writablesFinish = [awaitFinish(currentWritable)];
+    return new Writable({
+        objectMode: true,
+        highWaterMark,
+        write(chunk, _, cb) {
+            // pipe will take care of piping the data through the different streams correctly
+            transforms[0].write(chunk, cb);
+            if (splitPredicate(chunk, ++indexWritten)) {
+                logger.log(`writableChunk: splitting at index ${currentSplitIndex}`);
+                currentSplitIndex++;
+                transforms[0].end();
+                currentWritable = writableFactory(currentSplitIndex);
+                transforms = transformFactories.map(f => f());
+                generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2));
+                _last(transforms).pipe(currentWritable);
+                writablesFinish.push(awaitFinish(currentWritable));
+            }
+        },
+        async final(cb) {
+            try {
+                transforms[0].end();
+                await Promise.all(writablesFinish);
+                logger.log('writableChunk: all writables are finished');
+                cb();
+            }
+            catch (err) {
+                cb(err);
+            }
+        },
+    });
+}
+/**
+ * This is a helper function to create a promise which resolves when the stream emits a 'finish'
+ * event.
+ * This is used to await all the writables in the final method of the writableChunk
+ */
+async function awaitFinish(stream) {
+    return await new Promise(resolve => {
+        stream.on('finish', resolve);
+    });
+}
+/**
+ * Generates an array of [arr[i], arr[i+1]] tuples from the input array.
+ * The resulting array will have a length of `arr.length - 1`.
+ * ```ts
+ * generateTuples([1, 2, 3, 4]) // [[1, 2], [2, 3], [3, 4]]
+ * ```
+ */
+function generateTuples(arr) {
+    const tuples = [];
+    const arrCopy = _deepCopy(arr);
+    for (let i = 1; i < arrCopy.length; i++) {
+        tuples.push([arrCopy[i - 1], arrCopy[i]]);
+    }
+    return tuples;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.35.0",
+  "version": "15.37.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/index.ts

@@ -18,6 +18,7 @@ export * from './transform/transformLogProgress.js'
 export * from './transform/transformMap.js'
 export * from './transform/transformMapSimple.js'
 export * from './transform/transformMapSync.js'
+export * from './transform/transformMultiFork.js'
 export * from './transform/transformNoOp.js'
 export * from './transform/transformOffset.js'
 export * from './transform/transformSplit.js'
@@ -26,5 +27,6 @@ export * from './transform/transformThrottle.js'
 export * from './transform/worker/baseWorkerClass.js'
 export * from './transform/worker/transformMultiThreaded.js'
 export * from './transform/worker/transformMultiThreaded.model.js'
+export * from './writable/writableChunk.js'
 export * from './writable/writablePushToArray.js'
 export * from './writable/writableVoid.js'

package/src/stream/transform/transformMultiFork.ts

@@ -0,0 +1,97 @@
+import { Transform } from 'node:stream'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
+import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import type { Predicate } from '@naturalcycles/js-lib/types'
+import { Pipeline } from '../pipeline.js'
+import { createReadable } from '../readable/createReadable.js'
+import type { ReadableTyped, TransformOptions, TransformTyped } from '../stream.model.js'
+
+/**
+ * Like transformFork, but allows to fork multiple times,
+ * aka "split the stream" into chunks, and attach a Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
+ */
+export function transformMultiFork<T>(
+  splitPredicate: Predicate<T>,
+  fn: (pipeline: Pipeline<T>) => Promise<void>,
+  opt: TransformOptions = {},
+): TransformTyped<T, T> {
+  const { objectMode = true, highWaterMark } = opt
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+  let indexWritten = 0
+  let splitIndex = 0
+
+  let lock: DeferredPromise | undefined
+  let fork = createNewFork()
+
+  return new Transform({
+    objectMode,
+    highWaterMark,
+    async transform(chunk: T, _, cb) {
+      // pass through to the "main" pipeline
+      // Main pipeline should handle backpressure "automatically",
+      // so, we're not maintaining a Lock for it
+      this.push(chunk)
+
+      if (lock) {
+        // Forked pipeline is locked - let's wait for it to call _read
+        await lock
+        // lock is undefined at this point
+      }
+
+      // pass to the "forked" pipeline
+      const shouldContinue = fork.push(chunk)
+      if (!shouldContinue && !lock) {
+        // Forked pipeline indicates that we should Pause
+        lock = pDefer()
+        logger.debug(`TransformMultiFork(${splitIndex}): pause`)
+      }
+
+      if (splitPredicate(chunk, ++indexWritten)) {
+        logger.log(`TransformMultiFork(${splitIndex}): splitting to ${splitIndex + 1}`)
+        splitIndex++
+        fork.push(null)
+        lock?.resolve()
+        lock = undefined
+        fork = createNewFork()
+      }
+
+      // acknowledge that we've finished processing the input chunk
+      cb()
+    },
+    async final(cb) {
+      logger.log(`TransformMultiFork: final`)
+
+      // Pushing null "closes"/ends the secondary pipeline correctly
+      fork.push(null)
+
+      // Acknowledge that we've received `null` and passed it through to the fork
+      cb()
+    },
+  })
+
+  function createNewFork(): ReadableTyped<T> {
+    const mySplitIndex = splitIndex
+
+    const readable = createReadable<T>([], {}, () => {
+      // `_read` is called
+      if (!lock) return
+      // We had a lock - let's Resume
+      logger.debug(`TransformMultiFork(${mySplitIndex}): resume`)
+      const lockCopy = lock
+      lock = undefined
+      lockCopy.resolve()
+    })
+
+    void fn(Pipeline.from<T>(readable)).then(() => {
+      logger.log(`TransformMultiFork(${mySplitIndex}): done`)
+    })
+
+    return readable
+  }
+}

package/src/stream/writable/writableChunk.ts

@@ -0,0 +1,104 @@
+import { Writable } from 'node:stream'
+import { _first, _last } from '@naturalcycles/js-lib/array'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
+import { _deepCopy } from '@naturalcycles/js-lib/object'
+import type { Predicate } from '@naturalcycles/js-lib/types'
+import {
+  transformNoOp,
+  type TransformOptions,
+  type TransformTyped,
+  type WritableTyped,
+} from '../index.js'
+
+export interface WritableChunkOptions<T> extends TransformOptions {
+  splitPredicate: Predicate<T>
+  transformFactories?: (() => TransformTyped<T, T>)[]
+  writableFactory: (splitIndex: number) => WritableTyped<T>
+}
+
+/**
+ * Allows to split the output to multiple files by splitting into chunks
+ * based on `shouldSplitFn`.
+ * `transformFactories` are used to create a chain of transforms for each chunk.
+ * It was meant to be used with createGzip, which needs a proper start and end for each chunk
+ * for the output file to be a valid gzip file.
+ */
+export function writableChunk<T>(opt: WritableChunkOptions<T>): WritableTyped<T> {
+  const { highWaterMark, splitPredicate, transformFactories = [], writableFactory } = opt
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+
+  let indexWritten = 0
+  let currentSplitIndex = 0
+  // We don't want to have an empty chain, so we add a no-op transform
+  if (transformFactories.length === 0) {
+    transformFactories.push(transformNoOp<T>)
+  }
+
+  // Create the transforms as well as the Writable, and pipe them together
+  let currentWritable = writableFactory(currentSplitIndex)
+  let transforms = transformFactories.map(f => f())
+  generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2))
+  _last(transforms).pipe(currentWritable)
+
+  // We keep track of all the pending writables, so we can await them in the final method
+  const writablesFinish: Promise<void>[] = [awaitFinish(currentWritable)]
+
+  return new Writable({
+    objectMode: true,
+    highWaterMark,
+    write(chunk: T, _, cb) {
+      // pipe will take care of piping the data through the different streams correctly
+      transforms[0]!.write(chunk, cb)
+
+      if (splitPredicate(chunk, ++indexWritten)) {
+        logger.log(`writableChunk: splitting at index ${currentSplitIndex}`)
+        currentSplitIndex++
+        transforms[0]!.end()
+
+        currentWritable = writableFactory(currentSplitIndex)
+        transforms = transformFactories.map(f => f())
+        generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2))
+        _last(transforms).pipe(currentWritable)
+
+        writablesFinish.push(awaitFinish(currentWritable))
+      }
+    },
+    async final(cb) {
+      try {
+        transforms[0]!.end()
+        await Promise.all(writablesFinish)
+        logger.log('writableChunk: all writables are finished')
+        cb()
+      } catch (err) {
+        cb(err as Error)
+      }
+    },
+  })
+}
+
+/**
+ * This is a helper function to create a promise which resolves when the stream emits a 'finish'
+ * event.
+ * This is used to await all the writables in the final method of the writableChunk
+ */
+async function awaitFinish(stream: Writable): Promise<void> {
+  return await new Promise(resolve => {
+    stream.on('finish', resolve)
+  })
+}
+
+/**
+ * Generates an array of [arr[i], arr[i+1]] tuples from the input array.
+ * The resulting array will have a length of `arr.length - 1`.
+ * ```ts
+ * generateTuples([1, 2, 3, 4]) // [[1, 2], [2, 3], [3, 4]]
+ * ```
+ */
+function generateTuples<T>(arr: T[]): [T, T][] {
+  const tuples: [T, T][] = []
+  const arrCopy = _deepCopy(arr)
+  for (let i = 1; i < arrCopy.length; i++) {
+    tuples.push([arrCopy[i - 1]!, arrCopy[i]!])
+  }
+  return tuples
+}