@naturalcycles/nodejs-lib 15.36.0 → 15.37.0

package/dist/stream/index.d.ts

@@ -18,6 +18,7 @@ export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
+export * from './transform/transformMultiFork.js';
 export * from './transform/transformNoOp.js';
 export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';

package/dist/stream/index.js

@@ -18,6 +18,7 @@ export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
 export * from './transform/transformMapSimple.js';
 export * from './transform/transformMapSync.js';
+export * from './transform/transformMultiFork.js';
 export * from './transform/transformNoOp.js';
 export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';

package/dist/stream/transform/transformMultiFork.d.ts

@@ -0,0 +1,14 @@
+import type { Predicate } from '@naturalcycles/js-lib/types';
+import { Pipeline } from '../pipeline.js';
+import type { TransformOptions, TransformTyped } from '../stream.model.js';
+/**
+ * Like transformFork, but allows to fork multiple times,
+ * aka "split the stream" into chunks, and attach a Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
+ */
+export declare function transformMultiFork<T>(splitPredicate: Predicate<T>, fn: (pipeline: Pipeline<T>) => Promise<void>, opt?: TransformOptions): TransformTyped<T, T>;

package/dist/stream/transform/transformMultiFork.js

@@ -0,0 +1,79 @@
+import { Transform } from 'node:stream';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+import { Pipeline } from '../pipeline.js';
+import { createReadable } from '../readable/createReadable.js';
+/**
+ * Like transformFork, but allows to fork multiple times,
+ * aka "split the stream" into chunks, and attach a Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
+ */
+export function transformMultiFork(splitPredicate, fn, opt = {}) {
+    const { objectMode = true, highWaterMark } = opt;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+    let indexWritten = 0;
+    let splitIndex = 0;
+    let lock;
+    let fork = createNewFork();
+    return new Transform({
+        objectMode,
+        highWaterMark,
+        async transform(chunk, _, cb) {
+            // pass through to the "main" pipeline
+            // Main pipeline should handle backpressure "automatically",
+            // so, we're not maintaining a Lock for it
+            this.push(chunk);
+            if (lock) {
+                // Forked pipeline is locked - let's wait for it to call _read
+                await lock;
+                // lock is undefined at this point
+            }
+            // pass to the "forked" pipeline
+            const shouldContinue = fork.push(chunk);
+            if (!shouldContinue && !lock) {
+                // Forked pipeline indicates that we should Pause
+                lock = pDefer();
+                logger.debug(`TransformMultiFork(${splitIndex}): pause`);
+            }
+            if (splitPredicate(chunk, ++indexWritten)) {
+                logger.log(`TransformMultiFork(${splitIndex}): splitting to ${splitIndex + 1}`);
+                splitIndex++;
+                fork.push(null);
+                lock?.resolve();
+                lock = undefined;
+                fork = createNewFork();
+            }
+            // acknowledge that we've finished processing the input chunk
+            cb();
+        },
+        async final(cb) {
+            logger.log(`TransformMultiFork: final`);
+            // Pushing null "closes"/ends the secondary pipeline correctly
+            fork.push(null);
+            // Acknowledge that we've received `null` and passed it through to the fork
+            cb();
+        },
+    });
+    function createNewFork() {
+        const mySplitIndex = splitIndex;
+        const readable = createReadable([], {}, () => {
+            // `_read` is called
+            if (!lock)
+                return;
+            // We had a lock - let's Resume
+            logger.debug(`TransformMultiFork(${mySplitIndex}): resume`);
+            const lockCopy = lock;
+            lock = undefined;
+            lockCopy.resolve();
+        });
+        void fn(Pipeline.from(readable)).then(() => {
+            logger.log(`TransformMultiFork(${mySplitIndex}): done`);
+        });
+        return readable;
+    }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.36.0",
+  "version": "15.37.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/index.ts

@@ -18,6 +18,7 @@ export * from './transform/transformLogProgress.js'
 export * from './transform/transformMap.js'
 export * from './transform/transformMapSimple.js'
 export * from './transform/transformMapSync.js'
+export * from './transform/transformMultiFork.js'
 export * from './transform/transformNoOp.js'
 export * from './transform/transformOffset.js'
 export * from './transform/transformSplit.js'

package/src/stream/transform/transformMultiFork.ts

@@ -0,0 +1,97 @@
+import { Transform } from 'node:stream'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
+import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import type { Predicate } from '@naturalcycles/js-lib/types'
+import { Pipeline } from '../pipeline.js'
+import { createReadable } from '../readable/createReadable.js'
+import type { ReadableTyped, TransformOptions, TransformTyped } from '../stream.model.js'
+
+/**
+ * Like transformFork, but allows to fork multiple times,
+ * aka "split the stream" into chunks, and attach a Pipeline to
+ * each of the chunks.
+ *
+ * Example use case: you want to write to Cloud Storage, 1000 rows per file,
+ * each file needs its own destination Pipeline.
+ *
+ * @experimental
+ */
+export function transformMultiFork<T>(
+  splitPredicate: Predicate<T>,
+  fn: (pipeline: Pipeline<T>) => Promise<void>,
+  opt: TransformOptions = {},
+): TransformTyped<T, T> {
+  const { objectMode = true, highWaterMark } = opt
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+  let indexWritten = 0
+  let splitIndex = 0
+
+  let lock: DeferredPromise | undefined
+  let fork = createNewFork()
+
+  return new Transform({
+    objectMode,
+    highWaterMark,
+    async transform(chunk: T, _, cb) {
+      // pass through to the "main" pipeline
+      // Main pipeline should handle backpressure "automatically",
+      // so, we're not maintaining a Lock for it
+      this.push(chunk)
+
+      if (lock) {
+        // Forked pipeline is locked - let's wait for it to call _read
+        await lock
+        // lock is undefined at this point
+      }
+
+      // pass to the "forked" pipeline
+      const shouldContinue = fork.push(chunk)
+      if (!shouldContinue && !lock) {
+        // Forked pipeline indicates that we should Pause
+        lock = pDefer()
+        logger.debug(`TransformMultiFork(${splitIndex}): pause`)
+      }
+
+      if (splitPredicate(chunk, ++indexWritten)) {
+        logger.log(`TransformMultiFork(${splitIndex}): splitting to ${splitIndex + 1}`)
+        splitIndex++
+        fork.push(null)
+        lock?.resolve()
+        lock = undefined
+        fork = createNewFork()
+      }
+
+      // acknowledge that we've finished processing the input chunk
+      cb()
+    },
+    async final(cb) {
+      logger.log(`TransformMultiFork: final`)
+
+      // Pushing null "closes"/ends the secondary pipeline correctly
+      fork.push(null)
+
+      // Acknowledge that we've received `null` and passed it through to the fork
+      cb()
+    },
+  })
+
+  function createNewFork(): ReadableTyped<T> {
+    const mySplitIndex = splitIndex
+
+    const readable = createReadable<T>([], {}, () => {
+      // `_read` is called
+      if (!lock) return
+      // We had a lock - let's Resume
+      logger.debug(`TransformMultiFork(${mySplitIndex}): resume`)
+      const lockCopy = lock
+      lock = undefined
+      lockCopy.resolve()
+    })
+
+    void fn(Pipeline.from<T>(readable)).then(() => {
+      logger.log(`TransformMultiFork(${mySplitIndex}): done`)
+    })
+
+    return readable
+  }
+}