@naturalcycles/nodejs-lib 15.35.0 → 15.36.0

package/dist/stream/index.d.ts

@@ -26,5 +26,6 @@ export * from './transform/transformThrottle.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';
+export * from './writable/writableChunk.js';
 export * from './writable/writablePushToArray.js';
 export * from './writable/writableVoid.js';

package/dist/stream/index.js

@@ -26,5 +26,6 @@ export * from './transform/transformThrottle.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';
+export * from './writable/writableChunk.js';
 export * from './writable/writablePushToArray.js';
 export * from './writable/writableVoid.js';

package/dist/stream/writable/writableChunk.d.ts

@@ -0,0 +1,15 @@
+import type { Predicate } from '@naturalcycles/js-lib/types';
+import { type TransformOptions, type TransformTyped, type WritableTyped } from '../index.js';
+export interface WritableChunkOptions<T> extends TransformOptions {
+    splitPredicate: Predicate<T>;
+    transformFactories?: (() => TransformTyped<T, T>)[];
+    writableFactory: (splitIndex: number) => WritableTyped<T>;
+}
+/**
+ * Allows to split the output to multiple files by splitting into chunks
+ * based on `shouldSplitFn`.
+ * `transformFactories` are used to create a chain of transforms for each chunk.
+ * It was meant to be used with createGzip, which needs a proper start and end for each chunk
+ * for the output file to be a valid gzip file.
+ */
+export declare function writableChunk<T>(opt: WritableChunkOptions<T>): WritableTyped<T>;

package/dist/stream/writable/writableChunk.js

@@ -0,0 +1,83 @@
+import { Writable } from 'node:stream';
+import { _first, _last } from '@naturalcycles/js-lib/array';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
+import { _deepCopy } from '@naturalcycles/js-lib/object';
+import { transformNoOp, } from '../index.js';
+/**
+ * Allows to split the output to multiple files by splitting into chunks
+ * based on `shouldSplitFn`.
+ * `transformFactories` are used to create a chain of transforms for each chunk.
+ * It was meant to be used with createGzip, which needs a proper start and end for each chunk
+ * for the output file to be a valid gzip file.
+ */
+export function writableChunk(opt) {
+    const { highWaterMark, splitPredicate, transformFactories = [], writableFactory } = opt;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+    let indexWritten = 0;
+    let currentSplitIndex = 0;
+    // We don't want to have an empty chain, so we add a no-op transform
+    if (transformFactories.length === 0) {
+        transformFactories.push((transformNoOp));
+    }
+    // Create the transforms as well as the Writable, and pipe them together
+    let currentWritable = writableFactory(currentSplitIndex);
+    let transforms = transformFactories.map(f => f());
+    generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2));
+    _last(transforms).pipe(currentWritable);
+    // We keep track of all the pending writables, so we can await them in the final method
+    const writablesFinish = [awaitFinish(currentWritable)];
+    return new Writable({
+        objectMode: true,
+        highWaterMark,
+        write(chunk, _, cb) {
+            // pipe will take care of piping the data through the different streams correctly
+            transforms[0].write(chunk, cb);
+            if (splitPredicate(chunk, ++indexWritten)) {
+                logger.log(`writableChunk: splitting at index ${currentSplitIndex}`);
+                currentSplitIndex++;
+                transforms[0].end();
+                currentWritable = writableFactory(currentSplitIndex);
+                transforms = transformFactories.map(f => f());
+                generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2));
+                _last(transforms).pipe(currentWritable);
+                writablesFinish.push(awaitFinish(currentWritable));
+            }
+        },
+        async final(cb) {
+            try {
+                transforms[0].end();
+                await Promise.all(writablesFinish);
+                logger.log('writableChunk: all writables are finished');
+                cb();
+            }
+            catch (err) {
+                cb(err);
+            }
+        },
+    });
+}
+/**
+ * This is a helper function to create a promise which resolves when the stream emits a 'finish'
+ * event.
+ * This is used to await all the writables in the final method of the writableChunk
+ */
+async function awaitFinish(stream) {
+    return await new Promise(resolve => {
+        stream.on('finish', resolve);
+    });
+}
+/**
+ * Generates an array of [arr[i], arr[i+1]] tuples from the input array.
+ * The resulting array will have a length of `arr.length - 1`.
+ * ```ts
+ * generateTuples([1, 2, 3, 4]) // [[1, 2], [2, 3], [3, 4]]
+ * ```
+ */
+function generateTuples(arr) {
+    const tuples = [];
+    const arrCopy = _deepCopy(arr);
+    for (let i = 1; i < arrCopy.length; i++) {
+        tuples.push([arrCopy[i - 1], arrCopy[i]]);
+    }
+    return tuples;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.35.0",
+  "version": "15.36.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/index.ts

@@ -26,5 +26,6 @@ export * from './transform/transformThrottle.js'
 export * from './transform/worker/baseWorkerClass.js'
 export * from './transform/worker/transformMultiThreaded.js'
 export * from './transform/worker/transformMultiThreaded.model.js'
+export * from './writable/writableChunk.js'
 export * from './writable/writablePushToArray.js'
 export * from './writable/writableVoid.js'

package/src/stream/writable/writableChunk.ts

@@ -0,0 +1,104 @@
+import { Writable } from 'node:stream'
+import { _first, _last } from '@naturalcycles/js-lib/array'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
+import { _deepCopy } from '@naturalcycles/js-lib/object'
+import type { Predicate } from '@naturalcycles/js-lib/types'
+import {
+  transformNoOp,
+  type TransformOptions,
+  type TransformTyped,
+  type WritableTyped,
+} from '../index.js'
+
+export interface WritableChunkOptions<T> extends TransformOptions {
+  splitPredicate: Predicate<T>
+  transformFactories?: (() => TransformTyped<T, T>)[]
+  writableFactory: (splitIndex: number) => WritableTyped<T>
+}
+
+/**
+ * Allows to split the output to multiple files by splitting into chunks
+ * based on `shouldSplitFn`.
+ * `transformFactories` are used to create a chain of transforms for each chunk.
+ * It was meant to be used with createGzip, which needs a proper start and end for each chunk
+ * for the output file to be a valid gzip file.
+ */
+export function writableChunk<T>(opt: WritableChunkOptions<T>): WritableTyped<T> {
+  const { highWaterMark, splitPredicate, transformFactories = [], writableFactory } = opt
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+
+  let indexWritten = 0
+  let currentSplitIndex = 0
+  // We don't want to have an empty chain, so we add a no-op transform
+  if (transformFactories.length === 0) {
+    transformFactories.push(transformNoOp<T>)
+  }
+
+  // Create the transforms as well as the Writable, and pipe them together
+  let currentWritable = writableFactory(currentSplitIndex)
+  let transforms = transformFactories.map(f => f())
+  generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2))
+  _last(transforms).pipe(currentWritable)
+
+  // We keep track of all the pending writables, so we can await them in the final method
+  const writablesFinish: Promise<void>[] = [awaitFinish(currentWritable)]
+
+  return new Writable({
+    objectMode: true,
+    highWaterMark,
+    write(chunk: T, _, cb) {
+      // pipe will take care of piping the data through the different streams correctly
+      transforms[0]!.write(chunk, cb)
+
+      if (splitPredicate(chunk, ++indexWritten)) {
+        logger.log(`writableChunk: splitting at index ${currentSplitIndex}`)
+        currentSplitIndex++
+        transforms[0]!.end()
+
+        currentWritable = writableFactory(currentSplitIndex)
+        transforms = transformFactories.map(f => f())
+        generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2))
+        _last(transforms).pipe(currentWritable)
+
+        writablesFinish.push(awaitFinish(currentWritable))
+      }
+    },
+    async final(cb) {
+      try {
+        transforms[0]!.end()
+        await Promise.all(writablesFinish)
+        logger.log('writableChunk: all writables are finished')
+        cb()
+      } catch (err) {
+        cb(err as Error)
+      }
+    },
+  })
+}
+
+/**
+ * This is a helper function to create a promise which resolves when the stream emits a 'finish'
+ * event.
+ * This is used to await all the writables in the final method of the writableChunk
+ */
+async function awaitFinish(stream: Writable): Promise<void> {
+  return await new Promise(resolve => {
+    stream.on('finish', resolve)
+  })
+}
+
+/**
+ * Generates an array of [arr[i], arr[i+1]] tuples from the input array.
+ * The resulting array will have a length of `arr.length - 1`.
+ * ```ts
+ * generateTuples([1, 2, 3, 4]) // [[1, 2], [2, 3], [3, 4]]
+ * ```
+ */
+function generateTuples<T>(arr: T[]): [T, T][] {
+  const tuples: [T, T][] = []
+  const arrCopy = _deepCopy(arr)
+  for (let i = 1; i < arrCopy.length; i++) {
+    tuples.push([arrCopy[i - 1]!, arrCopy[i]!])
+  }
+  return tuples
+}