@naturalcycles/nodejs-lib 13.10.0 → 13.12.0

package/dist/index.d.ts

@@ -31,8 +31,6 @@ export * from './stream/pipeline/pipeline';
 export * from './stream/readable/readableCreate';
 export * from './stream/readable/readableForEach';
 export * from './stream/readable/readableFromArray';
-export * from './stream/readable/readableMap';
-export * from './stream/readable/readableMapToArray';
 export * from './stream/readable/readableToArray';
 export * from './stream/stream.model';
 export * from './stream/progressLogger';

package/dist/index.js

@@ -35,8 +35,6 @@ tslib_1.__exportStar(require("./stream/pipeline/pipeline"), exports);
 tslib_1.__exportStar(require("./stream/readable/readableCreate"), exports);
 tslib_1.__exportStar(require("./stream/readable/readableForEach"), exports);
 tslib_1.__exportStar(require("./stream/readable/readableFromArray"), exports);
-tslib_1.__exportStar(require("./stream/readable/readableMap"), exports);
-tslib_1.__exportStar(require("./stream/readable/readableMapToArray"), exports);
 tslib_1.__exportStar(require("./stream/readable/readableToArray"), exports);
 tslib_1.__exportStar(require("./stream/stream.model"), exports);
 tslib_1.__exportStar(require("./stream/progressLogger"), exports);

package/dist/stream/progressLogger.d.ts

@@ -1,5 +1,6 @@
 import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
-export interface ProgressLoggerCfg<IN = any> {
+import { ReadableMapper } from './stream.model';
+export interface ProgressLoggerCfg<T = any> {
     /**
      * Progress metric
      *
@@ -74,7 +75,7 @@ export interface ProgressLoggerCfg<IN = any> {
      *
      * chunk is undefined for "final" stats, otherwise is defined.
      */
-    extra?: (chunk: IN | undefined, index: number) => AnyObject;
+    extra?: (chunk: T | undefined, index: number) => AnyObject;
     /**
      * If specified - will multiply the counter by this number.
      * Useful e.g when using `transformBuffer({ batchSize: 500 })`, so
@@ -115,9 +116,9 @@ export interface ProgressLogItem extends AnyObject {
     rps10?: number;
     rpsTotal?: number;
 }
-export declare class ProgressLogger<IN> implements Disposable {
-    constructor(cfg?: ProgressLoggerCfg<IN>);
-    cfg: ProgressLoggerCfg<IN> & {
+export declare class ProgressLogger<T> implements Disposable {
+    constructor(cfg?: ProgressLoggerCfg<T>);
+    cfg: ProgressLoggerCfg<T> & {
         logEvery: number;
         logSizesBuffer: number;
         batchSize: number;
@@ -134,7 +135,7 @@ export declare class ProgressLogger<IN> implements Disposable {
     private sizes?;
     private sizesZipped?;
     private start;
-    log(chunk?: IN): void;
+    log(chunk?: T): void;
     done(): void;
     [Symbol.dispose](): void;
     private logStats;
@@ -142,4 +143,10 @@ export declare class ProgressLogger<IN> implements Disposable {
 /**
  * Create new ProgressLogger.
  */
-export declare function progressLogger<IN>(cfg?: ProgressLoggerCfg<IN>): ProgressLogger<IN>;
+export declare function progressLogger<T>(cfg?: ProgressLoggerCfg<T>): ProgressLogger<T>;
+/**
+ * Limitation: I don't know how to catch the `final` callback to log final stats.
+ *
+ * @experimental
+ */
+export declare function progressReadableMapper<T>(cfg?: ProgressLoggerCfg<T>): ReadableMapper<T, T>;

package/dist/stream/progressLogger.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.progressLogger = exports.ProgressLogger = void 0;
+exports.progressReadableMapper = exports.progressLogger = exports.ProgressLogger = void 0;
 const node_util_1 = require("node:util");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const colors_1 = require("../colors/colors");
@@ -118,3 +118,16 @@ function progressLogger(cfg = {}) {
     return new ProgressLogger(cfg);
 }
 exports.progressLogger = progressLogger;
+/**
+ * Limitation: I don't know how to catch the `final` callback to log final stats.
+ *
+ * @experimental
+ */
+function progressReadableMapper(cfg = {}) {
+    const progress = new ProgressLogger(cfg);
+    return chunk => {
+        progress.log(chunk);
+        return chunk;
+    };
+}
+exports.progressReadableMapper = progressReadableMapper;

package/dist/stream/readable/readableCreate.d.ts

@@ -19,4 +19,4 @@ export declare function readableCreate<T>(items?: Iterable<T>, opt?: ReadableOpt
 /**
  * Convenience type-safe wrapper around Readable.from() that infers the Type of input.
  */
-export declare function readableFrom<T>(items: Iterable<T> | AsyncIterable<T>, opt?: ReadableOptions): ReadableTyped<T>;
+export declare function readableFrom<T>(iterable: Iterable<T> | AsyncIterable<T>, opt?: ReadableOptions): ReadableTyped<T>;

package/dist/stream/readable/readableCreate.js

@@ -31,7 +31,7 @@ exports.readableCreate = readableCreate;
 /**
  * Convenience type-safe wrapper around Readable.from() that infers the Type of input.
  */
-function readableFrom(items, opt) {
-    return node_stream_1.Readable.from(items, opt);
+function readableFrom(iterable, opt) {
+    return node_stream_1.Readable.from(iterable, opt);
 }
 exports.readableFrom = readableFrom;

package/dist/stream/readable/readableToArray.d.ts

@@ -2,5 +2,8 @@ import { ReadableTyped } from '../stream.model';
 /**
  * Convenience function to read the whole Readable stream into Array (in-memory)
  * and return that array.
+ *
+ * Native `await readable.toArray()` can be used instead.
+ * This helper is kept for type-safery support.
  */
 export declare function readableToArray<T>(readable: ReadableTyped<T>): Promise<T[]>;

package/dist/stream/readable/readableToArray.js

@@ -4,12 +4,18 @@ exports.readableToArray = void 0;
 /**
  * Convenience function to read the whole Readable stream into Array (in-memory)
  * and return that array.
+ *
+ * Native `await readable.toArray()` can be used instead.
+ * This helper is kept for type-safery support.
  */
 async function readableToArray(readable) {
-    const a = [];
-    for await (const item of readable) {
-        a.push(item);
-    }
-    return a;
+    return await readable.toArray();
+    // const a: T[] = []
+    //
+    // for await (const item of readable) {
+    //   a.push(item)
+    // }
+    //
+    // return a
 }
 exports.readableToArray = readableToArray;

package/dist/stream/stream.model.d.ts

@@ -1,10 +1,42 @@
 /// <reference types="node" />
-import { Readable, Transform, Writable } from 'node:stream';
+import type { Readable, Transform, Writable } from 'node:stream';
+import type { Promisable } from '@naturalcycles/js-lib';
+export interface ReadableSignalOptions {
+    /** allows destroying the stream if the signal is aborted. */
+    signal?: AbortSignal;
+}
+export interface ReadableArrayOptions {
+    /** the maximum concurrent invocations of `fn` to call on the stream at once. **Default: 1**. */
+    concurrency?: number;
+    /** allows destroying the stream if the signal is aborted. */
+    signal?: AbortSignal;
+}
+export type ReadableMapper<IN, OUT> = (data: IN, opt?: ReadableSignalOptions) => Promisable<OUT>;
+export type ReadableFlatMapper<IN, OUT> = (data: IN, opt?: ReadableSignalOptions) => Promisable<OUT[]>;
+export type ReadableVoidMapper<IN> = (data: IN, opt?: ReadableSignalOptions) => void | Promise<void>;
+export type ReadablePredicate<IN> = (data: IN, opt?: ReadableSignalOptions) => boolean | Promise<boolean>;
 export interface ReadableTyped<T> extends Readable {
+    toArray: (opt?: ReadableSignalOptions) => Promise<T[]>;
+    map: <OUT>(mapper: ReadableMapper<T, OUT>, opt?: ReadableArrayOptions) => ReadableTyped<OUT>;
+    flatMap: <OUT>(mapper: ReadableFlatMapper<T, OUT>, opt?: ReadableArrayOptions) => ReadableTyped<OUT>;
+    filter: (predicate: ReadablePredicate<T>, opt?: ReadableArrayOptions) => ReadableTyped<T>;
+    forEach: (mapper: ReadableVoidMapper<T>, opt?: ReadableArrayOptions) => Promise<void>;
+    take: (limit: number, opt?: ReadableSignalOptions) => ReadableTyped<T>;
+    drop: (limit: number, opt?: ReadableSignalOptions) => ReadableTyped<T>;
 }
 export interface WritableTyped<T> extends Writable {
 }
-export interface TransformTyped<IN, OUT = IN> extends Transform {
+/**
+ * Type alias that indicates that the Readable is not in objectMode,
+ * e.g returns a binary stream (like a gzip stream).
+ */
+export type ReadableBinary = Readable;
+/**
+ * Type alias that indicates that the Writable is not in objectMode,
+ * e.g reads a binary stream (like a gzip stream).
+ */
+export type WritableBinary = Writable;
+export interface TransformTyped<IN, OUT> extends Transform {
 }
 export interface TransformOptions {
     /**

package/dist/stream/transform/transformTee.d.ts

@@ -10,5 +10,5 @@ type AnyStream = NodeJS.WritableStream | NodeJS.ReadWriteStream;
  *
  * @experimental
  */
-export declare function transformTee<T>(streams: AnyStream[]): TransformTyped<T>;
+export declare function transformTee<T>(streams: AnyStream[]): TransformTyped<T, T>;
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "13.10.0",
+  "version": "13.12.0",
   "scripts": {
     "prepare": "husky",
     "docs-serve": "vuepress dev docs",

package/src/index.ts

@@ -41,8 +41,6 @@ export * from './stream/pipeline/pipeline'
 export * from './stream/readable/readableCreate'
 export * from './stream/readable/readableForEach'
 export * from './stream/readable/readableFromArray'
-export * from './stream/readable/readableMap'
-export * from './stream/readable/readableMapToArray'
 export * from './stream/readable/readableToArray'
 export * from './stream/stream.model'
 export * from './stream/progressLogger'

package/src/stream/progressLogger.ts

@@ -10,8 +10,9 @@ import {
 } from '@naturalcycles/js-lib'
 import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors'
 import { SizeStack } from './sizeStack'
+import { ReadableMapper } from './stream.model'
 
-export interface ProgressLoggerCfg<IN = any> {
+export interface ProgressLoggerCfg<T = any> {
   /**
    * Progress metric
    *
@@ -98,7 +99,7 @@ export interface ProgressLoggerCfg<IN = any> {
    *
    * chunk is undefined for "final" stats, otherwise is defined.
    */
-  extra?: (chunk: IN | undefined, index: number) => AnyObject
+  extra?: (chunk: T | undefined, index: number) => AnyObject
 
   /**
    * If specified - will multiply the counter by this number.
@@ -150,8 +151,8 @@ const inspectOpt: InspectOptions = {
   breakLength: 300,
 }
 
-export class ProgressLogger<IN> implements Disposable {
-  constructor(cfg: ProgressLoggerCfg<IN> = {}) {
+export class ProgressLogger<T> implements Disposable {
+  constructor(cfg: ProgressLoggerCfg<T> = {}) {
     this.cfg = {
       metric: 'progress',
       rss: true,
@@ -170,7 +171,7 @@ export class ProgressLogger<IN> implements Disposable {
     this.logStats() // initial
   }
 
-  cfg!: ProgressLoggerCfg<IN> & {
+  cfg!: ProgressLoggerCfg<T> & {
     logEvery: number
     logSizesBuffer: number
     batchSize: number
@@ -201,7 +202,7 @@ export class ProgressLogger<IN> implements Disposable {
       : undefined
   }
 
-  log(chunk?: IN): void {
+  log(chunk?: T): void {
     this.progress++
     this.processedLastSecond++
 
@@ -223,7 +224,7 @@ export class ProgressLogger<IN> implements Disposable {
     this.done()
   }
 
-  private logStats(chunk?: IN, final = false, tenx = false): void {
+  private logStats(chunk?: T, final = false, tenx = false): void {
     if (!this.cfg.logProgress) return
 
     const {
@@ -304,6 +305,20 @@ export class ProgressLogger<IN> implements Disposable {
 /**
  * Create new ProgressLogger.
  */
-export function progressLogger<IN>(cfg: ProgressLoggerCfg<IN> = {}): ProgressLogger<IN> {
+export function progressLogger<T>(cfg: ProgressLoggerCfg<T> = {}): ProgressLogger<T> {
   return new ProgressLogger(cfg)
 }
+
+/**
+ * Limitation: I don't know how to catch the `final` callback to log final stats.
+ *
+ * @experimental
+ */
+export function progressReadableMapper<T>(cfg: ProgressLoggerCfg<T> = {}): ReadableMapper<T, T> {
+  const progress = new ProgressLogger(cfg)
+
+  return chunk => {
+    progress.log(chunk)
+    return chunk
+  }
+}

package/src/stream/readable/readableCreate.ts

@@ -34,8 +34,8 @@ export function readableCreate<T>(
  * Convenience type-safe wrapper around Readable.from() that infers the Type of input.
  */
 export function readableFrom<T>(
-  items: Iterable<T> | AsyncIterable<T>,
+  iterable: Iterable<T> | AsyncIterable<T>,
   opt?: ReadableOptions,
 ): ReadableTyped<T> {
-  return Readable.from(items, opt)
+  return Readable.from(iterable, opt)
 }

package/src/stream/readable/readableToArray.ts

@@ -3,13 +3,17 @@ import { ReadableTyped } from '../stream.model'
 /**
  * Convenience function to read the whole Readable stream into Array (in-memory)
  * and return that array.
+ *
+ * Native `await readable.toArray()` can be used instead.
+ * This helper is kept for type-safery support.
  */
 export async function readableToArray<T>(readable: ReadableTyped<T>): Promise<T[]> {
-  const a: T[] = []
-
-  for await (const item of readable) {
-    a.push(item)
-  }
-
-  return a
+  return await readable.toArray()
+  // const a: T[] = []
+  //
+  // for await (const item of readable) {
+  //   a.push(item)
+  // }
+  //
+  // return a
 }

package/src/stream/stream.model.ts

@@ -1,13 +1,66 @@
-import { Readable, Transform, Writable } from 'node:stream'
+import type { Readable, Transform, Writable } from 'node:stream'
+import type { Promisable } from '@naturalcycles/js-lib'
 
-// eslint-disable-next-line unused-imports/no-unused-vars
-export interface ReadableTyped<T> extends Readable {}
+export interface ReadableSignalOptions {
+  /** allows destroying the stream if the signal is aborted. */
+  signal?: AbortSignal
+}
+
+export interface ReadableArrayOptions {
+  /** the maximum concurrent invocations of `fn` to call on the stream at once. **Default: 1**. */
+  concurrency?: number
+  /** allows destroying the stream if the signal is aborted. */
+  signal?: AbortSignal
+}
+
+export type ReadableMapper<IN, OUT> = (data: IN, opt?: ReadableSignalOptions) => Promisable<OUT>
+
+export type ReadableFlatMapper<IN, OUT> = (
+  data: IN,
+  opt?: ReadableSignalOptions,
+) => Promisable<OUT[]>
+
+export type ReadableVoidMapper<IN> = (data: IN, opt?: ReadableSignalOptions) => void | Promise<void>
+
+export type ReadablePredicate<IN> = (
+  data: IN,
+  opt?: ReadableSignalOptions,
+) => boolean | Promise<boolean>
+
+export interface ReadableTyped<T> extends Readable {
+  toArray: (opt?: ReadableSignalOptions) => Promise<T[]>
+
+  map: <OUT>(mapper: ReadableMapper<T, OUT>, opt?: ReadableArrayOptions) => ReadableTyped<OUT>
+
+  flatMap: <OUT>(
+    mapper: ReadableFlatMapper<T, OUT>,
+    opt?: ReadableArrayOptions,
+  ) => ReadableTyped<OUT>
+
+  filter: (predicate: ReadablePredicate<T>, opt?: ReadableArrayOptions) => ReadableTyped<T>
+
+  forEach: (mapper: ReadableVoidMapper<T>, opt?: ReadableArrayOptions) => Promise<void>
+
+  take: (limit: number, opt?: ReadableSignalOptions) => ReadableTyped<T>
+  drop: (limit: number, opt?: ReadableSignalOptions) => ReadableTyped<T>
+}
 
 // eslint-disable-next-line unused-imports/no-unused-vars
 export interface WritableTyped<T> extends Writable {}
 
+/**
+ * Type alias that indicates that the Readable is not in objectMode,
+ * e.g returns a binary stream (like a gzip stream).
+ */
+export type ReadableBinary = Readable
+/**
+ * Type alias that indicates that the Writable is not in objectMode,
+ * e.g reads a binary stream (like a gzip stream).
+ */
+export type WritableBinary = Writable
+
 // eslint-disable-next-line unused-imports/no-unused-vars
-export interface TransformTyped<IN, OUT = IN> extends Transform {}
+export interface TransformTyped<IN, OUT> extends Transform {}
 
 export interface TransformOptions {
   /**

package/src/stream/transform/transformTee.ts

@@ -14,7 +14,7 @@ type AnyStream = NodeJS.WritableStream | NodeJS.ReadWriteStream
  *
  * @experimental
  */
-export function transformTee<T>(streams: AnyStream[]): TransformTyped<T> {
+export function transformTee<T>(streams: AnyStream[]): TransformTyped<T, T> {
   const readable = readableCreate<T>()
 
   const secondPipelinePromise = _pipeline([readable, ...streams])

package/dist/stream/readable/readableMap.d.ts

@@ -1,3 +0,0 @@
-import { AbortableAsyncMapper } from '@naturalcycles/js-lib';
-import { ReadableTyped } from '../stream.model';
-export declare function readableMap<IN, OUT>(readable: ReadableTyped<IN>, mapper: AbortableAsyncMapper<IN, OUT>): ReadableTyped<OUT>;

package/dist/stream/readable/readableMap.js

@@ -1,31 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.readableMap = void 0;
-const node_stream_1 = require("node:stream");
-const js_lib_1 = require("@naturalcycles/js-lib");
-function readableMap(readable, mapper) {
-    let i = -1;
-    const stream = readable
-        .on('error', err => stream.emit('error', err))
-        .pipe(new node_stream_1.Transform({
-        objectMode: true,
-        async transform(chunk, _enc, cb) {
-            try {
-                const r = await mapper(chunk, ++i);
-                if (r === js_lib_1.SKIP) {
-                    cb();
-                }
-                else {
-                    // _assert(r !== END, `readableMap END not supported`)
-                    cb(null, r);
-                }
-            }
-            catch (err) {
-                console.error(err);
-                cb(err);
-            }
-        },
-    }));
-    return stream;
-}
-exports.readableMap = readableMap;

package/dist/stream/readable/readableMapToArray.d.ts

@@ -1,12 +0,0 @@
-import { AsyncMapper } from '@naturalcycles/js-lib';
-import { ReadableTyped } from '../stream.model';
-import { TransformMapOptions } from '../transform/transformMap';
-/**
- * Map Readable items to array of results (in memory),
- * passing each result via `transformMap`.
- *
- * Warning! All results are stored in memory (no backpressure).
- *
- * Try native readable.toArray instead!
- */
-export declare function readableMapToArray<IN, OUT = IN>(stream: ReadableTyped<IN>, mapper?: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): Promise<OUT[]>;

package/dist/stream/readable/readableMapToArray.js

@@ -1,18 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.readableMapToArray = void 0;
-const index_1 = require("../../index");
-/**
- * Map Readable items to array of results (in memory),
- * passing each result via `transformMap`.
- *
- * Warning! All results are stored in memory (no backpressure).
- *
- * Try native readable.toArray instead!
- */
-async function readableMapToArray(stream, mapper = item => item, opt) {
-    const res = [];
-    await (0, index_1._pipeline)([stream, (0, index_1.transformMap)(mapper, opt), (0, index_1.writablePushToArray)(res)]);
-    return res;
-}
-exports.readableMapToArray = readableMapToArray;

package/src/stream/readable/readableMap.ts

@@ -1,34 +0,0 @@
-import { Transform } from 'node:stream'
-import { AbortableAsyncMapper, SKIP } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '../stream.model'
-
-export function readableMap<IN, OUT>(
-  readable: ReadableTyped<IN>,
-  mapper: AbortableAsyncMapper<IN, OUT>,
-): ReadableTyped<OUT> {
-  let i = -1
-
-  const stream: ReadableTyped<OUT> = readable
-    .on('error', err => stream.emit('error', err))
-    .pipe(
-      new Transform({
-        objectMode: true,
-        async transform(chunk, _enc, cb) {
-          try {
-            const r = await mapper(chunk, ++i)
-            if (r === SKIP) {
-              cb()
-            } else {
-              // _assert(r !== END, `readableMap END not supported`)
-              cb(null, r)
-            }
-          } catch (err) {
-            console.error(err)
-            cb(err as Error)
-          }
-        },
-      }),
-    )
-
-  return stream
-}

package/src/stream/readable/readableMapToArray.ts

@@ -1,24 +0,0 @@
-import { AsyncMapper } from '@naturalcycles/js-lib'
-import { transformMap, writablePushToArray, _pipeline } from '../../index'
-import { ReadableTyped } from '../stream.model'
-import { TransformMapOptions } from '../transform/transformMap'
-
-/**
- * Map Readable items to array of results (in memory),
- * passing each result via `transformMap`.
- *
- * Warning! All results are stored in memory (no backpressure).
- *
- * Try native readable.toArray instead!
- */
-export async function readableMapToArray<IN, OUT = IN>(
-  stream: ReadableTyped<IN>,
-  mapper: AsyncMapper<IN, OUT> = item => item as any,
-  opt?: TransformMapOptions<IN, OUT>,
-): Promise<OUT[]> {
-  const res: OUT[] = []
-
-  await _pipeline([stream, transformMap(mapper, opt), writablePushToArray(res)])
-
-  return res
-}