@naturalcycles/nodejs-lib 15.111.0 → 15.113.0

package/dist/exec2/exec2.d.ts

@@ -37,7 +37,7 @@ declare class Exec2 {
      * shell: true
      * log: true
      */
-    spawn(cmd: string, opt?: SpawnOptions): void;
+    spawn(cmd: string, opt?: SpawnSyncOptions): void;
     /**
      * Reasons to use it:
      *
@@ -105,6 +105,12 @@ export interface SpawnOutput {
     stdout: string;
     stderr: string;
 }
+export interface SpawnSyncOptions extends SpawnOptions {
+    /**
+     * Defaults to 10 MiB (higher than Node's own default of 1 MiB).
+     */
+    maxBuffer?: number;
+}
 export interface SpawnAsyncOptions extends SpawnOptions {
     /**
      * Defaults to true.
@@ -202,5 +208,9 @@ export interface ExecOptions {
      * beware that stdio: 'inherit', means we don't get the output returned.
      */
     stdio?: StdioOptions;
+    /**
+     * Defaults to 10 MiB (higher than Node's own default of 1 MiB).
+     */
+    maxBuffer?: number;
 }
 export {};

package/dist/exec2/exec2.js

@@ -3,6 +3,15 @@ import { _since } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { AppError } from '@naturalcycles/js-lib/error/error.util.js';
 import { _substringAfterLast } from '@naturalcycles/js-lib/string/string.util.js';
 import { dimGrey, dimRed, hasColors, white } from '../colors/colors.js';
+/**
+ * Default `maxBuffer` (in bytes) for the synchronous `exec`/`spawn`.
+ * Set to 10 MiB, which is higher than Node's own default of 1 MiB.
+ * Node's 1 MiB is too low for common large outputs (e.g. `git log`, `git diff`,
+ * `tsc`/`eslint` reports), while 10 MiB still bounds runaway processes to a
+ * memory level that won't OOM constrained deploy/CI environments.
+ * Can be overridden per-call via the `maxBuffer` option.
+ */
+const defaultMaxBuffer = 10 * 1024 * 1024;
 /**
  * Set of utility functions to work with Spawn / Exec.
  *
@@ -40,7 +49,7 @@ class Exec2 {
      * log: true
      */
     spawn(cmd, opt = {}) {
-        const { shell = true, cwd, env, passProcessEnv = true, forceColor = hasColors, stdio = 'inherit', } = opt;
+        const { shell = true, cwd, env, passProcessEnv = true, forceColor = hasColors, stdio = 'inherit', maxBuffer = defaultMaxBuffer, } = opt;
         opt.log ??= true; // by default log should be true, as we are printing the output
         opt.logStart ??= opt.log;
         opt.logFinish ??= opt.log;
@@ -51,6 +60,7 @@ class Exec2 {
             stdio,
             shell,
             cwd,
+            maxBuffer,
             env: {
                 ...(passProcessEnv ? process.env : {}),
                 ...(forceColor ? { FORCE_COLOR: '1' } : {}),
@@ -81,7 +91,7 @@ class Exec2 {
      * log: false
      */
     exec(cmd, opt = {}) {
-        const { cwd, env, passProcessEnv = true, timeout, stdio } = opt;
+        const { cwd, env, passProcessEnv = true, timeout, stdio, maxBuffer = defaultMaxBuffer } = opt;
         opt.logStart ??= opt.log ?? false;
         opt.logFinish ??= opt.log ?? false;
         const started = Date.now();
@@ -93,6 +103,7 @@ class Exec2 {
                 // shell: undefined,
                 cwd,
                 timeout,
+                maxBuffer,
                 env: {
                     ...(passProcessEnv ? process.env : {}),
                     ...env,

package/dist/stream/pipeline.d.ts

@@ -1,7 +1,7 @@
 import type { Transform } from 'node:stream';
 import type { ReadableStream as WebReadableStream } from 'node:stream/web';
 import type { ZlibOptions, ZstdOptions } from 'node:zlib';
-import type { AbortableAsyncMapper, AsyncIndexedMapper, AsyncPredicate, END, IndexedMapper, Integer, NonNegativeInteger, PositiveInteger, Predicate, SKIP } from '@naturalcycles/js-lib/types';
+import type { AbortableAsyncMapper, AbortableMapper, AsyncIndexedMapper, AsyncPredicate, END, IndexedMapper, Integer, NonNegativeInteger, PositiveInteger, Predicate, SKIP } from '@naturalcycles/js-lib/types';
 import type { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream.model.js';
 import type { TransformLogProgressOptions } from './transform/transformLogProgress.js';
 import type { TransformMapOptions } from './transform/transformMap.js';
@@ -68,8 +68,8 @@ export declare class Pipeline<T = unknown> {
     filter(asyncPredicate: AsyncPredicate<T>, opt?: TransformMapOptions): this;
     filterSync(predicate: Predicate<T>, opt?: TransformOptions): this;
     offset(opt: TransformOffsetOptions): this;
-    tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformOptions): this;
-    tapSync(fn: IndexedMapper<T, any>, opt?: TransformOptions): this;
+    tap(fn: AsyncIndexedMapper<T, void>, opt?: TransformOptions): this;
+    tapSync(fn: IndexedMapper<T, void>, opt?: TransformOptions): this;
     throttle(opt: TransformThrottleOptions): this;
     throttleByRSS(opt: TransformThrottleByRSSOptions): this;
     /**
@@ -113,7 +113,7 @@ export declare class Pipeline<T = unknown> {
      */
     toNDJsonFile(outputFilePath: string, level?: Integer): Promise<void>;
     to(destination: WritableTyped<T>): Promise<void>;
-    forEach(fn: AsyncIndexedMapper<T, void>, opt?: TransformMapOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
-    forEachSync(fn: IndexedMapper<T, void>, opt?: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
+    forEach(fn: AbortableAsyncMapper<T, void>, opt?: TransformMapOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
+    forEachSync(fn: AbortableMapper<T, void>, opt?: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
     run(): Promise<void>;
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.111.0",
+  "version": "15.113.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@standard-schema/spec": "^1",
@@ -18,7 +18,7 @@
   },
   "devDependencies": {
     "@typescript/native-preview": "beta",
-    "@naturalcycles/dev-lib": "20.49.0"
+    "@naturalcycles/dev-lib": "18.4.2"
   },
   "exports": {
     ".": "./dist/index.js",

package/src/exec2/exec2.ts

@@ -10,6 +10,16 @@ import type {
 } from '@naturalcycles/js-lib/types'
 import { dimGrey, dimRed, hasColors, white } from '../colors/colors.js'
 
+/**
+ * Default `maxBuffer` (in bytes) for the synchronous `exec`/`spawn`.
+ * Set to 10 MiB, which is higher than Node's own default of 1 MiB.
+ * Node's 1 MiB is too low for common large outputs (e.g. `git log`, `git diff`,
+ * `tsc`/`eslint` reports), while 10 MiB still bounds runaway processes to a
+ * memory level that won't OOM constrained deploy/CI environments.
+ * Can be overridden per-call via the `maxBuffer` option.
+ */
+const defaultMaxBuffer = 10 * 1024 * 1024
+
 /**
  * Set of utility functions to work with Spawn / Exec.
  *
@@ -46,7 +56,7 @@ class Exec2 {
    * shell: true
    * log: true
    */
-  spawn(cmd: string, opt: SpawnOptions = {}): void {
+  spawn(cmd: string, opt: SpawnSyncOptions = {}): void {
     const {
       shell = true,
       cwd,
@@ -54,6 +64,7 @@ class Exec2 {
       passProcessEnv = true,
       forceColor = hasColors,
       stdio = 'inherit',
+      maxBuffer = defaultMaxBuffer,
     } = opt
     opt.log ??= true // by default log should be true, as we are printing the output
     opt.logStart ??= opt.log
@@ -66,6 +77,7 @@ class Exec2 {
       stdio,
       shell,
       cwd,
+      maxBuffer,
       env: {
         ...(passProcessEnv ? process.env : {}),
         ...(forceColor ? { FORCE_COLOR: '1' } : {}),
@@ -99,7 +111,7 @@ class Exec2 {
    * log: false
    */
   exec(cmd: string, opt: ExecOptions = {}): string {
-    const { cwd, env, passProcessEnv = true, timeout, stdio } = opt
+    const { cwd, env, passProcessEnv = true, timeout, stdio, maxBuffer = defaultMaxBuffer } = opt
     opt.logStart ??= opt.log ?? false
     opt.logFinish ??= opt.log ?? false
     const started = Date.now() as UnixTimestampMillis
@@ -112,6 +124,7 @@ class Exec2 {
         // shell: undefined,
         cwd,
         timeout,
+        maxBuffer,
         env: {
           ...(passProcessEnv ? process.env : {}),
           ...env,
@@ -355,6 +368,13 @@ export interface SpawnOutput {
   stderr: string
 }
 
+export interface SpawnSyncOptions extends SpawnOptions {
+  /**
+   * Defaults to 10 MiB (higher than Node's own default of 1 MiB).
+   */
+  maxBuffer?: number
+}
+
 export interface SpawnAsyncOptions extends SpawnOptions {
   /**
    * Defaults to true.
@@ -462,4 +482,9 @@ export interface ExecOptions {
    * beware that stdio: 'inherit', means we don't get the output returned.
    */
   stdio?: StdioOptions
+
+  /**
+   * Defaults to 10 MiB (higher than Node's own default of 1 MiB).
+   */
+  maxBuffer?: number
 }

package/src/stream/pipeline.ts

@@ -8,6 +8,7 @@ import { createAbortableSignal } from '@naturalcycles/js-lib'
 import { _passthroughPredicate } from '@naturalcycles/js-lib/types'
 import type {
   AbortableAsyncMapper,
+  AbortableMapper,
   AsyncIndexedMapper,
   AsyncPredicate,
   END,
@@ -236,12 +237,12 @@ export class Pipeline<T = unknown> {
     return this
   }
 
-  tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformOptions): this {
+  tap(fn: AsyncIndexedMapper<T, void>, opt?: TransformOptions): this {
     this.transforms.push(transformTap(fn, opt))
     return this
   }
 
-  tapSync(fn: IndexedMapper<T, any>, opt?: TransformOptions): this {
+  tapSync(fn: IndexedMapper<T, void>, opt?: TransformOptions): this {
     this.transforms.push(transformTapSync(fn, opt))
     return this
   }
@@ -420,7 +421,7 @@ export class Pipeline<T = unknown> {
   }
 
   async forEach(
-    fn: AsyncIndexedMapper<T, void>,
+    fn: AbortableAsyncMapper<T, void>,
     opt: TransformMapOptions<T, void> & TransformLogProgressOptions<T> = {},
   ): Promise<void> {
     this.transforms.push(
@@ -437,7 +438,7 @@ export class Pipeline<T = unknown> {
   }
 
   async forEachSync(
-    fn: IndexedMapper<T, void>,
+    fn: AbortableMapper<T, void>,
     opt: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T> = {},
   ): Promise<void> {
     this.transforms.push(