@naturalcycles/nodejs-lib 13.18.0 → 13.20.0

package/dist/index.d.ts

@@ -37,6 +37,7 @@ export * from './stream/progressLogger';
 export * from './stream/transform/transformChunk';
 export * from './stream/transform/transformFilter';
 export * from './stream/transform/transformLimit';
+export * from './stream/transform/transformOffset';
 export * from './stream/transform/transformLogProgress';
 export * from './stream/transform/transformMap';
 export * from './stream/transform/transformMapSimple';

package/dist/index.js

@@ -41,6 +41,7 @@ tslib_1.__exportStar(require("./stream/progressLogger"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformChunk"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformFilter"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformLimit"), exports);
+tslib_1.__exportStar(require("./stream/transform/transformOffset"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformLogProgress"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformMap"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformMapSimple"), exports);

package/dist/stream/transform/transformLimit.d.ts

@@ -34,7 +34,4 @@ export interface TransformLimitOptions extends TransformOptions {
  */
 export declare class TransformLimit extends AbortableTransform {
 }
-/**
- * 0 or falsy value means "no limit"
- */
-export declare function transformLimit<IN>(opt?: TransformLimitOptions): TransformTyped<IN, IN>;
+export declare function transformLimit<IN>(opt: TransformLimitOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformLimit.js

@@ -10,10 +10,7 @@ const stream_util_1 = require("../stream.util");
 class TransformLimit extends index_1.AbortableTransform {
 }
 exports.TransformLimit = TransformLimit;
-/**
- * 0 or falsy value means "no limit"
- */
-function transformLimit(opt = {}) {
+function transformLimit(opt) {
     const { logger = console, limit, debug } = opt;
     if (!limit) {
         // No limit - returning pass-through transform

package/dist/stream/transform/transformMap.d.ts

@@ -18,7 +18,10 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
      * Number of concurrently pending promises returned by `mapper`.
      *
-     * @default 16 (to match default highWatermark option for objectMode streams)
+     * Default is 32.
+     * It was recently changed up from 16, after some testing that shown that
+     * for simple low-cpu mapper functions 32 produces almost 2x throughput.
+     * For example, in scenarios like streaming a query from Datastore.
      */
     concurrency?: number;
     /**
@@ -83,7 +86,7 @@ export interface TransformMapStatsSummary extends TransformMapStats {
  *
  * Only works in objectMode (due to through2Concurrent).
  *
- * Concurrency defaults to 16.
+ * Concurrency defaults to 32.
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */

package/dist/stream/transform/transformMap.js

@@ -15,12 +15,12 @@ const stream_util_1 = require("../stream.util");
  *
  * Only works in objectMode (due to through2Concurrent).
  *
- * Concurrency defaults to 16.
+ * Concurrency defaults to 32.
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 function transformMap(mapper, opt = {}) {
-    const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
+    const { concurrency = 32, predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, onDone, metric = 'stream', logger = console, } = opt;
     const started = Date.now();
     let index = -1;

package/dist/stream/transform/transformOffset.d.ts

@@ -0,0 +1,10 @@
+import { TransformOptions, TransformTyped } from '../stream.model';
+export interface TransformOffsetOptions extends TransformOptions {
+    /**
+     * How many items to skip (offset) in the stream.
+     *
+     * Nullish value (e.g 0 or undefined) would mean "no offset".
+     */
+    offset?: number;
+}
+export declare function transformOffset<IN>(opt: TransformOffsetOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformOffset.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.transformOffset = void 0;
+const node_stream_1 = require("node:stream");
+const index_1 = require("../../index");
+function transformOffset(opt) {
+    const { offset } = opt;
+    if (!offset) {
+        // No offset - returning pass-through transform
+        return (0, index_1.transformNoOp)();
+    }
+    let i = 0; // so we start first chunk with 1
+    return new node_stream_1.Transform({
+        objectMode: true,
+        ...opt,
+        transform(chunk, _, cb) {
+            if (++i <= offset) {
+                return cb(); // skip
+            }
+            cb(null, chunk);
+        },
+    });
+}
+exports.transformOffset = transformOffset;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "13.18.0",
+  "version": "13.20.0",
   "scripts": {
     "prepare": "husky",
     "docs-serve": "vuepress dev docs",

package/src/index.ts

@@ -47,6 +47,7 @@ export * from './stream/progressLogger'
 export * from './stream/transform/transformChunk'
 export * from './stream/transform/transformFilter'
 export * from './stream/transform/transformLimit'
+export * from './stream/transform/transformOffset'
 export * from './stream/transform/transformLogProgress'
 export * from './stream/transform/transformMap'
 export * from './stream/transform/transformMapSimple'

package/src/stream/transform/transformLimit.ts

@@ -40,10 +40,7 @@ export interface TransformLimitOptions extends TransformOptions {
  */
 export class TransformLimit extends AbortableTransform {}
 
-/**
- * 0 or falsy value means "no limit"
- */
-export function transformLimit<IN>(opt: TransformLimitOptions = {}): TransformTyped<IN, IN> {
+export function transformLimit<IN>(opt: TransformLimitOptions): TransformTyped<IN, IN> {
   const { logger = console, limit, debug } = opt
 
   if (!limit) {

package/src/stream/transform/transformMap.ts

@@ -40,7 +40,10 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
   /**
    * Number of concurrently pending promises returned by `mapper`.
    *
-   * @default 16 (to match default highWatermark option for objectMode streams)
+   * Default is 32.
+   * It was recently changed up from 16, after some testing that shown that
+   * for simple low-cpu mapper functions 32 produces almost 2x throughput.
+   * For example, in scenarios like streaming a query from Datastore.
    */
   concurrency?: number
 
@@ -117,7 +120,7 @@ export interface TransformMapStatsSummary extends TransformMapStats {
  *
  * Only works in objectMode (due to through2Concurrent).
  *
- * Concurrency defaults to 16.
+ * Concurrency defaults to 32.
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
@@ -126,7 +129,7 @@ export function transformMap<IN = any, OUT = IN>(
   opt: TransformMapOptions<IN, OUT> = {},
 ): TransformTyped<IN, OUT> {
   const {
-    concurrency = 16,
+    concurrency = 32,
     predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     flattenArrayOutput,

package/src/stream/transform/transformOffset.ts

@@ -0,0 +1,34 @@
+import { Transform } from 'node:stream'
+import { transformNoOp } from '../../index'
+import { TransformOptions, TransformTyped } from '../stream.model'
+
+export interface TransformOffsetOptions extends TransformOptions {
+  /**
+   * How many items to skip (offset) in the stream.
+   *
+   * Nullish value (e.g 0 or undefined) would mean "no offset".
+   */
+  offset?: number
+}
+
+export function transformOffset<IN>(opt: TransformOffsetOptions): TransformTyped<IN, IN> {
+  const { offset } = opt
+
+  if (!offset) {
+    // No offset - returning pass-through transform
+    return transformNoOp()
+  }
+
+  let i = 0 // so we start first chunk with 1
+  return new Transform({
+    objectMode: true,
+    ...opt,
+    transform(chunk: IN, _, cb) {
+      if (++i <= offset) {
+        return cb() // skip
+      }
+
+      cb(null, chunk)
+    },
+  })
+}