@naturalcycles/nodejs-lib 15.123.1 → 15.125.0

package/dist/log/log.util.d.ts

@@ -1,4 +1,11 @@
+import type { LogEntry } from '@naturalcycles/js-lib/log';
+import type { InspectAnyOptions } from '../string/inspect.js';
 /**
  * CommonLogger that logs to process.stdout directly (bypassing console.log)!
  */
 export declare const stdoutLogger: any;
+/**
+ * Renders a LogEntry as one line: the msg, the inspected err and the inspected remaining fields,
+ * separated by a space.
+ */
+export declare function logEntryToString({ level: _level, msg, err, ...fields }: LogEntry, opt?: InspectAnyOptions): string;

package/dist/log/log.util.js

@@ -3,6 +3,20 @@ import { _inspect } from '../string/inspect.js';
 /**
  * CommonLogger that logs to process.stdout directly (bypassing console.log)!
  */
-export const stdoutLogger = commonLoggerCreate((_level, args) => {
-    process.stdout.write(args.map(a => _inspect(a)).join(' ') + '\n');
+export const stdoutLogger = commonLoggerCreate(entry => {
+    process.stdout.write(logEntryToString(entry) + '\n');
 });
+/**
+ * Renders a LogEntry as one line: the msg, the inspected err and the inspected remaining fields,
+ * separated by a space.
+ */
+export function logEntryToString({ level: _level, msg, err, ...fields }, opt = {}) {
+    const parts = [];
+    if (msg !== undefined)
+        parts.push(msg);
+    if (err !== undefined)
+        parts.push(_inspect(err, { includeErrorStack: true, ...opt }));
+    if (Object.keys(fields).length)
+        parts.push(_inspect(fields, opt));
+    return parts.join(' ');
+}

package/dist/script/runScript.d.ts

@@ -7,7 +7,7 @@ export interface RunScriptOptions {
      */
     noExit?: boolean;
     /**
-     * Default to `console`
+     * Default to `consoleLogger`
      */
     logger?: CommonLogger;
     /**

package/dist/script/runScript.js

@@ -1,5 +1,6 @@
 import os from 'node:os';
 import { AsyncManager } from '@naturalcycles/js-lib';
+import { consoleLogger } from '@naturalcycles/js-lib/log';
 import { setGlobalStringifyFunction } from '@naturalcycles/js-lib/string/stringify.js';
 import { dimGrey } from '../colors/colors.js';
 import { loadEnvFileIfExists } from '../node.util.js';
@@ -27,13 +28,13 @@ const { DEBUG_RUN_SCRIPT } = process.env;
 export function runScript(fn, opt = {}) {
     checkAndlogEnvironment();
     setGlobalStringifyFunction(inspectStringifyFn);
-    const { logger = console, noExit, registerUncaughtExceptionHandlers = true } = opt;
+    const { logger = consoleLogger, noExit, registerUncaughtExceptionHandlers = true } = opt;
     if (registerUncaughtExceptionHandlers || DEBUG_RUN_SCRIPT) {
         process.on('uncaughtException', err => {
-            logger.error('runScript uncaughtException:', err);
+            logger.error({ msg: 'runScript uncaughtException:', err });
         });
         process.on('unhandledRejection', err => {
-            logger.error('runScript unhandledRejection:', err);
+            logger.error({ msg: 'runScript unhandledRejection:', err });
         });
     }
     if (DEBUG_RUN_SCRIPT) {
@@ -54,7 +55,7 @@ export function runScript(fn, opt = {}) {
             }
         }
         catch (err) {
-            logger.error('runScript error:', err);
+            logger.error({ msg: 'runScript error:', err });
             process.exitCode = 1;
             if (!noExit) {
                 setImmediate(() => process.exit(1));

package/dist/slack/slack.service.js

@@ -1,7 +1,7 @@
 import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { getFetcher } from '@naturalcycles/js-lib/http';
-import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
-import { _omit } from '@naturalcycles/js-lib/object/object.util.js';
+import { commonLoggerCreate, consoleLogger, createCommonLoggerAtLevel, } from '@naturalcycles/js-lib/log';
+import { _filterFalsyValues, _omit } from '@naturalcycles/js-lib/object/object.util.js';
 import { PQueue } from '@naturalcycles/js-lib/promise/pQueue.js';
 import { _inspect } from '../index.js';
 const GAE = !!process.env['GAE_INSTANCE'];
@@ -31,7 +31,7 @@ export class SlackService {
     constructor(cfg) {
         this.cfg = {
             messagePrefixHook: slackDefaultMessagePrefixHook,
-            logger: console,
+            logger: consoleLogger,
             ...cfg,
             inspectOptions: {
                 ...INSPECT_OPT,
@@ -60,7 +60,8 @@ export class SlackService {
         if (ctx !== undefined) {
             Object.assign(msg, { ctx });
         }
-        this.cfg.logger.log(...[msg.items, msg.kv, msg.attachments, msg.mentions].filter(Boolean));
+        const { items, kv, attachments, mentions } = msg;
+        this.cfg.logger.log(_filterFalsyValues({ msg: _inspect(items, inspectOptions), kv, attachments, mentions }));
         if (!webhookUrl)
             return;
         // Transform msg.kv into msg.attachments
@@ -122,12 +123,22 @@ export class SlackService {
         const q = new PQueue({
             concurrency: 1,
         });
-        return createCommonLoggerAtLevel({
-            debug: (...args) => q.push(() => this.send({ items: args, channel: debugChannel || defaultChannel })),
-            log: (...args) => q.push(() => this.send({ items: args, channel: logChannel || defaultChannel })),
-            warn: (...args) => q.push(() => this.send({ items: args, channel: warnChannel || defaultChannel })),
-            error: (...args) => q.push(() => this.send({ items: args, channel: errorChannel || defaultChannel })),
-        }, minLogLevel);
+        const channelByLevel = {
+            debug: debugChannel || defaultChannel,
+            log: logChannel || defaultChannel,
+            warn: warnChannel || defaultChannel,
+            error: errorChannel || defaultChannel,
+        };
+        return createCommonLoggerAtLevel(commonLoggerCreate(({ level, msg, err, ...fields }) => {
+            const items = [];
+            if (msg !== undefined)
+                items.push(msg);
+            if (err !== undefined)
+                items.push(err);
+            if (Object.keys(fields).length)
+                items.push(fields);
+            void q.push(() => this.send({ items, channel: channelByLevel[level] }));
+        }), minLogLevel);
     }
 }
 export function slackDefaultMessagePrefixHook(msg) {

package/dist/stream/progressLogger.js

@@ -1,6 +1,7 @@
 import { inspect } from 'node:util';
 import { _hc, _mb } from '@naturalcycles/js-lib';
 import { _since, localTime } from '@naturalcycles/js-lib/datetime';
+import { consoleLogger } from '@naturalcycles/js-lib/log';
 import { SimpleMovingAverage } from '@naturalcycles/js-lib/math';
 import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js';
 const inspectOpt = {
@@ -17,7 +18,7 @@ export class ProgressLogger {
             logEvery: 1000,
             logSizesBuffer: 100_000,
             chunkSize: 1,
-            logger: console,
+            logger: consoleLogger,
             logProgress: cfg.logProgress !== false && cfg.logEvery !== 0,
             ...cfg,
         };
@@ -100,7 +101,7 @@ export class ProgressLogger {
                 this.cfg.onProgressDone?.(o);
             }
             catch (err) {
-                logger.error(err);
+                logger.error({ err });
             }
         }
     }

package/dist/stream/stream.model.d.ts

@@ -38,7 +38,7 @@ export interface TransformOptions {
      */
     highWaterMark?: number;
     /**
-     * Defaults to `console`.
+     * Defaults to `consoleLogger`.
      */
     logger?: CommonLogger;
     /**

package/dist/stream/transform/transformMap.js

@@ -85,7 +85,7 @@ export function transformMap(mapper, opt = {}) {
                 }
             }
             catch (err) {
-                logger.error(err);
+                logger.error({ err });
                 errors++;
                 logErrorStats();
                 if (onError) {
@@ -167,7 +167,7 @@ export function transformMap(mapper, opt = {}) {
             });
         }
         catch (err) {
-            logger.error(err);
+            logger.error({ err });
         }
     }
 }

package/dist/stream/transform/transformMapSimple.js

@@ -1,5 +1,6 @@
 import { Transform } from 'node:stream';
 import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
+import { consoleLogger } from '@naturalcycles/js-lib/log';
 /**
  * Simplest version of `transformMap`.
  * errorMode: IMMEDIATE
@@ -11,7 +12,7 @@ import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
  */
 export function transformMapSimple(mapper, opt = {}) {
     let index = -1;
-    const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = console, objectMode = true, highWaterMark = 1, } = opt;
+    const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = consoleLogger, objectMode = true, highWaterMark = 1, } = opt;
     return new Transform({
         objectMode,
         highWaterMark,
@@ -20,7 +21,7 @@ export function transformMapSimple(mapper, opt = {}) {
                 cb(null, mapper(chunk, ++index));
             }
             catch (err) {
-                logger.error(err);
+                logger.error({ err });
                 if (errorMode === ErrorMode.SUPPRESS) {
                     cb(); // suppress the error
                 }

package/dist/stream/transform/transformMapSync.js

@@ -48,7 +48,7 @@ export function transformMapSync(mapper, opt = {}) {
                 cb(); // done processing
             }
             catch (err) {
-                logger.error(err);
+                logger.error({ err });
                 errors++;
                 logErrorStats();
                 if (onError) {
@@ -70,7 +70,7 @@ export function transformMapSync(mapper, opt = {}) {
                         });
                     }
                     catch (err) {
-                        logger.error(err);
+                        logger.error({ err });
                     }
                     // Emit error immediately
                     return cb(err);
@@ -96,7 +96,7 @@ export function transformMapSync(mapper, opt = {}) {
                     });
                 }
                 catch (err) {
-                    logger.error(err);
+                    logger.error({ err });
                 }
                 // emit Aggregated error
                 cb(new AggregateError(collectedErrors, `transformMapSync resulted in ${collectedErrors.length} error(s)`));
@@ -114,7 +114,7 @@ export function transformMapSync(mapper, opt = {}) {
                     });
                 }
                 catch (err) {
-                    logger.error(err);
+                    logger.error({ err });
                 }
                 cb();
             }

package/dist/stream/transform/transformSplit.d.ts

@@ -4,15 +4,16 @@ import type { TransformTyped } from '../stream.model.js';
  *
  * Useful for reading NDJSON files from fs.
  *
- * Same as binarySplit, but optimized (hard-coded) to split on NEWLINE (aka `\n`).
- * (+5-10% _pipeline speedup measured, compared to generic `binarySplit` on variable length delimiter)
+ * Same as `transformSplit('\n')`.
+ * (It used to be a separate hard-coded implementation, +5-10% faster than the generic one,
+ * but since the generic one searches 1-byte separators by byte value there's no difference anymore)
  */
 export declare function transformSplitOnNewline(): TransformTyped<Buffer, Buffer>;
 /**
  * Input: stream (objectMode=false) of arbitrary string|Buffer chunks, like when read from fs
  * Output: stream (objectMode=true) or string|Buffer chunks split by `separator` (@default to `\n`)
  *
- * Please use slightly more optimized `transformSplitOnNewline` for NDJSON file parsing.
- * (+5-10% _pipeline speedup measured!)
+ * Empty lines (consecutive separators) are skipped.
+ * Trailing data without a separator is emitted on flush.
  */
 export declare function transformSplit(separator?: string): TransformTyped<Buffer, Buffer>;

package/dist/stream/transform/transformSplit.js

@@ -1,133 +1,90 @@
 import { Transform } from 'node:stream';
 // The code below is carefully adopted from: https://github.com/max-mapper/binary-split
+// (including the v2.0.0 performance improvements: native Buffer#indexOf and linear-time long lines)
 /**
  * Transforms input Buffer/string stream into Buffer chunks (objectMode: true) split by newLine.
  *
  * Useful for reading NDJSON files from fs.
  *
- * Same as binarySplit, but optimized (hard-coded) to split on NEWLINE (aka `\n`).
- * (+5-10% _pipeline speedup measured, compared to generic `binarySplit` on variable length delimiter)
+ * Same as `transformSplit('\n')`.
+ * (It used to be a separate hard-coded implementation, +5-10% faster than the generic one,
+ * but since the generic one searches 1-byte separators by byte value there's no difference anymore)
  */
 export function transformSplitOnNewline() {
-    let buffered;
-    return new Transform({
-        writableObjectMode: false,
-        writableHighWaterMark: 64 * 1024,
-        readableObjectMode: true,
-        transform(buf, _enc, cb) {
-            let offset = 0;
-            let lastMatch = 0;
-            if (buffered) {
-                buf = Buffer.concat([buffered, buf]);
-                offset = buffered.length;
-                buffered = undefined;
-            }
-            while (true) {
-                const idx = firstNewlineMatch(buf, offset);
-                if (idx !== -1 && idx < buf.length) {
-                    if (lastMatch !== idx) {
-                        this.push(buf.slice(lastMatch, idx));
-                    }
-                    offset = idx + 1;
-                    lastMatch = offset;
-                }
-                else {
-                    buffered = buf.slice(lastMatch);
-                    break;
-                }
-            }
-            cb();
-        },
-        flush(done) {
-            if (buffered && buffered.length > 0)
-                this.push(buffered);
-            done();
-        },
-    });
+    return transformSplit('\n');
 }
 /**
  * Input: stream (objectMode=false) of arbitrary string|Buffer chunks, like when read from fs
  * Output: stream (objectMode=true) or string|Buffer chunks split by `separator` (@default to `\n`)
  *
- * Please use slightly more optimized `transformSplitOnNewline` for NDJSON file parsing.
- * (+5-10% _pipeline speedup measured!)
+ * Empty lines (consecutive separators) are skipped.
+ * Trailing data without a separator is emitted on flush.
  */
 export function transformSplit(separator = '\n') {
     const matcher = Buffer.from(separator);
-    let buffered;
+    if (matcher.length === 0) {
+        throw new Error('transformSplit: separator must not be empty');
+    }
+    const matcherLength = matcher.length;
+    // indexOf with a byte value is much faster than with a 1-byte Buffer, which matters for short lines
+    const needle = matcherLength === 1 ? matcher[0] : matcher;
+    // a multi-byte separator may straddle a chunk boundary, so we keep that many trailing bytes around
+    const overlap = matcherLength - 1;
+    // Unterminated data is collected as a list of chunks and concatenated only once a separator arrives,
+    // so a long line spanning many chunks is copied once rather than on every chunk (linear, not quadratic)
+    let pending = [];
+    let pendingLength = 0;
+    let edge; // last `overlap` bytes of pending data, only tracked when overlap > 0
     return new Transform({
-        readableObjectMode: true,
+        writableObjectMode: false,
         writableHighWaterMark: 64 * 1024,
-        transform(buf, _enc, done) {
+        readableObjectMode: true,
+        transform(chunk, _enc, cb) {
+            let buf = chunk;
             let offset = 0;
-            let lastMatch = 0;
-            if (buffered) {
-                buf = Buffer.concat([buffered, buf]);
-                offset = buffered.length;
-                buffered = undefined;
-            }
-            while (true) {
-                const idx = firstMatch(buf, offset - matcher.length + 1, matcher);
-                if (idx !== -1 && idx < buf.length) {
-                    if (lastMatch !== idx) {
-                        this.push(buf.slice(lastMatch, idx));
+            if (pendingLength > 0) {
+                const straddles = overlap > 0 && Buffer.concat([edge, chunk.subarray(0, overlap)]).includes(matcher);
+                const idxInChunk = straddles ? -1 : chunk.indexOf(needle);
+                if (!straddles && idxInChunk === -1) {
+                    // No separator yet: keep accumulating without copying
+                    pending.push(chunk);
+                    pendingLength += chunk.length;
+                    if (overlap > 0) {
+                        edge = Buffer.concat([edge, chunk.subarray(-overlap)]).subarray(-overlap);
                     }
-                    offset = idx + matcher.length;
-                    lastMatch = offset;
+                    cb();
+                    return;
                 }
-                else {
-                    buffered = buf.slice(lastMatch);
-                    break;
+                pending.push(chunk);
+                buf = Buffer.concat(pending, pendingLength + chunk.length);
+                // Start scanning where the first separator can possibly be
+                offset = straddles ? Math.max(0, pendingLength - overlap) : pendingLength + idxInChunk;
+                pending = [];
+                pendingLength = 0;
+            }
+            let start = 0;
+            let idx = buf.indexOf(needle, offset);
+            while (idx !== -1) {
+                if (idx > start) {
+                    this.push(buf.subarray(start, idx));
                 }
+                start = idx + matcherLength;
+                idx = buf.indexOf(needle, start);
             }
-            done();
+            if (start < buf.length) {
+                const rest = buf.subarray(start);
+                pending.push(rest);
+                pendingLength = rest.length;
+                if (overlap > 0)
+                    edge = rest.subarray(-overlap);
+            }
+            cb();
         },
-        flush(done) {
-            if (buffered && buffered.length > 0)
-                this.push(buffered);
-            done();
+        flush(cb) {
+            if (pendingLength > 0) {
+                this.push(Buffer.concat(pending, pendingLength));
+            }
+            cb();
         },
     });
 }
-// const NEWLINE = Buffer.from('\n')
-// const NEWLINE_CODE = NEWLINE[0]! // it is `10`
-const NEWLINE_CODE = 10;
-/**
- * Same as firstMatch, but optimized (hard-coded) to find NEWLINE (aka `\n`).
- */
-function firstNewlineMatch(buf, offset) {
-    const bufLength = buf.length;
-    if (offset >= bufLength)
-        return -1;
-    for (let i = offset; i < bufLength; i++) {
-        if (buf[i] === NEWLINE_CODE) {
-            return i;
-        }
-    }
-    return -1; // this code is unreachable, because i is guaranteed to be found in the loop above
-}
-function firstMatch(buf, offset, matcher) {
-    if (offset >= buf.length)
-        return -1;
-    let i;
-    for (i = offset; i < buf.length; i++) {
-        if (buf[i] === matcher[0]) {
-            if (matcher.length > 1) {
-                let fullMatch = true;
-                let j = i;
-                for (let k = 0; j < i + matcher.length; j++, k++) {
-                    if (buf[j] !== matcher[k]) {
-                        fullMatch = false;
-                        break;
-                    }
-                }
-                if (fullMatch)
-                    return j - matcher.length;
-            }
-            else {
-                break;
-            }
-        }
-    }
-    return i + matcher.length - 1;
-}

package/dist/stream/transform/transformTap.js

@@ -1,4 +1,5 @@
 import { Transform } from 'node:stream';
+import { consoleLogger } from '@naturalcycles/js-lib/log';
 /**
  * Similar to RxJS `tap` - allows to run a function for each stream item, without affecting the result.
  * Item is passed through to the output.
@@ -6,7 +7,7 @@ import { Transform } from 'node:stream';
  * Can also act as a counter, since `index` is passed to `fn`
  */
 export function transformTap(fn, opt = {}) {
-    const { logger = console, highWaterMark = 1 } = opt;
+    const { logger = consoleLogger, highWaterMark = 1 } = opt;
     let index = -1;
     return new Transform({
         objectMode: true,
@@ -16,7 +17,7 @@ export function transformTap(fn, opt = {}) {
                 await fn(chunk, ++index);
             }
             catch (err) {
-                logger.error(err);
+                logger.error({ err });
                 // suppressed error
             }
             cb(null, chunk);
@@ -27,7 +28,7 @@ export function transformTap(fn, opt = {}) {
  * Sync version of transformTap
  */
 export function transformTapSync(fn, opt = {}) {
-    const { logger = console, highWaterMark = 1 } = opt;
+    const { logger = consoleLogger, highWaterMark = 1 } = opt;
     let index = -1;
     return new Transform({
         objectMode: true,
@@ -37,7 +38,7 @@ export function transformTapSync(fn, opt = {}) {
                 fn(chunk, ++index);
             }
             catch (err) {
-                logger.error(err);
+                logger.error({ err });
                 // suppressed error
             }
             cb(null, chunk);

package/dist/stream/transform/transformThrottle.js

@@ -34,7 +34,8 @@ export function transformThrottle(opt) {
             if (!start) {
                 start = localTime.nowUnixMillis();
                 timeout = setTimeout(() => onInterval(), interval * 1000);
-                logger.log(`${localTime.now().toPretty()} transformThrottle started with`, {
+                logger.log({
+                    msg: `${localTime.now().toPretty()} transformThrottle started with`,
                     throughput,
                     interval,
                     rps: Math.round(throughput / interval),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.123.1",
+  "version": "15.125.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@standard-schema/spec": "^1",

package/src/log/log.util.ts

@@ -1,9 +1,26 @@
 import { commonLoggerCreate } from '@naturalcycles/js-lib/log'
+import type { LogEntry } from '@naturalcycles/js-lib/log'
 import { _inspect } from '../string/inspect.js'
+import type { InspectAnyOptions } from '../string/inspect.js'
 
 /**
  * CommonLogger that logs to process.stdout directly (bypassing console.log)!
  */
-export const stdoutLogger = commonLoggerCreate((_level, args) => {
-  process.stdout.write(args.map(a => _inspect(a)).join(' ') + '\n')
+export const stdoutLogger = commonLoggerCreate(entry => {
+  process.stdout.write(logEntryToString(entry) + '\n')
 })
+
+/**
+ * Renders a LogEntry as one line: the msg, the inspected err and the inspected remaining fields,
+ * separated by a space.
+ */
+export function logEntryToString(
+  { level: _level, msg, err, ...fields }: LogEntry,
+  opt: InspectAnyOptions = {},
+): string {
+  const parts: string[] = []
+  if (msg !== undefined) parts.push(msg)
+  if (err !== undefined) parts.push(_inspect(err, { includeErrorStack: true, ...opt }))
+  if (Object.keys(fields).length) parts.push(_inspect(fields, opt))
+  return parts.join(' ')
+}

package/src/script/runScript.ts

@@ -1,5 +1,6 @@
 import os from 'node:os'
 import { AsyncManager } from '@naturalcycles/js-lib'
+import { consoleLogger } from '@naturalcycles/js-lib/log'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { setGlobalStringifyFunction } from '@naturalcycles/js-lib/string/stringify.js'
 import type { AnyObject } from '@naturalcycles/js-lib/types'
@@ -18,7 +19,7 @@ export interface RunScriptOptions {
   noExit?: boolean
 
   /**
-   * Default to `console`
+   * Default to `consoleLogger`
    */
   logger?: CommonLogger
 
@@ -53,14 +54,14 @@ export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {
   checkAndlogEnvironment()
   setGlobalStringifyFunction(inspectStringifyFn)
 
-  const { logger = console, noExit, registerUncaughtExceptionHandlers = true } = opt
+  const { logger = consoleLogger, noExit, registerUncaughtExceptionHandlers = true } = opt
 
   if (registerUncaughtExceptionHandlers || DEBUG_RUN_SCRIPT) {
     process.on('uncaughtException', err => {
-      logger.error('runScript uncaughtException:', err)
+      logger.error({ msg: 'runScript uncaughtException:', err })
     })
     process.on('unhandledRejection', err => {
-      logger.error('runScript unhandledRejection:', err)
+      logger.error({ msg: 'runScript unhandledRejection:', err })
     })
   }
 
@@ -85,7 +86,7 @@ export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {
         setImmediate(() => process.exit(0))
       }
     } catch (err) {
-      logger.error('runScript error:', err)
+      logger.error({ msg: 'runScript error:', err })
       process.exitCode = 1
       if (!noExit) {
         setImmediate(() => process.exit(1))

package/src/slack/slack.service.ts

@@ -1,9 +1,13 @@
 import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { getFetcher } from '@naturalcycles/js-lib/http'
 import type { Fetcher } from '@naturalcycles/js-lib/http'
-import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
+import {
+  commonLoggerCreate,
+  consoleLogger,
+  createCommonLoggerAtLevel,
+} from '@naturalcycles/js-lib/log'
 import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log'
-import { _omit } from '@naturalcycles/js-lib/object/object.util.js'
+import { _filterFalsyValues, _omit } from '@naturalcycles/js-lib/object/object.util.js'
 import { PQueue } from '@naturalcycles/js-lib/promise/pQueue.js'
 import type { AnyObject } from '@naturalcycles/js-lib/types'
 import { _inspect } from '../index.js'
@@ -45,7 +49,7 @@ export class SlackService<CTX = any> {
   constructor(cfg: Partial<SlackServiceCfg<CTX>>) {
     this.cfg = {
       messagePrefixHook: slackDefaultMessagePrefixHook,
-      logger: console,
+      logger: consoleLogger,
       ...cfg,
       inspectOptions: {
         ...INSPECT_OPT,
@@ -82,7 +86,10 @@ export class SlackService<CTX = any> {
       Object.assign(msg, { ctx })
     }
 
-    this.cfg.logger.log(...[msg.items, msg.kv, msg.attachments, msg.mentions].filter(Boolean))
+    const { items, kv, attachments, mentions } = msg
+    this.cfg.logger.log(
+      _filterFalsyValues({ msg: _inspect(items, inspectOptions), kv, attachments, mentions }),
+    )
 
     if (!webhookUrl) return
 
@@ -163,17 +170,21 @@ export class SlackService<CTX = any> {
       concurrency: 1,
     })
 
+    const channelByLevel: Record<CommonLogLevel, string> = {
+      debug: debugChannel || defaultChannel,
+      log: logChannel || defaultChannel,
+      warn: warnChannel || defaultChannel,
+      error: errorChannel || defaultChannel,
+    }
+
     return createCommonLoggerAtLevel(
-      {
-        debug: (...args) =>
-          q.push(() => this.send({ items: args, channel: debugChannel || defaultChannel })),
-        log: (...args) =>
-          q.push(() => this.send({ items: args, channel: logChannel || defaultChannel })),
-        warn: (...args) =>
-          q.push(() => this.send({ items: args, channel: warnChannel || defaultChannel })),
-        error: (...args) =>
-          q.push(() => this.send({ items: args, channel: errorChannel || defaultChannel })),
-      },
+      commonLoggerCreate(({ level, msg, err, ...fields }) => {
+        const items: unknown[] = []
+        if (msg !== undefined) items.push(msg)
+        if (err !== undefined) items.push(err)
+        if (Object.keys(fields).length) items.push(fields)
+        void q.push(() => this.send({ items, channel: channelByLevel[level] }))
+      }),
       minLogLevel,
     )
   }

package/src/stream/progressLogger.ts

@@ -2,6 +2,7 @@ import type { InspectOptions } from 'node:util'
 import { inspect } from 'node:util'
 import { _hc, _mb } from '@naturalcycles/js-lib'
 import { _since, localTime } from '@naturalcycles/js-lib/datetime'
+import { consoleLogger } from '@naturalcycles/js-lib/log'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { SimpleMovingAverage } from '@naturalcycles/js-lib/math'
 import type { AnyObject, PositiveInteger, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
@@ -140,7 +141,7 @@ export class ProgressLogger<T> implements Disposable {
       logEvery: 1000,
       logSizesBuffer: 100_000,
       chunkSize: 1,
-      logger: console,
+      logger: consoleLogger,
       logProgress: cfg.logProgress !== false && cfg.logEvery !== 0,
       ...cfg,
     }
@@ -258,7 +259,7 @@ export class ProgressLogger<T> implements Disposable {
       try {
         this.cfg.onProgressDone?.(o)
       } catch (err) {
-        logger.error(err)
+        logger.error({ err })
       }
     }
   }

package/src/stream/stream.model.ts

@@ -66,7 +66,7 @@ export interface TransformOptions {
   highWaterMark?: number
 
   /**
-   * Defaults to `console`.
+   * Defaults to `consoleLogger`.
    */
   logger?: CommonLogger
 

package/src/stream/transform/transformMap.ts

@@ -186,7 +186,7 @@ export function transformMap<IN = any, OUT = IN>(
           this.push(res)
         }
       } catch (err) {
-        logger.error(err)
+        logger.error({ err })
         errors++
         logErrorStats()
 
@@ -278,7 +278,7 @@ export function transformMap<IN = any, OUT = IN>(
         started,
       })
     } catch (err) {
-      logger.error(err)
+      logger.error({ err })
     }
   }
 }

package/src/stream/transform/transformMapSimple.ts

@@ -1,5 +1,6 @@
 import { Transform } from 'node:stream'
 import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js'
+import { consoleLogger } from '@naturalcycles/js-lib/log'
 import type { IndexedMapper } from '@naturalcycles/js-lib/types'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
 
@@ -28,7 +29,7 @@ export function transformMapSimple<IN = any, OUT = IN>(
   let index = -1
   const {
     errorMode = ErrorMode.THROW_IMMEDIATELY,
-    logger = console,
+    logger = consoleLogger,
     objectMode = true,
     highWaterMark = 1,
   } = opt
@@ -40,7 +41,7 @@ export function transformMapSimple<IN = any, OUT = IN>(
       try {
         cb(null, mapper(chunk, ++index))
       } catch (err) {
-        logger.error(err)
+        logger.error({ err })
 
         if (errorMode === ErrorMode.SUPPRESS) {
           cb() // suppress the error

package/src/stream/transform/transformMapSync.ts

@@ -120,7 +120,7 @@ export function transformMapSync<IN = any, OUT = IN>(
 
         cb() // done processing
       } catch (err) {
-        logger.error(err)
+        logger.error({ err })
         errors++
 
         logErrorStats()
@@ -144,7 +144,7 @@ export function transformMapSync<IN = any, OUT = IN>(
               started,
             })
           } catch (err) {
-            logger.error(err)
+            logger.error({ err })
           }
 
           // Emit error immediately
@@ -174,7 +174,7 @@ export function transformMapSync<IN = any, OUT = IN>(
             started,
           })
         } catch (err) {
-          logger.error(err)
+          logger.error({ err })
         }
 
         // emit Aggregated error
@@ -197,7 +197,7 @@ export function transformMapSync<IN = any, OUT = IN>(
             started,
           })
         } catch (err) {
-          logger.error(err)
+          logger.error({ err })
         }
 
         cb()

package/src/stream/transform/transformSplit.ts

@@ -2,142 +2,101 @@ import { Transform } from 'node:stream'
 import type { TransformTyped } from '../stream.model.js'
 
 // The code below is carefully adopted from: https://github.com/max-mapper/binary-split
+// (including the v2.0.0 performance improvements: native Buffer#indexOf and linear-time long lines)
 
 /**
  * Transforms input Buffer/string stream into Buffer chunks (objectMode: true) split by newLine.
  *
  * Useful for reading NDJSON files from fs.
  *
- * Same as binarySplit, but optimized (hard-coded) to split on NEWLINE (aka `\n`).
- * (+5-10% _pipeline speedup measured, compared to generic `binarySplit` on variable length delimiter)
+ * Same as `transformSplit('\n')`.
+ * (It used to be a separate hard-coded implementation, +5-10% faster than the generic one,
+ * but since the generic one searches 1-byte separators by byte value there's no difference anymore)
  */
 export function transformSplitOnNewline(): TransformTyped<Buffer, Buffer> {
-  let buffered: Buffer | undefined
-
-  return new Transform({
-    writableObjectMode: false,
-    writableHighWaterMark: 64 * 1024,
-    readableObjectMode: true,
-
-    transform(buf: Buffer, _enc, cb) {
-      let offset = 0
-      let lastMatch = 0
-      if (buffered) {
-        buf = Buffer.concat([buffered, buf])
-        offset = buffered.length
-        buffered = undefined
-      }
-
-      while (true) {
-        const idx = firstNewlineMatch(buf, offset)
-        if (idx !== -1 && idx < buf.length) {
-          if (lastMatch !== idx) {
-            this.push(buf.slice(lastMatch, idx))
-          }
-          offset = idx + 1
-          lastMatch = offset
-        } else {
-          buffered = buf.slice(lastMatch)
-          break
-        }
-      }
-
-      cb()
-    },
-
-    flush(done) {
-      if (buffered && buffered.length > 0) this.push(buffered)
-      done()
-    },
-  })
+  return transformSplit('\n')
 }
 
 /**
  * Input: stream (objectMode=false) of arbitrary string|Buffer chunks, like when read from fs
  * Output: stream (objectMode=true) or string|Buffer chunks split by `separator` (@default to `\n`)
  *
- * Please use slightly more optimized `transformSplitOnNewline` for NDJSON file parsing.
- * (+5-10% _pipeline speedup measured!)
+ * Empty lines (consecutive separators) are skipped.
+ * Trailing data without a separator is emitted on flush.
  */
 export function transformSplit(separator = '\n'): TransformTyped<Buffer, Buffer> {
   const matcher = Buffer.from(separator)
-  let buffered: Buffer | undefined
+  if (matcher.length === 0) {
+    throw new Error('transformSplit: separator must not be empty')
+  }
+  const matcherLength = matcher.length
+  // indexOf with a byte value is much faster than with a 1-byte Buffer, which matters for short lines
+  const needle = matcherLength === 1 ? matcher[0]! : matcher
+  // a multi-byte separator may straddle a chunk boundary, so we keep that many trailing bytes around
+  const overlap = matcherLength - 1
+
+  // Unterminated data is collected as a list of chunks and concatenated only once a separator arrives,
+  // so a long line spanning many chunks is copied once rather than on every chunk (linear, not quadratic)
+  let pending: Buffer[] = []
+  let pendingLength = 0
+  let edge: Buffer | undefined // last `overlap` bytes of pending data, only tracked when overlap > 0
 
   return new Transform({
-    readableObjectMode: true,
+    writableObjectMode: false,
     writableHighWaterMark: 64 * 1024,
+    readableObjectMode: true,
 
-    transform(buf: Buffer, _enc, done) {
+    transform(chunk: Buffer, _enc, cb) {
+      let buf = chunk
       let offset = 0
-      let lastMatch = 0
-      if (buffered) {
-        buf = Buffer.concat([buffered, buf])
-        offset = buffered.length
-        buffered = undefined
-      }
 
-      while (true) {
-        const idx = firstMatch(buf, offset - matcher.length + 1, matcher)
-        if (idx !== -1 && idx < buf.length) {
-          if (lastMatch !== idx) {
-            this.push(buf.slice(lastMatch, idx))
+      if (pendingLength > 0) {
+        const straddles =
+          overlap > 0 && Buffer.concat([edge!, chunk.subarray(0, overlap)]).includes(matcher)
+        const idxInChunk = straddles ? -1 : chunk.indexOf(needle)
+        if (!straddles && idxInChunk === -1) {
+          // No separator yet: keep accumulating without copying
+          pending.push(chunk)
+          pendingLength += chunk.length
+          if (overlap > 0) {
+            edge = Buffer.concat([edge!, chunk.subarray(-overlap)]).subarray(-overlap)
           }
-          offset = idx + matcher.length
-          lastMatch = offset
-        } else {
-          buffered = buf.slice(lastMatch)
-          break
+          cb()
+          return
         }
+        pending.push(chunk)
+        buf = Buffer.concat(pending, pendingLength + chunk.length)
+        // Start scanning where the first separator can possibly be
+        offset = straddles ? Math.max(0, pendingLength - overlap) : pendingLength + idxInChunk
+        pending = []
+        pendingLength = 0
       }
 
-      done()
-    },
-
-    flush(done) {
-      if (buffered && buffered.length > 0) this.push(buffered)
-      done()
-    },
-  })
-}
+      let start = 0
+      let idx = buf.indexOf(needle, offset)
+      while (idx !== -1) {
+        if (idx > start) {
+          this.push(buf.subarray(start, idx))
+        }
+        start = idx + matcherLength
+        idx = buf.indexOf(needle, start)
+      }
 
-// const NEWLINE = Buffer.from('\n')
-// const NEWLINE_CODE = NEWLINE[0]! // it is `10`
-const NEWLINE_CODE = 10
+      if (start < buf.length) {
+        const rest = buf.subarray(start)
+        pending.push(rest)
+        pendingLength = rest.length
+        if (overlap > 0) edge = rest.subarray(-overlap)
+      }
 
-/**
- * Same as firstMatch, but optimized (hard-coded) to find NEWLINE (aka `\n`).
- */
-function firstNewlineMatch(buf: Buffer, offset: number): number {
-  const bufLength = buf.length
-  if (offset >= bufLength) return -1
-  for (let i = offset; i < bufLength; i++) {
-    if (buf[i] === NEWLINE_CODE) {
-      return i
-    }
-  }
-  return -1 // this code is unreachable, because i is guaranteed to be found in the loop above
-}
+      cb()
+    },
 
-function firstMatch(buf: Buffer, offset: number, matcher: Buffer): number {
-  if (offset >= buf.length) return -1
-  let i: number
-  for (i = offset; i < buf.length; i++) {
-    if (buf[i] === matcher[0]) {
-      if (matcher.length > 1) {
-        let fullMatch = true
-        let j = i
-        for (let k = 0; j < i + matcher.length; j++, k++) {
-          if (buf[j] !== matcher[k]) {
-            fullMatch = false
-            break
-          }
-        }
-        if (fullMatch) return j - matcher.length
-      } else {
-        break
+    flush(cb) {
+      if (pendingLength > 0) {
+        this.push(Buffer.concat(pending, pendingLength))
       }
-    }
-  }
-
-  return i + matcher.length - 1
+      cb()
+    },
+  })
 }

package/src/stream/transform/transformTap.ts

@@ -1,4 +1,5 @@
 import { Transform } from 'node:stream'
+import { consoleLogger } from '@naturalcycles/js-lib/log'
 import type { AsyncIndexedMapper, IndexedMapper } from '@naturalcycles/js-lib/types'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
 
@@ -12,7 +13,7 @@ export function transformTap<IN>(
   fn: AsyncIndexedMapper<IN, any>,
   opt: TransformOptions = {},
 ): TransformTyped<IN, IN> {
-  const { logger = console, highWaterMark = 1 } = opt
+  const { logger = consoleLogger, highWaterMark = 1 } = opt
   let index = -1
 
   return new Transform({
@@ -22,7 +23,7 @@ export function transformTap<IN>(
       try {
         await fn(chunk, ++index)
       } catch (err) {
-        logger.error(err)
+        logger.error({ err })
         // suppressed error
       }
 
@@ -38,7 +39,7 @@ export function transformTapSync<IN>(
   fn: IndexedMapper<IN, any>,
   opt: TransformOptions = {},
 ): TransformTyped<IN, IN> {
-  const { logger = console, highWaterMark = 1 } = opt
+  const { logger = consoleLogger, highWaterMark = 1 } = opt
   let index = -1
 
   return new Transform({
@@ -48,7 +49,7 @@ export function transformTapSync<IN>(
       try {
         fn(chunk, ++index)
       } catch (err) {
-        logger.error(err)
+        logger.error({ err })
         // suppressed error
       }
 

package/src/stream/transform/transformThrottle.ts

@@ -56,7 +56,8 @@ export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTy
       if (!start) {
         start = localTime.nowUnixMillis()
         timeout = setTimeout(() => onInterval(), interval * 1000)
-        logger.log(`${localTime.now().toPretty()} transformThrottle started with`, {
+        logger.log({
+          msg: `${localTime.now().toPretty()} transformThrottle started with`,
           throughput,
           interval,
           rps: Math.round(throughput / interval),