@naturalcycles/nodejs-lib 13.31.1 → 13.33.0

package/dist/csv/csvReader.js

@@ -34,14 +34,14 @@ function csvStringParse(str, cfg = {}) {
     });
 }
 function csvStringToArray(str) {
-    const objPattern = new RegExp(String.raw `(,|\r?\n|\r|^)(?:"([^"]*(?:""[^"]*)*)"|([^,\r\n]*))`, 'gi');
+    const objPattern = /(,|\r?\n|\r|^)(?:"([^"]*(?:""[^"]*)*)"|([^,\r\n]*))/gi;
     let matches;
     const arr = [[]];
     while ((matches = objPattern.exec(str))) {
         if (matches[1].length && matches[1] !== ',') {
             arr.push([]);
         }
-        arr[arr.length - 1].push(matches[2] ? matches[2].replaceAll(new RegExp('""', 'g'), '"') : matches[3]);
+        arr[arr.length - 1].push(matches[2] ? matches[2].replaceAll('""', '"') : matches[3]);
     }
     return arr;
 }

package/dist/fs/fs2.js

@@ -188,8 +188,10 @@ class FS2 {
         }
         catch (err) {
             // If the stat call above failed because the directory doesn't exist, create it
-            if (err?.code === 'ENOENT')
-                return this.ensureDir(dir);
+            if (err?.code === 'ENOENT') {
+                this.ensureDir(dir);
+                return;
+            }
             throw err;
         }
         node_fs_1.default.writeFileSync(filePath, '');
@@ -230,7 +232,8 @@ class FS2 {
             items = node_fs_1.default.readdirSync(dirPath);
         }
         catch {
-            return this.ensureDir(dirPath);
+            this.ensureDir(dirPath);
+            return;
         }
         items.forEach(item => this.removePath(node_path_1.default.join(dirPath, item)));
     }

package/dist/index.d.ts

@@ -62,8 +62,8 @@ export * from './stream/writable/writableVoid';
 export * from './string/inspect';
 export * from './util/buildInfo.util';
 export * from './util/env.util';
-export * from './util/exec.util';
-export * from './util/git.util';
+export * from './util/exec2';
+export * from './util/git2';
 export * from './util/lruMemoCache';
 export * from './util/zip.util';
 export * from './validation/ajv/ajv.util';

package/dist/index.js

@@ -66,8 +66,8 @@ tslib_1.__exportStar(require("./stream/writable/writableVoid"), exports);
 tslib_1.__exportStar(require("./string/inspect"), exports);
 tslib_1.__exportStar(require("./util/buildInfo.util"), exports);
 tslib_1.__exportStar(require("./util/env.util"), exports);
-tslib_1.__exportStar(require("./util/exec.util"), exports);
-tslib_1.__exportStar(require("./util/git.util"), exports);
+tslib_1.__exportStar(require("./util/exec2"), exports);
+tslib_1.__exportStar(require("./util/git2"), exports);
 tslib_1.__exportStar(require("./util/lruMemoCache"), exports);
 tslib_1.__exportStar(require("./util/zip.util"), exports);
 tslib_1.__exportStar(require("./validation/ajv/ajv.util"), exports);

package/dist/slack/slack.service.js

@@ -65,7 +65,7 @@ class SlackService {
         if (msg.kv) {
             ;
             (msg.attachments ||= []).push({ fields: this.kvToFields(msg.kv) });
-            delete msg.kv; // to not pass it all the way to Slack Api
+            msg.kv = undefined; // to not pass it all the way to Slack Api
         }
         let text;
         // Array has a special treatment here
@@ -101,9 +101,7 @@ class SlackService {
             if (msg.throwOnError) {
                 throw err;
             }
-            else {
-                console.log(err);
-            }
+            console.log(err);
         });
     }
     kvToFields(kv) {

package/dist/stream/ndjson/transformJsonParse.js

@@ -43,7 +43,7 @@ function transformJsonParse(opt = {}) {
     });
 }
 // Based on: https://stackoverflow.com/a/34557997/4919972
-const bufferReviver = (k, v) => {
+const bufferReviver = (_k, v) => {
     if (v !== null && typeof v === 'object' && v.type === 'Buffer' && Array.isArray(v.data)) {
         return Buffer.from(v.data);
     }

package/dist/stream/transform/transformMap.d.ts

@@ -18,10 +18,12 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
      * Number of concurrently pending promises returned by `mapper`.
      *
-     * Default is 32.
+     * Default is 16.
      * It was recently changed up from 16, after some testing that shown that
      * for simple low-cpu mapper functions 32 produces almost 2x throughput.
      * For example, in scenarios like streaming a query from Datastore.
+     * UPD: changed back from 32 to 16, "to be on a safe side", as 32 sometimes
+     * causes "Datastore timeout errors".
      */
     concurrency?: number;
     /**
@@ -86,7 +88,7 @@ export interface TransformMapStatsSummary extends TransformMapStats {
  *
  * Only works in objectMode (due to through2Concurrent).
  *
- * Concurrency defaults to 32.
+ * Concurrency defaults to 16.
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */

package/dist/stream/transform/transformMap.js

@@ -16,12 +16,12 @@ const stream_util_1 = require("../stream.util");
  *
  * Only works in objectMode (due to through2Concurrent).
  *
- * Concurrency defaults to 32.
+ * Concurrency defaults to 16.
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 function transformMap(mapper, opt = {}) {
-    const { concurrency = 32, predicate, // we now default to "no predicate" (meaning pass-everything)
+    const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, onDone, metric = 'stream', logger = console, } = opt;
     const started = Date.now();
     let index = -1;

package/dist/stream/transform/transformSplit.js

@@ -17,7 +17,7 @@ function transformSplitOnNewline() {
     return new node_stream_1.Transform({
         readableObjectMode: true,
         writableHighWaterMark: 64 * 1024,
-        transform(buf, enc, done) {
+        transform(buf, _enc, done) {
             let offset = 0;
             let lastMatch = 0;
             if (buffered) {
@@ -61,7 +61,7 @@ function transformSplit(separator = '\n') {
     return new node_stream_1.Transform({
         readableObjectMode: true,
         writableHighWaterMark: 64 * 1024,
-        transform(buf, enc, done) {
+        transform(buf, _enc, done) {
             let offset = 0;
             let lastMatch = 0;
             if (buffered) {

package/dist/stream/transform/worker/workerClassProxy.js

@@ -1,6 +1,6 @@
 const started = Date.now()
-const { workerData, parentPort } = require('worker_threads')
-const { inspect } = require('util')
+const { workerData, parentPort } = require('node:worker_threads')
+const { inspect } = require('node:util')
 const { workerFile, workerIndex, logEvery = 1000, metric = 'worker' } = workerData || {}
 
 if (!workerFile) {

package/dist/stream/writable/writableLimit.js

@@ -11,7 +11,7 @@ function writableLimit(readable, limit) {
     let i = 0;
     return new node_stream_1.Writable({
         objectMode: true,
-        write(chunk, _, cb) {
+        write(_chunk, _, cb) {
             if (limit === 0)
                 return cb(); // no limit, just passthrough
             i++;

package/dist/stream/writable/writableVoid.js

@@ -11,7 +11,7 @@ function writableVoid(opt = {}) {
     return new node_stream_1.Writable({
         objectMode: true,
         ...opt,
-        write(chunk, _, cb) {
+        write(_chunk, _, cb) {
             cb();
         },
         final(cb) {

package/dist/util/buildInfo.util.js

@@ -3,14 +3,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.generateBuildInfo = generateBuildInfo;
 const js_lib_1 = require("@naturalcycles/js-lib");
 const fs2_1 = require("../fs/fs2");
-const git_util_1 = require("./git.util");
+const git2_1 = require("./git2");
 function generateBuildInfo(opt = {}) {
     const now = js_lib_1.localTime.orNow(opt.overrideTimestamp);
     const ts = now.unix;
-    const rev = (0, git_util_1.gitCurrentCommitSha)();
-    const branchName = (0, git_util_1.gitCurrentBranchName)();
-    const repoName = (0, git_util_1.gitCurrentRepoName)();
-    const tsCommit = (0, git_util_1.gitCurrentCommitTimestamp)();
+    const rev = git2_1.git2.gitCurrentCommitSha();
+    const branchName = git2_1.git2.gitCurrentBranchName();
+    const repoName = git2_1.git2.gitCurrentRepoName();
+    const tsCommit = git2_1.git2.gitCurrentCommitTimestamp();
     const ver = [now.toStringCompact(), repoName, branchName, rev].join('_');
     let { APP_ENV: env } = process.env;
     if (!env) {

package/dist/util/exec2.d.ts

@@ -0,0 +1,167 @@
+import { AnyObject, AppError, NumberOfMilliseconds } from '@naturalcycles/js-lib';
+/**
+ * Set of utility functions to work with Spawn / Exec.
+ *
+ * How to decide between Spawn and Exec?
+ *
+ * Long-running job that prints output, and no need to return the output - use Spawn.
+ *
+ * Short-running job, no need to print the output, might want to return the output - use Exec.
+ *
+ * Need to both print and return the output - use SpawnAsync.
+ *
+ * ***
+ *
+ * Spawn is good for long-running large-output processes, that continuously output data.
+ * E.g running `jest`.
+ *
+ * Exec is the opposite - good for short-running processes that output small data.
+ * Exec allows to return the output as a string.
+ * Exec doesn't stream data during execution, so the output/error will only be printed
+ * at the end.
+ * Exec always uses the shell (there's no option to disable it).
+ */
+declare class Exec2 {
+    /**
+     * Advanced/async version of Spawn.
+     * Consider simpler `spawn` or `exec` first, which are also sync.
+     *
+     * spawnAsync features:
+     *
+     * 1. Async
+     * 2. Allows to collect the output AND print it while running.
+     * 3. Returns SpawnOutput with stdout, stderr and exitCode.
+     * 4. Allows to not throw on error, but just return SpawnOutput for further inspection.
+     *
+     * Defaults:
+     *
+     * shell: true
+     * printWhileRunning: true
+     * collectOutputWhileRunning: true
+     * throwOnNonZeroCode: true
+     * log: true
+     */
+    spawnAsync(cmd: string, opt?: SpawnAsyncOptions): Promise<SpawnOutput>;
+    /**
+     * Reasons to use it:
+     * - Sync
+     * - Need to print output while running
+     *
+     * Limitations:
+     * - Cannot return stdout/stderr (use exec or spawnAsync for that)
+     *
+     * Defaults:
+     *
+     * shell: true
+     * log: true
+     */
+    spawn(cmd: string, opt?: SpawnOptions): void;
+    /**
+     * Reasons to use it:
+     *
+     * - Sync
+     * - Need to return output
+     *
+     * Limitations:
+     * - Cannot print while running (use spawn or spawnAsync for that)
+     *
+     * Defaults:
+     *
+     * shell: true
+     * log: true
+     */
+    exec(cmd: string, opt?: ExecOptions): string;
+    throwOnNonZeroExitCode(o: SpawnOutput): void;
+    private logStart;
+    private logFinish;
+}
+export declare const exec2: Exec2;
+export declare class SpawnError extends AppError<SpawnErrorData> {
+    constructor(message: string, data: SpawnErrorData);
+}
+export interface SpawnErrorData extends SpawnOutput {
+}
+export interface SpawnOutput {
+    /**
+     * Exit code of the spawned process.
+     * 0 means success, anything else means failure.
+     */
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+}
+export interface SpawnAsyncOptions extends SpawnOptions {
+    /**
+     * Defaults to true.
+     * If true - prints both stdout and stderr to console while running,
+     * otherwise runs "silently".
+     * Returns SpawnOutput in the same way, regardless of `printWhileRunning` setting.
+     */
+    printWhileRunning?: boolean;
+    /**
+     * Defaults to true.
+     * If true - collects stdout and stderr while running, and return it in the end.
+     * stdout/stderr are collected and returned regardless if it returns with error or not.
+     * On success - stdout/stderr are available from `SpawnOutput`.
+     * On error - stdout/stderr are available from `SpawnError.data`.
+     */
+    collectOutputWhileRunning?: boolean;
+    /**
+     * Defaults to true.
+     * If true - throws SpawnError if non-zero code is returned.
+     * SpawnError conveniently contains .data.stdout and .data.strerr for inspection.
+     * If false - will not throw, but return SpawnOutput with stdout, stderr and exitCode.
+     */
+    throwOnNonZeroCode?: boolean;
+}
+export interface SpawnOptions {
+    args?: string[];
+    /**
+     * Defaults to true.
+     */
+    logStart?: boolean;
+    /**
+     * Defaults to true.
+     */
+    logFinish?: boolean;
+    /**
+     * Defaults to true.
+     * Controls/overrides both logStart and logFinish simultaneously.
+     */
+    log?: boolean;
+    /**
+     * Defaults to true.
+     */
+    shell?: boolean;
+    /**
+     * If specified - will be used as "command name" for logging purposes,
+     * instead of "cmd + args"
+     */
+    name?: string;
+    cwd?: string;
+    env?: AnyObject;
+}
+export interface ExecOptions {
+    /**
+     * Defaults to false.
+     */
+    logStart?: boolean;
+    /**
+     * Defaults to false.
+     */
+    logFinish?: boolean;
+    /**
+     * Defaults to false.
+     * Controls/overrides both logStart and logFinish simultaneously.
+     */
+    log?: boolean;
+    /**
+     * If specified - will be used as "command name" for logging purposes,
+     * instead of "cmd + args"
+     */
+    name?: string;
+    cwd?: string;
+    timeout?: NumberOfMilliseconds;
+    env?: AnyObject;
+}
+export {};

package/dist/util/exec2.js

@@ -0,0 +1,204 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SpawnError = exports.exec2 = void 0;
+const tslib_1 = require("tslib");
+const node_child_process_1 = tslib_1.__importDefault(require("node:child_process"));
+const js_lib_1 = require("@naturalcycles/js-lib");
+const colors_1 = require("../colors/colors");
+/**
+ * Set of utility functions to work with Spawn / Exec.
+ *
+ * How to decide between Spawn and Exec?
+ *
+ * Long-running job that prints output, and no need to return the output - use Spawn.
+ *
+ * Short-running job, no need to print the output, might want to return the output - use Exec.
+ *
+ * Need to both print and return the output - use SpawnAsync.
+ *
+ * ***
+ *
+ * Spawn is good for long-running large-output processes, that continuously output data.
+ * E.g running `jest`.
+ *
+ * Exec is the opposite - good for short-running processes that output small data.
+ * Exec allows to return the output as a string.
+ * Exec doesn't stream data during execution, so the output/error will only be printed
+ * at the end.
+ * Exec always uses the shell (there's no option to disable it).
+ */
+class Exec2 {
+    /**
+     * Advanced/async version of Spawn.
+     * Consider simpler `spawn` or `exec` first, which are also sync.
+     *
+     * spawnAsync features:
+     *
+     * 1. Async
+     * 2. Allows to collect the output AND print it while running.
+     * 3. Returns SpawnOutput with stdout, stderr and exitCode.
+     * 4. Allows to not throw on error, but just return SpawnOutput for further inspection.
+     *
+     * Defaults:
+     *
+     * shell: true
+     * printWhileRunning: true
+     * collectOutputWhileRunning: true
+     * throwOnNonZeroCode: true
+     * log: true
+     */
+    async spawnAsync(cmd, opt = {}) {
+        const started = Date.now();
+        this.logStart(cmd, opt);
+        const { shell = true, printWhileRunning = true, collectOutputWhileRunning = true, throwOnNonZeroCode = true, cwd, env, } = opt;
+        let stdout = '';
+        let stderr = '';
+        return await new Promise((resolve, reject) => {
+            const p = node_child_process_1.default.spawn(cmd, opt.args || [], {
+                shell,
+                cwd,
+                env,
+                // ...process.env, // not passing by default for security reasons
+            });
+            p.stdout.on('data', data => {
+                if (collectOutputWhileRunning) {
+                    stdout += data.toString();
+                    // console.log('stdout:', data.toString())
+                }
+                if (printWhileRunning) {
+                    process.stdout.write(data);
+                    // console.log('stderr:', data.toString())
+                }
+            });
+            p.stderr.on('data', data => {
+                if (collectOutputWhileRunning) {
+                    stderr += data.toString();
+                }
+                if (printWhileRunning) {
+                    process.stderr.write(data);
+                }
+            });
+            p.on('close', code => {
+                this.logFinish(cmd, opt, started);
+                const exitCode = code || 0;
+                const o = {
+                    exitCode,
+                    stdout: stdout.trim(),
+                    stderr: stderr.trim(),
+                };
+                if (throwOnNonZeroCode && code) {
+                    return reject(new SpawnError(`spawnAsync exited with code ${code}: ${cmd}`, o));
+                }
+                resolve(o);
+            });
+        });
+    }
+    /**
+     * Reasons to use it:
+     * - Sync
+     * - Need to print output while running
+     *
+     * Limitations:
+     * - Cannot return stdout/stderr (use exec or spawnAsync for that)
+     *
+     * Defaults:
+     *
+     * shell: true
+     * log: true
+     */
+    spawn(cmd, opt = {}) {
+        const started = Date.now();
+        this.logStart(cmd, opt);
+        const { shell = true, cwd, env } = opt;
+        const r = node_child_process_1.default.spawnSync(cmd, opt.args, {
+            encoding: 'utf8',
+            stdio: 'inherit',
+            shell,
+            cwd,
+            env,
+            // ...process.env, // not passing by default for security reasons
+        });
+        this.logFinish(cmd, opt, started);
+        if (r.error) {
+            throw r.error;
+        }
+        if (r.status) {
+            throw new Error(`spawn exited with code ${r.status}: ${cmd}`);
+        }
+    }
+    /**
+     * Reasons to use it:
+     *
+     * - Sync
+     * - Need to return output
+     *
+     * Limitations:
+     * - Cannot print while running (use spawn or spawnAsync for that)
+     *
+     * Defaults:
+     *
+     * shell: true
+     * log: true
+     */
+    exec(cmd, opt = {}) {
+        const started = Date.now();
+        this.logStart(cmd, opt);
+        const { cwd, env, timeout } = opt;
+        try {
+            return node_child_process_1.default
+                .execSync(cmd, {
+                encoding: 'utf8',
+                // stdio: 'inherit', // no, otherwise we don't get the output returned
+                stdio: undefined,
+                // shell: undefined,
+                cwd,
+                timeout,
+                env,
+                // ...process.env, // not passing by default for security reasons
+            })
+                .trim();
+        }
+        catch (err) {
+            // Not logging stderr, as it's printed by execSync by default (somehow)
+            // stdout is not printed by execSync though, therefor we print it here
+            // if ((err as any).stderr) {
+            //   process.stderr.write((err as any).stderr)
+            // }
+            if (err.stdout) {
+                process.stdout.write(err.stdout);
+            }
+            throw new Error(`exec exited with code ${err.status}: ${cmd}`);
+        }
+        finally {
+            this.logFinish(cmd, opt, started);
+        }
+    }
+    throwOnNonZeroExitCode(o) {
+        if (o.exitCode) {
+            throw new SpawnError(`spawn exited with code ${o.exitCode}`, o);
+        }
+    }
+    logStart(cmd, opt) {
+        if (!opt.logStart && !opt.log)
+            return;
+        console.log([
+            (0, colors_1.dimGrey)(...Object.entries(opt.env || {}).map(([k, v]) => [k, v].join('='))),
+            (0, colors_1.white)(opt.name || cmd),
+            ...((!opt.name && opt.args) || []),
+        ]
+            .filter(Boolean)
+            .join(' '));
+    }
+    logFinish(cmd, opt, started) {
+        if (!opt.logFinish && !opt.log)
+            return;
+        console.log([(0, colors_1.white)(opt.name || cmd), (0, colors_1.dimGrey)('took ' + (0, js_lib_1._since)(started))].join(' '));
+    }
+}
+exports.exec2 = new Exec2();
+class SpawnError extends js_lib_1.AppError {
+    constructor(message, data) {
+        super(message, data, { name: 'SpawnError' });
+    }
+}
+exports.SpawnError = SpawnError;

package/dist/util/git2.d.ts

@@ -0,0 +1,25 @@
+import type { UnixTimestampNumber } from '@naturalcycles/js-lib';
+/**
+ * Set of utility functions to work with git.
+ */
+declare class Git2 {
+    getLastGitCommitMsg(): string;
+    commitMessageToTitleMessage(msg: string): string;
+    gitHasUncommittedChanges(): boolean;
+    /**
+     * Returns true if there were changes
+     */
+    gitCommitAll(msg: string): boolean;
+    /**
+     * @returns true if there are not pushed commits.
+     */
+    gitIsAhead(): boolean;
+    gitPull(): void;
+    gitPush(): void;
+    gitCurrentCommitSha(full?: boolean): string;
+    gitCurrentCommitTimestamp(): UnixTimestampNumber;
+    gitCurrentBranchName(): string;
+    gitCurrentRepoName(): string;
+}
+export declare const git2: Git2;
+export {};