@vbarbarosh/node-helpers 3.71.0 → 3.72.0

package/src/cli-apps/watchdog.d/connect_no_ping.js

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+
+// ❌ 8. Client connects but NEVER sends PING
+// Simulates misbehaving client.
+
+const net = require('net');
+
+net.connect(process.env.WATCHDOG_SOCKET, () => {
+    console.log('connected but no ping');
+    // do nothing
+});
+
+setInterval(() => {}, 1000);

package/src/cli-apps/watchdog.d/early_exit.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+// ❌ 4. Client that exits immediately
+// Watchdog must detect child exit first.
+
+console.log('bye');
+process.exit(0);

package/src/cli-apps/watchdog.d/flood.js

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+
+// ❌ 7. Client that floods watchdog with 1GB of data
+// Outcome: watchdog must handle MASSIVE reads and still time out.
+
+const fs = require('fs');
+const net = require('net');
+
+const socket = net.connect(process.env.WATCHDOG_SOCKET, () => {
+    console.log('connected');
+    const stream = fs.createReadStream('/dev/zero');
+    stream.pipe(socket);
+});

package/src/cli-apps/watchdog.d/flood2.js

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+
+// ❌ 7. Client that floods watchdog with 1GB of data
+// Outcome: watchdog must handle MASSIVE reads and still time out.
+
+const fs = require('fs');
+const net = require('net');
+
+const socket = net.connect(process.env.WATCHDOG_SOCKET, () => {
+    console.log('connected');
+    const stream = fs.createReadStream('/dev/zero');
+    stream.pipe(socket).on('error', () => 0);
+});
+
+setInterval(() => {}, 1000);

package/src/cli-apps/watchdog.d/hang.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+// ❌ 5. Client that freezes (event loop blocked)
+// Watchdog will not receive PINGs.
+
+while (true) {
+    // infinite CPU loop
+}

package/src/cli-apps/watchdog.d/huge_delay.js

@@ -0,0 +1,18 @@
+#!/usr/bin/env node
+
+// ❌ 10. Client sends heartbeats but with huge delay
+// Tests delayed heartbeat behavior.
+// Expected: watchdog should kill before that.
+
+const net = require('net');
+
+const INTERVAL = Number(process.env.WATCHDOG_INTERVAL || 1000);
+
+const socket = net.connect(process.env.WATCHDOG_SOCKET, () => {
+    console.log('connected');
+
+    setTimeout(() => {
+        console.log('sending late heartbeat');
+        socket.end('PING');
+    }, INTERVAL * 10);
+});

package/src/cli-apps/watchdog.d/ignore_sigterm.js

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+
+// 1️⃣ Client that ignores SIGTERM
+// Goal: process stays alive after SIGTERM and only dies on SIGKILL (exactly what pid_kill_grace should handle).
+//
+// What this tests for watchdog:
+//   - watchdog sends SIGTERM → process prints “ignored” and keeps running.
+//   - pid_kill_grace waits grace_timeout_ms, sees PID still alive, sends SIGKILL.
+//   - Process must die after SIGKILL.
+//   - Watchdog should log the whole sequence correctly.
+// This is a realistic scenario.
+
+console.log('PID:', process.pid);
+
+// Ignore SIGTERM
+process.on('SIGTERM', function () {
+    console.log('SIGTERM received, but intentionally ignored');
+});
+
+// Still react to SIGINT (Ctrl+C) so you can stop it manually if needed
+process.on('SIGINT', function () {
+    console.log('SIGINT received, exiting');
+    process.exit(130);
+});
+
+// Keep process alive
+setInterval(() => {
+    // simulate work
+}, 1000);

package/src/cli-apps/watchdog.d/late_heartbeat.js

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+
+// ❌ 11. Client sends heartbeats sporadically (sometimes missing)
+// Simulates flaky client.
+
+const net = require('net');
+
+function randomDelay() {
+    return Math.floor(Math.random() * 5000);
+}
+
+setTimeout(ping, randomDelay());
+
+function ping()
+{
+    const socket = net.connect(process.env.WATCHDOG_SOCKET);
+    socket.on('connect', () => {
+        socket.end('PING');
+        setTimeout(ping, randomDelay());
+    });
+}

package/src/cli-apps/watchdog.d/my-exit-after-2-pings.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+const Promise = require('bluebird');
+const cli = require('../../cli');
+const ping_socket = require('../../ping_socket');
+
+cli(main);
+
+async function main()
+{
+    for (let iter = 1, last = 2; iter <= last; ++iter) {
+        await ping_socket(process.env.WATCHDOG_SOCKET);
+        await Promise.delay(1000);
+    }
+    console.log('Bye!');
+}

package/src/cli-apps/watchdog.d/my-happy-path.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+const Promise = require('bluebird');
+const cli = require('@vbarbarosh/node-helpers/src/cli');
+const ping_socket = require('@vbarbarosh/node-helpers/src/ping_socket');
+
+cli(main);
+
+async function main()
+{
+    for (let i = 0; i < 100; ++i) {
+        console.log('[client]', i);
+        await ping_socket(process.env.WATCHDOG_SOCKET);
+        await Promise.delay(1000);
+    }
+}

package/src/cli-apps/watchdog.d/my-ignore-sigterm.js

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+
+const Promise = require('bluebird');
+const cli = require('@vbarbarosh/node-helpers/src/cli');
+const now_human = require('@vbarbarosh/node-helpers/src/now_human');
+
+cli(main);
+
+async function main()
+{
+    process.on('SIGTERM', function () {
+        console.log(`[${now_human()}][ignore-sigterm] SIGTERM, ignoring...`);
+    });
+    process.on('SIGINT', function () {
+        console.log(`[${now_human()}][ignore-sigterm] SIGINT, ignoring...`);
+    });
+
+    for (let iter = 1, last = 100; iter <= last; ++iter) {
+        console.log(`[${now_human()}][ignore-sigterm] ${iter} of ${last}`);
+        await Promise.delay(100);
+    }
+}

package/src/cli-apps/watchdog.d/my-never-ping.js

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+
+const Promise = require('bluebird');
+const cli = require('@vbarbarosh/node-helpers/src/cli');
+const now_human = require('@vbarbarosh/node-helpers/src/now_human');
+
+cli(main);
+
+async function main()
+{
+    for (let iter = 1, last = 100; iter <= last; ++iter) {
+        console.log(`[${now_human()}][never-ping] ${iter} of ${last}`);
+        await Promise.delay(100);
+    }
+}

package/src/cli-apps/watchdog.d/no_heartbeat.js

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+
+// ❌ 2. Client that never sends heartbeat (expected: watchdog kills it)
+
+setInterval(() => {}, 1000); // alive but no PINGs

package/src/cli-apps/watchdog.d/ok.js

@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+
+// ✅ 1. Client that works normally (sends correct PINGs)
+
+const net = require('net');
+
+const SOCKET = process.env.WATCHDOG_SOCKET;
+const INTERVAL = Number(process.env.WATCHDOG_INTERVAL || 1000);
+
+if (!SOCKET) throw new Error('WATCHDOG_SOCKET missing');
+
+setInterval(ping, INTERVAL / 2);
+
+function ping()
+{
+    const socket = net.connect(SOCKET, () => socket.end('PING'));
+}

package/src/cli-apps/watchdog.d/spawn_children.js

@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+
+// ❌ 6. Client that spawns children
+// This tests your process group killing.
+// If watchdog does not kill the entire group, the sleeps will remain after watchdog exits.
+
+const {spawn} = require('child_process');
+
+spawn('sleep', ['9999'], {detached: false});
+spawn('sleep', ['9999'], {detached: false});
+
+console.log('spawned 2 children');
+
+setInterval(() => {}, 1000);

package/src/cli-apps/watchdog.js

@@ -0,0 +1,146 @@
+#!/usr/bin/env node
+
+const ExitCodeError = require('../errors/ExitCodeError');
+const HeartbeatServer = require('../HeartbeatServer');
+const Promise = require('bluebird');
+const cli = require('../cli');
+const cuid = require('@paralleldrive/cuid2');
+const make = require('@vbarbarosh/type-helpers/src/make');
+const now_human = require('../now_human');
+const pgid_exists = require('../pgid_exists');
+const pgid_kill_grace = require('../pgid_kill_grace');
+const shell_spawn = require('../shell_spawn');
+
+const LOGS_ROOT_UID = cuid.createId();
+
+// 💎 Only the lack of a valid PING within WATCHDOG_INTERVAL is fatal
+
+// Ctrl-C should gracefully terminate a child process and close heartbeat server
+// Test
+// - heartbeat server starts failed
+// - heartbeat server does not close connection
+// - heartbeat server sends 1G of data
+// - child spawn failed
+// - child receives no valid WATCHDOG_SOCKET
+// - child does not send heartbeat
+// - child spawns several children (watchdog → child → subchild → subsubchild)
+// - watchdog has global timeout, child terminated earlier: watchdog should return immediately
+
+cli(main, report);
+
+function report(error)
+{
+    log(`[heartbeat_end_error] ${error.message}`);
+}
+
+async function main()
+{
+    const args = process.argv.slice(2);
+    if (!args.length) {
+        usage();
+        process.exit(1);
+    }
+
+    log('[heartbeat_begin]');
+
+    const WATCHDOG_INTERVAL = make(process.env.WATCHDOG_INTERVAL, {type: 'int', min: 1000, default: 5000});
+    const heartbeat_server = new HeartbeatServer(WATCHDOG_INTERVAL);
+    const WATCHDOG_SOCKET = heartbeat_server.socket_path;
+    heartbeat_server.on('heartbeat', () => log('[heartbeat_ping] ❤️'))
+    heartbeat_server.on('warning', error => log(`[heartbeat_warn] ⚠️ ${error}`))
+
+    try {
+        log(`[heartbeat_socket_path] ${heartbeat_server.socket_path}`);
+        const env = {...process.env, WATCHDOG_INTERVAL, WATCHDOG_SOCKET};
+        const proc = await shell_spawn(args, {stdio: 'inherit', env, detached: true}).init();
+        const pgid = proc.pid; // group id == leader pid
+
+        const signal = new Promise(function (resolve, reject) {
+            process.on('SIGTERM', function () {
+                log('[heartbeat_sigterm]');
+                reject(new ExitCodeError(143, 'SIGTERM'));
+            });
+            process.on('SIGINT', function () {
+                log('[heartbeat_sigint]');
+                reject(new ExitCodeError(130, 'SIGINT'));
+            });
+        });
+
+        try {
+            await Promise.race([signal, heartbeat_server.promise(), proc.promise()]);
+            // happy path
+        }
+        finally {
+            if (pgid_exists(pgid)) {
+                await pgid_kill_grace(pgid, {
+                    log: s => log(`[heartbeat_pgid_kill_grace] ${s}`),
+                });
+            }
+        }
+    }
+    finally {
+        log('[heartbeat_dispose] Closing heartbeat server...');
+        await Promise.resolve(heartbeat_server.dispose()).timeout(1000);
+    }
+
+    log('[heartbeat_end_ok]');
+    process.exit(0);
+}
+
+function log(s)
+{
+    const ss = JSON.stringify(s).slice(1, -1).replaceAll('\\"', '"');
+    if (ss[0] === '[') {
+        console.log(`[${LOGS_ROOT_UID}][${now_human()}]${ss}`);
+    }
+    else {
+        console.log(`[${LOGS_ROOT_UID}][${now_human()}] ${ss}`);
+    }
+}
+
+function usage()
+{
+    process.stdout.write(`
+usage: watchdog <cmd> [args...]
+
+Runs <cmd> under supervision.  Watchdog starts a heartbeat server,
+spawns the child process, and expects the child to periodically send
+PING messages to the provided UNIX socket.
+
+Environment variables passed to the child:
+
+  WATCHDOG_SOCKET   Path to the UNIX socket that accepts heartbeat
+                    messages.  The child must connect and send 'PING'
+                    frames at regular intervals.
+
+  WATCHDOG_INTERVAL Heartbeat interval in milliseconds.  The child must
+                    send at least one PING within each interval.  If no
+                    PING is received before the timeout expires, the
+                    child is considered unresponsive.
+
+Behavior:
+
+  1) Watchdog creates a heartbeat server bound to WATCHDOG_SOCKET.
+  2) Watchdog spawns <cmd> with WATCHDOG_SOCKET and WATCHDOG_INTERVAL
+     injected into the child's environment.
+  3) Watchdog waits for either:
+        - a heartbeat timeout (no PING received in time), or
+        - the child process exiting.
+  4) If the heartbeat timeout expires, watchdog terminates the child
+     process (SIGTERM → grace period → SIGKILL).
+  5) Watchdog shuts down the heartbeat server and exits with a
+     non-zero status on failure or with the child's exit code on
+     normal termination.
+
+Exit Codes:
+
+------------------------+---------
+child exit 0            | 0
+child exit non-zero     | child exit code
+heartbeat timeout       | 124
+SIGINT                  | 130
+SIGTERM                 | 143
+watchdog internal error | 1
+-----------------------------------
+`.trim() + '\n');
+}

package/src/cli.js

@@ -1,7 +1,10 @@
+const ExitCodeError = require('./errors/ExitCodeError');
+const Promise = require('bluebird');
+
 /**
  * Entry point for Node CLI apps
  */
-function cli(main)
+function cli(main, report = error => console.error(error))
 {
     // https://stackoverflow.com/a/46916601/1478566
     const timer = setInterval(v => v, 1E9);
@@ -13,8 +16,13 @@ function cli(main)
     }
     function reject(error) {
         clearInterval(timer);
-        console.error(error);
-        process.exit(1);
+        report(error);
+        if (error instanceof ExitCodeError) {
+            process.exit(error.exit_code);
+        }
+        else {
+            process.exit(1);
+        }
     }
 }
 

package/src/errors/ExitCodeError.js

@@ -0,0 +1,9 @@
+class ExitCodeError extends Error
+{
+    constructor(exit_code, message = '') {
+        super(message);
+        this.exit_code = exit_code;
+    }
+}
+
+module.exports = ExitCodeError;

package/src/fs_assert_file_socket.js

@@ -0,0 +1,11 @@
+const fs_fi = require('./fs_fi');
+
+async function fs_assert_file_socket(file)
+{
+    const fi = await fs_fi(file);
+    if (!fi.isSocket()) {
+        throw new Error(`Not a socket: ${file}`);
+    }
+}
+
+module.exports = fs_assert_file_socket;

package/src/math_gcd.js

@@ -0,0 +1,10 @@
+// https://stackoverflow.com/a/39764792/1478566
+function math_gcd(a, b)
+{
+    if (b) {
+        return math_gcd(b, a % b);
+    }
+    return a;
+}
+
+module.exports = math_gcd;

package/src/math_gcd.md

@@ -0,0 +1,2 @@
+Returns the greatest common divisor (GCD) of two integers a and b
+using the Euclidean algorithm.

package/src/math_lcm.js

@@ -0,0 +1,8 @@
+const math_gcd = require('./math_gcd');
+
+function math_lcm(a, b)
+{
+    return Math.abs(a*b) / math_gcd(a, b);
+}
+
+module.exports = math_lcm;

package/src/math_lcm.md

@@ -0,0 +1,2 @@
+Returns the least common multiple (LCM) of two integers a and b
+using the relationship `LCM(a, b) = |a * b| / GCD(a, b)`.

package/src/parse_hms.js

@@ -0,0 +1,43 @@
+function parse_hms(expr)
+{
+    if (!expr && expr !== 0) {
+        return 0;
+    }
+
+    expr = String(expr).trim();
+    if (!expr) {
+        return 0;
+    }
+
+    const parts = expr.split(':');
+
+    let h = 0, m = 0, s = 0;
+
+    if (parts.length === 1) {
+        // "SS(.fff)"
+        s = parseFloat(parts[0]);
+    }
+    else if (parts.length === 2) {
+        // "MM:SS(.fff)"
+        m = parseFloat(parts[0]);
+        s = parseFloat(parts[1]);
+    }
+    else if (parts.length === 3) {
+        // "HH:MM:SS(.fff)"
+        h = parseFloat(parts[0]);
+        m = parseFloat(parts[1]);
+        s = parseFloat(parts[2]);
+    }
+    else {
+        return Number.NaN;
+    }
+
+    // Any failed numeric parse ⇒ NaN
+    if ([h, m, s].some(n => Number.isNaN(n))) {
+        return Number.NaN;
+    }
+
+    return h*3600 + m*60 + s;
+}
+
+module.exports = parse_hms;

package/src/parse_hms.md

@@ -0,0 +1,9 @@
+Parses a time expression into seconds.
+
+**Supported formats:**
+- `SS(.fff)` — seconds
+- `MM:SS(.fff)` — minutes and seconds
+- `HH:MM:SS(.fff)` — hours, minutes, and seconds
+
+**Returns:**  
+A number representing total seconds, or `NaN` if the input is invalid.

package/src/parse_hms.test.js

@@ -0,0 +1,32 @@
+const assert = require('assert');
+const parse_hms = require('./parse_hms');
+
+const items = [
+    ['', 0],
+    [null, 0],
+    [undefined, 0],
+    ['0', 0],
+    ['0:5', 5],
+    ['00:05', 5],
+    ['0:15', 15],
+    ['0:15.123', 15.123],
+    ['0:75.123', 75.123],
+    ['   5', 5],
+    [' 0:05 ', 5],
+    [100, 100],
+    ['1:2:3:4', NaN, 'Too many segments'],
+    ['abc', NaN, 'Non-numeric'],
+    ['0:ab', NaN, 'Non-numeric'],
+    ['1:x:3', NaN, 'Non-numeric'],
+];
+
+describe('parse_hms', function () {
+    items.forEach(function ([input, expected, description]) {
+        const title = description
+            ? `${input} → ${expected} | ${description}`
+            : `${input} → ${expected}`;
+        it(title, function () {
+            assert.strictEqual(expected, parse_hms(input));
+        });
+    });
+});

package/src/pgid_exists.js

@@ -0,0 +1,14 @@
+// group id == leader pid
+// keep in sync with pid_exists
+function pgid_exists(pgid)
+{
+    try {
+        process.kill(-pgid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+
+module.exports = pgid_exists;

package/src/pgid_kill_grace.js

@@ -0,0 +1,55 @@
+const Promise = require('bluebird');
+const format_thousands = require('./format_thousands');
+const ignore = require('./ignore');
+const pgid_exists = require('./pgid_exists');
+
+// group id == leader pid
+// keep in sync with pid_kill_grace
+async function pgid_kill_grace(pgid, {grace_timeout_ms = 5000, log = ignore} = {})
+{
+    log(`Terminating process group ${pgid}: sending SIGTERM`);
+    try {
+        process.kill(-pgid, 'SIGTERM');
+    }
+    catch (error) {
+        throw new Error(`Failed to send SIGTERM to process group ${pgid}: ${error.message}`);
+    }
+
+    const end = Date.now() + grace_timeout_ms;
+    while (true) {
+        if (!pgid_exists(pgid)) {
+            return;
+        }
+        if (Date.now() >= end) {
+            break;
+        }
+        const remain = end - Date.now();
+        log(`Process group ${pgid} still alive; grace time remaining: ${format_thousands(remain)}ms`);
+        await Promise.delay(Math.max(0, Math.min(100, remain)));
+    }
+
+    try {
+        log(`SIGTERM grace period expired for group ${pgid}; sending SIGKILL`);
+        process.kill(-pgid, 'SIGKILL');
+    }
+    catch (error) {
+        if (error.code === 'ESRCH' && error.syscall === 'kill') {
+            log(`Process group ${pgid} already terminated`);
+            return;
+        }
+        throw error;
+    }
+
+    // Wait for kernel to deliver SIGKILL
+    const final_deadline = Date.now() + 200;
+    while (Date.now() < final_deadline) {
+        if (!pgid_exists(pgid)) {
+            return;
+        }
+        await Promise.delay(10);
+    }
+
+    throw new Error(`Process group ${pgid} survived SIGKILL`);
+}
+
+module.exports = pgid_kill_grace;

package/src/pid_exists.js

@@ -1,3 +1,4 @@
+// keep in sync with pgid_exists
 function pid_exists(pid)
 {
     try {

package/src/pid_kill_grace.d/ignore-sigterm-for-10ms.js

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+
+const Promise = require('bluebird');
+const cli = require('../cli');
+const now_human = require('../now_human');
+
+// Child ignores SIGTERM for 10 ms, but dies naturally before the grace loop ends
+
+cli(main);
+
+async function main()
+{
+    let timer;
+
+    process.on('SIGTERM', function () {
+        console.log(`[${now_human()}][ignore-sigterm-for-10ms] SIGTERM_ignoring...`);
+        timer ??= setTimeout(terminate, 10);
+    });
+    process.on('SIGINT', function () {
+        console.log(`[${now_human()}][ignore-sigterm-for-10ms] SIGINT_ignoring...`);
+    });
+
+    for (let iter = 1, last = 100; iter <= last; ++iter) {
+        console.log(`[${now_human()}][ignore-sigterm-for-10ms] ${iter} of ${last}`);
+        await Promise.delay(10);
+    }
+}
+
+function terminate()
+{
+    console.log(`[${now_human()}][ignore-sigterm-for-10ms] TERMINATE_AFTER_10MS`);
+    process.exit(0);
+}