dcp-worker 4.1.1 → 4.2.1

package/lib/blessed-components/log.js

@@ -50,11 +50,21 @@ class Log extends Box
     this.setContent(this.lastLogContent);
     if (!this.paused)
       this.setScrollPerc(100);
+    if (this.pendingUpdate)
+    {
+      this.pendingUpdate = true;
+      setImmediate(() => {
+        this.pendingUpdate = false;
+        this.screen.render();
+      });
+    }
+    
   }
 
   advanceThrob(char)
   {
     this.setContent(this.lastLogContent + char);
+    this.screen.render();
   }
 }
 

package/lib/check-scheduler-version.js

@@ -17,6 +17,7 @@ exports.check = function checkSchedulerVersion$$check(quiet)
 {
   const dcpConfig = require('dcp/dcp-config');
   const schedulerConfig = dcpConfig.scheduler;
+  const { errorCodes } = require('./consts');
 
   // Check for old versions of the config
   if (!schedulerConfig.worker)
@@ -50,9 +51,9 @@ exports.check = function checkSchedulerVersion$$check(quiet)
                 `This worker is:  ${currentWorkerType}, ${currentWorkerVersion}\n` +
                 `dcp-client: ${require('util').inspect(require('dcp/build'))}\n`);
     if (process.env.DCP_WORKER_CHECK_SEMVER !== 'false')
-      process.exit(1);
+      process.exit(exitCodes.invalidScheduler);
   }
 
   if (!quiet)
-    console.log(`The current scheduler supports worker type(s) ${Object.values(schedulerConfig.worker.types)} and operations ${schedulerConfig.worker.operations}`);
+    console.log(` . Scheduler supports worker type(s) ${Object.values(schedulerConfig.worker.types)} and operations ${schedulerConfig.worker.operations}`);
 }

package/lib/consts.js

@@ -0,0 +1,25 @@
+/**
+ * @file     consts.js - Constants for use within dcp-worker program
+ * @author   Wes Garland, wes@distributive.network
+ * @date     June 2025
+ */
+'use strict';
+
+require('dcp-client');
+
+const { ConstantGroup } = require('dcp/utils');
+exports.exitCodes = new ConstantGroup({
+  normal:            0, /* 0=no error, 1-15 used by Node.js */
+  error:            22, /* misc errors */
+  invalid:          24,
+  unhandled:        25, /* unhandled rejection or uncaught exception */
+  lostRef:          26, /* lost worker reference unexpectedly */
+  userError:        27, /* user made an error in cli opts */
+
+  invalidScheduler: 80, /* scheduler version check failed */
+  pidConflict:      81, /* pidfile library can't create unique pidfile */
+  workerError:      82, /* worker.on('error') emitted */
+});
+
+/* log levels, ordered from least to most important */
+exports.logLevels = new ConstantGroup(['debug', 'log', 'info', 'warn', 'error']);

package/lib/loggers.js

@@ -0,0 +1,256 @@
+/**
+ *  @file       startWorkerLogger.js
+ *              Start the DCP Worker logging subsystem. Sets console.log (etc) redirection, determine
+ *              the correct log target (screen, TUI, syslog, windows event log, file) and redirect the
+ *              output there.
+ *
+ * This module's exports are:
+ *  - hook()   - replace console with new console and multiplex its logs to loggers
+ *  - unhook() - go back to normal
+ *  - exists() - determine if a given logger is exists or not
+ *
+ * A logger module's exports are its API. The following functions are supported, in this order
+ * of severity:
+ * . debug: debug-level message
+ * . info:  informational message
+ * . log:   normal, but significant, condition
+ * . warn:  warning conditions
+ * . error: error conditions
+ *
+ * Additionally, generic functions may be used when one of the above is not defined:
+ * . at:    write a log message at a specific log level; the level is the first argument
+ * . raw:   same as at, but arguments are not formatted
+ * . any:   write a log message without regard to log level
+ *
+ * All of these functions, with the exception of raw, receive only string arguments.
+ *
+ * When this library is initialized, it initializes all of the loggers that the program is using (per
+ * options.loggers). Just before initialization, each logger module has a handleFatalError export added
+ * to it. Loggers should use this export to report errors which are fatal to the logger back to this
+ * module, so that the logger can be removing from the multiplex list.
+ *
+ * This is completely untested, however, it should be possible to specify an arbitrary logger which is
+ * outside of the dcp-worker package by specifying the absolute path to a module implementing the logger
+ * API as the logger name.
+ *
+ * @author     Ryan Rossiter, ryan@kingsds.network
+ * @date       April 2020
+ * @author     Wes Garland, wes@distributive.network
+ * @date       Jun 2025
+ */
+'use strict';
+
+const process = require('process');
+const path    = require('path');
+const util    = require('util');
+const debug   = require('debug');
+
+const { logLevels } = require('./consts');
+const logLevelValues = Object.values(logLevels);
+const loggers = [];
+const { Reference } = require('dcp/dcp-timers');
+var preHookConsole;
+
+/**
+ * Default options for util.inspect. Merged with each logger library for individual inspect preferences.
+ */
+const inspectOptions = {
+  colors: process.env.FORCE_COLOR ? true : 'auto',
+};
+
+/**
+ * @param {string} loggerName    name of the logger (eg syslog, logfile)
+ * @returns {object} logger's module exports
+ */
+function requireLogger(loggerName)
+{
+  try
+  {
+    debug('dcp-worker:logger')(`loading logger '${loggerName}'`);
+    return require(path.resolve(__dirname, 'worker-loggers', loggerName));
+  }
+  catch (error)
+  {
+    if (error.code === 'MODULE_NOT_FOUND')
+    {
+      const errorMessageStart = error.message.replace(/\n.*/g, '');
+      error.message = `Unknown logger '${loggerName}' (${errorMessageStart})`;
+    }
+    throw error;
+  }
+}
+
+/**
+ * Check to see if a given string is the name of a logger (eg syslog)
+ * @param {string} loggerName name of logger
+ * @returns true iff loggerName is the name of a logger
+ */
+exports.exists = function loggers$exists(loggerName)
+{
+  try
+  {
+    return Boolean(requireLogger(loggerName));
+  }
+  catch(error)
+  {
+    return false;
+  }
+}
+
+/**
+ * Replace globalThis.console with the new console object, after modifying it so that it logs to the
+ * loggers in addition to the console.
+ *
+ * Each logger (module in ./worker-loggers) represents a different type of non-console logging. This
+ * currently (jun 2025) includes syslog, a log file, and the windows event log. It is arbitrarily
+ * expandable to including additional loggger types in the future, however the architecture currently
+ * limits us to one log target (destination) per logger type.
+ *
+ * @param {object} options - loggingOptions from dcp-worker
+ */
+exports.hook = function loggers$$hook(console, options)
+{
+  debug('dcp-worker:log')('Initializing loggers:', options.loggers);
+
+  for (let loggerName of options.loggers)
+  {
+    const logger = requireLogger(loggerName);
+    loggers.push(logger);
+    logger.inspectOptions = Object.assign({}, inspectOptions, logger.inspectOptions);
+    logger.handleFatalError = function loggers$$handleFatalError(error) {
+      const idx = loggers.indexOf(logger);
+      if (idx === -1)
+        return;
+      debug('dcp-worker:logger')(`${loggerName} logger reports fatal error`, error);
+      loggers.splice(idx, 1);
+      const msg = `Disabling logger ${loggerName}: ${error?.message}` + (error?.code ? ` (${error.code})` : '');
+      console.error(msg);
+      multiplex(logLevels.error, msg);
+    };
+    try
+    {
+      logger.open(options);
+    }
+    catch(error)
+    {
+      logger.handleFatalError(error);
+    }
+  }
+
+  function multiplex(level, ...args)
+  {
+    for (let loggerName in loggers)
+    {
+      if (isLessImportant(level, options.minLevel[loggerName] || options.minLevel.all))
+        continue;
+      const logger = loggers[loggerName];
+      if (logger[level])
+        logger[level](...format(logger, ...args));
+      else if (logger.at)
+        logger.at(level, ...format(logger, ...args));
+      else if (logger.raw)
+        logger.raw(level, ...args);
+      else if (logger.any)
+        logger.any(`${level}:`, format(logger, ...args));
+      else
+        throw new Error(`logger ${loggerName} missing implementation for '${level}'`);
+    }
+  }
+
+  /* Replace the global console object with a console object which is derived from the console argument
+   * to this function, but modified so that it will multiplex its output to all loggers.
+   */
+  preHookConsole = globalThis.console;
+  globalThis.console = Object.create(console.constructor.prototype);
+  globalThis.console.constructor = console.constructor;
+  Object.assign(globalThis.console, console);
+
+  for (let level of Object.values(logLevels))
+  {
+    globalThis.console[level] = function loggers$$consoleHookWrapper(...args) {
+      multiplex(level, ...args);
+      if (!isLessImportant(level, options.minLevel.console || options.minLevel.all))
+        console[level](...args);
+    };
+  }
+
+  require('./telnetd').reintercept();
+
+  /**
+   * Console.throb is effectively console.info from the POV of remote loggers, but on the local console,
+   * it is slightly different because it can place the cursor and advances a throbber.
+   * (API - no args = advance throbber, else print message)
+   */
+  globalThis.console.throb = (...args) => {
+    console.throb(...args);
+    multiplex('info', ...args);
+  };
+}
+
+/**
+ * Format console.log arguments for use by a non-native logger, eg syslog.  All non-string arguments are
+ * converted into the best human-readable strings we can muster.
+ */
+function format(logger, ...argv)
+{
+  for (let i in argv)
+  {
+    try
+    {
+      if (typeof argv[i] === 'object' && argv[i] instanceof String)
+        argv[i] = String(argv[i]);
+      argv[i] = typeof argv[i] === 'string' ? argv[i] : util.inspect(argv[i], logger.inspectOptions);
+    }
+    catch(e)
+    {
+      argv[i] = '[encoding error: ' + e.message + ']';
+    }
+  }
+
+  return argv;
+}
+
+/**
+ * Returns true if log level1 is less important than log level2
+ */
+function isLessImportant(level1, level2)
+{
+  if (typeof level2 === 'undefined')
+    return false;
+  if (level1 === level2)
+    return false;
+
+  const idx1 = logLevelValues.indexOf(level1);
+  const idx2 = logLevelValues.indexOf(level2);
+
+  return idx1 < idx2;
+}
+
+/**
+ * Close all loggers that have close functions that were initialized during hook(), removing them
+ * from the multiplex list just before they are closed. An event loop reference is added for the
+ * duration of this function, so that we don't accidentally exit while waiting for loggers to flush.
+ */
+exports.unhook = async function loggers$$unhook()
+{
+  debug('dcp-worker:exit')('unhooking', loggers.length, 'loggers');
+  const elref = new Reference().ref();
+  for (let logger; (logger = loggers.pop());)
+  {
+    try
+    {
+      await logger.close();
+    }
+    catch(error)
+    {
+      debug('dcp-worker:exit')('error closing logger;', error);
+    }
+  }
+  elref.unref();
+
+  if (preHookConsole)
+  {
+    globalThis.console = preHookConsole;
+    require('./telnetd').reintercept();
+  }
+}

package/lib/node-version-check.js

@@ -1,7 +1,9 @@
 /**
- * @file     node-version-check.js - preload module
+ * @file     node-version-check.js - preload module, not part of dcp-worker
  * @author   Wes Garland, wes@distributive.network
  * @date     Sep 2024
+ *
+ * @example: node -r ./lib/node-version-check bin/dcp-worker
  */
 'use strict';
 
@@ -9,5 +11,5 @@ if (!(parseInt(process.versions.node, 10) >= 18))
 {
   console.error(`Your version of node (${process.version}) is far to old to run this program. Please upgrade to a supported version of Node.js.`);
   console.error('For more information, visit', require('../package').homepage);
-  process.exit(1);
+  process.exit(127);
 }

package/lib/pidfile.js

@@ -9,6 +9,7 @@
 'use strict';
 
 const fs   = require('fs');
+const os   = require('os');
 const path = require('path');
 
 /**
@@ -20,11 +21,13 @@ const path = require('path');
  * cleanup depends on the process<dcpExit> event fired by dcp-client during process tear-down.
  *
  * @param {string} filename           the location of the pid file
+ * @returns true iff writing the pid file was successful
  */
 exports.write = function pidfile$$write(filename)
 {
   var fd;
-  
+
+  console.debug(' . pidfile is', filename);
   if (fs.existsSync(filename))
   {
     console.warn(`Warning: found pidfile at ${filename}`);
@@ -48,20 +51,23 @@ exports.write = function pidfile$$write(filename)
     {
       console.error(`Process at PID ${oldPid} is still running; cannot start new process with pidfile ${filename}`);
       require('dcp/utils').sleep(3); /* put the brakes on a respawn loop */
-      process.exit(1);
+      return false;
     }
   }
 
   try
   {
+    fs.mkdirSync(path.dirname(filename), { recursive: true });
     fd = fs.openSync(filename, 'wx');
     fs.writeSync(fd, Buffer.from(process.pid + '\n'), 0);
     process.on('dcpExit', removePidFile);   // Cleanup PID file on exit
+    return true;
   }
   catch (error)
   {
     console.warn(`Warning: Could not create pidfile at ${filename} (${error.code || error.message})`);
     removePidFile();
+    return false;
   }
 
   function removePidFile()
@@ -74,7 +80,8 @@ exports.write = function pidfile$$write(filename)
     }
     catch (error)
     {
-      console.warn(`Warning: Could not remove pidfile at ${filename} (${error.code})`);
+      if (fd)
+        console.warn(`Warning: Could not remove pidfile at ${filename} (${error.code})`);
     }
   }
 }
@@ -90,7 +97,84 @@ exports.getDefaultPidFileName = function getDefaultPidFileName(basename)
   var defaultPidFileName = basename || path.basename(require.main.filename, '.js') + '.pid';
 
   if (!path.isAbsolute(defaultPidFileName))
-    defaultPidFileName = path.resolve(__dirname, '..', 'run', defaultPidFileName);
+  {
+    switch(os.platform())
+    {
+      case 'linux':
+        defaultPidFileName = path.resolve(process.env.XDG_RUNTIME_DIR || '/run/', defaultPidFileName);
+        break;
+      default:
+        defaultPidFileName = path.resolve(process.env.XDG_RUNTIME_DIR || '/var/run/', defaultPidFileName);
+        break;
+      case 'win32':
+        defaultPidFileName = path.resolve(process.env.LOCALAPPDATA || process.env.TEMP, 'Distributive/run');
+        break;
+    }
+  }
 
   return defaultPidFileName;
 }
+
+/**
+ * Returns true if the signal was delivered or false if it was not delivered because the process was not
+ * running. Throws if there was an error (eg perms) delivering the signal.
+ */
+function kill(pid, signal)
+{
+  try
+  {
+    process.kill(pid, signal);
+  }
+  catch(error)
+  {
+    if (error.code === 'ESRCH') /* no such process */
+      return false;
+    throw error;
+  }
+  return true;
+}
+
+/**
+ * Signal another process on this host. Handles the case of a dangnling pidfile by ignoring it.
+ * @param {string}              filename      the path to the pidfile, same semantics as exports.write()
+ * @param {string | number }    signal        the signal to send
+ * @param {number | undefined } timeout       the number of seconds to wait for the process to die before
+ *                                            rejecting with code ETIMEOUT
+ * @param {string | undefined}  doingWarning  console.throb() string to send just before signalling. If
+ *                                            the string contains %i, it will be replaced with the pid.
+ * @rejects {Error} if we can't deliver the signal for an unexpected reason (eg. permssions) or the
+ *                  process took more than timeout seconds to die.
+ * @returns {Promise} which resolves true if the signal was delivered and the process died
+ */
+exports.signal = async function pidfile$$signal(filename, signal, timeout, doingWarning)
+{
+  const { a$sleep } = require('dcp/utils');
+  if (!filename || !fs.existsSync(filename))
+    return false;
+
+  const pid = parseInt(fs.readFileSync(filename, 'utf8'), 10);
+  if (!pid || kill(pid, 0) === false)
+    return false;
+
+  if (doingWarning)
+    console.throb(doingWarning.replace(/%i/g, pid));
+
+  kill(pid, signal);
+
+  const loopUntil = Date.now() + (timeout * 1e3);
+  while (kill(pid, 0) === true)
+  {
+    await a$sleep(0.10);
+    if (doingWarning)
+      console.throb();
+    if (Date.now() >= loopUntil)
+    {
+      const error = new Error(`timeout waiting for process ${pid} to exit`);
+      error.code = 'ETIMEOUT';
+      return error;
+    }
+    await a$sleep(0.40);
+  }
+
+  return true;
+}

package/lib/reports.js

@@ -0,0 +1,95 @@
+#! /usr/bin/env node
+/**
+ * @file       reports.js
+ *             Interval reports for dcp-worker, based on code removed from dcp-worker
+ *
+ * @author     Wes Garland, wes@distributive.network
+ * @date       May 2025
+ */
+'use strict';
+
+exports.generateSliceReport = generateSliceReport;
+exports.printSliceReport    = printSliceReport;
+
+const hr = ('='.repeat(78)) + '\n';
+
+function printSliceReport(worker)
+{
+  const report = generateSliceReport(worker);
+  console.log(hr + report + hr);
+  return report;
+}
+
+/** generator a slice report screen */
+function generateSliceReport(worker)
+{
+  let report = '';
+
+  if (!worker)
+    return ' * worker not defined * ';
+  
+  const sbStates = {
+    WORKING: 0,
+    ASSIGNED: 0,
+    READY: 0,
+    TERMINATED: 0,
+  };
+  const stateNames = {
+    WORKING: 'Working',
+    ASSIGNED: 'Assigned',
+    READY: 'Ready',
+    TERMINATED: 'Terminated',
+  };
+  worker.sandboxes?.forEach(sb => {
+    const { state } = sb;
+    if (!sbStates[state])
+      sbStates[state] = 0;
+    sbStates[state]++;
+  });
+
+  report += (Date()) + '\n';
+  report += ('Sandboxes:') + '\n';
+  Object.keys(sbStates).forEach(state => {
+    const stateName = stateNames[state] || state;
+    report += (`  ${(stateName + ':').padEnd(12)} ${sbStates[state]}`) + '\n';
+  })
+  report += (`  * ALL:       ${worker.sandboxes?.length}`) + '\n';
+
+  report += ('Progress:') + '\n';
+  worker.workingSandboxes?.forEach(sb => {
+    const jobName = sb.job?.public?.name || `idek (${sb.jobId})`;
+    let el = Date.now() - sb.sliceStartTime;
+    const t = el < 1000000
+          ? toInterval(el)
+          : 'new';
+
+    el = sb.progressReports && sb.progressReports.last
+      ? Date.now() - (sb.sliceStartTime + (sb.progressReports.last?.timestamp ?? 0))
+      : 0;
+    const pct = (typeof sb.progress) === 'number'
+          ? `${Number(sb.progress).toFixed(0).padStart(2)}%`
+          : 'ind';
+    const stale = (el < 2000) ? '' : `(stale: ${toInterval(el)})`;
+
+    report += (` ${String(sb.id).padStart(4)}: ${sb.jobId} ${jobName.padEnd(34)} `+ `${t} ${pct} ${stale}`.padStart(13)) + '\n';
+  });
+
+  report += ('Slices:') + '\n';
+  report += (`  working:     ${worker.workingSlices?.length}`) + '\n';
+  report += (`  queued:      ${worker.queuedSlices?.length}`) + '\n';
+
+  return report
+}
+
+/**
+ *  Convert a timespan in ms to a human-readable interval in minutes and seconds
+ *
+ *  @param  {number} el Milliseconds to convert
+ *  @return {string}    Timespan formatted as `m:ss`
+ */
+function toInterval(el)
+{
+  const m = Math.floor((el / 1000) / 60).toString(10);
+  const s = Math.floor((el / 1000) % 60).toString(10).padStart(2, '0');
+  return `${m}:${s}`;
+}

package/lib/show.js

@@ -0,0 +1,54 @@
+/**
+ * @file        show.js
+ *              Implementation for --show-* command-line options
+ * @author      Wes Garland
+ * @date        July 2025
+ */
+'use strict';
+
+exports.show = show;
+
+async function show(worker, mode)
+{
+  const identity = require('dcp/identity');
+  const { DistributiveWorker } = require('dcp/worker');
+  
+  switch (mode)
+  {
+    default:
+    {
+      var value, path;
+      
+      if (!mode.match(/^[a-zA-Z0-9_$.-]*$/))
+        throw new Error('invalid show mode ' + mode);
+      for (value = worker.config, path = mode.split('.');
+           path.length;
+           value = value[path.shift()])
+      ; /* walk the dot path to the end */
+      return value;
+    }
+    case 'config':
+      return worker.config;
+    case 'compute-groups':
+      return worker.inspect.computeGroups;
+    case 'worker-id':
+      return worker.id;
+    case 'allowed-origins':
+      return worker.originManager.origins;
+    case 'identity':
+      return identity.get().address;
+    case 'owner':
+    {
+      if (worker.config.unmanaged)
+        return undefined;
+      const db = await DistributiveWorker.jnuConnect();
+      const cursor = await db.select({
+        table: 'w$worker', prototype: {
+          id: worker.id,
+        }
+      });
+      const row = await cursor.getFirst();
+      return row?.owner;
+    }
+  }
+}