dcp-worker 4.1.1 → 4.2.1

package/lib/{dashboard-tui.js → worker-consoles/dashboard-console.js}

@@ -1,23 +1,31 @@
 /**
- *  @file       worker-loggers/dashboard.js
- *  @author     Ryan Rossiter, ryan@kingsds.network
- *  @date       April 2020
- *  @author     Wes Garland, wes@distributive.network
- *  @date       June 2023
+ * @file       dashboard-console.js
+ * @author     Ryan Rossiter, ryan@kingsds.network
+ * @date       April 2020
+ * @author     Wes Garland, wes@distributive.network
+ * @date       June 2023, June 2025
  *
- *  This module uses the blessed library to create a monitoring dashboard for the worker.
- *  A corresponding worker-logger, dashboard.js, knows how to log to this dashboard.
+ * This module uses the blessed library to create a monitoring dashboard for the worker. The init
+ * function returns a console object which knows how to log to this dashboard. The console's close
+ * method destroys the dashboard and dumps recent log messages to stdout/stderr.
  */
 'use strict';
 
+const process    = require('process');
+const debug      = require('debug');
 const dcpConfig  = require('dcp/dcp-config');
+const wallet     = require('dcp/wallet');
 const chalk      = require('chalk');
 const blessed    = require('blessed');
 const contrib    = require('blessed-contrib');
-const components = require('./blessed-components');
-const utils      = require('../lib/utils');
+const components = require('../blessed-components');
+const utils      = require('../utils');
 
-const { replaceWorkerEventHandler, replaceSandboxEventHandler, newSandboxCallbacks }  = require('./default-ui-events');
+const { Console }    = require('console');
+const { RingBuffer } = require('dcp/utils');
+const { logLevels }  = require('../consts');
+
+const { replaceWorkerEventHandler, replaceSandboxEventHandler, newSandboxCallbacks }  = require('../default-ui-events')
 const systemStateInfo = globalThis.systemStateInfo;
 
 const SLICE_FETCH_STATUS = {
@@ -31,38 +39,55 @@ const usingDebugger = require('module')._cache.niim instanceof require('module')
 const screenConf = {
   input:  usingDebugger ? new (require('events').EventEmitter) : undefined,
   output: usingDebugger ? new (require('events').EventEmitter) : undefined,
+  debug:  false,
+  log:    process.env.DCP_WORKER_BLESSED_DASHBOARD_LOG || undefined,
+  dump:   Boolean(process.env.DCP_WORKER_BLESSED_DASHBOARD_LOG),
+  smartCSR: true,
 };
+const closeLogDumpSize = Number(process.env.DCP_WORKER_DASHBOARD_PRESERVE_LOG_LINES) || 30; /* Dump this many log messages when .close method invoked */
+
+var screen;         /* top-level tui handle */
+var logPane;        /* where console.log etc messages are displayed */
+var sandboxPane;    /* where live info about sandboxes is displayed */
+var workerInfoPane; /* where mostly-static info about the worker is displayed */
+var sliceFetchStatus = SLICE_FETCH_STATUS.IDLE;
+var totalDCCs = 0;
+var lastTask;
+
 /**
- *  Initialize the blessed dashboard
- *  @param  {Worker}    worker    Reference to the DCP Worker
- *  @param {object} options Options which may affect behaviour. Not currently used.
+ * Send a signal to the caller
+ * @param {number|string} sig    the signal to raise
  */
-exports.init = function dashboard$$init(worker, options, bannerLogCumulative)
+function raise(sig)
 {
-  var sliceFetchStatus = SLICE_FETCH_STATUS.IDLE;
-  var totalDCCs = 0;
-  var screen = blessed.screen(screenConf);
+  if (sig === 'SIGESC') /* synthetic */
+    process.emit(sig, sig);
+  else
+    process.kill(process.pid, sig);
+}
 
-  worker.on('end', () =>  {
-    if (screen?.destroy)
-      screen.destroy();
-    screen.destroy = false;
-  });
-  process.on('exit', () => {
-    if (screen?.destroy)
-      screen.destroy();
-  });
+/**
+ * Initialize the blessed dashboard console.
+ */
+function createDashboard()
+{
+  globalThis.screen = screen = blessed.screen(screenConf);
+
+  /* Add redraw method - move cursor to home position, replay screenshot without trailing \n */
+  screen.redraw = function screen$$redraw() {
+    process.stdout.write('\u001b[H' + this.screenshot().slice(0,-1));
+  }
 
   const grid = new contrib.grid({ rows: 3, cols: 6, screen }); // eslint-disable-line new-cap
-  const workerInfoPane = grid.set(2, 0, 1, 6, components.log, {
+  workerInfoPane = grid.set(2, 0, 1, 6, components.log, {
     label: 'Worker Status',
     scrollbar: { bg: 'blue' },
   });
-  const logPane = grid.set(0, 2, 2, 4, components.log, {
+  logPane = grid.set(0, 2, 2, 4, components.log, {
     label: 'Worker Log',
     scrollbar: { bg: 'blue' },
   });
-  const sandboxPane = grid.set(0, 0, 2, 2, components.sandboxPaneFactory, {
+  sandboxPane = grid.set(0, 0, 2, 2, components.sandboxPaneFactory, {
     label: 'Sandboxes',
     scrollable: true,
     alwaysScroll: true,
@@ -87,8 +112,8 @@ exports.init = function dashboard$$init(worker, options, bannerLogCumulative)
     hidden: true,
   });
 
-  global.tui = { workerInfoPane, logPane, sandboxPane, screen, grid, passwordBox };
-  let lastTask;
+  if (usingDebugger)
+    globalThis.tui = { workerInfoPane, logPane, sandboxPane, screen, grid, passwordBox };
 
   function askPassword(promptMessage)
   {
@@ -116,20 +141,32 @@ exports.init = function dashboard$$init(worker, options, bannerLogCumulative)
     return askPassword(promptMessage);
   };
 
-  delete exports.init; /* singleton */
-  console.log(' . Starting dashboard and redirecting logs');
-  if (!usingDebugger)
-    exports.logPane = logPane; /* now dashboard log can find the pane */
-  for (let args of bannerLogCumulative) /* re-draw previously-emited log lines as screen has cleared */
-    logPane.log(...args);
-  setInterval(() => screen.render(), 2000).unref(); /* ensure we didn't forget to render an important update */
-  updateWorkerInfo();
-
   /* Apply key bindings which mimic canonical input mode */
-  screen.key(['C-c'], ()    => raise('SIGINT'));
-  screen.key(['C-z'], ()    => raise('SIGTSTP'));
+  screen.key(['C-c'],    () => raise('SIGINT'));
+  screen.key(['C-z'],    () => raise('SIGTSTP'));
   screen.key(['\u001c'], () => raise('SIGQUIT')); /* C-\ */
-  screen.key(['escape'], () => raise('SIGINT'));
+  screen.key(['escape'], () => raise('SIGESC'));
+  screen.key(['C-l'],    () => screen.redraw());
+
+  /* Try to restore the dashboard after ^Z */
+  process.on('SIGCONT', () => {
+    if (!usingDebugger)
+    {
+      process.stdin.setRawMode(false);
+      process.stdin.setRawMode(true);
+    }
+    screen.render();
+    screen.redraw();
+  });
+}
+
+function setWorker(worker)
+{
+  const identity = require('dcp/identity');
+  
+  console.log(' . Starting dashboard');
+  setInterval(() => screen.render(), 2000).unref(); /* ensure we didn't forget to render an important update */
+  updateWorkerInfo();
 
   setInterval(updateWorkerInfo, 1000).unref();
   function updateWorkerInfo(fetchInfo)
@@ -151,7 +188,7 @@ exports.init = function dashboard$$init(worker, options, bannerLogCumulative)
         gpuInfo = chalk.cyan(systemStateInfo.gpu.device);
     }
     else
-      gpuInfo = chalk.red(`error detecting`);
+      gpuInfo = chalk.red('error detecting');
 
     workerInfoPane.setLabel(`Worker Status [${sliceFetchStatus}]`);
     workerInfoPane.setContent([
@@ -159,19 +196,19 @@ exports.init = function dashboard$$init(worker, options, bannerLogCumulative)
       '',
       `           Scheduler: ${chalk.yellow(dcpConfig.scheduler.location.href)}`,
       `                Bank: ${chalk.yellow(dcpConfig.bank.location.href)}`,
-      `        Bank Account: ${chalk.yellow(worker.paymentAddress || 'Starting...')}`,
-      `            Identity: ${chalk.yellow(worker.identityKeystore? worker.identityKeystore.address : 'Starting...')}`,
+      `        Bank Account: ${chalk.yellow(worker.config.paymentAddress)}`,
+      `            Identity: ${chalk.yellow(identity.get().address)}`,
       `                 GPU: ${gpuInfo}`,
       `                Jobs: ${worker.config.jobIds?.length ? worker.config.jobIds.join(', ') : '<any>'}`,
-      `      Compute Groups: ${Object.keys(worker.config.computeGroups).length + (worker.config.leavePublicGroup ? 0 : 1)}`,
-      `Global Compute Group: ${worker.config.leavePublicGroup ? 'no' : 'yes'}`,
+      `      Compute Groups: ${Object.keys(worker.config.computeGroups).length + (worker.config.leaveGlobalGroup ? 0 : 1)}`,
+      `Global Compute Group: ${worker.config.leaveGlobalGroup ? 'no' : 'yes'}`,
       `           Worker Id: ${worker.id}`,
       `        Current Time: ${new Date(Date.now()).toUTCString()}`,
-      `   Last Task Request: ${lastTask ? new Date(lastTask.fetchStart).toUTCString() : 0}`,
-      `      Slices in task: ${lastTask ? Object.values(lastTask.slices).reduce((acc, val) => acc + val, 0) : 0}`,
-      `        Jobs in task: ${lastTask ? Object.keys(lastTask.jobs).length : 0}`,
+      `   Last Task Request: ${lastTask?.fetchStart ? new Date(lastTask.fetchStart).toUTCString() : '-'}`,
+      `      Slices in task: ${lastTask?.slices ? Object.values(lastTask.slices).reduce((acc, val) => acc + val, 0) : '-'}`,
+      `        Jobs in task: ${lastTask?.jobs ? Object.keys(lastTask.jobs).length : '-'}`,
     ].join('\n'));
-    screen.render();
+    screen && screen.render();
   }
 
   /* Override default event behaviour to work better with the Dashboard. */
@@ -270,13 +307,86 @@ exports.init = function dashboard$$init(worker, options, bannerLogCumulative)
     sandboxPane.update();
     updateWorkerInfo();
   });
-};
+}
 
 /**
- * Send a signal to the caller
- * @param {number|string} sig    the signal to raise
+ * Initialize the the TUI, then make a new console object for stdio and override the individual methods
+ * so that they display on the TUI log pane. Console messages which go to the TUI are also buffered in
+ * ring buffer, which is allows us to dump the most recent log messages to stdout/stderr when destroying
+ * the console.
  */
-function raise(sig)
+exports.init = function dashboardConsole$$Init()
+{
+  const console = new Console(process);
+  const logHistory = new RingBuffer(closeLogDumpSize);
+  const inspect = Symbol.for('nodejs.util.inspect.custom');
+
+  createDashboard();
+
+  function consoleMethodFactory(logLevel)
+  {
+    return function dashboardConsoleMethodImpl() {
+      const argv = Array.from(arguments);
+      logHistory.push({ logLevel, argv: argv.slice() });
+      for (let i in argv)
+      {
+        if (argv[i] instanceof wallet.Address)
+          argv[i] = chalk.grey(String(argv[i]));
+        else if (argv[i] instanceof Error || (typeof argv[i] === 'object' && argv[i][inspect]))
+          argv[i] = require('node:util').inspect(argv[i]);
+        else if (logLevel === 'error' && typeof argv[i] === 'string')
+          argv[i] = chalk.red(argv[i]);
+      }
+      logPane.log(...argv);
+    }
+  }
+
+  for (let level of Object.values(logLevels))
+    console[level] = consoleMethodFactory(level);
+
+  /**
+   * throb API: calls with arguments set facility and message.  First call without argument emits the
+   *            message. All calls without arguments advance the throbber.
+   */
+  console.throb = function dashboardConsole$$throb(...args)
+  {
+    const throb = dashboardConsole$$throb;
+    var throbPos = throb.pos || 0;
+    if (args.length)
+      logPane.log(...args);
+    const throbChars = '/-\\|';
+    throb.pos = throbPos = (throbPos + 1) % throbChars.length;
+    logPane.advanceThrob(throbChars[throbPos]);
+  }
+
+  /**
+   * Destroy the dashboard, then dump the most recent lines back to stdout/stderr.
+   */
+  console.close = function dashboardConsole$$close()
+  {
+    if (screen?.destroy)
+      screen.destroy();
+    screen = false;
+
+    const stdioConsole = new (require('console').Console)(process);
+    if (logHistory.count > logHistory.size)
+      process.stdout.write(chalk.grey('---' + ' \u2702 -'.repeat(15) + '--\n'));
+    logHistory.forEach(entry => {
+      const { logLevel, argv } = entry;
+      stdioConsole[logLevel](...argv);
+    });
+  }
+
+  console.setWorker = setWorker;
+
+  debug('dcp-worker:console')('Initialized dashboard console');
+  return console;
+}
+
+/**
+ * Symbols shared only with "friends" of dashboard-console
+ */
+exports.getFriendSymbols = function dashboardConsole$$getFriendSymbols()
 {
-  process.kill(process.pid, sig);
+  return { screen, logPane }
 }

package/lib/worker-consoles/index.js

@@ -0,0 +1,44 @@
+/**
+ * @file        worker-consoles/index.js
+ *              Implementation of the setConsoleType method, which replaces the global console object.
+ *              That, in turn, allows dcp library code (eg the supervisor) to have access to the user
+ *              operating and troubleshooting the worker. 
+ *
+ *              The console object is an abstraction of the "right in front of me" user interface. In
+ *              addition to logging functions, it also has the following methods:
+ *               - setWorker: tell the console about the worker object
+ *               - throb:     display/advance a throbber
+ *               - close:     restore the default console
+ *
+ * @author      Wes Garland, wes@distributive.network
+ * @date        June 2025
+ */
+'use strict';
+
+const path    = require('path');
+const debug   = require('debug');
+
+exports.setConsoleType = setConsoleType;
+
+function setConsoleType(consoleType)
+{
+  var consoleImpl;
+  debug('dcp-worker:console')('set console type to', consoleType);
+  
+  try
+  {
+    consoleType = path.basename(consoleType); /* ensure no .. */
+    consoleImpl = require(`./${consoleType}-console`);
+  }
+  catch (error)
+  {
+    if (error.code === 'MODULE_NOT_FOUND')
+    {
+      const errorMessageStart = error.message.replace(/\n.*/g, '');
+      error.message = `Unknown console type '${consoleType}' (${errorMessageStart})`;
+    }
+    throw error;
+  }
+
+  return consoleImpl.init();
+}

package/lib/worker-consoles/none-console.js

@@ -0,0 +1,23 @@
+/**
+ * @file        none-console.js
+ *              Implementation of the "none" console type
+ *
+ * @author      Wes Garland, wes@distributive.network
+ * @date        June 2025
+ */
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const { Console } = require('console');
+
+exports.init = function noneConsole$$init()
+{
+  const nullStream = fs.createWriteStream(os.devNull);
+  const console = new Console({ stdout: nullStream, stderr: nullStream });
+
+  console.setWorker = console.throb = console.close = () => 1;
+  return console;
+}
+
+

package/lib/worker-consoles/stdio-console.js

@@ -0,0 +1,70 @@
+/**
+ * @file        stdio-console.js
+ *              Implementation of the "stdio" console type. This is the default in Node.js, so we
+ *
+ * @author      Wes Garland, wes@distributive.network
+ * @date        June 2023, June 2025
+ */
+'use strict';
+
+const process = require('process');
+const util    = require('util');
+
+const { Console } = require('console');
+const { logLevels } = require('../consts');
+
+const usingDebugger = require('module')._cache.niim instanceof require('module').Module;
+
+/**
+ *  Initialize the stdio console
+ *
+ *  @param  {object} options Any options affecting console behaviour. Not presently used.
+ */
+exports.init = function stdioConsole$$init(options)
+{
+  const { stderr, stdout } = process;
+  const console = new Console({ stderr, stdout });
+  var lastWasThrobber = false;
+  var throbIdx = 0;
+
+  console.setWorker = console.close = () => 1;
+
+  for (let logLevel in logLevels)
+  {
+    console[`__${logLevel}`] = console[logLevel];
+    console[logLevel] = function stdioConsoleLogWrapper() {
+      if (lastWasThrobber)
+      {
+        stdout.write('\n');
+        lastWasThrobber = false;
+      }
+      console[`__${logLevel}`].apply(this, arguments);
+    };
+  }
+
+  const throbChars = '/-\\|';
+  /**
+   * throb API: Writes message to stdout with a newline. Throb char is appended to message. If no message,
+   *            advance the previous throb char.
+   */
+  console.throb = function throb(...args) {
+    if (args.length && lastWasThrobber)
+      stdout.write(' \n');
+
+    for (let i=0; i < args.length; i++)
+    {
+      if (i)
+        stdout.write(' ');
+      stdout.write(typeof args[i] === 'string' ? args[i] : util.inspect(args[i]));
+    }
+
+    if (stdout.isTTY && !usingDebugger)
+    {
+      throbIdx = (throbIdx + 1) % throbChars.length;
+      stdout.write(throbChars[throbIdx] + '\u0008');
+      lastWasThrobber = true;
+    }
+  }
+
+  return console;
+}

package/lib/worker-info.js

@@ -1,18 +1,20 @@
 /**
- *  @file       worker-reporting.js
+ *  @file       worker-info.js
  *              Check evaluator to determine if certain information including:
  *                - if webgpu is enabled, and what capabilities it has.
  *                - supported worktimes for the evaluator
- * 
+ *
  *  @author     Ryan Saweczko, ryansaweczko@distributive.network
  *
  *  @date       Sept 2024
  */
 'use strict';
 
-const kvin = require('kvin');
 const evaluatorId = 'reportingCheck';
 
+/**
+ * This function is serialized and executed inside the evaluator.
+ */
 async function eval$$getInfo()
 {
   const info = {};
@@ -47,7 +49,7 @@ async function eval$$getInfo()
       const adapterInfo = await adapter.requestAdapterInfo();
       for (let key of properties)
         webgpuInfo[key] = adapterInfo[key];
-      info.webgpu = { enabled: true, info: webgpuInfo };  
+      info.webgpu = { enabled: true, info: webgpuInfo };
     }
   }
   catch (err)
@@ -63,31 +65,38 @@ async function eval$$getInfo()
 /**
  * Connect to an evaluator instance and return back information on the evaluator
  * @param {object} evaluatorConfig - Object containing the hostname and port to connect to the evaluator instance on
- * @returns {Promise} which resolves with undefined detection failed (ie couldn't connect to evaluator), or the information object
+ * @returns {Promise} which resolves with
+ *                          - false when we can't connect to the evaluator, or
+ *                          - the evaluator's information object
+ *                 or which rejects with an error delivered by StandaloneWorker<error>
  */
 function getEvaluatorInformation(evaluatorConfig)
 {
   const StandaloneWorker = require('dcp-client/lib/standaloneWorker').workerFactory(evaluatorConfig);
-  const { a$sleep } = require('dcp/utils');
-
   const evaluatorHandle = new StandaloneWorker({ name: 'DCP Sandbox #1', });
-
-  var noResponse, resolve;
-  const p$workerInfo = new Promise((res, rej) => { resolve = res; }).finally(() => {
+  var resolve, reject;
+  const p$workerInfo = new Promise((res, rej) => {
+    resolve = res;
+    reject = rej;
+  }).finally(() => {
     evaluatorHandle.terminate();
-    noResponse.intr();
+    evaluatorHandle.removeAllListeners();
   });
+  evaluatorHandle.addEventListener('end', () => resolve(false));
+  evaluatorHandle.addEventListener('error', reject);
+  evaluatorHandle.ref();
 
-  noResponse = a$sleep(2);
-  noResponse.then(resolve); /* no answer */
-  noResponse.finally(() => process.off('dcpBeforeExit', noResponse.intr));
-  process.on('dcpBeforeExit', noResponse.intr);
-  
-  evaluatorHandle.onmessage = function onmessage(event)
+  evaluatorHandle.onmessage = function onmessage(__event)
   {
-    const reply = kvin.unmarshal(event.data).value;
+    const event = StandaloneWorker.decodeMessageEvent(__event); /* should eventually be superflous /wg may 2025 */
+    const reply = event.data.value;
     if (reply.request === `evalResult::${evaluatorId}`)
-      resolve(reply.data /* { enabled: false|true } */);
+    {
+      const workerInfo = reply.data;
+      if (typeof workerInfo !== 'object' || !workerInfo.worktimes || !workerInfo.webgpu)
+        throw new Error(`malformed response from evaluator`);
+      resolve(workerInfo);
+    }
   }
 
   const message = {
@@ -95,7 +104,7 @@ function getEvaluatorInformation(evaluatorConfig)
     data: '(' + eval$$getInfo.toString() + ')()',
     msgId: evaluatorId,
   }
-  evaluatorHandle.postMessage(kvin.marshal(message));
+  evaluatorHandle.postMessage(StandaloneWorker.encodeMessage(message));
   return p$workerInfo;
 }
 

package/lib/worker-loggers/event-log.js

@@ -8,14 +8,17 @@
  */
 'use strict';
 
+const assert = require('assert');
 const os = require('os');
+var eventLogHnd;
+
 if (os.platform() !== 'win32')
   throw new Error(`Windows Event Log module is not supported on ${os.platform()}`);
 
-const { EventLog } = require('node-eventlog');
+const { logLevels } = require('../consts');
+const { EventLog }  = require('node-eventlog');
+const processName = require('path').basename(process.mainModule.filename);
 
-// Copy the original global console object's properties onto a backup
-const _console = Object.assign({}, console);
 
 /**
  *  Initialize the eventlog worker logger
@@ -23,35 +26,30 @@ const _console = Object.assign({}, console);
  *  @param  {object} options          Options for logger behaviour (passed
  *    through to consoleLogger)
  */
-exports.init = function eventLog$$init(options)
+exports.open = function eventLog$$open(options)
 {
-  exports._processName = require('path').basename(process.mainModule.filename || process.argv0);
-  const source = options.source || exports._processName || 'dcp-worker';
-  exports._eventLog = new EventLog(source);
-  require('../startWorkerLogger').inspectOptions.colors = false;
-  exports.at = log;
+  const source = processName || 'dcp-worker';
+  eventLogHnd = new EventLog(source);
 }
 
 /**
- *  Emit a message to the Windows event-log
+ * Emit a message to the Windows event-log
  * @param {string}    level    The node log level to log at
  * @param {[string]}  items    An array of strings to log as a single message
  */
-function log(level, ...items)
+exports.at = function eventLog$$at(level, ...items)
 {
-  {    
-    // Use the string log-level to look up the severity number:
-    let severity = {
-      error: 'error',
-      warn:  'warn',
-      log:   'info',
-      info:  'info',
-      debug: 'info',
-    }[level];
-
-    return exports._eventLog.log(items.join(' '), severity).catch(error => {
-      if (error)
-        _console.error('255: Unexpected error writing to event log:', error);
-    });
-  }
+  // Use the string log-level to look up the severity number:
+  let severity = {
+    [logLevels.error]:  'error',
+    [logLevels.warn]:   'warn',
+    [logLevels.notice]: 'info',
+    [logLevels.info]:   'info',
+    [logLevels.debug]:  'info',
+  }[level];
+
+  assert(severity);
+  return eventLogHnd.log(items.join(' '), severity || 'error').catch(exports.handleFatalError);
 }
+
+exports.inspectOptions = { colors: false };