dcp-worker 3.3.8 → 3.3.9-0

package/bin/dcp-worker

@@ -21,7 +21,8 @@ const path    = require('path');
 const crypto  = require('crypto');
 const chalk   = require('chalk');
 const telnetd = require('../lib/remote-console');
-const utils   = require('../lib/utils');
+
+const { slicesFetched, debugging, displayMaxDiagInfo } = require('../lib/utils');
 
 const configName = process.env.DCP_CONFIG || '../etc/dcp-worker-config';
 const EXIT_UNHANDLED = 5;
@@ -74,6 +75,12 @@ function parseCliArgs()
         describe: 'default proportion of CPU,GPU to use when cores not specified',
         type: 'string',
       },
+      maxSandboxes: {
+        alias: 'm',
+        describe: 'Maximum number of sandboxes',
+        type: 'number',
+        default: undefined,
+      },
       verbose: {
         alias: 'v',
         describe: 'Enable verbose output',
@@ -294,7 +301,7 @@ async function main()
   process.on('SIGQUIT', handleSigDeath);
   process.on('unhandledRejection', handleUnhandled);
   process.on('uncaughtException', handleUnhandled);
-  
+
   const getOpts = {};
   if (cliArgs.defaultPaymentAddressToDCP)
     getOpts.oAuth = false;
@@ -343,7 +350,6 @@ async function main()
   const dcpWorkerOptions = dcpConfig.worker;
   const forceOptions = {
     paymentAddress,
-    maxWorkingSandboxes: cliArgs.cores,
   };
   const defaultOptions = {
     sandboxOptions: {
@@ -357,7 +363,7 @@ async function main()
     forceOptions.publicGroupFallback = mkBool(cliArgs.publicGroupFallback);
 
   addConfig(dcpWorkerOptions, defaultOptions, dcpConfig.worker, forceOptions);
-  processCoresAndDensity(dcpWorkerOptions, cliArgs);
+  processCoresAndMaxSandboxes(dcpWorkerOptions, cliArgs);
 
   /* Support magic value used by Windows screensaver configuration /wg June 2023 */
   if (dcpWorkerOptions.leavePublicGroup === 'fallback')
@@ -398,10 +404,23 @@ async function main()
     dcpWorkerOptions.watchdogInterval = cliArgs.watchdogInterval;
 
   worker = new DCPWorker(identityKeystore, dcpWorkerOptions);
-  worker.on('error',   (...payload) => console.error(...payload));
   worker.on('warning', (...payload) => console.warn (...payload));
   worker.on('stop', () => { console.log('Worker is stopping') });
   worker.on('end',  () => { logClosing('log', 'Worker has stopped') });
+  // Display clean diagnostic when not debugging and env var
+  //   DCP_SUPERVISOR_DEBUG_DISPLAY_MAX_INFO isn't set.
+  worker.on('error',   (error) => {
+    if (displayMaxDiagInfo())
+      console.error(error);
+    else
+    {
+      const errorCode = error.code ?? error.errorCode;
+      if (errorCode)
+        console.error(`Error: ${error.message}: error.code ${errorCode}`);
+      else
+        console.error(`Error: ${error.message}`);
+    }
+  });
   require('../lib/default-ui-events').hook(worker, cliArgs);
 
   if (cliArgs.outputMode === 'dashboard')
@@ -431,7 +450,7 @@ async function main()
         if (ev instanceof Error)
           console.error('Error fetching task:', ev);
         else
-          dcpWorkerOptions.leavePublicGroup = Boolean(utils.slicesFetched(ev) > 0);
+          dcpWorkerOptions.leavePublicGroup = Boolean(slicesFetched(ev) > 0);
       }
     }
   }
@@ -466,8 +485,8 @@ async function main()
     introBanner += ' . Leaving the public compute group' + '\n';
   if (dcpWorkerOptions.cores)
     introBanner += ` . Configured Cores: ${dcpWorkerOptions.cores.cpu},${dcpWorkerOptions.cores.gpu}\n`
-  else
-    introBanner += ` . Target core density: ${JSON.stringify(dcpWorkerOptions.defaultCoreDensity)}\n`;
+  if (typeof dcpWorkerOptions.maxSandboxes !== 'undefined')
+    introBanner += ` . Maximum Sandboxes: ${dcpWorkerOptions.maxSandboxes}\n`
   if (cliArgs.verbose)
     introBanner += ` + Verbosity level: ${cliArgs.verbose}` + '\n';
   if (telnetd.hasOwnProperty('port'))
@@ -480,7 +499,7 @@ async function main()
     for (const wt of worktimes)
       introBanner += `    -\t${wt.name}@${wt.versions.join(';')}\n`;
   }
-  
+
   introBanner += ' . Supervisor version: ' + worker.supervisorVersion;
   introBanner += ' . Output mode: ' + cliArgs.outputMode + '\n';
   introBanner += ' * Ready' + '\n';
@@ -501,7 +520,8 @@ async function main()
 }
 
 /**
- * Process the cores and density cli arguments.
+ * Process the cores and maxSandboxes cli arguments.
+ * Also processes density, but that isn't currently used and may be deprecated in the future.
  *
  * cliArgs.cores is the core count of the hardware to use.
  * It can be specified with only the cpu or gpu component, or both.
@@ -509,8 +529,10 @@ async function main()
  * E.g. -c 2,1  => cores = { cpu: 2,         gpu: 1 }
  *      -c 10   => cores = { cpu: 10,        gpu: <default> }
  *      -c ,10  => cores = { cpu: <default>, gpu: 10 }
+ *      -m 5    => maxSandboxes = 5
+ *      default maxSandboxes := Math.max( 1, Math.round((System RAM) / 2.5GB) );
  */
-function processCoresAndDensity (dcpWorkerOptions, cliArgs)
+function processCoresAndMaxSandboxes (dcpWorkerOptions, cliArgs)
 {
   const DCPWorker = require('dcp/worker').Worker;
   const defaultTargets = {
@@ -532,10 +554,12 @@ function processCoresAndDensity (dcpWorkerOptions, cliArgs)
   parseArg('density');
   parseArg('cores');
 
-  if (dcpWorkerOptions.cores)
-    debugging() && console.debug('dcp-worker: cores =', dcpWorkerOptions.cores);
-  if (dcpWorkerOptions.density)
-    debugging() && console.debug('dcp-worker: core density =', dcpWorkerOptions.density);
+  if (typeof cliArgs['maxSandboxes'] !== 'undefined')
+    dcpWorkerOptions.maxSandboxes = Number(cliArgs['maxSandboxes']);
+  else
+    DCPWorker.defaultMaxSandboxes(dcpWorkerOptions);
+  debugging() && console.debug(`dcp-worker: cores = { ${dcpWorkerOptions.cores.cpu}, ${dcpWorkerOptions.cores.gpu} }`);
+  debugging() && console.debug('dcp-worker: core density =', dcpWorkerOptions.density);
 }
 
 /**
@@ -688,24 +712,44 @@ function sliceReport()
 /**
  * Handle a signal which requests our the death of the Worker by
  *  - stopping the worker
- *  - unregistering the handler (this allows a second signal to forcibly terminate the process
- *    if that is the default behaviour)
+ *  - unregistering the signal handler and replacing it with a call to process.exit() after
+ *    two tries.
+ *    -> this allows a third signal to forcibly terminate the process
  *  - set a long timeout (dcpConfig.worker.cleanupTimeout seconds), after which the process
  *    exits forcibly with a non-zero exit code (unix standard for various signals)
+ *  - the worker is soft-stopped after the first signal if it is not SIGQUIT but hard stopped
+ *    all other times.
  */
 function handleSigDeath(signalName, signal)
 {
-  process.off(signalName, handleSigDeath);
+  handleSigDeath.count = Number(handleSigDeath.count || 0) + 1;
+
+  if (handleSigDeath.count > 2)
+  {
+    process.on(signalName, () => {
+      console.error(signalName);
+      process.exit(signal - 128)
+    });
+  }
 
   if (!worker)
     console.warn(`trapped ${signalName}, signal ${signal}`);
   else
   {
-    console.warn(`trapped ${signalName}, signal ${signal} -- stopping worker`);
-    worker.stop(signalName === 'SIGQUIT');
+    const immediate = signalName === 'SIGQUIT' || handleSigDeath.count > 1;
+    console.warn(`trapped ${signalName}, signal ${signal} -- stopping worker`,
+                 immediate ? 'immediately' : `after ${handleSigDeath.count} slices have finished`);
+    worker.stop(immediate);
   }
 
-  setTimeout(() => processExit(signal - 128), getCleanupTimeoutMs()).unref();
+  function die()
+  {
+    processExit(signal - 128);
+  }
+
+  setTimeout(die, getCleanupTimeoutMs()).unref();
+  if (handleSigDeath.count === 3)
+    die();
 }
 
 /**
@@ -714,7 +758,7 @@ function handleSigDeath(signalName, signal)
 function getCleanupTimeoutMs()
 {
   const defaultCT = 60;
-  var cleanupTimeout = dcpConfig.worker.cleanupTimeout;
+  var cleanupTimeout = dcpConfig.worker?.cleanupTimeout;
 
   if (typeof cleanupTimeout === 'undefined')
     cleanupTimeout = defaultCT;
@@ -783,14 +827,6 @@ function verifyDefaultConfigIntegrity()
   }
 }
 
-/* thunk - ensures global debugging() symbol always available even if called before dcp-client init */
-function debugging()
-{
-  require('dcp-client');
-  debugging = require('dcp/internal/debugging').scope('dcp-worker');
-  return debugging.apply(this, arguments);
-}
-
 /**
  * Cast b to boolean such that 'false' becomes false, falsey things become false, and everything else
  * becomes true.

package/docs/CODEOWNERS

@@ -1,4 +1,4 @@
-# @file   CODEOWNERS - machine-generated via dcp-docs-wes git@gitlab.com:Distributed-Compute-Protocol/dcp-docs-wes.git 5eb6bcbbb494c50f0a8d4162a11f3eb1af2da7d9
+# @file   CODEOWNERS - machine-generated via dcp-docs-wes git@gitlab.com:Distributed-Compute-Protocol/dcp-docs-wes.git 0b99b9719e7523d036bbea5b3c3f069ad2f5579c
 # @author wes@people
 
 [Brandon]

package/lib/blessed-components/log.js

@@ -2,7 +2,7 @@
  *  @file       blessed-components/log.js
  *  @author     Ryan Rossiter, ryan@kingsds.network
  *  @date       April 2020
- * 
+ *
  *  This blessed component is based on the blessed-contrib
  *  log component, but features text wrapping and behaves
  *  closer to the built-in console log methods.
@@ -11,29 +11,47 @@
 
 const { Box } = require('blessed');
 
-class Log extends Box {
-  constructor(options={}) {
+const defaultOptions = {
+  scrollable: true,
+  bufferLength: 1000,
+  mouse: true,
+  keys: true,
+  vi: true
+};
+
+class Log extends Box
+{
+  paused = false;
+
+  constructor(options={})
+  {
+    options = Object.assign({}, defaultOptions, options);
     super(options);
-
-    options.bufferLength = options.bufferLength || 35;
     this.options = options;
-
     this.logLines = [];
+
+    this.screen.key(['C-s'], () => this.paused = true);
+    this.screen.key(['C-q'], () => this.paused = false);
+    this.screen.key([' '], () => {
+      this.paused = false;
+      this.setScrollPerc(100);
+    });
   }
 
-  log(...args) {
-    let str = args.reduce(
+  log(...args)
+  {
+    const str = args.reduce(
       (s, arg) => (s += `${typeof arg === 'string'? arg : JSON.stringify(arg, null, 2)} `),
     '');
 
     this.logLines.push(str);
-    
-    if (this.logLines.length > this.options.bufferLength) {
+
+    if (this.logLines.length > this.options.bufferLength)
       this.logLines.shift();
-    }
 
     this.setContent(this.logLines.join('\n'));
-    this.setScrollPerc(100);
+    if (!this.paused)
+      this.setScrollPerc(100);
   }
 }
 

package/lib/blessed-components/sandboxes.js

@@ -1,95 +1,116 @@
 /**
- *  @file       blessed-components/sandboxes.js
- *  @author     Ryan Rossiter, ryan@kingsds.network
- *  @date       April 2020
- * 
- *  This blessed component produces labeled progress bars
- *  to render the progress of worker sandboxes.
+ * @file       blessed-components/sandboxes.js
+ * @author     Ryan Rossiter, ryan@kingsds.network
+ * @date       April 2020
+ * @author     Wes Garland, wes@distributive.network
+ * @date       May 2024
+ *
+ * This blessed component produces labeled progress bars to render the progress of worker sandboxes.
+ * These are called sandboxRows.
  */
 'use strict';
 
-const { Box, ProgressBar, Text } = require('blessed');
+const blessed = require('blessed');
 
-class Sandboxes extends Box {
-  constructor(options) {
+class SandboxPane extends blessed.Box
+{
+  constructor(options)
+  {
     super(options);
-
-    this.data = options.data || [];
-    this.progressBars = [];
-    if (options.defaultProgressBars) {
-      [...Array(options.defaultProgressBars)].map(
-        () => this.createProgressBar());
-    }
-
+    this.sandboxRows = [];
     this.update();
   }
 
-  createProgressBar({ progress=0, label='IDLE', indeterminate=true }={}) {
-    let progressBar = new ProgressBar({
+  /**
+   * Create a new progress bar, and associate it with the sandboxData object that it will read to
+   * update its state.
+   */
+  createSandboxRow(sandboxData)
+  {
+    let progressBar = new blessed.ProgressBar({
       parent: this,
       orientation: 'horizontal',
-      filled: progress,
-      top: this.progressBars.length,
-      left: 0, right: 1,
+      top: this.sandboxRows.length,
+      left: 0,
+      right: 0,
       height: 1,
-      style: {
-        bg: 'black',
-        bar: {
-          bg: indeterminate? 'blue' : 'green',
-        },
-      }
+      style: { bg: 'black' },
     });
-    let labelElement = new Text({
+
+    const text = new blessed.Text({
       parent: this,
-      content: label + ' ',
-      top: this.progressBars.length,
-      left: 0, right: 1,
-      style: {
-        transparent: true,
-      }
+      content: '<initializing>X', /* bug: last char truncates */
+      wrap: false,
+      top: this.sandboxRows.length,
+      left: 0,
+      right: 0,
+      style: { transparent: true }
     });
 
-    this.progressBars.push({
-      progressBar,
-      label: labelElement,
-    });
-    return progressBar;
+    const sandboxRow = { progressBar, text, sandboxData };
+    this.sandboxRows.push(sandboxRow);
+    this.updateSandboxRow(sandboxRow);
+    sandboxRow.update = () => this.updateSandboxRow(sandboxRow);
+    sandboxRow.delete = () => this.deleteSandboxRow(sandboxRow);
+    return sandboxRow;
   }
 
-  updateProgressBar(i, { progress=0, label='IDLE', indeterminate=true }={}) {
-    this.progressBars[i].progressBar.setProgress(progress);
-    // Add a space because for some reason the last char gets truncated
-    this.progressBars[i].label.setContent(label + ' ');
+  /* Update a single progress bar's appearance, based on sandboxData */
+  updateSandboxRow(sandboxRow)
+  {
+    const { progressBar, text, sandboxData } = sandboxRow;
 
-    this.progressBars[i].progressBar.style.bar.bg = indeterminate? 'blue' : 'green';
-  }
+    if (sandboxData.label)
+      text.setContent(sandboxData.label + ' ' /* bug - last char gets trunc'd */);
 
-  update(data=this.data) {
-    this.data = data;
-    for (let i = 0; i < this.data.length; i++) {
-      this.updateProgressBar(i, this.data[i]);
+    if (!sandboxData.slice.number)
+    {
+      progressBar.style.bar.bg = 'black';
+      text.style.fg = 'cyan';
+      progressBar.setProgress(100);
     }
+    else if (sandboxData.slice.progress >= 0)
+    {
+      progressBar.style.bar.bg = 'green';
+      text.style.fg = 'white';
+      progressBar.setProgress((sandboxData.slice.progress * 0.99) + 1); /* draw the first tick asap */
+    }
+    else /* indeterminate progress */
+    {
+      progressBar.style.bar.bg = 'blue';
+      text.style.fg = 'white';
+      progressBar.setProgress(100);
+    }
+  }
 
-    if (this.data.length < this.progressBars.length) {
-      for (let i = this.data.length; i < this.progressBars.length; i++) {
-        this.updateProgressBar(i);
-      }
+  deleteSandboxRow(sandboxRow)
+  {
+    const idx = this.sandboxRows.indexOf(sandboxRow);
+    if (idx === -1)
+      return;
+    this.sandboxRows.splice(idx, 1);
+    for (let i=idx; i < this.sandboxRows.length; i++)
+    {
+      this.sandboxRows[i].progressBar.position.top -= 1;
+      this.sandboxRows[i].text.position.top -= 1;
     }
 
-    this.setLabel(`${this.options.label} (${this.progressBars.length})`);
+    sandboxRow.text.destroy();
+    sandboxRow.progressBar.destroy();
   }
 
-  // Deletes last progress bar
-  deleteProgressBar() {
-    let i = this.progressBars.length - 1;
-    this.progressBars[i].label.destroy()
-    this.progressBars[i].progressBar.destroy()
-    this.progressBars.pop(i)
+  /* Update all everything in the progress bar pane */
+  update()
+  {
+    this.sandboxRows.forEach(sandboxRow => this.updateSandboxRow(sandboxRow));
+    this.setLabel(`${this.options.label} (${this.sandboxRows.length})`);
+    this.screen.render();
   }
 }
 
-Object.assign(exports, {
-  sandboxes(...args) {
-    return new Sandboxes(...args);
-  },
-});
+/**
+ * Factory function for returning a sandbox pane. Wraps the SandboxPane ctor.
+ */
+exports.sandboxPaneFactory = function sandboxPaneFactory(...args) {
+  return new SandboxPane(...args);
+};

package/lib/check-scheduler-version.js

@@ -19,15 +19,14 @@ exports.check = function checkSchedulerVersion$$check(quiet)
   const schedulerConfig = dcpConfig.scheduler;
 
   // Check for old versions of the config
-  if (!schedulerConfig.worker) {
+  if (!schedulerConfig.worker)
+  {
     schedulerConfig.worker = {};
     schedulerConfig.worker.operations = '1.0.0';
-    if (dcpConfig.worker.nativeEvaluator){
+    if (dcpConfig.worker.nativeEvaluator)
       schedulerConfig.worker.types = ['v4'];
-    } 
-    else {
+    else
       schedulerConfig.worker.types = ['v3'];
-    } 
   }
 
   // Check if scheduler supports current worker version
@@ -37,12 +36,13 @@ exports.check = function checkSchedulerVersion$$check(quiet)
   let currentWorkerType = require('dcp/build').workerType || 'v4';
   let currentWorkerVersion = require('dcp/build').workerVersion || '1.0.0';
   
-  if (parseInt(currentWorkerVersion, 10) !== 1) {
+  if (parseInt(currentWorkerVersion, 10) !== 1)
     throw new Error('This version of dcp-client is not compatible with the standalone worker implemention in this dcp-worker package');
-  }
 
-  if (!Object.values(schedulerConfig.worker.types).includes(currentWorkerType) || 
-      !require('semver').satisfies(schedulerConfig.worker.operations, '^'+currentWorkerVersion)) {
+  if (false
+      || !Object.values(schedulerConfig.worker.types).includes(currentWorkerType) 
+      || !require('semver').satisfies(schedulerConfig.worker.operations, '^'+currentWorkerVersion))
+  {
     console.error('\b**** Please update ****\b');
     console.error('The selected scheduler is not capable of running this worker version.');
     console.log(`Scheduler href:  ${dcpConfig.scheduler.location.href}\n` +

package/lib/dashboard-tui.js

@@ -17,21 +17,21 @@ const contrib    = require('blessed-contrib');
 const components = require('./blessed-components');
 const utils      = require('../lib/utils');
 
-const { replaceWorkerEvent, replaceSandboxEvent }  = require('./default-ui-events');
+const { replaceWorkerEventHandler, replaceSandboxEventHandler, newSandboxCallbacks }  = require('./default-ui-events');
 
 const SLICE_FETCH_STATUS = {
   IDLE: chalk.yellow('Idle'),
   FETCHING: chalk.blue('Fetching Work...'),
   WORKING: chalk.green('Working'),
   NO_WORK: chalk.red('No Work Available'),
-}
+};
 
 const usingDebugger = require('module')._cache.niim instanceof require('module').Module;
 const screenConf = {
   input:  usingDebugger ? new (require('events').EventEmitter) : undefined,
   output: usingDebugger ? new (require('events').EventEmitter) : undefined,
 };
-/** 
+/**
  *  Initialize the blessed dashboard
  *  @param  {Worker}    worker    Reference to the DCP Worker
  *  @param {object} options Options which may affect behaviour. Not currently used.
@@ -40,23 +40,25 @@ exports.init = function dashboard$$init(worker, options)
 {
   var sliceFetchStatus = SLICE_FETCH_STATUS.IDLE;
   var totalDCCs = 0;
-  const screen = blessed.screen(screenConf);
-  const grid = new contrib.grid({rows: 3, cols: 5, screen});
+  var screen = blessed.screen(screenConf);
+
+  worker.on('end', () =>  {
+    screen.destroy();
+    screen = false;
+  });
+  process.on('exit', () => {
+    if (screen)
+      screen.destroy();
+  });
+
+  const grid = new contrib.grid({ rows: 3, cols: 5, screen }); // eslint-disable-line new-cap
   const workerInfoPane = grid.set(2, 0, 1, 5, blessed.text);
-  
   const logPane = grid.set(0, 2, 2, 3, components.log, {
     label: 'Worker Log',
-    scrollable: true,
-    alwaysScroll: true,
-    mouse: true,
-    scrollbar: {
-      bg: 'blue',
-    },
+    scrollbar: { bg: 'blue' },
   });
-
-  const sandboxPane = grid.set(0, 0, 2, 2, components.sandboxes, {
+  const sandboxPane = grid.set(0, 0, 2, 2, components.sandboxPaneFactory, {
     label: 'Sandboxes',
-    defaultProgressBars: 0,
     scrollable: true,
     alwaysScroll: true,
     mouse: true,
@@ -64,7 +66,6 @@ exports.init = function dashboard$$init(worker, options)
       bg: 'blue',
     },
   });
-
   const passwordBox = blessed.textbox({
     parent: screen,
     border: 'line',
@@ -81,6 +82,8 @@ exports.init = function dashboard$$init(worker, options)
     hidden: true,
   });
 
+  global.tui = { workerInfoPane, logPane, sandboxPane, screen, grid, passwordBox };
+
   function askPassword(promptMessage)
   {
     return new Promise((resolve, reject) => {
@@ -91,6 +94,7 @@ exports.init = function dashboard$$init(worker, options)
       function passwordSubmitFn(value)
       {
         passwordBox.hide();
+        screen.render();
         passwordBox.removeListener('submit', passwordSubmitFn);
         passwordBox.setValue('');
         resolve(value);
@@ -110,24 +114,19 @@ exports.init = function dashboard$$init(worker, options)
 
   if (!usingDebugger)
     exports.logPane = logPane; /* now dashboard log can find the pane */
-  setInterval(() => screen.render(), 50).unref(); /* 50ms = 20 fps */
+  setInterval(() => screen.render(), 2000).unref(); /* ensure we didn't forget to render an important update */
   updateWorkerInfo();
-  screen.render();
-  
+
   /* Apply key bindings which mimic canonical input mode */
   screen.key(['C-c'], ()    => raise('SIGINT'));
   screen.key(['C-z'], ()    => raise('SIGTSTP'));
   screen.key(['\u001c'], () => raise('SIGQUIT')); /* C-\ */
-  
-  screen.key(['escape'], () => {
-    console.log('Stopping worker...');
-    worker.stop();
-  });
+  screen.key(['escape'], () => raise('SIGINT'));
 
   function updateWorkerInfo()
   {
     const workerOptions = worker.workerOptions;
-    
+
     workerInfoPane.setLabel(`Worker Status [${sliceFetchStatus}]`);
     workerInfoPane.setContent([
       chalk.green(` DCCs Earned: ${chalk.bold(totalDCCs.toFixed(3))}`),
@@ -140,64 +139,48 @@ exports.init = function dashboard$$init(worker, options)
       ` Priv Groups: ${Object.keys(workerOptions.computeGroups).length}`,
       `   Pub Group: ${workerOptions.leavePublicGroup ? 'no' : 'yes'}`,
     ].join('\n'));
+    screen.render();
   }
 
   /* Override default event behaviour to work better with the Dashboard. */
-
-  /** XXXpfr @todo Is this correct?  Or should we init progressData inside 'slice' like we used to.  */
-  replaceSandboxEvent('ready', function dashboard$$job(sandbox, sandboxData, ev) {
-    sandboxData.progressData = {
-      indeterminate: true,
-      progress: 0,
-      label: sandbox?.public ? sandbox.public.name: '<no-label>',
-    };
+  replaceSandboxEventHandler('ready', function dashboard$$sandboxReady(sandbox, sandboxData) {
+    if (!sandboxData.label)
+      sandboxData.label = '<ready>';
+    sandboxData.slice.number = 0;
+    sandboxPane.update();
   });
-  
-  replaceSandboxEvent('slice', function dashboard$$slice(sandbox, sandboxData, ev) {
-    sandboxPane.data.push(sandboxData.progressData);
+
+  replaceSandboxEventHandler('job', function dashboard$$sandboxJob(sandbox, sandboxData, job) {
+    sandboxData.job = job;
+    sandboxData.label = job.name ? `${job.name} ${job.address.slice(0,8)}` : `<job ${job.address}>`;
+    if (job.description)
+      sandboxData.label += ': ' + job.description;
     sandboxPane.update();
   });
 
-  replaceSandboxEvent('progress', function dashboard$$progress(sandbox, sandboxData, ev) {
-    if (!ev)
-    {
-      sandboxData.progressData.progress = 100;
-      setTimeout(() => {
-        if (sandboxData.progressData.indeterminate)
-        {
-          sandboxData.progressData.progress = 0;
-          sandboxPane.update();
-        }
-      }, 500).unref();
-    }
-    else
-    {
-      sandboxData.progressData.progress = ev;
-      sandboxData.progressData.indeterminate = false;
-    }
+  replaceSandboxEventHandler('slice', function dashboard$$slice(sandbox, sandboxData, sliceNumber) {
+    sandboxData.slice.number = sliceNumber;
+  });
 
-    sandboxPane.update();
+  replaceSandboxEventHandler('progress', function dashboard$$progress(sandbox, sandboxData, progress) {
+    sandboxData.slice.progress = progress;
   });
-  
-  replaceSandboxEvent('sliceEnd', function dashboard$$sliceEnd(sandbox, sandboxData, ev) {
-    sandboxPane.data = sandboxPane.data.filter(d => d != sandboxData.progressData);
-    sandboxData.progressData.progress = 0;
-    sandboxPane.update();
+
+  replaceSandboxEventHandler('sliceEnd', function dashboard$$sliceEnd(sandbox, sandboxData, sliceNumber) {
+    sandboxData.slice.progress = 100;
   });
 
-  replaceSandboxEvent('end', function dashboard$$end(sandbox, sandboxData, ev) {
-    sandboxPane.data = sandboxPane.data.filter(d => d != sandboxData.progressData);
-    sandboxPane.deleteProgressBar();
-    sandboxData.progressData.progress = 0;
+  replaceSandboxEventHandler('end', function dashboard$$end(sandbox, sandboxData) {
+    sandboxPane.deleteSandboxRow(sandboxData.sandboxRow);
     sandboxPane.update();
   });
 
-  replaceWorkerEvent('beforeFetch', function dashboard$$beforeFetch(ev) {
+  replaceWorkerEventHandler('beforeFetch', function dashboard$$beforeFetch(ev) {
     sliceFetchStatus = SLICE_FETCH_STATUS.FETCHING;
     updateWorkerInfo();
   });
-  
-  replaceWorkerEvent('fetch', function dashboard$$fetch(ev) {
+
+  replaceWorkerEventHandler('fetch', function dashboard$$fetch(ev) {
     sliceFetchStatus = SLICE_FETCH_STATUS.NO_WORK;
     if (ev instanceof Error)
       console.error('Error fetching slices:', ev);
@@ -206,16 +189,50 @@ exports.init = function dashboard$$init(worker, options)
     updateWorkerInfo();
   });
 
-  worker.on('end', () =>  { screen.destroy(); });
+  /* Sandbox Data
+   * label - text that describes the sandbox
+   * slice - defined on first slice
+   *   .number - holds current slice
+   *   .progress - last progress update for this slice, 0 on new slice if last slice had numeric progress
+   * sandboxRow - TUI element from lib/blessed-components/sandboxes.js
+   *   .text - text element
+   *   .progressBar - progress bar element
+   *   .sandboxData - this sandbox data object
+   */
+  function initSandboxData(_worker, sandbox, sandboxData)
+  {
+    sandboxData.seq = initSandboxData.seq = (initSandboxData.seq || 0) + 1;
+    sandboxData.slice = {};
+    sandboxData.sandboxRow = sandboxPane.createSandboxRow(sandboxData);
+
+    /* Updating an autoUpdate property cases that change to be immediately reflected in the
+     * data that blessed uses to draw the TUI dashboard. We build this object after initializing
+     * the sandbox row to keep race conditions at bay.
+     */
+    function autoUpdate(prop)
+    {
+      Object.defineProperty(autoUpdate.object, prop, {
+        get: () => autoUpdate.storage[prop],
+        set: (value) => {
+          autoUpdate.storage[prop] = value;
+          sandboxData.sandboxRow.update();
+        },
+        enumerable: true,
+      });
+    }
 
-  worker.on('sandbox', function dashboard$$sandbox(ev) {
-    sandboxPane.createProgressBar();
-    sandboxPane.update();
-  });
+    autoUpdate.storage = sandboxData.slice;
+    sandboxData.slice = autoUpdate.object = {};
+    autoUpdate('progress');
+    autoUpdate('number');
+    sandboxData.sandboxRow.update();
+  }
+  newSandboxCallbacks.push(initSandboxData);
 
+  worker.on('job', job => console.log(`new job: ${job.name} ${job.address.slice(0,8)} ${job.description || ''} ${job.link || ''}`));
   worker.on('payment', function dashboard$$payment(ev) {
     const payment = parseFloat(ev);
-    
+
     if (!isNaN(payment))
       totalDCCs += payment;
 

package/lib/default-ui-events.js

@@ -26,23 +26,23 @@
  */
 'use strict';
 
-const utils = require('../lib/utils');
+const { slicesFetched, debugging } = require('./utils');
 
 const sandboxEventHandlers = {};
 const workerEventHandlers = {};
 
-/**
- *   Sandbox   1:  Slice Started:   slice 1, 0x5b5214D48F0428669c4E: Simple Job
- *   Sandbox   1:  Slice Completed: slice 1, 0x5b5214D48F0428669c4E: Simple Job: dt 114ms
- */
+function placeholder()
+{
+  /* event target for events not used by the default loggers */
+}
 
 /**
- *  Hook a worker's events
+ *  Hook a worker's events and the events of the sandboxes it creates.
  *  
  * @param    worker    The instance of Worker to hook
  * @param    options   cliArgs from worker
  */
-exports.hook = function hookWorkerEvents$$hook(worker, options)
+exports.hook = function defaultUiEvents$$hook(worker, options)
 {
   const sliceMap = {};          // jobAddress --> ( sliceNumber, t0 )
   const truncationLength = 22;  // Extra 2 for '0x'
@@ -71,9 +71,8 @@ exports.hook = function hookWorkerEvents$$hook(worker, options)
     console.log(` . Sandbox ${sandboxData.shortId}:  Slice Started:   ${makeSliceId(sandbox)}`);
   };
 
-  sandboxEventHandlers.progress = function progressHandler(sandbox, sandboxdData, ev) {
-    // Overridden in dashboard-tui.js
-  };
+  sandboxEventHandlers.progress = placeholder;
+  sandboxEventHandlers.job      = placeholder;
 
   sandboxEventHandlers.sliceEnd = function sliceEndHandler(sandbox, sandboxData, ev) {
     const sliceInfo = sliceMap[sandbox.id];
@@ -104,19 +103,18 @@ exports.hook = function hookWorkerEvents$$hook(worker, options)
 
   workerEventHandlers.fetch = function fetchHandler(ev) {
     if (ev instanceof Error)
-      console.error(' ! Failed to fetch slices:', ev);
+      debugging() && options.verbose && console.error(' ! Failed to fetch slices:', ev); // redundant
     else
-      options.verbose && console.log(' . Fetched', utils.slicesFetched(ev), 'slices');
+      options.verbose && console.log(' . Fetched', slicesFetched(ev), 'slices');
   };
 
-
   workerEventHandlers.beforeResult = function beforeResultHandler() {
     options.verbose >= 2 && console.log(' * Submitting results...');
   };
 
   workerEventHandlers.result = function resultHandler(ev) {
     if (ev instanceof Error)
-      console.error(" ! Failed to submit results:", ev);
+      debugging() && options.verbose >= 2 && console.error(" ! Failed to submit results:", ev); // redundant
     else
       options.verbose >= 2 && console.log(' . Submitted');
   };
@@ -133,11 +131,11 @@ exports.hook = function hookWorkerEvents$$hook(worker, options)
     worker.on(eventName, (...args) => workerEventHandlers[eventName](...args));
 
   worker.on('sandbox', function newSandboxHandler(sandbox) {
-    const sandboxData = {
-      shortId: sandbox.id.toString(10).padStart(3)
-    };
+    const sandboxData = { shortId: sandbox.id.toString(10).padStart(3) };
     for (let eventName in sandboxEventHandlers)
       sandbox.on(eventName, (...args) => sandboxEventHandlers[eventName](sandbox, sandboxData, ...args));
+    for (let callback of exports.newSandboxCallbacks)
+      callback(worker, sandbox, sandboxData);
   });
   
   exports.sandboxEventHandlers = sandboxEventHandlers;
@@ -150,7 +148,7 @@ exports.hook = function hookWorkerEvents$$hook(worker, options)
  * @param {string}   eventName          name of the event to replace
  * @param {function} eventHandler       new event handler
  */
-exports.replaceWorkerEvent = function hookWorkerEvents$$replace(eventName, eventHandler)
+exports.replaceWorkerEventHandler = function defaultUiEvents$$replaceWorkerEventHandler(eventName, eventHandler)
 {
   if (!workerEventHandlers.hasOwnProperty(eventName))
     throw new Error('unknown worker event: ' + eventName + `(${Object.keys(workerEventHandlers).join(', ')})`);
@@ -164,10 +162,16 @@ exports.replaceWorkerEvent = function hookWorkerEvents$$replace(eventName, event
  * @param {string}   eventName          name of the event to replace
  * @param {function} eventHandler       new event handler
  */
-exports.replaceSandboxEvent = function hookSandboxEvents$$replace(eventName, eventHandler)
+exports.replaceSandboxEventHandler = function defaultUiEvents$$replaceSandboxEventHandler(eventName, eventHandler)
 {
   if (!sandboxEventHandlers.hasOwnProperty(eventName))
     throw new Error('unknown sandbox event: ' + eventName + `(${Object.keys(sandboxEventHandlers).join(', ')})`);
 
   sandboxEventHandlers[eventName] = eventHandler;
 }
+
+/**
+ * Functions which are fired every time the worker creates a new sandbox.
+ * Each function receives as its arguments worker, sandbox, sandboxData.
+ */
+exports.newSandboxCallbacks = [];

package/lib/remote-console.js

@@ -25,28 +25,29 @@
 
 const path = require('path');
 const fs   = require('fs');
+const { debugging } = require('./utils');
 var dcpConfig;
 var mainEval;
 var ci;
 
 require('dcp-client'); /* plumb in modules from bundle even if library has not been initialized */
-const debugging = require('dcp/internal/debugging').scope('dcp-worker');
 
 function daemonEval()
 {
-  try {
+  try
+  {
     if (typeof dcpConfig === 'undefined')
       dcpConfig = require('dcp/dcp-config');
   }
-  catch(e)
-  {}
+  catch(e) {} // eslint-disable-line no-empty
 
   if (mainEval)
     return mainEval(arguments[0]);
   return eval(arguments[0]);  /* eslint-disable-line no-eval */
 }
 
-function callbackTelnet(port, client, registry) {
+function callbackTelnet(port, client, registry)
+{
   client.unref();
   debugging() && console.notice(' ! telnetd - listening on port', port);
 }

package/lib/utils.js

@@ -8,6 +8,8 @@
  */
 'use strict';
 
+const process = require('process');
+
 /**
  * Figure out #slices fetched from the different forms of the 'fetch' event.
  * @param {*|string|number} task
@@ -15,10 +17,6 @@
  */
 function slicesFetched (task)
 {
-  if (typeof task === 'number') /* <= June 2023 Worker events: remove ~ Sep 2023 /wg */
-    return task;
-  if (typeof task === 'string') /* <= June 2023 Worker events: remove ~ Sep 2023 /wg */
-    return parseInt(task, 10) || 0;
   /* eslint-disable-next-line no-shadow */
   let slicesFetched = 0;
   for (const job in task.slices)
@@ -26,4 +24,23 @@ function slicesFetched (task)
   return slicesFetched;
 }
 
+/** thunk - ensures global debugging() symbol always available even if called before dcp-client init */
+function debugging()
+{
+  require('dcp-client');
+  debugging = require('dcp/internal/debugging').scope('dcp-worker'); // eslint-disable-line no-func-assign
+  return debugging.apply(this, arguments); // eslint-disable-line no-invalid-this
+}
+
+/**
+ * Flag to display detailed debug info in diagnostics.
+ * @return {boolean}
+ */
+function displayMaxDiagInfo ()
+{
+  return Boolean(process.env.DCP_SUPERVISOR_DEBUG_DISPLAY_MAX_INFO) || debugging() || false;
+}
+
 exports.slicesFetched = slicesFetched;
+exports.debugging = debugging;
+exports.displayMaxDiagInfo = displayMaxDiagInfo;

package/lib/worker-loggers/dashboard.js

@@ -9,7 +9,6 @@
 'use strict';
 
 const chalk = require('chalk');
-const _console = new (require('console').Console)(process);
 
 /**
  *  Initialize the Blessed dashboard

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcp-worker",
-  "version": "3.3.8",
+  "version": "3.3.9-0",
   "description": "JavaScript portion of DCP Workers for Node.js",
   "main": "bin/dcp-worker",
   "keywords": [
@@ -12,7 +12,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git@github.com:Distributed-Compute-Labs/dcp-worker.git"
+    "url": "git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-worker.git"
   },
   "license": "MIT",
   "author": "Kings Distributed Systems",
@@ -40,7 +40,7 @@
     "blessed": "^0.1.81",
     "blessed-contrib": "4.11.0",
     "chalk": "^4.1.0",
-    "dcp-client": "4.4.8",
+    "dcp-client": "4.4.9-0",
     "kvin": "^1.2.7",
     "posix-getopt": "^1.2.1",
     "semver": "^7.3.8",