dcp-worker 3.2.10 → 3.2.12-b

package/bin/dcp-worker

@@ -12,15 +12,30 @@ const process = require('process');
 const os = require('os');
 const fs = require('fs');
 
-const DEFAULT_CORES = os.cpus().length - 1;
+// Set to true to try to understand unhandled rejection.
+const ANALYZE_UNHANDELD_REJECTION = false;
+
+const TOTAL_CPU_VCORES = os.cpus().length;
+const DEFAULT_CORES = TOTAL_CPU_VCORES - 1;
 var worker, dcpConfig;
 
+const EXIT_CLEAN           = 0;   // normal exit, no error
+const EXIT_SIGQUIT         = 2;   // received SIGQUIT
+const EXIT_ERROR_STOPPING  = 3;   // failed to stop worker, no other error
+const EXIT_TIMED_OUT       = 4;   // failed to exit after requesting graceful exit
+const EXIT_UNHANDLED       = 5;   // unhandled rejection
+
+
+process.on('unhandledRejection', (error) => {
+  console.error('Unhandled rejection:', error);
+});
+
 async function main () {
-  process.on('unhandledRejection', unhandledRejectionHandler);
+  if (!ANALYZE_UNHANDELD_REJECTION)
+    process.on('unhandledRejection', unhandledRejectionHandler);
 
   await require('dcp-client').init(require(process.env.DCP_CONFIG || '../etc/dcp-config'));
   dcpConfig = require('dcp/dcp-config');
-  require('../lib/check-scheduler-version').check();
 
   const cliArgs = require('dcp/cli')
   .base('Standalone NodeJS DCP Worker')
@@ -38,15 +53,17 @@ async function main () {
       verbose: {
         alias: 'v',
         describe: 'Enable verbose output',
-        type: 'boolean',
+        type: 'count',
         default: false,
+        group: 'Output options',
       },
       outputMode: {
-        alias: 'o',
+        alias: ['o', 'output'],
         describe: 'Set the output mode',
         type: 'string',
         default: 'detect',
-        choices: ['detect', 'console', 'dashboard'],
+        choices: ['detect', 'console', 'dashboard', 'event-log', 'syslog', 'logfile'],
+        group: 'Output options',
       },
       hostname: {
         alias: 'H',
@@ -71,130 +88,220 @@ async function main () {
         describe: 'Restrict worker to a specific job (use N times for N jobs)',
         type: 'array',
       },
-      'leave-public-group': {
-        type: 'boolean',
-        hidden: true,
-        default: false
-      },
-      'join': {
+
+      join: {
         alias: 'g',
-        describe: 'Join compute group',
+        describe: 'Join compute group; the format is "joinKey,joinSecret" or "joinKey,eh1-joinHash"',
         type: 'array'
       },
-      'join-keystore': {
+      joinKeystore: {
         hidden: true,
         /* future */
       },
-      'allowed-origins': {
+
+      leavePublicGroup: {
+        type: 'boolean',
+        describe: 'Do not fetch slices from public compute group',
+        default: false,
+      },
+      publicGroupFallback: {
+        describe: 'If set, worker will prefer private groups but fall back on the public group if no preferred work is available',
+        type: 'boolean',
+        default: false,
+      },
+
+      identityKey: {
+        describe: 'Identity key, in hex format',
+        type: 'string',
+        group: 'Identity options',
+      },
+      identityKeystore: {
+        describe: 'Identity keystore, in json format',
+        type: 'string',
+        group: 'Identity options',
+      },
+
+      reportInterval: {
+        describe: 'If set, output a status summary every [interval] seconds in console output mode',
+        type: 'number',
+        group: 'Output options',
+      },
+      eventDebug: {
+        hide: true,
+        describe: 'If set, dump all sandbox and worker events',
+      },
+
+      logfile: {
+        describe: 'Path to log file (if --output=file)',
+        type: 'string',
+        group: 'Log File output options',
+      },
+      syslogAddress: {
+        describe: 'Address of rsyslog server (if --output=syslog)',
+        type: 'string',
+        group: 'Syslog output options',
+      },
+      syslogTransport: {
+        describe: 'Transport to connect to rsyslog daemon (if --output=syslog)',
+        type: 'string',
+        choices: ['udp','tcp'],
+        group: 'Syslog output options',
+      },
+      syslogPort: {
+        describe: 'UDP/TCP port of rsyslog server',
+        type: 'number',
+        group: 'Syslog output options',
+      },
+
+      allowedOrigins: {
         alias: 'a',
-        describe: 'origins where the worker is allowed to fetch input data from',
+        describe: 'modify the \'any\' allow origins of dcpConfig',
         type: 'array'
       },
+
+      replPort: {
+        describe: 'If set, open a REPL on specified TCP port',
+        type: 'number',
+        default: undefined,
+      },
+      watchdogInterval: {
+        alias: 'W',
+        describe: 'Number of milliseconds between watchdog cycles',
+        type: 'number',
+        hidden: true,
+      },
+      dumpConfig: {
+        describe: 'If set, dump the configuration and exit',
+        type: 'boolean',
+        hidden: true,
+      }
     })
   .strict()
   .wrap(process.stdout.columns || 80)
   .argv;
 
-  await startWorking(cliArgs);
+  if (cliArgs.dumpConfig)
+  {
+    console.debug(JSON.stringify(require('dcp/dcp-config'), null, 2));
+    process.exit(1);
+  }
+
+  return startWorking(cliArgs);
 }
 
 // Preserve console.error, the dashboard replaces it with a custom logger
 const logError = console.error;
 main()
-.then(() => process.exit(0))
+.then(exitcode => process.exit(exitcode || 0))
 .catch(e => {
   logError("Script failed:");
   logError(e);
   process.exit(1);
 });
 
+// Imperfect, but handles CG { joinKey, joinHash }.
+function isHash(b) {
+  return b && b.length === 68 && b.startsWith('eh1-');
+}
+
 async function startWorking(cliArgs) {
+  //console.log('cliArgs', cliArgs);
   const wallet = require('dcp/wallet');
   const DCPWorker = require('dcp/worker').Worker;
   const { startWorkerLogger } = require('../lib/startWorkerLogger');
-  const identityKeystore = await wallet.getId();
-  /** @type {string[]} */
-  var dcpWorkerOptions;
-  var paymentAddress;
-  var sawOptions = {
+  const sawOptions = {
     hostname: cliArgs.hostname,
     port:     cliArgs.port
   };
 
-  if (cliArgs.paymentAddress) {
+  let paymentAddress;
+  if (cliArgs.paymentAddress)
     paymentAddress = new wallet.Address(cliArgs.paymentAddress);
-  } else {
+  else
     paymentAddress = (await wallet.get()).address;
-  }
 
-  dcpWorkerOptions = {
+
+  // Different ways to get the identity: 
+  let identityKeystore = false;
+
+  if (cliArgs.identityKey)
+    identityKeystore = await new wallet.IdKeystore(cliArgs.identityKey, '');
+  else if (cliArgs.identityKeystore)
+    identityKeystore = await new wallet.IdKeystore(JSON.parse(cliArgs.identityKeystore), '');
+  else
+    identityKeystore = await wallet.getId();
+
+  // Set the provided identity as the wallet's default
+  await wallet.addId(identityKeystore);
+
+
+  if (typeof dcpConfig.worker.unhandledRejectionCleanupTimeout !== 'undefined')
+    unhandledRejectionHandler.timeout = dcpConfig.worker.unhandledRejectionCleanupTimeout;
+
+  // Leave the public compute group, if desired
+  if (cliArgs.leavePublicGroup || cliArgs.publicGroupFallback)
+    dcpConfig.worker.leavePublicGroup = true;
+
+
+  // The exitGuard will hold an "exit" method, and a Promise to await for
+  // the exit code passed to exitGuard.exit()
+  let exitcode = EXIT_CLEAN;
+  const exitGuard = {
+    promise: Promise.resolve(0),                     // will be overwritten when worker starts
+    exit(code) { process.exit(code||exitcode||0) },  // will be overwritten when worker starts
+  };
+  process.on('SIGQUIT', () => {
+    exitcode = EXIT_SIGQUIT;
+    cliArgs.verbose >= 1 && console.info(`240: Caught SIGQUIT; exiting worker with exitcode ${exitcode}`);
+    exitGuard.exit(exitcode);
+  });
+
+
+  /** @type {string[]} */
+  const dcpWorkerOptions = {
     paymentAddress,
+    identity: identityKeystore,
     maxWorkingSandboxes: cliArgs.cores,
-    priorityOnly:        cliArgs.priorityOnly,
+    cores:               { cpu: TOTAL_CPU_VCORES, gpu: undefined }, /** XXXpfr @todo: Figure out how many gpus. */
+    targetLoad:          { cpu: 1.0, gpu: 1.0 }, /** Use 100%: XXXpfr @todo Allow command-line override. */
     sandboxOptions: {
       SandboxConstructor: require('dcp-client/lib/standaloneWorker').workerFactory(sawOptions)
     },
     computeGroups: [], /* public group is implied */
-    leavePublicGroup: cliArgs.leavePublicGroup
+    leavePublicGroup: cliArgs.leavePublicGroup || dcpConfig.worker.leavePublicGroup,
   };
 
   /* cliArgs.join is the list of compute groups to join */
   if (cliArgs.join && cliArgs.join.length)
   {
     dcpWorkerOptions.computeGroups = cliArgs.join
-      .map   ((el) => { var [a,b] = el.split(','); return { joinKey: a, joinSecret: b }})  /* Map cliArgs.join to give us [{ joinKey, joinSecret }...] */
-      .filter((el) => el.joinKey)                                                          /* Filter out entries with no joinKey */
-    ;
+      .map((el) => {
+        /* Map cliArgs.join to give us [{ joinKey, joinSecret/joinHash }...] */
+        const [a, b] = el.split(',');
+        return isHash(b) ? { joinKey: a, joinHash: b } : { joinKey: a, joinSecret: b };
+      })
+      .filter((el) => el.joinKey); /* Filter out entries with no joinKey */
+    //console.log(dcpWorkerOptions.computeGroups);
   }
   
-  if (typeof dcpConfig.worker.unhandledRejectionCleanupTimeout !== 'undefined')
-    unhandledRejectionHandler.timeout = dcpConfig.worker.unhandledRejectionCleanupTimeout;
-
-  if (cliArgs.jobId) {
+  if (cliArgs.jobId)
+  {
     dcpWorkerOptions.jobAddresses = cliArgs.jobId;
+    dcpWorkerOptions.priorityOnly = true;
   }
-  if (cliArgs.allowedOrigins) {
-    dcpWorkerOptions.allowedOrigins = cliArgs.allowedOrigins;
-  }
+  if (cliArgs.allowedOrigins)
+    dcpConfig.worker.allowOrigins.any = cliArgs.allowedOrigins;
+  if (cliArgs.watchdogInterval)
+    dcpWorkerOptions.watchdogInterval = cliArgs.watchdogInterval;
 
-  worker = new DCPWorker(dcpWorkerOptions);
-
-  if (process.env.TEST_HARNESS) {
-    const { bindToTestHarness } = require('../lib/bindToTestHarness');
-    bindToTestHarness(worker);
-  }
 
-  /* Delay log messages so that they appear in TUI window */
-  setImmediate(() => {
-    console.log(` * Starting DCP Worker`);
-    console.log(` . Configured for scheduler ${dcpConfig.scheduler.location}`);
-    console.log(` . Bank is ${dcpConfig.bank.location}`);
-    console.log(` . Earned funds will be deposited in account ${paymentAddress}`);
-    console.log(` . Identity is ${identityKeystore.address}`);
-
-    function qty(amount, singular, plural) /* XXX i18n */
-    {
-      if (Array.isArray(amount))
-        amount = amount.length;
-      if (!plural)
-        plural = singular + 's';
-      if (!amount)
-        return plural;
-      if (amount == 1)
-        return singular;
-      return plural;
-    }
+  worker = new DCPWorker(dcpWorkerOptions);
 
-    if (dcpWorkerOptions.jobAddresses)
-      console.log(` * Processing only ${qty(dcpWorkerOptions.jobAddresses, 'job')}`, dcpWorkerOptions.jobAddresses.join(', '));
-    if (dcpWorkerOptions.computeGroups.length)
-      console.log(` * Joining compute ${qty(dcpWorkerOptions.computeGroups, 'group')}`, dcpWorkerOptions.computeGroups.map(el => el.joinKey).join(', '));
-    if (dcpWorkerOptions.leavePublicGroup)
-      console.log(' * Leaving the public compute group');
-    console.log(' . ready');
-  });
-  
   /**
+   * NOTE: In Supervisor2 this function is a NOOP.
+   * When (and if) we stop using Supevisor1, delete this reference to setDefaultIdentityKeystore
+   * and delete the corresponding fucntion from Supervisor2.
+   *
    * startWorkerLogger needs to be called before the worker is started so that
    * it can attach event listeners before the events fire, else UI events for
    * things such as progress will never get attached.
@@ -207,15 +314,208 @@ async function startWorking(cliArgs) {
    * implementation of the worker that should be addressed in Supervisor 2
    */
   await worker.supervisor.setDefaultIdentityKeystore();
+
+
+  if (cliArgs.eventDebug)
+  {
+    worker.debug = true;
+    worker.supervisor.debug = true;
+  }
+
+
+  // if the worker stops internally (eg. schedmsg stop), then exit without
+  // changing the saved exitcode
+  worker.on('stop', () => {
+    exitGuard.exit();
+  });
+
+
   startWorkerLogger(worker, {
+    exitGuard,
     verbose: cliArgs.verbose,
     outputMode: cliArgs.outputMode,
+
+    logfile: cliArgs.logfile,
+
+    syslogAddress: cliArgs.syslogAddress,
+    syslogTransport: cliArgs.syslogTransport,
+    syslogPort: cliArgs.syslogPort,
+  });
+
+  require('../lib/remote-console').init(cliArgs.replPort, {
+    help: {
+      report: 'Print a worker status & slice report',
+      kill: 'Kill the worker',
+    },
+    commands: {
+      report: printReport,
+      kill: exitcode => exitGuard.exit(exitcode),
+    },
+  });
+  require('../lib/remote-console').setMainEval(function mainEval() { return eval(arguments[0]) });
+
+  console.log(` * Starting DCP Worker`);
+  console.log(` . Configured for scheduler ${dcpConfig.scheduler.location}`);
+  console.log(` . Bank is ${dcpConfig.bank.location}`);
+  console.log(` . Earned funds will be deposited in account ${paymentAddress}`);
+  console.log(` . Identity is ${identityKeystore.address}`);
+
+  function qty(amount, singular, plural) /* XXX i18n */
+  {
+    if (Array.isArray(amount))
+      amount = amount.length;
+    if (!plural)
+      plural = singular + 's';
+    if (!amount)
+      return plural;
+    if (amount == 1)
+      return singular;
+    return plural;
+  }
+
+  if (dcpWorkerOptions.jobAddresses)
+    console.log(` * Processing only ${qty(dcpWorkerOptions.jobAddresses, 'job')}`, dcpWorkerOptions.jobAddresses.join(', '));
+  if (dcpWorkerOptions.computeGroups.length)
+    console.log(` * Joining compute ${qty(dcpWorkerOptions.computeGroups, 'group')}`, dcpWorkerOptions.computeGroups.map(el => el.joinKey).join(', '));
+  if (dcpWorkerOptions.publicGroupFallback)
+    console.log(' * Falling back on public group when preferred groups have no work');
+  else if (dcpWorkerOptions.leavePublicGroup)
+    console.log(' * Leaving the public compute group');
+  if (cliArgs.verbose)
+    console.log(` + Verbosity level: ${cliArgs.verbose}`);
+  if (cliArgs.eventDebug)
+    console.log(' + Event debug on');
+  console.log(' . output mode: ' + cliArgs.output);
+  
+  require('../lib/check-scheduler-version').check();
+
+  console.log(' . ready');
+
+
+  /** print the slice report via console.log */
+  function printReport()
+  {
+    console.log(sliceReport());
+  }
+
+  /** retrieve a slice report screen */
+  function sliceReport()
+  {
+    const sup = worker.supervisor;
+    let report = '';
+
+    report += ('='.repeat(78)) + '\n';
+
+    const sbStates = {
+      WORKING: 0,
+      ASSIGNED: 0,
+      READY: 0,
+      TERMINATED: 0,
+    };
+    const stateNames = {
+      WORKING: 'Working',
+      ASSIGNED: 'Assigned',
+      READY: 'Ready',
+      TERMINATED: 'Terminated',
+    };
+    sup.sandboxes.forEach(sb => {
+      const { state } = sb;
+      if (!sbStates[state])
+        sbStates[state] = 0;
+      sbStates[state]++;
+    });
+
+    report += (Date()) + '\n';
+    report += ('Sandboxes:') + '\n';
+    Object.keys(sbStates).forEach(state => {
+      const stateName = stateNames[state] || state;
+      report += (`  ${(stateName + ':').padEnd(12)} ${sbStates[state]}`) + '\n';
+    })
+    report += (`  * ALL:       ${sup.sandboxes.length}`) + '\n';
+
+    report += ('Progress:') + '\n';
+    sup.workingSandboxes.forEach(sb => {
+      const jobName = sb.job && sb.job.public && sb.job.public.name || `idek (${sb.jobAddress})`;
+      let el = Date.now() - sb.sliceStartTime;
+      const t = el < 1000000
+        ? toInterval(el)
+        : 'new';
+
+      el = sb.progressReports && sb.progressReports.last
+        ? Date.now() - (sb.sliceStartTime + sb.progressReports.last.timestamp)
+        : 0;
+      const pct = (typeof sb.progress) === 'number'
+        ? `${Number(sb.progress).toFixed(0).padStart(2)}%`
+        : 'ind';
+      const stale = (el < 2000) ? '' : `(stale: ${toInterval(el)})`;
+
+      report += (` ${String(sb.id).padStart(4)}: ${sb.jobAddress} ${jobName.padEnd(34)} `+ `${t} ${pct} ${stale}`.padStart(13)) + '\n';
+    });
+
+    report += ('Slices:') + '\n';
+    report += (`  working:     ${sup.allocatedSlices.length}`) + '\n';
+    report += (`  queued:      ${sup.queuedSlices.length}`) + '\n';
+
+    report += ('='.repeat(78)) + '\n';
+
+    return report;
+  }
+
+  /**
+   *  Convert a timespan in ms to a human-readable interval in minutes and seconds
+   *
+   *  @param  {number} el Milliseconds to convert
+   *  @return {string}    Timespan formatted as `m:ss`
+   */
+  function toInterval(el)
+  {
+    const m = Math.floor((el / 1000) / 60).toString(10);
+    const s = Math.floor((el / 1000) % 60).toString(10).padStart(2, '0');
+    return `${m}:${s}`;
+  }
+
+  if (parseFloat(cliArgs.reportInterval))
+  {
+    if (cliArgs.outputMode !== 'dashboard')
+      setInterval(printReport, parseFloat(cliArgs.reportInterval) * 1000);
+    else
+      console.log('Ignoring --reportInterval in dashboard output mode');
+  }
+
+
+  // Set the exit guard - this method can be called by signal and exception
+  // handlers
+  exitGuard.promise = new Promise(resolve => {
+    exitGuard.exit = resolve;
   });
 
+
   await worker.start();
-  await new Promise(resolve => process.on('SIGQUIT', resolve));
-  console.log('\n*** caught SIGQUIT; exiting...\n');
-  await worker.stop(true);
+
+  exitcode = await exitGuard.promise;
+
+  cliArgs.verbose >= 1 && console.log(`418: exit guard called with ${exitcode}`);
+
+  const exitTimeAllowed = 30;   // seconds to allow for worker to stop gracefully
+  const forceExitTimeout = setTimeout(() => {
+    console.error(`396: Worker failed to exit within ${exitTimeAllowed} seconds; terminating forcibly.`);
+    process.exit(exitcode || EXIT_TIMED_OUT)
+  }, exitTimeAllowed * 1000);
+
+  await worker.stop(true)
+  .catch(error => {
+    if (error.message.includes('Already stopped'))
+      return;
+    console.error('255: Unexpected error stopping worker:',
+      error.code
+      ? `${error.code}: ${error.message}`
+      : error.message);
+    exitcode = exitcode || EXIT_ERROR_STOPPING;
+  });
+
+  clearTimeout(forceExitTimeout);
+
+  return exitcode;
 }
 
 /** 
@@ -234,7 +534,7 @@ async function unhandledRejectionHandler (error) {
 
   try {
     let log = dcpConfig && dcpConfig.worker && dcpConfig.worker.unhandledRejectionLog;
-    log = process.env.DCP_WORKER_UNHANDLED_REJECTION_LOG;
+    if (!log) log = process.env.DCP_WORKER_UNHANDLED_REJECTION_LOG;
     if (log) {
       fs.appendFileSync(process.env.DCP_WORKER_UNHANDLED_REJECTION_LOG,
                         `${Date.now()}: ${error.message}\n${error.stack}\n\n`);
@@ -247,6 +547,7 @@ async function unhandledRejectionHandler (error) {
     if (screen) {
       screen.log(error.message + '\n' + error.stack);
       screen.destroy();
+      logError(error.message + '\n' + error.stack);
     } else {
       console.error('Unhandled rejection - preparing to exit:', error.message);
     }
@@ -260,7 +561,7 @@ async function unhandledRejectionHandler (error) {
     } catch(e) {
       console.error(error);
     }
-    process.exit(exitCode || 3);
+    process.exit(exitCode || EXIT_UNHANDLED);
   }
   setTimeout(bail, 1000 * unhandledRejectionHandler.timeout);
 
@@ -270,6 +571,6 @@ async function unhandledRejectionHandler (error) {
     console.log('Error during worker.stop:', e);
   }
 
-  setImmediate(() => bail(33));
+  setImmediate(() => bail(EXIT_UNHANDLED));
 };
 unhandledRejectionHandler.timeout = 5;

package/etc/dcp-config.js

@@ -30,6 +30,14 @@ const dcpConfig =
     },
     reloadBehaviour: 'process.exit(12)',
   },
+  worker:
+  {
+    sandbox: {
+      progressTimeout: 30 * 1000,
+    },
+    leavePublicGroup: false,
+    allowConsoleAccess: false,
+  },
 }
 
 if (!dcpConfig.evaluator.location)

package/lib/remote-console.js

@@ -0,0 +1,107 @@
+/**
+ * @file        remote-console.js
+ *              DCP Service Worker support for a remote console, accessible via telnet.
+ *
+ *              * SECURITY NOTICE *
+ *
+ *              This feature is turned off by default and should not be turned on except for
+ *              troubleshooting, as any attacker on the network will have the same OS-level
+ *              priviledges as the DCP Service Worker.
+ *
+ *              To enable this feature, create a file in ../etc name enabled-debug-console. That
+ *              file should contain a number which identities the port number this service will
+ *              listen on.
+ *
+ *              The port file could also contain the string "false", which is just an explicit
+ *              way of turning this feature off.
+ *
+ *              A history file will also be created in the etc directory, assuming the service
+ *              worker has permission to write there.
+ *
+ * @author      Wes Garland, wes@kingsds.network
+ * @date        Dec 2021
+ */
+'use strict';
+
+const path = require('path');
+const fs   = require('fs');
+var dcpConfig;
+var mainEval;
+var ci;
+
+function daemonEval()
+{
+  try {
+    if (typeof dcpConfig === 'undefined')
+      dcpConfig = require('dcp/dcp-config');
+  }
+  catch(e){};
+
+  if (mainEval)
+    return mainEval(arguments[0]);
+  return eval(arguments[0]);
+}
+
+function callbackTelnet(port, client, registry) {
+  console.log(' ! telnetd - listening on port', port);
+}
+
+/**
+ * Function to let this module know about eval within the context of the main function in
+ * the service worker.
+ */
+exports.setMainEval = function removeConsole$$setMainEval()
+{
+  mainEval = arguments[0];
+}
+
+exports.init = function remoteConsole$$init(port, ...commands)
+{
+  try
+  {
+    let historyFilename;
+    
+    if (!port)
+    {
+      /* invoked before main */
+      const edcFilename = path.resolve(__dirname, '..', 'etc', 'enable-debug-console');
+      if (!fs.existsSync(edcFilename))
+        return;
+
+      port = JSON.parse(fs.readFileSync(edcFilename, 'ascii').trim());
+      if (!port)
+        return;
+      historyFilename = edcFilename + '.history';
+    }
+    else
+    {
+      historyFilename = path.resolve(__dirname, '..', 'etc', `telnetd-repl-port-${port}.history`);
+    }
+    
+    if (port)
+    {
+      console.warn(`*** Enabling telnet daemon on port ${port} (security risk) ***`);
+      ci = require('telnet-console').start({
+        port,
+        callbackTelnet,
+        eval: daemonEval,
+        histfile: historyFilename,
+      }, ...commands);
+      exports.init = () => {
+        throw new Error('remote console has already been initialized');
+      };
+    }
+  }
+  catch(e)
+  {
+    console.warn(' ! Failed to enable telnet daemon:', e.message);
+  }
+}
+
+exports.reintercept = function remoteConsole$$reintercept()
+{
+  if (typeof ci === 'undefined')
+    return; /* no interception => no reinterception */
+
+  ci.reintercept();
+}

package/lib/startWorkerLogger.js

@@ -22,14 +22,36 @@ function getLogger({ outputMode='detect' }) {
     }
   }
 
-  switch (outputMode) {
-    case 'console':
-      return require('./worker-loggers/console');
-    case 'dashboard':
-      return require('./worker-loggers/dashboard');
-    default:
-      throw new Error(`Unknown outputMode "${outputMode}"`);
+  try
+  {
+    const om = require('path').basename(outputMode);
+    return require('./worker-loggers/' + om);
   }
+  catch (error)
+  {
+    console.error(`032: Failed to load worker logger "${outputMode}":`, error);
+    throw new Error(`Unknown outputMode "${outputMode}"`);
+  }
+}
+
+const workerEvents = {
+  fetchStart:     'onFetchingSlices',
+  fetchEnd:       'onFetchedSlices',
+  fetchError:     'onFetchSlicesFailed',
+  submit:         'onSubmit',
+  submitError:    'onSubmitError',
+  payment:        'onPayment',
+  error:          'onError',
+  warning:        'onWarning',
+}
+const supervisorEvents = {
+  submittingResult: 'onSubmitStart',
+}
+const sandboxEvents = {
+  start:          'sandbox$onSliceStart',
+  sliceProgress:  'sandbox$onSliceProgress',
+  sliceFinish:    'sandbox$onSliceFinish',
+  terminated:     'sandbox$onWorkerStop',
 }
 
 Object.assign(exports, {
@@ -40,28 +62,35 @@ Object.assign(exports, {
    * 
    * @param {Worker} worker 
    * @param {object} options
-   * @param {boolean} options.verbose
+   * @param {number} options.verbose
    * @param {boolean} options.outputMode - which logger to use (default='detect')
    */
   startWorkerLogger(worker, options={}) {
     const logger = getLogger(options);
     logger.init(worker, options);
 
-    worker.on('fetchStart', logger.onFetchingSlices.bind(logger));
-    worker.on('fetch', logger.onFetchedSlices.bind(logger));
-    worker.on('fetchError', logger.onFetchSlicesFailed.bind(logger));
+    for (const [ev, handler] of Object.entries(workerEvents))
+    {
+      if (typeof logger[handler] === 'function')
+        worker.on(ev, logger[handler].bind(logger));
+    }
+    for (const [ev, handler] of Object.entries(supervisorEvents))
+    {
+      if (typeof logger[handler] === 'function')
+        worker.supervisor.on(ev, logger[handler].bind(logger));
+    }
+
     worker.on('sandbox', (sandbox) => {
       /**
        * logger.onSandboxStart can return a data object that will be provided to
        * the other sandbox event handlers
        */
       const data = logger.onSandboxReady(sandbox) || {};
-      sandbox.on('sliceStart', logger.sandbox$onSliceStart.bind(logger, sandbox, data));
-      sandbox.on('sliceProgress', logger.sandbox$onSliceProgress.bind(logger, sandbox, data));
-      sandbox.on('sliceFinish', logger.sandbox$onSliceFinish.bind(logger, sandbox, data));
-      sandbox.on('workerStop', logger.sandbox$onWorkerStop.bind(logger, sandbox, data));
+      for (const [ev, handler] of Object.entries(sandboxEvents))
+      {
+        if (typeof logger[handler] === 'function')
+          sandbox.on(ev, logger[handler].bind(logger, sandbox, data));
+      }
     });
-
-    worker.on('payment', logger.onPayment.bind(logger));
   }
 });

package/lib/worker-loggers/common-types.js

@@ -5,6 +5,8 @@
  * @property {function} onFetchingSlices
  * @property {function} onFetchedSlices
  * @property {function} onFetchSlicesFailed
+ * @property {function} onError
+ * @property {function} onWarning
  * @property {SandboxCallback} sandbox$onSliceStart
  * @property {SandboxCallback} sandbox$onSliceProgress
  * @property {SandboxCallback} sandbox$onSliceFinish

package/lib/worker-loggers/console.js

@@ -14,24 +14,23 @@ const consoleLogger = {
   init(worker, options) {
     this.worker = worker;
     this.supervisor = worker.supervisor;
-    this.options = options;
+    this.options = Object.assign({}, options);
   },
 
   onSandboxReady(sandbox) {
-    const zeroes = '000';
-    const shortId = (zeroes + sandbox.id.toString(16).toUpperCase()).slice(-zeroes.length);
+    const shortId = sandbox.id.toString(10).padStart(3);
 
     const sandboxData = {
       shortId,
     };
 
-    console.log(` ~ [Sandbox 0x${sandboxData.shortId}] Initialized`);
+    console.log(` * Sandbox ${sandboxData.shortId}:  Initialized`);
 
     return sandboxData;
   },
 
   sandbox$onSliceStart(sandbox, sandboxData, slice) {
-    console.log(` ~ [Sandbox 0x${sandboxData.shortId}] Slice Started - "${sandbox.public.name}"`);
+    console.log(` * Sandbox ${sandboxData.shortId}:  Slice Started:   ${sandbox.jobAddress} ${sandbox.public.name}`);
   },
 
   sandbox$onSliceProgress(sandbox, sandboxData, ev) {
@@ -39,38 +38,66 @@ const consoleLogger = {
   },
   
   sandbox$onSliceFinish(sandbox, sandboxData, ev) {
-    console.log(` ~ [Sandbox 0x${sandboxData.shortId}] Slice Completed - "${sandbox.public.name}"`);
+    console.log(` * Sandbox ${sandboxData.shortId}:  Slice Completed: ${sandbox.jobAddress} ${sandbox.public.name}`);
   },
 
   sandbox$onWorkerStop(sandbox, sandboxData, _event) {
-    const job = sandbox.job ? sandbox.job.address : sandbox.job;
-    console.log(
-      ` ~ [Sandbox 0x${sandboxData.shortId}] Terminated - Job: ${job}`,
-    );
+    const job = sandbox.public ? `${sandbox.jobAddress} ${sandbox.public.name}` : sandbox.job;
+    console.log(` * Sandbox ${sandboxData.shortId}:  Terminated:      ${job}`);
   },
 
   onPayment({ payment }) {
     try {
       payment = parseFloat(payment);
     } catch (e) {
-      console.error("Failed to parse payment float:", payment);
+      console.error(" ! Failed to parse payment:", payment);
       return;
     }
 
-    console.log(`DCC Credit: ${payment}`);
+    if (payment > 0) console.log(` * DCC Credit: ${payment.toFixed(3)}`);
   },
 
   onFetchingSlices() {
-    this.options.verbose && console.log("Fetching slices...");
+    this.options.verbose && console.log(" * Fetching slices...");
   },
 
   onFetchedSlices(ev) {
-    this.options.verbose && console.log("Finished fetching slices", ev);
+    this.options.verbose && console.log(" * Fetched", ev, 'slices');
   },
 
   onFetchSlicesFailed(ev) {
-    this.options.verbose && console.log("Failed to fetch slices", ev);
+    console.log(" ! Failed to fetch slices:", ev);
   },
+
+  onSubmitStart() {
+    this.options.verbose >= 2 && console.log(" * Submitting results...");
+  },
+
+  onSubmit() {
+    this.options.verbose >= 2 && console.log(" * Submitted");
+  },
+
+  onSubmitError(ev) {
+    console.log(" ! Failed to submit results:", ev);
+  },
+
+  onError(ev) {
+    this.options.verbose && console.error(" ! Worker error:", ev);
+  },
+
+  onWarning(ev) {
+    this.options.verbose >= 2 && console.warn(" ! Worker warning:", ev);
+  },
+
+  onError(ev)
+  {
+    this.options.verbose && console.error("Worker Error:", ev);
+  },
+
+  onWarning(ev)
+  {
+    this.options.verbose && console.warn("Worker Warning:", ev);
+  }
 };
 
 Object.assign(exports, consoleLogger);

package/lib/worker-loggers/dashboard.js

@@ -25,8 +25,9 @@ const SLICE_FETCH_STATUS = {
 
 /** @type {WorkerLogger} */
 const dashboardLogger = {
-  init(worker) {
+  init(worker, options) {
     this.worker = worker;
+    this.options = options;
     this.supervisor = worker.supervisor;
     this.totalDCCs = 0;
     this.sliceFetchStatus = SLICE_FETCH_STATUS.IDLE;
@@ -72,7 +73,11 @@ const dashboardLogger = {
     setInterval(() => this.screen.render(), 1000);
 
     this.screen.key(['escape', 'C-c'], () => {
-      process.exit(0);
+      console.log('076: Exit requested...');
+      if (typeof options.exitGuard?.exit === 'function')
+        options.exitGuard.exit(0);
+      else
+        process.exit(0);
     });
   },
 
@@ -168,6 +173,14 @@ const dashboardLogger = {
     this.sliceFetchStatus = SLICE_FETCH_STATUS.NO_WORK;
     this.updateWorkerInfo();
   },
+
+  onError(error) {
+    this.options.verbose && console.error('Unexpected error from worker:', error);
+  },
+
+  onWarning(error) {
+    this.options.verbose >= 2 && console.warn('Unexpected warning from worker:', error);
+  },
 };
 
 Object.assign(exports, dashboardLogger);

package/lib/worker-loggers/event-log.js

@@ -0,0 +1,88 @@
+/**
+ *  @file       worker-loggers/event-log.js
+ *  @author     Eddie Roosenmaallen <eddie@kingsds.network>
+ *  @date       August 2022
+ *
+ *  This logger module emits events to the Windows event-log, writing all
+ *  console logs to it. Most worker events are passed through to be handled
+ *  by the console logger.
+ *
+ *  @TODO: This could likely be improved by handling worker events directly
+ *  and emitting events to the event log more deliberately than just
+ *  redirecting the base console output. ~ER20220831
+ */
+
+require('./common-types');
+const consoleLogger = require('./console');
+
+const { EventLog } = require('node-eventlog');
+
+// Copy the original global console object's properties onto a backup
+const _console = Object.assign({}, console);
+
+
+const eventlogLogger = {
+  /**
+   *  Initialize the eventlog worker logger
+   *
+   *  @param  {Worker} worker           DCP Worker object to log
+   *  @param  {object} options          Options for logger behaviour (passed
+   *    through to consoleLogger)
+   */
+  init(worker, options) {
+    consoleLogger.init(worker, options);
+
+    this._processName = require('path').basename(process.mainModule.filename || process.argv0);
+    const source = options.source || this._processName || 'dcp-worker';
+    // _console.log(`036: creating new EventLog(${source}) client...`);
+    this._eventLog = new EventLog(source);
+
+    ['debug','error','info','log','warn'].forEach(level => {
+      console[level] = (...args) => this._log(level, ...args);
+    });
+  },
+
+  _log(level, ...items) {
+    const strBuilder = [`${this._processName}[${process.pid}]:`];
+    
+    items.forEach(i => {
+      try {
+        switch (typeof i) {
+          case 'object':
+            strBuilder.push(JSON.stringify(i));
+            break;
+          default:
+            strBuilder.push(String(i));
+        }
+      }
+      catch (e) {
+        if (e instanceof TypeError) {
+          strBuilder.push('[encoding error: ' + e.message + ']');
+        }
+      }
+    });
+    
+    // Use the string log-level to look up the severity number:
+    let severity = {
+      error: 'error',
+      warn:  'warn',
+      log:   'info',
+      info:  'info',
+      debug: 'info',
+    }[level];
+
+    // _console.debug(`074: about to actually log a line:`, strBuilder, severity);
+    return this._eventLog.log(strBuilder.join(' '), severity).catch(error => {
+      if (error)
+        _console.error('255: Unexpected error writing to event log:', error);
+    });
+  }
+};
+
+// necessary to keep `this` pointing at the correct thing when we call 
+for (const [prop, value] of Object.entries(consoleLogger))
+{
+  if (typeof value === 'function')
+    exports[prop] = value.bind(consoleLogger);
+}
+Object.assign(exports, eventlogLogger);

package/lib/worker-loggers/logfile.js

@@ -0,0 +1,125 @@
+/**
+ *  @file       worker-loggers/logfile.js
+ *  @author     Eddie Roosenmaallen <eddie@kingsds.network>
+ *  @date       August 2022
+ *
+ *  This logger module maintains a log file, writing all console logs to it.
+ *  Most worker events are passed through to the console logger.
+ */
+
+require('./common-types');
+const consoleLogger = require('./console');
+
+// Copy the original global console object's properties onto a backup
+const _console = Object.assign({}, console);
+
+const logfileLogger = {
+  /**
+   *  Initialize the logfile worker logger
+   *
+   *  @param  {Worker}  worker           DCP Worker object to log
+   *  @param  {object}  options          Options for logger behaviour
+   *  @param  {string}  options.filepath Path to worker file; default: ./dcp-worker.log
+   *  @param  {boolean} options.truncate If true, logfile will be cleared at worker startup
+   */
+  init(worker, options) {
+    consoleLogger.init(worker, options);
+    
+    this._filepath = options.filepath || './dcp-worker.log';
+    this.options = options;
+    
+    options.verbose >= 3 && console.debug('050: constructing LogfileConsole', this._filepath, options);
+    
+    this._getLogFile();
+    
+    // on SIGHUP, close the output stream and open a new one
+    process.on('SIGHUP', () => {
+      this._getLogFile(true);
+    });
+
+    ['debug','error','info','log','warn'].forEach(level => {
+      console[level] = (...args) => this._log(level, ...args);
+    });
+  },
+
+  /**
+   *  Return a handle to the WritableStream for this logger, creating one if
+   *  necessary.
+   *
+   *  @param  {boolean} forceNew If truthy, close any existing stream and open
+   *                             a new one
+   *  @return {fs.WriteStream}
+   */
+  _getLogFile(forceNew = false) {
+    const fs = require('fs');
+    
+    if (this._file)
+    {
+      if (!forceNew)
+        return this._file;
+
+      try
+      {
+        this._file.end();
+      }
+      catch (err)
+      {
+        console.error('061: failed to close old log file:', err);
+      }
+      this._file = false;
+    }
+    
+    const options = {
+      flags: this.options.truncate ? 'w' : 'a',  // NYI: cli --truncate 
+    }
+    
+    const file = this._file = fs.createWriteStream(this._filepath, options);
+    
+    // On error, close & recreate the log file
+    file.on('error', err => {
+      _console.error('082: console-patch::LogFileConsole write error:', err);
+      
+      this._getLogFile(true);
+    });
+    
+    return file;
+  },
+  
+  /** Write a log line to the output file. Items will be converted to strings as
+   *  possible and concatenated after the log-level and timestamp, then written to the
+   *  current outputStream.
+   *  @param {string} level Log level.
+   *  @param {any} ...items Items to log.
+   */
+  _log(level = 'none', ...items) {
+    const strBuilder = [
+      (new Date()).toISOString(),
+      level,
+    ];
+    
+    items.forEach(i => {
+      try {
+        switch (typeof i) {
+          case 'object':
+            strBuilder.push(JSON.stringify(i));
+          default:
+            strBuilder.push(String(i));
+        }
+      }
+      catch (e) {
+        if (e instanceof TypeError) {
+          strBuilder.push('[encoding error: ' + e.message + ']');
+        }
+      }
+    });
+    
+    try {
+      this._file.write(strBuilder.join(' ') + '\n');
+    }
+    catch (error) {
+      _console.error('131: Unexpected error writing to log file:', error);
+    }
+  },
+};
+
+Object.assign(exports, consoleLogger, logfileLogger);

package/lib/worker-loggers/syslog.js

@@ -0,0 +1,91 @@
+/**
+ *  @file       worker-loggers/syslog.js
+ *  @author     Eddie Roosenmaallen <eddie@kingsds.network>
+ *  @date       August 2022
+ *
+ *  This logger module emits log lines to a remote syslogd, writing all
+ *  console logs to it. Most worker events are passed through to be handled
+ *  by the console logger.
+ *
+ *  @TODO: This could likely be improved by handling worker events directly
+ *  and emitting events to the event log more deliberately than just
+ *  redirecting the base console output. ~ER20220831
+ */
+
+require('./common-types');
+const consoleLogger = require('./console');
+
+const syslog = require('syslog-client');
+
+// Copy the original global console object's properties onto a backup
+const _console = Object.assign({}, console);
+
+
+const eventlogLogger = {
+  /**
+   *  Initialize the syslog worker logger
+   *
+   *  @param  {Worker} worker           DCP Worker object to log
+   *  @param  {object} options          Options for logger behaviour (passed
+   *    through to consoleLogger)
+   */
+  init(worker, options) {
+    consoleLogger.init(worker, options);
+
+    this.options = Object.assign({}, options);
+    const syslogOptions = {
+      transport: options.syslogTransport || 'udp', // tcp, udp, unix, tls
+      port: options.syslogPort || 514,
+    }
+
+    this._syslog = syslog.createClient(options.syslogAddress || '127.0.0.1', options);
+    this._processName = require('path').basename(process.mainModule.filename || process.argv0);
+
+    ['debug','error','info','log','warn'].forEach(level => {
+      console[level] = (...args) => this._log(level, ...args);
+    });
+  },
+
+  _log(level, ...items) {
+    const strBuilder = [`${this._processName}[${process.pid}]:`];
+    
+    items.forEach(i => {
+      try {
+        switch (typeof i) {
+          case 'object':
+            strBuilder.push(JSON.stringify(i));
+          default:
+            strBuilder.push(String(i));
+        }
+      }
+      catch (e) {
+        if (e instanceof TypeError) {
+          strBuilder.push('[encoding error: ' + e.message + ']');
+        }
+      }
+    });
+    
+    // Use the string log-level to look up the severity number:
+    let severity = {
+      error: syslog.Severity.Error,
+      warn:  syslog.Severity.Warning,
+      log:   syslog.Severity.Notice,
+      info:  syslog.Severity.Informational,
+      debug: syslog.Severity.Debug,
+    }[level];
+
+    this._syslog.log(strBuilder.join(' '), {
+      severity,
+    }, error => {
+      if (error)
+        _console.error('168: Unexpected error writing to syslog:', error);
+    });
+  }
+};
+
+for (const [prop, value] of Object.entries(consoleLogger))
+{
+  if (typeof value === 'function')
+    exports[prop] = value.bind(consoleLogger);
+}
+Object.assign(exports, eventlogLogger);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcp-worker",
-  "version": "3.2.10",
+  "version": "3.2.12-b",
   "description": "JavaScript portion of DCP Workers for Node.js",
   "main": "bin/dcp-worker",
   "keywords": [
@@ -36,7 +36,7 @@
     "blessed-contrib": "^4.11.0",
     "bravojs": "^1.0.15",
     "chalk": "^4.1.1",
-    "dcp-client": "^4.2.10",
+    "dcp-client": "^4.2.12-a",
     "kvin": "^1.2.7",
     "semver": "^7.3.5"
   },

package/lib/bindToTestHarness.js

@@ -1,55 +0,0 @@
-/**
- *  @file       bindToTestHarness.js
- *  @author     Ryan Rossiter, ryan@kingsds.network
- *  @date       May 2020
- */
-
-function setObjProps(obj, source) {
-  for (let p in source) {
-    if (typeof source[p] === 'object') setObjProps(obj[p], source[p]);
-    else obj[p] = source[p];
-  }
-}
-
-exports.bindToTestHarness = function (worker) {
-  worker.on('start', () => {
-    process.send({
-      request: 'Process Started'
-    });
-  });
-
-  worker.on('fetch', () => {
-    process.send({
-      request: 'fetchedSlices'
-    });
-  });
-
-  worker.on('sandbox', (sandbox) => {
-    sandbox.on('sliceStart', () => {
-      process.send({
-        request: 'sliceStart'
-      });
-    });
-  });
-
-  worker.supervisor.once('capabilitiesCalculated', (originalCaps) => {
-    let reportedCaps = originalCaps;
-
-    Object.defineProperty(worker.supervisor, 'capabilities', {
-      get: () => reportedCaps,
-      set: v => reportedCaps = v,
-    });
-
-    process.on('message', ({ request, data }) => {
-      const [namespace, method] = request.split('::');
-      if (namespace !== 'dcpWorker') return
-      if (method === 'setCaps') {
-        // Deep copy the original caps
-        reportedCaps = JSON.parse(JSON.stringify(originalCaps));
-        setObjProps(reportedCaps, data);
-      }
-      else if (method === 'resetCaps') reportedCaps = originalCaps;
-      // else if (method === 'reportCaps') console.error(reportedCaps);
-    });
-  });
-}