dcp-worker 3.2.30-7 → 3.2.30-9

package/.gitlab-ci.yml

@@ -28,5 +28,4 @@ tidelift:
   script:
     - echo "Running alignment and saving to Tidelift"
     - ./tidelift alignment save --wait
-  allow_failure: true
   cache: []

package/bin/dcp-worker

@@ -8,16 +8,21 @@
  */
 'use strict';
 
-var worker;
-
 const process = require('process');
 const fs      = require('fs');
-const path    = require('path');
 const crypto  = require('crypto');
 const chalk   = require('chalk');
-const telnetd = require('../lib/remote-console');
 
 const configName = process.env.DCP_CONFIG || '../etc/dcp-worker-config';
+var worker, dcpConfig, debugging;
+
+const debug = (...args) => {
+  if (!debugging)
+    debugging = require('dcp/internal/debugging').scope('dcp-worker');
+  if (debugging())
+    console.debug('dcp-worker:', ...args);
+};
+
 const EXIT_UNHANDLED = 5;
 
 /* Setup the telnet REPL up early to ensure early-failure log messages are captured */
@@ -29,11 +34,11 @@ const replHelpers = {
   },
   commands: {
     report: printReport,
-    kill:   processExit,
+    kill:   process.exit,
     die:    () => worker && worker.stop()
   },
 };
-telnetd.init(replHelpers);
+require('../lib/remote-console').init(replHelpers);
 
 /* Initialize dcp-client with local config defaults and run the main function. DCP_CONFIG_COOKIE becomes dcpConfig.cookie. */
 process.env.DCP_CONFIG_COOKIE = (Math.random().toString(16)).slice(2) + '-' + process.pid + '-' + Date.now();
@@ -43,6 +48,7 @@ function parseCliArgs()
 {
   var defaultPidFileName;
   
+  dcpConfig = require('dcp/dcp-config');
   defaultPidFileName = require('../lib/pidfile').getDefaultPidFileName(dcpConfig.worker.pidfile);
   
   const cliArgs = require('dcp/cli')
@@ -92,21 +98,18 @@ function parseCliArgs()
       },
       priorityOnly: {
         alias: 'P',
-        hidden: true,
         describe: 'Set the priority mode [deprecated]',
         type: 'boolean',
         default: false
       },
       'job-id': {
         alias: 'j',
-        hidden: true,
         describe: 'Restrict worker to a specific job (use N times for N jobs)',
         type: 'array',
       },
 
       join: {
         alias: 'g',
-        hidden: true,
         describe: 'Join compute group; the format is "joinKey,joinSecret" or "joinKey,eh1-joinHash"',
         type: 'array'
       },
@@ -117,27 +120,22 @@ function parseCliArgs()
 
       leavePublicGroup: {
         type: 'boolean',
-        hidden: true,
-        describe: 'Do not fetch slices from public compute group.',
-        default: undefined,
+        describe: 'Do not fetch slices from public compute group. Ignored if --publicGroupFallback is set',
+        default: false,
       },
-
       publicGroupFallback: {
-        hidden: true,
         describe: 'If set, worker will prefer private groups but fall back on the public group if no preferred work is available',
         type: 'boolean',
-        default: 'undefined',
-        defaultDescription: undefined,
+        default: 'false',
+        defaultDescription: false,
       },
 
       identityKey: {
-        hidden: true,
         describe: 'Identity key, in hex format',
         type: 'string',
         group: 'Identity options',
       },
       identityKeystore: {
-        hidden: true,
         describe: 'Identity keystore, in json format',
         type: 'string',
         group: 'Identity options',
@@ -149,40 +147,30 @@ function parseCliArgs()
         group: 'Output options',
       },
       eventDebug: {
-        hidden: true,
+        hide: true,
         describe: 'If set, dump all sandbox and worker events',
       },
 
       logfile: {
-        describe: 'Path to log file',
+        describe: 'Path to log file (if --output=file)',
         type: 'string',
         group: 'Log File output options',
-        default: path.resolve('../log/dcp-worker.log'),
       },
       syslogAddress: {
-        describe: 'Address of syslog server',
+        describe: 'Address of rsyslog server (if --output=syslog)',
         type: 'string',
         group: 'Syslog output options',
-        default: 'loghost',
-      },
-      syslogFacility: {
-        describe: 'Name of syslog facility',
-        type: 'string',
-        group: 'Syslog output options',
-        default: 'local7',
       },
       syslogTransport: {
-        describe: 'Transport to connect to use for syslog',
+        describe: 'Transport to connect to rsyslog daemon (if --output=syslog)',
         type: 'string',
-        choices: ['udp','tcp','unix','tls'],
+        choices: ['udp','tcp'],
         group: 'Syslog output options',
-        default: 'udp',
       },
       syslogPort: {
-        describe: 'UDP/TCP port to use for syslog',
+        describe: 'UDP/TCP port of rsyslog server',
         type: 'number',
         group: 'Syslog output options',
-        default: 514,
       },
 
       allowedOrigins: {
@@ -214,8 +202,8 @@ function parseCliArgs()
 
   if (cliArgs.dumpConfig)
   {
-    console.log(JSON.stringify(require('dcp/dcp-config'), null, 2));
-    processExit(0);
+    console.debug(JSON.stringify(require('dcp/dcp-config'), null, 2));
+    process.exit(0);
   }
 
   return cliArgs;
@@ -240,20 +228,6 @@ function addConfig(target, ...objs)
   Object.assign(target, tmp);
 }
 
-/** 
- * Replacement for process.exit() that tries to increase the probability
- * that remote log messages will make it out over the network.
-*/
-function processExit()
-{
-  logClosing('debug', 'Exit Code:', process.exitCode || 0);
-  if (console.close)
-    console.close();
-  setImmediate(() => {
-    process.exit.apply(null, arguments);
-  });
-}
-
 /**
  * Main program entry point. Assumes DCP client is already initialized and console logging is ready.
  */
@@ -261,22 +235,21 @@ async function main()
 {
   const wallet = require('dcp/wallet');
   const DCPWorker = require('dcp/worker').Worker;
+  const { startWorkerLogger } = require('../lib/startWorkerLogger');
   const cliArgs = parseCliArgs();
   const sawOptions = {
     hostname: cliArgs.hostname,
     port:     cliArgs.port
   };
 
-  telnetd.setMainEval(function mainEval() { return eval(arguments[0]) }); // eslint-disable-line no-eval
-  require('../lib/startWorkerLogger').init(cliArgs); /* Start remote logger as early as possible */
-  verifyDefaultConfigIntegrity();                    /* Bail before TUI & as early as possible if bad conf */
+  verifyDefaultConfigIntegrity();
 
   process.on('SIGINT', handleSigDeath);
   process.on('SIGTERM', handleSigDeath);
   process.on('SIGQUIT', handleSigDeath);
   process.on('unhandledRejection', handleUnhandled);
   process.on('uncaughtException', handleUnhandled);
-  
+
   let paymentAddress = false
       || cliArgs.paymentAddress
       || dcpConfig.worker.paymentAddress
@@ -287,7 +260,7 @@ async function main()
   if (cliArgs.pidFile)
     require('../lib/pidfile').write(cliArgs.pidFile);
 
-  /* Figure out the worker's identity and put that keystore in the wallet */
+  /* Figure out of the worker's identity and put that keystore in the wallet */
   let identityKeystore = false;
   if (cliArgs.identityKey)
     identityKeystore = await new wallet.IdKeystore(cliArgs.identityKey, '');
@@ -305,14 +278,11 @@ async function main()
    * which were derived from dcpConfig in the first place. defaultOptions are overrideable by the usual
    * dcpConfig mechanisms, but since they are dynamic (or non-user-facing) they don't come from the
    * etc/dcp-worker-config.js file that ships with the work.
-   *
-   * It is important to never disable leavePublicGroup as a side effect of any other operation, or
-   * slight configuration errors could have large security impacts.
    */
   const dcpWorkerOptions = dcpConfig.worker;
   const forceOptions = {
     paymentAddress,
-    maxWorkingSandboxes: cliArgs.cores,
+    leavePublicGroup: cliArgs.leavePublicGroup || dcpConfig.worker.leavePublicGroup || cliArgs.publicGroupFallback || false,
   };
   const defaultOptions = {
     sandboxOptions: {
@@ -320,21 +290,9 @@ async function main()
     },
   };
 
-  if (cliArgs.leavePublicGroup !== undefined)
-    forceOptions.leavePublicGroup = mkBool(cliArgs.leavePublicGroup);
-  if (cliArgs.publicGroupFallback !== undefined)
-    forceOptions.publicGroupFallback = mkBool(cliArgs.publicGroupFallback);
-
   addConfig(dcpWorkerOptions, defaultOptions, dcpConfig.worker, forceOptions);
   processCoresAndDensity(dcpWorkerOptions, cliArgs);
 
-  /* Support magic value used by Windows screensaver configuration /wg June 2023 */
-  if (dcpWorkerOptions.leavePublicGroup === 'fallback')
-  {
-    dcpWorkerOptions.publicGroupFallback = true;
-    dcpWorkerOptions.leavePublicGroup = undefined;
-  }
-
   /* cliArgs.join is the list of compute groups to join */
   if (cliArgs.join && cliArgs.join.length)
   {
@@ -367,57 +325,47 @@ async function main()
     dcpWorkerOptions.watchdogInterval = cliArgs.watchdogInterval;
 
   worker = new DCPWorker(identityKeystore, dcpWorkerOptions);
-  worker.on('error',   (...payload) => console.error(...payload));
-  worker.on('warning', (...payload) => console.warn (...payload));
-  worker.on('stop', () => { console.log('Worker is stopping') });
-  worker.on('end',  () => { logClosing('log', 'Worker has stopped') });
-  require('../lib/default-ui-events').hook(worker, cliArgs);
+  worker.on('error', (...payload) => console.error(...payload));
+  worker.on('warning', (...payload) => console.warn(...payload));
 
-  if (cliArgs.outputMode === 'dashboard')
-    require('../lib/dashboard-tui').init(worker, cliArgs);
-  
   /* Let incorrect event-loop references keep us alive when linked with a debug library, but
    * exit quickly/accurately for production code even when the library isn't perfect.
    */
   if (require('dcp/build').config.build !== 'debug')
-    worker.on('end', processExit);
+    worker.on('end', process.exit);
   else
-    worker.on('end', () => setTimeout(processExit, getCleanupTimeoutMs()).unref());
+    worker.on('end', () => setTimeout(process.exit, getCleanupTimeoutMs()).unref());
 
   if (cliArgs.eventDebug)
     worker.enableDebugEvents = true;
 
-  if (dcpWorkerOptions.publicGroupFallback)
+  worker.on('stop', () => { console.log('Worker is stopping') });
+  worker.on('end',  () => { logClosing('log', 'Worker has stopped') });
+  startWorkerLogger(worker, cliArgs);
+
+  require('../lib/remote-console').setMainEval(function mainEval() { return eval(arguments[0]) });
+
+  // Activate public group fallback
+  // If requested by CLI
+  // OR if requested by dcpConfig and not forbidden by the cli
+  if (cliArgs.publicGroupFallback
+      || (dcpConfig.worker?.leavePublicGroup === 'fallback'
+          && typeof cliArgs.publicGroupFallback !== false))
   {
-    if (dcpWorkerOptions.leavePublicGroup)
-      console.warn(' ! Public Group fallback has been requested, but the public group is blocked by local configuration');
+    dcpWorkerOptions.publicGroupFallback = true;
+
+    // If local config blocks the public group, then complain instead of activating fallback
+    if (dcpConfig.worker?.leavePublicGroup === true)
+    {
+      console.warn('* Public Group fallback has been requested, but the public group is blocked by local configuration');
+    }
     else
     {
-      /* Enable public group fallback - this currently works by enabling or disabling the public group
-       * on the next fetch based on whether or not the most recent fetch was an empty task or not.
-       */
-      worker.on('fetch', fetchEventHandler);
-
-      function fetchEventHandler(ev)
-      {
-        var slicesFetched;
-
-        if (ev instanceof Error)
-        {
-          console.error('Error fetching task:', ev);
-          return;
-        }
-
-        if (typeof ev === 'number' || typeof ev === 'string') /* <= June 2023 Worker events: remove ~ Sep 2023 /wg */
-          slicesFetched = ev;
-        else
-        {
-          const task = ev;
-          slicesFetched = task.slices.length;
-        }
-
-        dcpWorkerOptions.leavePublicGroup = Boolean(slicesFetched > 0);
-      }
+      worker.on('fetchend', slicesFetched => {
+        // Iff we got work in this fetch, then leave the public group for the
+        // next fetch
+        dcpConfig.worker.leavePublicGroup = Boolean(slicesFetched > 0);
+      });
     }
   }
 
@@ -443,23 +391,21 @@ async function main()
 
   if (dcpWorkerOptions.jobAddresses?.length > 0)
     introBanner += ` * Processing only ${qty(dcpWorkerOptions.jobAddresses, 'job')} ` + dcpWorkerOptions.jobAddresses.join(', ') + '\n';
-  if (dcpWorkerOptions.computeGroups && Object.keys(dcpWorkerOptions.computeGroups).length > 0)
+  if (dcpWorkerOptions.computeGroups?.length > 0)
     introBanner += ` . Joining compute ${qty(dcpWorkerOptions.computeGroups, 'group')} ` + dcpWorkerOptions.computeGroups.map(el => el.joinKey).join(', ') + '\n';
   if (dcpWorkerOptions.publicGroupFallback)
     introBanner += ' . Falling back on public group when preferred groups have no work' + '\n';
-  if (dcpWorkerOptions.leavePublicGroup)
+  else if (dcpWorkerOptions.leavePublicGroup)
     introBanner += ' . Leaving the public compute group' + '\n';
   if (dcpWorkerOptions.cores)
-    introBanner += ` . Configured Cores: ${dcpWorkerOptions.cores.cpu},${dcpWorkerOptions.cores.gpu}\n`
+    introBanner += ` . Target cores: ${JSON.stringify(dcpWorkerOptions.cores)}\n`;
   else
     introBanner += ` . Target core density: ${JSON.stringify(dcpWorkerOptions.defaultCoreDensity)}\n`;
   if (cliArgs.verbose)
     introBanner += ` + Verbosity level: ${cliArgs.verbose}` + '\n';
   if (cliArgs.eventDebug)
     introBanner += ' + Event debug on' + '\n';
-  if (telnetd.hasOwnProperty('port'))
-    introBanner += ` ! telnetd listening on port ${telnetd.port}\n`;
-  
+
   introBanner += ' . Supervisor version: ' + worker.supervisorVersion;
   introBanner += ' . Output mode: ' + cliArgs.outputMode + '\n';
   introBanner += ' * Ready' + '\n';
@@ -472,7 +418,7 @@ async function main()
     if (cliArgs.outputMode !== 'dashboard')
       setInterval(printReport, parseFloat(cliArgs.reportInterval) * 1000).unref();
     else
-      console.warn('Ignoring --reportInterval in dashboard output mode');
+      console.log('Ignoring --reportInterval in dashboard output mode');
   }
 
   /* Start the worker. Normal process exit happens by virtue of the worker<end> event */
@@ -511,9 +457,9 @@ function processCoresAndDensity (dcpWorkerOptions, cliArgs)
   parseArg('cores');
 
   if (dcpWorkerOptions.cores)
-    debugging() && console.debug('dcp-worker: cores =', dcpWorkerOptions.cores);
+    debug('cores = ', dcpWorkerOptions.cores);
   else
-    debugging() && console.debug('dcp-worker: core density =', dcpWorkerOptions.defaultCoreDensity);
+    debug('core density = ', dcpWorkerOptions.defaultCoreDensity);
 }
 
 /**
@@ -524,7 +470,9 @@ function logClosing(facility, ...message)
 {
   var screen = require('../lib/worker-loggers/dashboard').screen;
 
-  if (screen)
+  if (!screen)
+    console[facility](message);
+  else
   {
     /* Turn off fullscreen TUI and resume "normal" console logging.
      * FUTURE: dashboard API should know how to unregister its hook so that we don't have to clobber
@@ -533,11 +481,10 @@ function logClosing(facility, ...message)
     screen.log(...message);
     screen.destroy();
     screen = false;
-    console = new (require('console').Console)(process); // eslint-disable-line no-global-assign
-    telnetd.reintercept();
+    console = new (require('console').Console)(process);
+    require('../lib/remote-console').reintercept();
+    console[facility].call(null, ...message);
   }
-
-  console[facility](...message);
 }
 
 /** 
@@ -546,7 +493,7 @@ function logClosing(facility, ...message)
  * the worker must be restarted. This handler does its best to report the rejection and give the worker a few
  * seconds in which to attempt to return slices to the scheduler before it gives up completely.
  */
-function handleUnhandled(error)
+async function handleUnhandled(error)
 {
   var _worker = worker;
   worker = false;
@@ -555,21 +502,21 @@ function handleUnhandled(error)
   
   try
   {
-    logClosing('error', error);
-  } catch(e) {} // eslint-disable-line no-empty
+    logClosing(error);
+  } catch(e) {};
 
   if (!_worker)
     console.error('trapped unhandled error:', error)
   else
   {
     console.error('trapped unhandled error -- stopping worker:', error);
-    _worker.on('end', processExit);
+    _worker.on('end', process.exit);
     _worker.stop();
   }
 
   setTimeout(() => {
     logClosing('error', 'handleFatalError timeout - exiting now');
-    processExit();
+    process.exit();
   }, getCleanupTimeoutMs()).unref();
 
   try {
@@ -579,7 +526,7 @@ function handleUnhandled(error)
       fs.appendFileSync(process.env.DCP_WORKER_UNHANDLED_REJECTION_LOG,
                         `${Date.now()}: ${error.message}\n${error.stack}\n\n`);
     }
-  } catch(e) {} // eslint-disable-line no-empty
+  } catch(e) {};
 }
 
 /** print the slice report via console.log */
@@ -676,14 +623,14 @@ function handleSigDeath(signalName, signal)
   process.off(signalName, handleSigDeath);
 
   if (!worker)
-    console.warn(`trapped ${signalName}, signal ${signal}`);
+    console.error(`trapped ${signalName}, signal ${signal}`);
   else
   {
-    console.warn(`trapped ${signalName}, signal ${signal} -- stopping worker`);
+    console.error(`trapped ${signalName}, signal ${signal} -- stopping worker`);
     worker.stop(signalName === 'SIGQUIT');
   }
 
-  setTimeout(() => processExit(signal - 128), getCleanupTimeoutMs()).unref();
+  setTimeout(() => process.exit(signal - 128), getCleanupTimeoutMs()).unref();
 }
 
 /**
@@ -703,7 +650,7 @@ function getCleanupTimeoutMs()
     cleanupTimeout = defaultCT;
     if (!getCleanupTimeoutMs.warned)
     {
-      console.error(`warning: dcpConfig.worker.cleanupTimeout is not a number (${dcpConfig.worker.cleanupTimeout})`);
+      console.warn(`warning: dcpConfig.worker.cleanupTimeout is not a number (${dcpConfig.worker.cleanupTimeout})`);
       getCleanupTimeoutMs.warned = true;
     }
   }
@@ -725,14 +672,14 @@ function verifyDefaultConfigIntegrity()
 
   if (!fs.existsSync(md5sumPath))
   {
-    console.error(chalk.bold.red(` ! warning: ${md5sumPath} not found; cannot verify configuration integrity`));
+    console.log(chalk.bold.red(` ! warning: ${md5sumPath} not found; cannot verify configuration integrity`));
     require('dcp/utils').sleep(2);
   }
   else
   {
     const originalMd5sum = fs.readFileSync(md5sumPath, 'ascii');
     const actualMd5sum = crypto.createHash('md5')
-          .update(fs.readFileSync(workerConfPath))
+          .update(fs.readFileSync(workerConfPath, 'ascii'))
           .digest('hex');
 
     if (!originalMd5sum.startsWith(actualMd5sum))
@@ -747,7 +694,7 @@ function verifyDefaultConfigIntegrity()
       console.warn('    - the Windows Registry');
 
       if (require('dcp/build').config.build !== 'debug')
-        processExit(1);
+        process.exit(1);
 
       console.log(chalk.bold.red.inverse("If this wasn't a debug build, the worker would exit now."));
       require('dcp/utils').sleep(2);
@@ -757,23 +704,6 @@ function verifyDefaultConfigIntegrity()
   if (dcpConfig.cookie !== process.env.DCP_CONFIG_COOKIE || !dcpConfig.cookie)
   {
     console.error(' ! DCP Worker default configuration was not loaded; exiting.');
-    processExit(1);
+    process.exit(1);
   }
 }
-
-/* thunk - ensures global debugging() symbol always available even if called before dcp-client init */
-function debugging()
-{
-  require('dcp-client');
-  debugging = require('dcp/internal/debugging').scope('dcp-worker');
-  return debugging.apply(this, arguments);
-}
-
-/**
- * Cast b to boolean such that 'false' becomes false, falsey things become false, and everything else
- * becomes true.
- */
-function mkBool(b)
-{
-  return Boolean(b) && (b !== 'false');
-}

package/docs/CODEOWNERS

@@ -18,11 +18,9 @@
 /package-lock.json @eroosenmaallen
 
 [Wes]
-/npm-hooks/ @wesgarland
 /README.md @wesgarland
 /docs/ @wesgarland
 /.eslintrc.json @wesgarland
-/.npmrc @wesgarland
 /.tidelift @wesgarland
 /lib/ @wesgarland
 /LICENSE.md @wesgarland

package/lib/blessed-components/index.js

@@ -1,4 +1,3 @@
-'use strict';
 
 Object.assign(exports, require('./log'));
 Object.assign(exports, require('./sandboxes'));

package/lib/blessed-components/log.js

@@ -7,7 +7,6 @@
  *  log component, but features text wrapping and behaves
  *  closer to the built-in console log methods.
  */
-'use strict';
 
 const { Box } = require('blessed');
 

package/lib/check-scheduler-version.js

@@ -6,7 +6,6 @@
  * @author      Sam Cantor
  * @date        Nov 2020
  */
-'use strict';
 
 exports.check = function checkSchedulerVersion$$check(quiet)
 {

package/lib/pidfile.js

@@ -21,7 +21,7 @@ const path = require('path');
  *
  * @param {string} filename           the location of the pid file
  */
-exports.write = function pidfile$$write(filename)
+exports.write = async function pidfile$$write(filename)
 {
   var fd;
   

package/lib/remote-console.js

@@ -47,7 +47,7 @@ function daemonEval()
 
 function callbackTelnet(port, client, registry) {
   client.unref();
-  debugging() && console.notice(' ! telnetd - listening on port', port);
+  debugging() && console.debug(' ! telnetd - listening on port', port);
 }
 
 /**
@@ -76,16 +76,8 @@ exports.init = function remoteConsole$$init(...commands)
 
     console.warn('*** Enabling telnet daemon on port', port, '(security risk) ***');
 
-    if (port !== 0)
-      exports.port = port;
-    else
-    {
-      /* telnet-console library does not properly support port 0 so we mostly work-around here */
-      exports.port = Math.floor(1023 + (Math.random() * (63 * 1024)));
-    }
-
     ci = require('telnet-console').start({
-      port: exports.port,
+      port,
       callbackTelnet,
       eval: daemonEval,
       histfile: edcFilename + '.history',