dcp-worker 4.3.8 → 4.4.1

package/bin/dcp-evaluator-manager

@@ -147,9 +147,11 @@ async function listenForConnections(config)
     {
       console.log('Using dynamic port', server.address().port);
       // Send info by IPC, if the spawn is setup for that. Probably don't need to try/catch.
+      /* WHAT IS THIS FOR? /wg feb 2026 */
       if (process.send)
         try { process.send(server.address().port); } catch (e) {}
     }
+
     console.log('Evaluator command is', config.proc, config.argv);
   });
 }

package/bin/dcp-worker

@@ -18,13 +18,14 @@ const process = require('process');
 const fs      = require('fs');
 const path    = require('path');
 const debug   = require('debug');
+const assert  = require('assert');
 const telnetd = require('../lib/telnetd');
-const httpd   = require('../lib/web-console');
+const httpd   = require('../lib/web-iface');
 const pidfile = require('../lib/pidfile');
 const reports = require('../lib/reports');
 const utils   = require('../lib/utils');
 const loggers = require('../lib/loggers');
-var wallet, identity, a$sleep, DistributiveWorker; /* populate in main() */
+var wallet, a$sleep, DistributiveWorker; /* populate in main() */
 
 const { exitCodes,
         logLevels }          = require('../lib/consts');
@@ -53,7 +54,7 @@ const replHelpers = {
   },
 };
 telnetd.init(replHelpers);
-const dcpClientOptions = { dcpConfig: { worker: { logging: { syslog: {} } } } };
+const dcpClientOptions = { dcpConfig: { worker: { logging: { } } } };
 require('dcp-client').init(dcpClientOptions).then(main).catch(handleUnhandled);
 
 function usage()
@@ -70,59 +71,63 @@ Options:
       --dcp-identity=filename        Name of file containing identity keystore
       --earnings-account=string      Bank account into which to deposit earnings
       --earnings-account=keystore    Keystore, in json format, for bank account
-      --earnings-account=filename    Name of file containing bank account keystore
+      --earnings-account=filename    Keystore filename for bank account
   -c, --cores=[#cpu][,#gpu]          Override detected number of CPU and GPU
                                      cores available to the evaluator.
-  -u, --utilization=[cpu][,gpu]      default proportion of CPU,GPU to utilize: eg use
-                                     -u 0.8,0.75 for 80% CPU and 75% GPU utilization
-  -m, --max-sandboxes=integer        Override the default Maximum number of sandboxes
-  -H, --hostname                     Override evaluator hostname
-  -p, --port                         Override evaluator port number
-  -o, --output=mode                  Set the console mode: text, dashboard, or none
-  -l, --logger=type                  Add an additional logger: syslog, logfile (all
-                                     platforms), event-log (win32)
+  -u, --utilization=[cpu][,gpu]      proportion of CPU,GPU to use; 0..1
+  -m, --max-sandboxes=integer        Override the maximum number of sandboxes
+  -H, --hostname=                    Override evaluator hostname
+  -p, --port=                        Override evaluator port number
+  -o, --output=mode                  Set the console mode: text|dashboard|none
+  -l, --logger=type                  Add an additional logger: syslog, logfile
+                                     (all platforms), event-log (win32)
       --logfile=filename             Change filename, implies --logger=logfile
-      --overwrite-logfile            Overwrite previous log files instead of appending
-      --syslog=url                   Change syslog url, implies --logger=syslog. Syslog
-                                     facility (eg. local7) is the pathname, structured
-                                     data is encoded in search params; id+name=value.
-      --syslog=facility              Change syslog facility, implies --logger=syslog.
-      --min-level=[logger,]level     Set the lowest level of log sent to a given logger
-      --min-level=[console,]level    Set the lowest level of log sent to the console
-  -j, --job                          Restrict worker to a specific set of jobs. Use more
-                                     than once for multiple jobs.
-  -g, --join=                        Join a compute group; the format is
-                                     "joinKey,joinSecret" or "joinKey,eh1-joinHash".
+      --overwrite-logfile            Overwrite log files instead of appending
+      --syslog=url|facility          Change syslog target config, implies
+                                     --logger=syslog. Facility (eg. local7) can
+                                     be specified alone or as the pathname
+                                     component of the URL.
+      --min-level=[logger,]level     Set lowest log level sent to a given logger
+      --min-level=[console,]level    Set lowest log level sent to the console
+                                     Levels: debug, info, notice, warn, error,
+                                             crit, alert, emerg
+  -j, --job                          Restrict worker to a specific set of jobs.
+                                     Use more than once for multiple jobs.
+  -g, --join=joinKey,joinSecret      Join a compute group via plaintext secret
+  -g, --join=joinKey,eh1-joinHash    Join a compute group via hashed secret.
                                      Use more than once to join multiple groups.
-  -a, --allow-origin=origin          Grant all jobs network access to specified origin
+  -a, --allow-origin=origin          Grant all jobs network access to origin
                                      Use more than once for multiple origins.
-      --zero-trust                   Do not load configuration data from scheduler
-      --no-global                    Do not fetch work from the Global Compute Group
-      --show=                        Show configuration data and exit. Valid arguments:
+      --zero-trust                   Do not load config data from scheduler,
+                                     send management statistics, etc.
+      --no-global                    Do not join the Global Compute Group
+      --show=type                    Show configuration data and exit. Types:
                                       - worker-id: the worker id
-                                      - identity: the address of the active DCP identity
-                                      - earnings-account: where funds will be deposited
-                                      - owner: the address of the registered owner
-                                      - allowed-origins: origins in origin manager
-                                      - compute-groups: credentials for task requests
+                                      - identity: identity address
+                                      - earnings-account: bank account address
+                                      - owner: registered owner address
+                                      - allowed-origins: data in origin manager
+                                      - compute-groups: credentials in request
                                       - config: entire worker.config
                                       - <dot-path>: a property of worker.config
       --kvin | --json                Use KVIN or JSON for show output
       --startup-delay=num            Delay for num seconds during startup
-      --startup-delay-rnd=num        Delay for a random amount of time up to num seconds
-      --disable-httpd                Disable web interface (${httpd.getConfig().listen})
-      --register=key                 Use the given key to register this worker with your
-                                     Distributive portal account and exit
-      --unmanaged                    Do not report statistics to or read configuration
+      --startup-delay-rnd=num        Delay for a random time up to num seconds
+      --disable-httpd                Disable management service
+      --management-hostname=         Override the management service hostname
+      --management-port=             Override the management service port
+      --register=key[,label]         Use the given key to register this worker
+                                     with your Distributive account and exit
+      --unmanaged                    Do not report statistics to or read config
                                      from scheduler; implies --no-claim-earnings
-      --no-claim-earnings            Do not transfer unclaimed earnings to the earnings
-                                     account when the worker starts
-      --claim-earnings=              Transfer unclaimed earnings to the specified
-                                     account and exit
+      --no-claim-earnings            Do not transfer unclaimed earnings to the
+                                      earnings account when the worker starts
+      --claim-earnings=              Transfer unclaimed earnings to the
+                                     specified account and exit
   -f, --pidfile=filename             Specify the location of the pid file
       --no-pidfile                   Do not write the pid file
-  -v, --verbose                      Enable verbose output. Use more than once to
-                                     increase verbosity.
+  -v, --verbose                      Enable verbose output. Use more than once
+                                     to increase verbosity.
 `);
 }
 
@@ -143,13 +148,20 @@ function* parseCliOptions(argv)
       + 'h(help)*:(earnings-account)*:(identity-keystore)'
       + 'c:(cores)u:(utilization)m:(max-sandboxes)H:(hostname)p:(port)'
       + 'o:(output)l:(logger)*:(logfile)*(overwrite-logfile)*:(syslog)*:(min-level)j:(job)g:(join)'
-      + 'a:(allow-origin)*(zero-trust)*(no-global)*(kvin)*(json)*:(show)*(unmanaged)'
+      + 'a:(allow-origin)*(zero-trust)*(no-global)*(kvin)*(json)*:(show)*'
+      + '(unmanaged)*:(management-hostname)*:(management-port)'
+      + '*(no-masquerade)*:(force-worker-id)*:(lookup-worker-id)'
       + '*(no-claim-earnings)*:(register)*:(claim-earnings)*(reset)f:(pidfile)*(no-pidfile)v(verbose)';
 
+  const ck = /(\([^)]*\):?){2}/.exec(optionsDefn);
+  if (ck)
+    throw new Error(`getopt - option character missing at offset ${ck.index} ${ck[1]}`);
+
   optionsDefn = optionsDefn /* patchup long opt * to > \u1000 chars */
     .match(/[A-Za-z*]:?(\([^)]*\))?/g)
     .map(s => s[0] === '*' ? String.fromCharCode(++longOptCount + 0x1000) + s.slice(1) : s).join('');
   const optionChars = optionsDefn.match(/[A-Za-z\u1000-\u1100/](:?\([^)]*\))?/g).map(s=>s[0]);
+
   if (optionChars.length !== utils.uniq(optionChars).length)
     throw new Error('getopt - option character used more than once');
 
@@ -225,8 +237,10 @@ function processCliOptsPhase1(argv, operatingMode)
       case 'zero-trust':
         operatingMode.zeroTrust = true;
         operatingMode.unmanaged = true;
+        operatingMode.startHttpd = false;
         break;
       case 'unmanaged':
+        operatingMode.startHttpd = false;
         operatingMode.unmanaged = true;
         break;
       case 'H':
@@ -278,9 +292,29 @@ function processCliOptsPhase1(argv, operatingMode)
       case '?':
         process.exit(exitCodes.error);
         break;
+      case 'lookup-worker-id':
+        operatingMode.lookupWorkerId = [ optarg ].concat(operatingMode.lookupWorkerId);
+        break;
+      case 'no-masquerade':
+        operatingMode.identityMasquerade = false;
+        break;
+      case 'force-worker-id':
+      {
+        const { schedulerConstants } = require('dcp/utils');
+        const idRe = new RegExp(`^[${schedulerConstants.workerIdAlphabet}]*`);
+        const match = optarg.match(idRe);
+        if (match[0].length !== optarg.length)
+          reportUserError(`--force-worker-id: invalid character at position ${match[0].length + 1}`);
+        else if (optarg.length !== schedulerConstants.workerIdLength)
+          reportUserError('--force-worker-id: invalid length');
+        DistributiveWorker.obtainWorkerId = () => optarg;
+        break;
+      }
+      /* options below this point are processed in both phase 1 and phase 2 */
       case 'claim-earnings':
       case 'register':
         operatingMode.zeroTrust = true;
+      case 'show':
         operatingMode.consoleType = 'stdio';
         /* fallthrough */
       default:
@@ -294,8 +328,8 @@ function processCliOptsPhase1(argv, operatingMode)
 
 async function processCliOptsPhase2(argv, operatingMode, loggingOptions)
 {
-  var pidfileName = dcpConfig.worker.pidfile || pidfile.getDefaultPidFileName();
-
+  if (!worker.config.pidfile)
+    worker.config.pidfile = pidfile.getDefaultPidFileName();
   for (let { option, optarg } of parseCliOptions(argv))
   {
     switch(option)
@@ -350,6 +384,7 @@ async function processCliOptsPhase2(argv, operatingMode, loggingOptions)
         break;
       case 'show':
         operatingMode.show = optarg;
+        operatingMode.consoleType = 'stdio';
         break;
       case 'startup-delay':
         await a$sleep(Number(optarg));
@@ -358,10 +393,10 @@ async function processCliOptsPhase2(argv, operatingMode, loggingOptions)
         await a$sleep(Number(optarg) * Math.random());
         break;
       case 'f':
-        pidfileName = optarg;
+        worker.config.pidfile = optarg;
         break;
       case 'no-pidfile':
-        pidfileName = false;
+        worker.config.pidfile = false;
         break;
       case 'unmanaged':
         worker.config.unmanaged = true;
@@ -371,24 +406,15 @@ async function processCliOptsPhase2(argv, operatingMode, loggingOptions)
         break;
       case 'register':
       {
-        if (!wallet.isPrivateKey(optarg))
-          await reportUserError('invalid registration key');
         try
         {
-          const reg = await worker.register(optarg);
+          const [ rkey, label ] = optarg.split(',');
+          const reg = await utils.registerWorker(worker, rkey, label, reportUserError);
           if (reg instanceof Error)
             console.error(reg);
           else
-          {
-            if (reg.status !== 'registered')
-              console.error(`Registration failure: ${reg.status}`);
-            else
-            {
-              console.log(`Registration complete; worker ${worker.id} now managed by ${reg.owner}`);
-              if (pidfileName)
-                await require('../lib/pidfile').signal(pidfileName, 'SIGQUIT', 30, 'stopping worker on pid %i');
-            }
-          }
+            await require('../lib/pidfile').signal(worker.config.pidfile, 'SIGQUIT', 30,
+                                                   'stopping worker on pid %i');
         }
         catch(error)
         {
@@ -428,6 +454,12 @@ async function processCliOptsPhase2(argv, operatingMode, loggingOptions)
         await cleanupThenExit(exitCodes.normal);
         break;
       }
+      case 'management-hostname':
+        httpd.config.hostname = optarg;
+        break;
+      case 'management-port':
+        httpd.config.port = Number(optarg);
+        break;
       case 'disable-httpd':
         operatingMode.startHttpd = false;
         break;
@@ -444,8 +476,6 @@ async function processCliOptsPhase2(argv, operatingMode, loggingOptions)
         break;
     }
   }
-
-  return { pidfileName };
 }
 
 /**
@@ -489,9 +519,10 @@ async function mergeDbAppConfig(remainArgv, operatingMode, loggingOptions)
  */
 async function main()
 {
+  var workerIsStopping = false;
+  const identity = require('dcp/identity');
   /* handle file-wide dcp-client imports */
-  identity = require('dcp/identity');
-  wallet   = require('dcp/wallet');
+  wallet = require('dcp/wallet');
   ({ a$sleep } = require('dcp/utils'));
   ({ DistributiveWorker } = require('dcp/worker'));
 
@@ -502,7 +533,6 @@ async function main()
     startHttpd:     true
   };
 
-  identity.interactive = false;
   process.exitCode = exitCodes.lostRef;
 
   /* Telnetd is started ASAP to make it possible to troubleshoot very early problems. Then set our
@@ -515,7 +545,6 @@ async function main()
     evaluatorOptions,   /* options for sandbox constructor factory */
     loggingOptions,     /* options to tell us where logs go besides the console */
   } = processCliOptsPhase1(process.argv, operatingMode);
-  await identity.login('$login');
 
   if (dcpConfig.worker.logging?.syslog)
     loggingOptions.loggers.push('syslog');
@@ -523,6 +552,21 @@ async function main()
     loggingOptions.loggers.push('logfile');
   if (dcpConfig.worker.eventLog)
     loggingOptions.loggers.push('event-log');
+  if (operatingMode.lookupWorkerId)
+    await utils.patchWorkerId(operatingMode.lookupWorkerId);
+
+  identity.interactive = false;
+  if (operatingMode.identityMasquerade === false)
+    await identity.login('$login');
+  else
+  {
+    /* Acquire a unique identity for this worker so that registration proxies will be unique even in
+     * case when the id.keystore is shared across a campus. The masquerade identity is composed from the
+     * worker id and the either the default id.keystore file or the --dcp-identity parameter.
+     */
+    await identity.masquerade(DistributiveWorker.obtainWorkerId(), '$login');
+  }
+
   if (!operatingMode.unmanaged)
     await mergeDbAppConfig(remainArgv, operatingMode, loggingOptions);
   loggingOptions.loggers = utils.uniq(loggingOptions.loggers);
@@ -534,8 +578,12 @@ async function main()
     trustScheduler: operatingMode.zeroTrust ? false : dcpConfig.worker.trustScheduler,
     unmanaged:      operatingMode.unmanaged ? true  : dcpConfig.worker.unmanaged,
   }), SandboxConstructor);
-  const { pidfileName } = await processCliOptsPhase2(remainArgv, operatingMode, loggingOptions);
+
   worker.runInfo.type = 'dcp-worker';
+  worker.on('warning', (...payload) => console.warn (...payload));
+  worker.on('claim', amount => console.log(` . transfered ${amount} ⊇ to ${worker.config.paymentAddress}`));
+
+  await processCliOptsPhase2(remainArgv, operatingMode, loggingOptions);
 
   if (operatingMode.show)
   {
@@ -547,6 +595,9 @@ async function main()
     await cleanupThenExit(exitCodes.normal);
   }
 
+  if (operatingMode.startHttpd && worker.config.unmanaged !== true)
+    await httpd.a$init(worker);
+
   const sandboxConfig = (new SandboxConstructor()).config;
   console.info(` * Starting Distributive Worker ${worker.id}`);
   console.info(` . Using evaluator at ${utils.shortLoc(sandboxConfig.location)}`);
@@ -557,16 +608,17 @@ async function main()
   else
     console.info(' . Earned funds will be deposited into an unclaimed earnings account - register worker to claim');
   console.info(` . Identity is ${identity.get()?.address}`);
+
   if (operatingMode.zeroTrust)
-     console.info(' ! Zero Trust mode enabled');
+    console.info(' ! Zero Trust mode enabled');
   else if (!worker.config.trustScheduler)
     console.info(' ! Scheduler Trust disabled');
-
-  worker.on('warning', (...payload) => console.warn (...payload));
-  worker.on('stop', () => { console.log(' - Worker is stopping') });
-  worker.on('end',  () => { console.log(' * Worker has stopped') });
-  worker.on('job',   job    => console.log(` . Job: ${job.name} ${job.id.slice(0,8)} ${job.description || ''} ${job.link || ''}`));
-  worker.on('claim', amount => console.log(` . transfered ${amount} ⊇ to ${worker.config.paymentAddress}`));
+  if (operatingMode.zeroTrust !== false)
+  {
+    assert(worker.config.trustScheduler === false);
+    assert(worker.config.unmanaged === true)
+    assert(operatingMode.startHttpd === false);
+  }
 
   require('../lib/default-ui-events').hook(worker, loggingOptions);
   console.setWorker(worker);
@@ -581,9 +633,6 @@ async function main()
   if (loggingOptions.verbose)
     console.info(` + Verbosity level: ${loggingOptions.verbose}`);
 
-  if (operatingMode.startHttpd)
-    await httpd.a$init().catch(e => e);
-
   /* Poll the evaluator forever until it accepts a connection */
   let workerInfo = false;
   console.throb(` . Waiting for dcp-evaluator on ${utils.shortLoc(sandboxConfig.location)} to start...`);
@@ -592,6 +641,8 @@ async function main()
     const ts = performance.now();
     if ((workerInfo = await require('../lib/worker-info').getEvaluatorInformation(sandboxConfig)))
       break;
+    if (i === 0 && operatingMode.unmanaged !== true)
+      utils.noteWorkerStart(worker);
     if (performance.now() - ts > 100)
       console.throb();
     await a$sleep(Math.max(i + 1, 10) / 4);
@@ -609,6 +660,12 @@ async function main()
   {
     console.info(' . WebGPU not enabled');
     systemStateInfo.gpu = false;
+    if (worker.runInfo)
+    {
+      /** @todo move GPU type reporting into  DistributiveWorker class /wg jun 2025 */
+      worker.runInfo.gpuType = null;
+      worker.runInfo.gpuCores = 0;
+    }
   }
   else
   {
@@ -634,16 +691,29 @@ async function main()
     }
 
     /** @todo move GPU type reporting into  DistributiveWorker class /wg jun 2025 */
-    worker.runInfo.gpuType = workerInfo.webgpu.info.device;
-    if (systemStateInfo.gpu.disabled)
-      worker.runInfo.gpuType += ' (disabled)';
+    if (worker.runInfo)
+    {
+      if (systemStateInfo.gpu.disabled)
+      {
+        worker.runInfo.gpuType += ' (disabled)';
+        worker.runInfo.gpuCores = 0;
+      }
+      else
+      {
+        worker.runInfo.gpuType = workerInfo.webgpu.info.device;
+        worker.runInfo.gpuCores = 1;
+      }
+    }
   }
 
-  console.info(` . Utilization Target:  cpu: ${String(Math.round(worker.config.utilization.cpu * 100)).padStart(3, ' ')}% `
-            + `gpu: ${Math.round(worker.config.utilization.gpu * 100)}%`);
-  console.info(` . Configured Cores:    cpu: ${String(worker.config.cores.cpu).padEnd(3, ' ')} `
-            + ` gpu: ${worker.config.cores.gpu}`);
-  console.info(` . Maximum Sandboxes:  ${worker.config.maxSandboxes}`);
+  if (!workerIsStopping)
+  {
+    console.info(` . Utilization Target:  cpu: ${String(Math.round(worker.config.utilization.cpu * 100)).padStart(3, ' ')}% `
+                 + `gpu: ${Math.round(worker.config.utilization.gpu * 100)}%`);
+    console.info(` . Configured Cores:    cpu: ${String(worker.config.cores.cpu).padEnd(3, ' ')} `
+                 + ` gpu: ${worker.config.cores.gpu}`);
+    console.info(` . Maximum Sandboxes:  ${worker.config.maxSandboxes}`);
+  }
   console.info(' . Supervisor version: ' + worker.supervisorVersion);
   console.info(' . Console type:       ' + operatingMode.consoleType);
   console.info(' . Logging:            ' + (loggingOptions.loggers.join(', ') || 'none'));
@@ -659,10 +729,10 @@ async function main()
     return;
   require('../lib/check-scheduler-version').check();
 
-  worker.on('end', () => {
+  worker.on('end', async () => {
     if (process.exitCode === exitCodes.lostRef)
       process.exitCode = exitCodes.normal;
-    cleanupThenExit(exitCodes.normal);
+    await cleanupThenExit(exitCodes.normal);
   });
 
   worker.on('error', function handleWorkerError(error) {
@@ -692,7 +762,7 @@ async function main()
     }
 
     /* Let the supervisor try to finish slices for up to three minutes after a single unrecoverable error. */
-    worker.stop(Boolean(handleWorkerError.count));
+    worker && worker.stop(Boolean(handleWorkerError.count));
     handleWorkerError.count = 1 + (handleWorkerError.count || 0);
     if (handleWorkerError.count === 1)
     {
@@ -701,7 +771,7 @@ async function main()
     }
   });
 
-  if (pidfileName && require('../lib/pidfile').write(pidfileName) !== true)
+  if (worker.config.pidfile && require('../lib/pidfile').write(worker.config.pidfile) !== true)
     await cleanupThenExit(exitCodes.pidConflict);
 
   if (worker.config.claimEarnings === false && !worker.config.unmanaged)
@@ -711,9 +781,11 @@ async function main()
       console.info(` ! not claming earnings in the amount of ${amount} ⊇ (can claim later)`);
   }
 
+  worker.on('stop', () => { console.log(' - Worker is stopping'); workerIsStopping = true });
+  worker.on('end',  () => { console.log(' * Worker has stopped') });
+  worker.on('job',   job    => console.log(` . Job: ${job.name} ${job.id.slice(0,8)} ${job.description || ''} ${job.link || ''}`));
   console.info(' * Ready.\n');
   await worker.start();
-  httpd.setWorker(worker);
 } /* main() */
 
 /**
@@ -730,12 +802,17 @@ async function main()
  */
 function parseResourceOption(option)
 {
+  function pctOrNumber(x)
+  {
+    return x.endsWith('%') ? parseFloat(x) * 100 : Number(x);
+  }
+
   const [cpu, gpu] = option.replace(/ /g, '').split(',');
   const res = {};
   if (cpu?.length > 0)
-    res.cpu = Number(cpu);
+    res.cpu = pctOrNumber(cpu);
   if (gpu?.length > 0)
-    res.gpu = Number(gpu);
+    res.gpu = pctOrNumber(gpu);
   return res;
 }
 
@@ -745,9 +822,11 @@ function parseResourceOption(option)
  * the worker must be restarted. This handler does its best to report the rejection and give the worker a few
  * seconds in which to attempt to return slices to the scheduler before it gives up completely.
  */
-function handleUnhandled(error)
+async function handleUnhandled(error)
 {
-  if (!error.request || !error.code?.startsWith('EHTTP_'))
+  if (error instanceof utils.UserError)
+    console.error('Error:', error?.message);
+  else if (!error.request || !error.code?.startsWith('EHTTP_'))
     console.error(' *** trapped unhandled', error);
   else
   {
@@ -769,7 +848,7 @@ function handleUnhandled(error)
                         `${Date.now()}: ${error.message}\n${error.stack}\n\n`);
   } catch(e) {} // eslint-disable-line @distributive/brace-style,no-empty
 
-  cleanupThenExit(exitCodes.unhandled);
+  await cleanupThenExit(exitCodes.unhandled);
 }
 
 /**
@@ -780,9 +859,13 @@ function handleUnhandled(error)
  * @returns {Promise} - a promise which never resolves, can be used to "block" execution of the current
  *                      "thread" while the console and loggers are being unhooked.
  */
-function cleanupThenExit(exitCode)
+async function cleanupThenExit(exitCode)
 {
   debug('dcp-worker:exit')('cleanup and exit, code', exitCode);
+  if (worker)
+    worker.stop(true); /* toggle internal worker state so it won't surprise us by starting sandboxes */
+
+  const p$note = utils.noteWorkerShutdown(worker);
   if (cleanupThenExit.busy)
   {
     if (!process.exitCode)
@@ -813,7 +896,11 @@ function cleanupThenExit(exitCode)
       console.close();
     loggers.unhook()
       .catch(error => { console.error('Error unhooking loggers:', error) })
-      .finally(() => process.exit());
+      .finally(async () => {
+        await p$note;
+        debug('dcp-worker:exit')('done cleanup, exitCode=' + process.exitCode);
+        process.exit();
+      });
   }
   catch(error)
   {
@@ -821,6 +908,9 @@ function cleanupThenExit(exitCode)
     process.exit(exitCodes.unhandled);
   }
 
+  /* Return a promise that never resolves so that async functions which want the program to exit can just
+   * await it while the log flushing above goes on.
+   */
   return new Promise(()=>1);
 }
 
@@ -841,9 +931,9 @@ async function handleSigDeath(signalName, signal)
   handleSigDeath.count = Number(handleSigDeath.count || 0) + 1;
   if (handleSigDeath.count > 2)
   {
-    process.on(signalName, () => {
+    process.on(signalName, async () => {
       console.error(signalName);
-      cleanupThenExit(exitCode);
+      await cleanupThenExit(exitCode);
     });
   }
 
@@ -854,7 +944,7 @@ async function handleSigDeath(signalName, signal)
   {
     console.warn(warning);
     await cleanupThenExit(exitCode);
-    return;
+    return; /* unreached */
   }
   else
   {
@@ -871,6 +961,12 @@ async function handleSigDeath(signalName, signal)
     await cleanupThenExit(exitCode);
 }
 
+/**
+ * Report a user error and then shutdown the process.  Await the return value to "hang" the thread until
+ * the process has ended.
+ *
+ * @returns promise from cleanupThenExit
+ */
 function reportUserError(message)
 {
   console.error(message);

package/lib/default-ui-events.js

@@ -26,7 +26,7 @@
  */
 'use strict';
 
-const { slicesFetched, debugging } = require('./utils');
+const { slicesFetched } = require('./utils');
 
 const sandboxEventHandlers = {};
 const workerEventHandlers = {};
@@ -101,7 +101,7 @@ exports.hook = function defaultUiEvents$$hook(worker, options)
 
   workerEventHandlers.fetch = function fetchHandler(ev) {
     if (ev instanceof Error)
-      debugging() && options.verbose && console.error(' ! Failed to fetch slices:', ev); // redundant
+      options.verbose >= 2 && console.error(' ! Failed to fetch slices:', ev); // redundant
     else
       options.verbose && console.log(' . Fetched', slicesFetched(ev), 'slices');
   };
@@ -112,7 +112,7 @@ exports.hook = function defaultUiEvents$$hook(worker, options)
 
   workerEventHandlers.result = function resultHandler(ev) {
     if (ev instanceof Error)
-      debugging() && options.verbose >= 2 && console.error(" ! Failed to submit results:", ev); // redundant
+      options.verbose >= 2 && console.error(" ! Failed to submit results:", ev); // redundant
     else
       options.verbose >= 2 && console.log(' . Submitted');
   };

package/lib/pidfile.js

@@ -135,7 +135,7 @@ function kill(pid, signal)
 }
 
 /**
- * Signal another process on this host. Handles the case of a dangnling pidfile by ignoring it.
+ * Signal another process on this host. Handles the case of a dangling pidfile by ignoring it.
  * @param {string}              filename      the path to the pidfile, same semantics as exports.write()
  * @param {string | number }    signal        the signal to send
  * @param {number | undefined } timeout       the number of seconds to wait for the process to die before
@@ -164,17 +164,21 @@ exports.signal = async function pidfile$$signal(filename, signal, timeout, doing
   const loopUntil = Date.now() + (timeout * 1e3);
   while (kill(pid, 0) === true)
   {
-    await a$sleep(0.10);
+    await a$sleep(0.10).ref();
     if (doingWarning)
       console.throb();
     if (Date.now() >= loopUntil)
     {
       const error = new Error(`timeout waiting for process ${pid} to exit`);
       error.code = 'ETIMEOUT';
+      if (doingWarning)
+        console.throb(null);
       return error;
     }
-    await a$sleep(0.40);
+    await a$sleep(0.40).ref();
   }
 
+  if (doingWarning)
+    console.throb(null);
   return true;
 }