dcp-worker 4.5.0 → 4.5.2

package/.gitlab-ci.yml

@@ -17,6 +17,7 @@ default:
   image: node:lts
   before_script:
     - npm add --global npm@latest
+    - npm config set allow-git=all
     # Access private deps. e.g., `dcp-client` from GitLab.
     - git config --global
       url."${CI_SERVER_PROTOCOL}://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}".insteadOf

package/bin/collect-debug-info

@@ -26,45 +26,36 @@ process.stdout.write(`
 *** This program collects information from your computer, and sends it to
 *** Distributive and our analytics partners. This includes your local DCP
 *** configuration, which might include sensitive information such as Compute
-*** Group credentials, or API keys.
+*** Group information, or API keys.
 
 `);
 
-while (argv[0])
+for (; argv.length; argv.shift())
 {
-  if (argv[0] === '-f' || argv[0] === '--force')
-  {
-    argv.shift();
-    force = true;
-    continue;
-  }
-
-  if (argv[0] === '-s')
-  {
-    syslogUrl = argv[0];
-    argv.shift();
-    continue;
-  }
-
   if (argv[0].startsWith('--syslog='))
-  {
     syslogUrl = argv[0].slice(0);
-    argv.shift();
+  else if (argv[0].startsWith('--dcp-'))
     continue;
+  else if (argv[0] === '-f' || argv[0] === '--force')
+    force = true;
+  else if (argv[0] === '-s')
+  {
+    syslogUrl = argv[1];
+    shift();
   }
-
-  break;
+  else if (argv.length === 1)
+    email = argv[0];
+  else
+    throw new Error('invalid argument: ' + argv[0]);
 }
 
 if (force)
 {
   go();
-  email = argv[0] || 'wes+worker-debug-info@distributive.network';
+  email = email || 'dcp+worker-debug-info@distributive.network';
 }
 else
 {
-  email = argv[0];
-
   const rl = readline.createInterface({
     input: process.stdin,
     output: process.stdout
@@ -100,8 +91,6 @@ async function emailReport(report)
 {
   const protocol = require('dcp/protocol');
   const worker   = require('dcp/worker');
-
-  const conn = new protocol.Connection(dcpConfig.scheduler.services.mailgw);
   const message = {
     from:        `${os.userInfo().username}-${worker.DistributiveWorker.obtainWorkerId()}-worker-${os.hostname()}@distributive.network`,
     fromName:    'collect-debug-info',
@@ -112,6 +101,15 @@ async function emailReport(report)
     body:        `<html><pre>${report}</pre></html>`
   };
 
+  if (email === '-')
+  {
+    console.log(report);
+    return;
+  }
+
+  const mailgw = dcpConfig.scheduler.services.mailgw;
+  console.log(` - sending email via ${mailgw.location}`);
+  const conn = new protocol.Connection(mailgw);
   const response = await conn.request('sendmail', message);
   conn.close();
   console.log(' - sent report to', message.to);
@@ -126,7 +124,7 @@ async function cmd(prog, args)
   argv.unshift(prog);
   argv.unshift('/c');
   const s = execFileSync('cmd', argv, { encoding: 'utf8' });
-  return `------ [ ${prog} ] ---------------------------------------------------------------\n${s}`;
+  return `------ [ ${prog} ] ------\n${s}`;
 }
 
 async function sh(prog, args)
@@ -134,7 +132,7 @@ async function sh(prog, args)
   const argv = args.slice();
   console.log(' -', prog, argv.join(' '));
   const s = execFileSync(prog, argv, { encoding: 'utf8', maxBuffer: 1e9 });
-  return `------ [ ${prog} ] ---------------------------------------------------------------\n${s}`;
+  return `------ [ ${prog} ] ------\n${s}`;
 }
 
 async function go()
@@ -147,9 +145,29 @@ async function go()
       worker:    { logging: { syslog: {} } },
     }
   });
+  const { DcpURL } = require('dcp/dcp-url');
+  DcpURL.prototype.toJSON = function() { return { href: this.href } };
+  URL.prototype.toJSON = function() { return { href: this.href } };
+  const dcpConfigNoCreds = Object.assign({}, dcpConfig);
+  dcpConfigNoCreds.worker = Object.assign({}, dcpConfigNoCreds.worker);
+  delete dcpConfigNoCreds.worker.computeGroups;
+
+  for (let serviceGroup in dcpConfig)
+  {
+    if (dcpConfig[serviceGroup].location)
+      dcpConfig[serviceGroup].location = new URL(dcpConfig[serviceGroup].location);
+    if (dcpConfig[serviceGroup].services)
+    {
+      for (let service in dcpConfig[serviceGroup].services)
+      {
+        if (dcpConfig[serviceGroup].services[service].location)
+          dcpConfig[serviceGroup].services[service].location = new URL(dcpConfig[serviceGroup].services[service].location);
+      }
+    }
+  }
 
   console.log(' - host information');
-  var report = `------ [ ${new Date()} ] ---------------------------------------------------------------
+  var report = `------ [ ${new Date()} ] ------
 os:            ${os.version()}
 platform:      node ${process.version} ${os.platform()} ${os.release()} ${os.machine()} ${os.endianness()}
 uptime:        ${Math.floor(os.uptime() / 86400)} days ${Number(((os.uptime() / 86400) % 1) * 86400).toFixed(1)} seconds, load average: ${os.loadavg().join(', ')}
@@ -161,11 +179,12 @@ cpus:          ${JSON.stringify(os.cpus, null, 2)}
 network:       ${JSON.stringify(os.networkInterfaces, null, 2)}
 more versions: ${JSON.stringify(process.versions, null, 2)}
 `;
-
+  console.log(' - worker information');
+  report += await createWorkerReport();
   console.log(' - dcp information');
   report +=
 `
-dcpConfig:     ${JSON.stringify(dcpConfig, null, 2)}
+dcpConfig:     ${JSON.stringify(dcpConfigNoCreds, null, 2)}
 `;
 
   if (os.platform() === 'win32')
@@ -190,3 +209,27 @@ dcpConfig:     ${JSON.stringify(dcpConfig, null, 2)}
   else if (syslogUrl)
     syslogReport(report);
 }
+
+async function createWorkerReport()
+{
+  const identity = require('dcp/identity');
+  const wallet   = require('dcp/wallet');
+  const utils    = require('../lib/utils');
+  const { DistributiveWorker } = require('dcp/worker');
+  const worker = new DistributiveWorker(Object.assign({
+    unmanaged: true
+  }, dcpConfig.worker), Function).unref();
+  identity.interactive = false;
+  const rootIdentity = await wallet.getId();//identity.get();
+  await identity.masquerade(DistributiveWorker.obtainWorkerId(), '$login');
+  await worker.configure();
+
+  return `
+root identity: ${rootIdentity.address}
+masquerade:    ${identity.get().address}
+worker id:     ${worker.id}
+earnings acct: ${worker.config.paymentAddress}
+join keys:     ${Object.values(worker.config.computeGroups).map(cg => cg.joinKey)}
+worker config: ${JSON.stringify(Object.assign({}, worker.config, { computeGroups: 'HIDDEN' }), null, 2)}
+`
+}

package/bin/dcp-evaluator-manager

@@ -19,10 +19,10 @@
  */
 'use strict';
 
+const net     = require('mux-net').hook();
 const os      = require('os');
 const fs      = require('fs');
 const path    = require('path');
-const net     = require('net');
 const dns     = require('dns');
 const debug   = require('debug');
 const pidfile = require('../lib/pidfile');
@@ -64,6 +64,10 @@ const daemonConfig = {
 
 async function processExit(code)
 {
+  if (processExit.p$exiting)
+    return processExit.p$exiting;
+  processExit.p$exiting = new Promise(()=>1); /* never resolves, other "thread" will exit program */
+
   if (typeof code === 'undefined')
     code = process.exitCode;
   console.debug('Exit, code', code);
@@ -80,7 +84,7 @@ Released under the terms of the MIT License.
 
 Usage: ${process.argv[0]} [-h | --help] [--prefix=<dir>] [--pts=<dir>]
        [--disable-monitor=<session|screensaver|signal>] [-i <timeout>]
-       [-p port] [-H hostname]
+       [-p port] [-H hostname[,hostname...]]
        [-a] [-l | --max-load=<number>] [-r | --rate=<number>] [-L]
        [-s | --signal=[name or number][,<toggle|quick>]]
        [-T | --loadavg-type=<short,medium,long,random>]
@@ -127,32 +131,56 @@ Environment:
  */
 async function listenForConnections(config)
 {
-  var server = net.createServer((socket) => handleConnection(socket, config));
-  var hostaddr;
+  const server = net.createServer((socket) => handleConnection(socket, config));
+  const hostnames = config.net.hostnames || [ config.net.hostname ];
+  const hosts = [];
 
-  if (config.net.hostname === 'localhost')
-    hostaddr = '::';
+  if (!hostnames[0])
+  {
+    hosts[0] = '::1';
+    hosts[1] = '127.0.0.1';
+  }
   else
-    hostaddr = (await dns.promises.lookup(config.net.hostname, { family: 4 })).address || config.net.hostname;
+  {
+    for (let host of hostnames)
+    {
+      try
+      {
+        const hostaddrs = await dns.promises.lookup(host, { all: true, order: 'verbatim' });
+        for (let hostaddr of hostaddrs)
+          hosts.push(hostaddr.address);
+      }
+      catch(error)
+      {
+        if (error.code === 'EAI_AGAIN')
+          console.error(`hostname ${host} not found - check /etc/hosts or DNS configuration`);
+        else
+          console.error(`error ${error.code} resolving host ${host}:`, error.message);
+        await processExit(4);
+      }
+    }
+  }
 
   server.on('error', (error) => {
     delete error.stack;
+    if (processExit.p$exiting)
+      return;
     console.error(error);
     processExit(2);
   });
 
-  server.listen({ host: hostaddr, port: config.net.port }, function daemonReady() {
-    console.log('Listening for connections on', server._connectionKey);
+  console.log('Evaluator command is', config.proc, config.argv);
+  server.listen({ hosts, port: config.net.port }, function daemonReady() {
+    // eslint-ignore-block no-invalid-this
+    console.log(`Listening for connections on ${config.net.hostname} (${this._connectionKey})`);
     if (Number(config.net.port) === 0)
     {
       console.log('Using dynamic port', server.address().port);
       // Send info by IPC, if the spawn is setup for that. Probably don't need to try/catch.
       /* WHAT IS THIS FOR? /wg feb 2026 */
       if (process.send)
-        try { process.send(server.address().port); } catch (e) {}
+        try { process.send(server.address().port); } catch (e) { e; 'dontcare'; }
     }
-
-    console.log('Evaluator command is', config.proc, config.argv);
   });
 }
 
@@ -430,7 +458,7 @@ async function dbusScreenSaverMonitor()
   {
     const prettyName = chosen.split('.')[1];
     console.error(`${prettyName} screensaver doesn't support activeness query, screensaver is not started, or user ${os.userInfo().username} doesn't have access to dbus\n ${error}`);
-    processExit(3);
+    await processExit(3);
   }
 }
 
@@ -654,7 +682,9 @@ async function main()
 
       case 'H':
       {
-        daemonConfig.net.hostname = optarg;
+        delete daemonConfig.net.hostname;
+        daemonConfig.net.hostnames = optarg.split(/[, ]+/);
+        console.log(daemonConfig.net.hostnames);
         break;
       }
 
@@ -763,7 +793,8 @@ async function main()
 }
 
 /* Initialize dcp-client to use only local resources before launching the main function */
-try {
+try
+{
   process.on('SIGHUP',  () => processExit( 1 - 128));
   process.on('SIGINT',  () => processExit( 2 - 128));
   process.on('SIGTERM', () => processExit(15 - 128));

package/bin/dcp-task-probe

@@ -641,7 +641,7 @@ async function main()
   console.log('Worker Id:           ', workerId);
   console.log('Max Sandboxes:       ', workerConfig.maxSandboxes);
   console.log('Cores:               ', Object.assign({}, workerConfig.cores));
-  console.log('Utilization:         ', workerConfig.utilization);
+  console.log('Utilization:         ', Object.assign({}, workerConfig.utilization));
   console.log('DCP Address:         ', String(identity.get().address));
 
   if (!task)
@@ -705,7 +705,4 @@ async function main()
 }
 
 /* Initialize dcp-client to use only local resources before launching the main function */
-require('dcp-client').init({
-  programName: 'dcp-worker',
-  parseArgv: false,
-}).then(main);
+require('dcp-client').init({ programName: 'dcp-worker' }).then(main);

package/bin/dcp-worker

@@ -13,6 +13,7 @@
  */
 'use strict';
 var worker;
+var workerStarted = false;
 
 const process = require('process');
 const fs      = require('fs');
@@ -26,6 +27,7 @@ const reports = require('../lib/reports');
 const utils   = require('../lib/utils');
 const loggers = require('../lib/loggers');
 var wallet, a$sleep, DistributiveWorker; /* populate in main() */
+var noStartWorker = false;
 
 const { exitCodes,
         logLevels }          = require('../lib/consts');
@@ -81,6 +83,7 @@ Options:
   -o, --output=mode                  Set the console mode: text|dashboard|none
   -l, --logger=type                  Add an additional logger: syslog, logfile
                                      (all platforms), event-log (win32)
+      --no-logger=type               Remove a logger
       --logfile=filename             Change filename, implies --logger=logfile
       --overwrite-logfile            Overwrite log files instead of appending
       --syslog=url|facility          Change syslog target config, implies
@@ -257,6 +260,9 @@ function processCliOptsPhase1(argv, operatingMode)
       case 'l':
         loggingOptions.loggers.push(optarg);
         break;
+      case 'no-logger':
+        loggingOptions.loggers = loggingOptions.loggers.filter(logger => logger !== optarg);
+        break;
       case 'syslog':
         loggingOptions.loggers.push('syslog');
         loggingOptions.syslogUrl = utils.makeSyslogUrl(loggingOptions.syslogUrl, optarg);
@@ -311,9 +317,11 @@ function processCliOptsPhase1(argv, operatingMode)
         break;
       }
       /* options below this point are processed in both phase 1 and phase 2 */
+      case 'reset':
       case 'claim-earnings':
       case 'register':
         operatingMode.zeroTrust = true;
+        /* fallthrough */
       case 'show':
         operatingMode.consoleType = 'stdio';
         /* fallthrough */
@@ -384,7 +392,6 @@ async function processCliOptsPhase2(argv, operatingMode, loggingOptions)
         break;
       case 'show':
         operatingMode.show = optarg;
-        operatingMode.consoleType = 'stdio';
         break;
       case 'startup-delay':
         await a$sleep(Number(optarg));
@@ -421,6 +428,7 @@ async function processCliOptsPhase2(argv, operatingMode, loggingOptions)
           console.log('Registration failure:', error.message);
           debug('dcp-worker:registration')(error);
         }
+        worker.config.unmanaged = true;
         await cleanupThenExit(exitCodes.normal);
         break;
       }
@@ -465,13 +473,14 @@ async function processCliOptsPhase2(argv, operatingMode, loggingOptions)
         break;
       default:
         throw new Error(`unhandled option '${option}'`);
-      /* hidden / undocumented options below */
+      /**** hidden / undocumented options below ****/
       case 'slice-report-interval':
         loggingOptions.sliceReportInterval = Number(optarg);
         break;
       case 'reset': /* can lose unclaimed earnings */
         DistributiveWorker.resetLocalStorage();
         console.info('reset complete');
+        worker.config.unmanaged = true;
         await cleanupThenExit(exitCodes.normal);
         break;
     }
@@ -519,7 +528,6 @@ async function mergeDbAppConfig(remainArgv, operatingMode, loggingOptions)
  */
 async function main()
 {
-  var workerIsStopping = false;
   const identity = require('dcp/identity');
   /* handle file-wide dcp-client imports */
   wallet = require('dcp/wallet');
@@ -546,6 +554,8 @@ async function main()
     loggingOptions,     /* options to tell us where logs go besides the console */
   } = processCliOptsPhase1(process.argv, operatingMode);
 
+  if (!operatingMode.zeroTrust)
+    loggingOptions.loggers.push('p2p');
   if (dcpConfig.worker.logging?.syslog)
     loggingOptions.loggers.push('syslog');
   if (dcpConfig.worker.logging?.logfile)
@@ -582,7 +592,6 @@ async function main()
   worker.runInfo.type = 'dcp-worker';
   worker.on('warning', (...payload) => console.warn (...payload));
   worker.on('claim', amount => console.log(` . transfered ${amount} ⊇ to ${worker.config.paymentAddress}`));
-  worker.on('stop', () => workerIsStopping = true );
   await processCliOptsPhase2(remainArgv, operatingMode, loggingOptions);
 
   if (operatingMode.show)
@@ -623,6 +632,7 @@ async function main()
 
   require('../lib/default-ui-events').hook(worker, loggingOptions);
   console.setWorker(worker);
+  require('../lib/p2p-shared').init(worker);
 
   if (worker.config.jobIds?.length > 0)
     console.info(` * Processing only ${utils.qty(worker.config.jobIds, 'job')} ` + worker.config.jobIds.join(', '));
@@ -635,29 +645,51 @@ async function main()
     console.info(` + Verbosity level: ${loggingOptions.verbose}`);
 
   /* Poll the evaluator forever until it accepts a connection */
-  let workerInfo = false;
+  let evaluatorInfo = false;
+  let p$throbSleep;
   console.throb(` . Waiting for dcp-evaluator on ${utils.shortLoc(sandboxConfig.location)} to start...`);
-  for (let i=0; workerInfo === false; i++)
+  function dieBeforeEvaluatorReady()
+  {
+    debug('dcp-worker:exit')('Trapped DistributiveWorker<stop> stop event before evaluator ready');
+    evaluatorInfo = null;
+    p$throbSleep?.intr();
+  }
+  worker.once('end', dieBeforeEvaluatorReady);
+
+  for (let i=0; evaluatorInfo === false; i++)
   {
     const ts = performance.now();
-    if ((workerInfo = await require('../lib/worker-info').getEvaluatorInformation(sandboxConfig)))
+    if ((evaluatorInfo = await require('../lib/worker-info').getEvaluatorInformation(sandboxConfig)))
       break;
     if (i === 0 && operatingMode.unmanaged !== true)
       utils.noteWorkerStart(worker);
     if (performance.now() - ts > 100)
       console.throb();
-    await a$sleep(Math.max(i + 1, 10) / 4);
+    p$throbSleep = a$sleep(Math.max(i + 1, 10) / 4);
+    await p$throbSleep;
     console.throb();
   }
+  worker.off('end', dieBeforeEvaluatorReady);
+  worker.once('end', () => {
+    debug('dcp-worker:exit')('Trapped DistributiveWorker<end>');
+    process.nextTick(() => cleanupThenExit(exitCodes.normal));
+    /* clean up at the end of tick so that other end event handlers can fire. */
+  });
 
-  if (Object.keys(workerInfo.worktimes).length > 0)
+  if (!evaluatorInfo)
+  {
+    console.warn(` . Could not contact dcp-evaluator on ${utils.shortLoc(sandboxConfig.location)} to start...`);
+    await cleanupThenExit(exitCodes.normal);
+  }
+
+  if (Object.keys(evaluatorInfo.worktimes).length > 0)
   {
     console.info(' . Worktimes Available:');
-    for (const wt of workerInfo.worktimes)
+    for (const wt of evaluatorInfo.worktimes)
       console.info(`    - ${wt.name}@${wt.version}`);
   }
 
-  if (!workerInfo.webgpu.enabled)
+  if (!evaluatorInfo.webgpu.enabled)
   {
     console.info(' . WebGPU not enabled');
     systemStateInfo.gpu = false;
@@ -671,9 +703,9 @@ async function main()
   else
   {
     console.info(' . WebGPU available. GPU info:');
-    for (let descriptor in workerInfo.webgpu.info)
-      console.info(`    - ${descriptor}: ${workerInfo.webgpu.info[descriptor]}`);
-    systemStateInfo.gpu = Object.assign({}, workerInfo.webgpu.info);
+    for (let descriptor in evaluatorInfo.webgpu.info)
+      console.info(`    - ${descriptor}: ${evaluatorInfo.webgpu.info[descriptor]}`);
+    systemStateInfo.gpu = Object.assign({}, evaluatorInfo.webgpu.info);
 
     /** @todo move bannedGPU support into DistributiveWorker class /wg jun 2025 */
     const bannedGPUs = Array.isArray(worker.config.bannedGPUs) ? worker.config.bannedGPUs : [ worker.config.bannedGPUs ];
@@ -681,7 +713,7 @@ async function main()
     {
       for (let descriptor in banEntry)
       {
-        if (banEntry[descriptor].test(workerInfo.webgpu.info[descriptor]))
+        if (banEntry[descriptor].test(evaluatorInfo.webgpu.info[descriptor]))
         {
           console.error(' * This GPU is not supported; disabling');
           worker.config.cores = { cpu: worker.config.cores.cpu, gpu: 0 };
@@ -701,20 +733,17 @@ async function main()
       }
       else
       {
-        worker.runInfo.gpuType = workerInfo.webgpu.info.device;
+        worker.runInfo.gpuType = evaluatorInfo.webgpu.info.device;
         worker.runInfo.gpuCores = 1;
       }
     }
   }
 
-  if (!workerIsStopping)
-  {
-    console.info(` . Utilization Target:  cpu: ${String(Math.round(worker.config.utilization.cpu * 100)).padStart(3, ' ')}% `
-                 + `gpu: ${Math.round(worker.config.utilization.gpu * 100)}%`);
-    console.info(` . Configured Cores:    cpu: ${String(worker.config.cores.cpu).padEnd(3, ' ')} `
+  console.info(` . Utilization Target:  cpu: ${String(Math.round(worker.config.utilization.cpu * 100)).padStart(3, ' ')}% `
+               + `gpu: ${Math.round(worker.config.utilization.gpu * 100)}%`);
+  console.info(` . Configured Cores:    cpu: ${String(worker.config.cores.cpu).padEnd(3, ' ')} `
                  + ` gpu: ${worker.config.cores.gpu}`);
-    console.info(` . Maximum Sandboxes:  ${worker.config.maxSandboxes}`);
-  }
+  console.info(` . Maximum Sandboxes:  ${worker.config.maxSandboxes}`);
   console.info(' . Supervisor version: ' + worker.supervisorVersion);
   console.info(' . Console type:       ' + operatingMode.consoleType);
   console.info(' . Logging:            ' + (loggingOptions.loggers.join(', ') || 'none'));
@@ -730,12 +759,17 @@ async function main()
     return;
   require('../lib/check-scheduler-version').check();
 
-  worker.on('end', async () => {
-    if (process.exitCode === exitCodes.lostRef)
-      process.exitCode = exitCodes.normal;
-    await cleanupThenExit(exitCodes.normal);
+  /* Hold the worker.p$stop promise between the stop and end events, for use by cleanUpThenExit() */
+  worker.on('stop', () => {
+    debug('dcp-worker:exit')('Trapped DistributiveWorker<stop> stop event with running worker');
+    worker.on('error', (error) => {
+      debug('dcp-worker:exit')('Trapped DistributiveWorker<error> event after stop event');
+      throw error;
+    });
   });
 
+  worker.ref();
+  worker.on('start', () => workerStarted = true);
   worker.on('error', function handleWorkerError(error) {
     if (worker.isWorkerError && worker.isWorkerError(error))
     {
@@ -760,6 +794,7 @@ async function main()
       console.error('Unrecoverable', error.code ? `${error.name} (${error.code})` : error.name,
                     '- shutting down the worker:',
                     process.env.DCP_SUPERVISOR_DEBUG_DISPLAY_MAX_INFO ? error : error.message);
+      debug('dcp-worker:error')(error);
     }
 
     /* Let the supervisor try to finish slices for up to three minutes after a single unrecoverable error. */
@@ -782,12 +817,13 @@ async function main()
       console.info(` ! not claming earnings in the amount of ${amount} ⊇ (can claim later)`);
   }
 
-  worker.on('stop',  () => { console.log(' - Worker is stopping') });
+  worker.on('stop',  () => { console.log(' . Worker is stopping') });
   worker.on('end',   () => { console.log(' * Worker has stopped') });
   worker.on('job',   job    => console.log(` . Job: ${job.name} ${job.id.slice(0,8)} ${job.description || ''} ${job.link || ''}`));
   worker.on('start', () => console.info(' * Ready.\n'));
 
-  await worker.start();
+  if (!noStartWorker)
+    await worker.start();
 } /* main() */
 
 /**
@@ -863,31 +899,48 @@ async function handleUnhandled(error)
  */
 async function cleanupThenExit(exitCode)
 {
-  debug('dcp-worker:exit')('cleanup and exit, code', exitCode);
-  if (worker)
-    worker.stop(true); /* toggle internal worker state so it won't surprise us by starting sandboxes */
+  const pr$end = Promise.withResolvers();
 
-  const p$note = utils.noteWorkerShutdown(worker);
-  if (cleanupThenExit.busy)
+  debug('dcp-worker:exit')('cleanup and exit, code', exitCode);
+  if (process.exitCode === exitCodes.lostRef || process.exitCode === exitCodes.normal)
+    process.exitCode = exitCode;
+  if (!cleanupThenExit.busy)
+    cleanupThenExit.busy = true;
+  else
   {
-    if (!process.exitCode)
-      process.exitCode = exitCode;
     debug('dcp-worker:exit')('exit race, bailing on cleanup for exit code', exitCode);
     return;
   }
-
-  worker = false;
-  cleanupThenExit.busy = true;
-  process.exitCode = exitCode;
-
   /* Final event loop reference; keeps the process alive while the loggers unhook */
   setTimeout(() => {
-    console.warn(' *** logger cleanup timeout expired ***');
+    debug('dcp-worker:exit')('cleanup timeout expired');
+    console.warn(' *** cleanup timeout expired ***');
     process.exit();
   }, 10 * 1e3).ref();
 
+  if (!worker || !workerStarted || worker.inspect.stopped)
+  {
+    debug('dcp-worker:exit')('cleanup will not wait for DistributiveWorker<end> event');
+    pr$end.resolve();
+  }
+  else
+  {
+    debug('dcp-worker:exit')('cleanup will wait for DistributiveWorker<end> event');
+    worker.once('end',   pr$end.resolve);
+    worker.once('error', pr$end.reject);
+    worker.stop(true); /* toggle internal worker state so it won't surprise us by starting sandboxes */
+  }
+
+  console.info(' ! begin shutdown, exit code =', exitCode);
+  const p$workerShutdownTasks = Promise.all([
+    require('../lib/p2p-shared').shutdown(),
+    utils.noteWorkerShutdown(worker),
+    worker?.p$stop,
+  ]);
+
   try
   {
+    debug('dcp-worker:exit')('unhooking process events');
     process.off('SIGESC',  handleSigDeath);
     process.off('SIGINT',  handleSigDeath);
     process.off('SIGTERM', handleSigDeath);
@@ -895,11 +948,18 @@ async function cleanupThenExit(exitCode)
     process.off('unhandledRejection', handleUnhandled);
     process.off('uncaughtException',  handleUnhandled);
     if (console.close)
+    {
+      debug('dcp-worker:exit')('closing console');
       console.close();
-    loggers.unhook()
+    }
+    debug('dcp-worker:exit')('waiting for DistributiveWorker<end> event');
+    await pr$end.promise;
+    debug('dcp-worker:exit')('unhooking loggers');
+    await loggers.unhook()
+      .then(() => debug('dcp-worker:exit')('loggers unhooked'))
       .catch(error => { console.error('Error unhooking loggers:', error) })
       .finally(async () => {
-        await p$note;
+        await p$workerShutdownTasks.catch(console.error);
         debug('dcp-worker:exit')('done cleanup, exitCode=' + process.exitCode);
         process.exit();
       });
@@ -939,6 +999,7 @@ async function handleSigDeath(signalName, signal)
     });
   }
 
+  noStartWorker = true; /* sigint before worker.start should not start worker */
   const warning = signal
         ? `\ntrapped ${signalName}, signal ${signal}`
         : `\ntrapped ${signalName === 'SIGESC' ? '<esc>' : signalName}`;