dcp-worker 4.1.0 → 4.2.0

package/.gitlab-ci.yml

@@ -149,11 +149,9 @@ publish_docs:
     AWS_REGION: 'us-east-1'
     TECHDOCS_S3_BUCKET_NAME: 'backstage.distributive.network'
     ENTITY_NAMESPACE: 'default'
-  stage: deploy
-  needs: []
   image: registry.gitlab.com/distributed-compute-protocol/backstage-ci-image:latest
   script:
-    - bin/publish-docs.sh
+    - ./publish-docs.sh
   cache:
     - key:
         prefix: node-modules

package/bin/dcp-evaluator-manager

@@ -41,8 +41,8 @@ var lastConnectionMs = 0;
 const defaultPrefix = '/usr';
 const daemonConfig = {
   net:  new URL('tcpip://localhost:9000/'),
-  proc: require.resolve('./dcp-evaluator-start'),
-  argv: [ '-s', '--prefix', `${path.resolve(defaultPrefix)}/` ],
+  proc: process.argv[0],
+  argv: [ require.resolve('./dcp-evaluator-start'), '-s', '--prefix', `${path.resolve(defaultPrefix)}/` ],
   pidfile: pidfile.getDefaultPidFilename,
   limits: {
     loadavgType: 'short',
@@ -65,9 +65,10 @@ function usage()
 DCP Evaluator Monitor - Copyright (c) 2022-2024 Distributive Corp.
 Released under the terms of the MIT License.
 
-Usage: ${process.argv[0]} [-h | --help] [--prefix=<dir>] [--pts=<dir>] [-a]
+Usage: ${process.argv[0]} [-h | --help] [--prefix=<dir>] [--pts=<dir>]
        [--disable-monitor=<session|screensaver|signal>] [-i <timeout>]
-       [-p port] [-l | --max-load=<number>] [-r | --rate=<number>] [-L]
+       [-p port] [-H hostname]
+       [-a] [-l | --max-load=<number>] [-r | --rate=<number>] [-L]
        [-s | --signal=[name or number][,<toggle|quick>]]
        [-T | --loadavg-type=<short,medium,long,random>]
        [-f | --pidfile=</path/to/file.pid>]
@@ -75,6 +76,7 @@ Usage: ${process.argv[0]} [-h | --help] [--prefix=<dir>] [--pts=<dir>] [-a]
 Where:
  -i sets the idle timeout in seconds for tty sessions (${daemonConfig.session.idleTimeout}s)
  -p specifies the port to listen on for dcpsaw: worker connections (${daemonConfig.net.port})
+ -H specifies the address to listen on for dcpsaw: worker connections (${daemonConfig.net.port})
  --prefix specifies the directory where the DCP evaluator package was installed
  --pidfile specifies an alternate location for the pid file (currently ${daemonConfig.pidfile})
  --pts specifies the location of the system's pseudo terminal (${ptsDir})
@@ -110,7 +112,7 @@ async function listenForConnections(config)
   if (config.net.hostname === 'localhost')
     hostaddr = '::';
   else
-    hostaddr = await dns.promises.lookup(config.net.hostname, { family: 4 });
+    hostaddr = (await dns.promises.lookup(config.net.hostname, { family: 4 })).address || config.net.hostname;
 
   server.on('error', (error) => {
     delete error.stack;
@@ -367,7 +369,7 @@ async function dbusScreenSaverMonitor()
     console.warn('Warning: could not open dbus session to any screensaver, tried', screensaverList);
     return;
   }
-  
+
   try
   {
     // Note this is almost guranteed to fail if (GNOME is installed + you're accessing it not on GNOME,
@@ -524,7 +526,8 @@ function odRequire(moduleId)
 /* Main program entry point */
 async function main()
 {
-  const parser = new getopt.BasicParser('h(help)P:(prefix)d:(disable-monitor)l:(max-load)r:(rate)Li:ap:s:(signal)T:(loadavg-type)f:(pidfile)', process.argv);
+  const parser = new getopt.BasicParser('h(help)P:(prefix)H:p:d:(disable-monitor)l:(max-load)r:(rate)'
+                                        + 'Li:as:(signal)T:(loadavg-type)f:(pidfile)', process.argv);
   var option;
 
   if (dcpConfig.evaluator.listen)
@@ -601,6 +604,12 @@ async function main()
         break;
       }
 
+      case 'H':
+      {
+        daemonConfig.net.hostname = option.optarg;
+        break;
+      }
+
       case 'l':
       {
         daemonConfig.limits.maxLoad = Number(option.optarg);
@@ -660,11 +669,20 @@ async function main()
 }
 
 /* Initialize dcp-client to use only local resources before launching the main function */
-require('dcp-client').init({
-  programName: 'dcp-worker',
-  parseArgv: false,
-  dcpConfig: {
-    scheduler: { configLocation: false },
-    bundle:    { location: false },
-  }
-}).then(main);
+try {
+  require('dcp-client').init({
+    programName: 'dcp-worker',
+    parseArgv: false,
+    dcpConfig: {
+      scheduler: { configLocation: false },
+      bundle:    { location: false },
+    }
+  }).then(main);
+}
+catch(error)
+{
+  if (error.name === 'SyntaxError')
+    console.error(`Unable to initialize dcp-client - Node.js version too old? (SyntaxError: ${error.message})`);
+  else
+    throw error;
+}

package/bin/dcp-evaluator-start

@@ -1,15 +1,15 @@
 #!/usr/bin/env node
-/** 
+/**
  * @file       dcp-evaluator-start
- * @author     Eddie Roosenmaallen, eddie@kingsds.network
- *             Levi Stringer, levi@kingsds.network
- * @date       September 2020
+ * @author     Wes Garland, wes@distributive.network
+ *             Brandon Christie, brandon@distributive.network
+ * @date       Feb 2021, June 2024
  *
  * This script starts a DCP Evaluator. It acts as a facade which takes a standardized command-line
  * interface and spawns the selected evaluator type with the correct corresponding options. Evaluator
  * support files (eg. those used to implement the host environment and sandboxes) are loaded from the
- * dcp-client module or it's dependencies, and which of those files to use are specified by the 
- * sandbox-definitions.json which is generated during a dcp-client build.
+ * dcp-client module or its dependencies, and which of those files to use are specified by the
+ * sandbox-definitions.json, which is generated during a dcp-client build.
  */
 'use strict';
 
@@ -19,103 +19,144 @@ function panic(...argv)
   process.exit(1);
 }
 
-function main() {
+function main()
+{
   const { clearLine, cursorTo } = require('readline');
+  const { BasicParser } = require('posix-getopt');
+  const { expandPath } = require('dcp/utils');
   const path = require('path');
+  const pidfile = require('../lib/pidfile')
   const dcpConfig = require('dcp/dcp-config');
   const dcpClientDir = path.dirname(require.resolve('dcp-client'));
-  const defaultPidFileName = require('../lib/pidfile').getDefaultPidFileName(dcpConfig.worker.pidfile);
-  var evaluator;
+  const parser = new BasicParser(
+    'h(help)P:(prefix)p:(port)s(stdio)e:(evaluator)r:(rEvaluator)\u1000:(sandbox-libexec-dir)' +
+    '\u1001:(sandbox-definitions)t:(sandbox-type)D(dump)T(dump-types)F(dump-files)J(json)v(verbose)' +
+    'f:(pidfile)d:(logLevel)',
+    process.argv
+  );
 
-  /* All evaluators must understand the following command-line options:
-   * -p - port to listen on; unspecified => stdio
-   */
-  const options = require('dcp/cli')
-        .base('Evaluator frontend')
-        .option('prefix', {
-          desc: 'Path under which native components are installed',
-          default: path.resolve(path.dirname(process.argv[1]), '..'),
-        })
-        .option('port', {
-          alias: ['p'],
-          desc: 'Port to listen on',
-          default: Number(dcpConfig.evaluator.listen.port),
-        })
-        .option('stdio', {
-          alias: ['s'],
-          desc: 'Use stdio pipeline (eg for inetd) instead of tcp port',
-          type: 'boolean',
-          default: false,
-        })
-        .option('evaluator', {
-          alias: ['e'],
-          desc: 'Path to evaluator (default: prefix/bin/dcp-evaluator)',
-        })
-        .option('rEvaluator', {
-          alias: ['r'],
-          desc: 'Path to evaluator as a module identitifer',
-        })
-        .option('sandbox-libexec-dir', {
-          desc: 'Location of sandbox setup files',
-          default: path.resolve(dcpClientDir, 'libexec', 'sandbox'),
-        })
-        .option('sandbox-definitions', {
-          desc: 'Index of sandbox setup files for different evaluator/sandbox types',
-          default: path.resolve(dcpClientDir, 'generated', 'sandbox-definitions.json'),
-        })
-        .option('sandbox-type', {
-          alias: ['t'],
-          desc: 'Type of evaluator/sandbox',
-          default: 'native'
-        })
-        .option('dump', {
-          alias: ['D'],
-          desc: 'Dump configuration to stdout and exit',
-        })
-        .option('dump-types', {
-          alias: ['T'],
-          desc: 'Dump list of known sandbox types to stdout and exit',
-        })
-        .option('dump-files', {
-          alias: ['F'],
-          desc: 'Dump list of startup files which would be passed to the evaluator and exit',
-        })
-        .option('json', {
-          alias: ['J'],
-          desc: 'Dump in JSON format',
-        })
-        .option('verbose', {
-          alias: ['v'],
-          desc: 'Generate verbose output',
-        })
-        .option('pidFile', {
-          alias: ['f'],
-          describe: `create a .pid file; value overrides default location (${defaultPidFileName})`,
-        })
-        .options('debugLogLevel', {
-          alias: ['d'],
-          desc: 'Log level for evaluator output (emerg, alert, crit, error, warn, notice, info, debug)',
-        })
-        .epilogue("To pass arguments to the evaluator, make sure to separate your arguments with a '--'. For example: 'dcp-evaluator-start -p 9000 -- --options=--max-old-space-size=2048'")
-        .wrap(process.stdout.columns || 80)
-        .strict()
-        .argv;
+  const options = {
+    prefix:             path.resolve(path.dirname(process.argv[1]), '..'),
+    port:               Number(dcpConfig.evaluator.listen.port),
+    sandboxLibexecDir:  path.resolve(dcpClientDir, 'libexec', 'sandbox'),
+    sandboxDefinitions: path.resolve(dcpClientDir, 'generated', 'sandbox-definitions.json'),
+    sandboxType:        'native',
+    evaluator:          path.join('bin', 'dcp-evaluator'), /* resolved against prefix later */
+    verbose:           0,
+  };
+
+  function help() {
+    const binName = path.basename(__filename);
+    console.log(`
+${binName} - Evaluator frontend
+Copyright (c) 2021-2025, Distributive Corp. Released under the terms of the MIT License.
 
-  if (!options.evaluator)
-    options.evaluator = path.join(options.prefix, 'bin', 'dcp-evaluator');
+Usage: ${binName} [options...]
+Options:
+  -h, --help                  Display this help and exit
+      --prefix=               Path under which native components are installed;
+                              default is ${options.prefix}
+  -p, --port=                 Port to listen on; default is ${options.port}
+  -s, --stdio                 Use stdio pipeline (eg for inetd) instead of tcp
+  -t, --sandbox-type=         Type of evaluator/sandbox; default is ${options.sandboxType}
+  -e, --evaluator=            Path to evaluator; default is
+                              ${options.prefix}/bin/dcp-evaluator
+  -m, --evaluator-module=     Path to evaluator as a module identitifer
+      --sandbox-definitions=  JSON file defining the sandbox setup files for the
+                              different evaluator/sandbox types; default is
+                              ${options.sandboxDefinitions}
+      --sandbox-libexec-dir=  Location of sandbox setup files; default is
+                              ${options.sandboxLibexecDir}
+  -D, --dump                  Dump configuration to stdout and exit
+  -T, --dump-types            Dump list of known sandbox types to stdout and exit
+  -F, --dump-files            Dump list of startup files which would be passed
+                              to the evaluator and exit
+  -J, --json                  Dump in JSON format
+  -v, --verbose               Generate verbose output
+  -f, --pidfile=              create a .pid file; default is
+                              ${pidfile.getDefaultPidFileName(dcpConfig.worker.pidfile)}
+  -d, --log-level=            Select minimum log level for evaluator output
+                              (debug, info, notice, warn, error, crit, alert, emerg)
+`);
+  }
 
-  if (options.stdio)
-    delete options.port;
+  for (let opthnd; (opthnd = parser.getopt()) !== undefined; /* while loop */)
+  {
+    let option;
 
-  const sandboxSetupDefs = require(options.sandboxDefinitions)[options.sandboxType]
+    /* To fake long-only-opts, we use short opts >= \u1000), and present only the long opt below */
+    if (opthnd.option < '\u1000')
+      option = opthnd.option;
+    else
+      option = (Object.entries(parser.gop_aliases).filter(longShort => longShort[1] === opthnd.option)[0] || ['?'])[0];
+
+    switch (option)
+    {
+      case '?':
+        process.exit(1);
+        break;
+      case 'h':
+        help();
+        process.exit(0);
+        break;
+      case 'P':
+        options.prefix = opthnd.optarg;
+        break;
+      case 'p':
+        options.port = opthnd.optarg;
+        break;
+      case 's':
+        options.stdio = true;
+        options.port = false;
+        break;
+      case 'e':
+        options.evaluator = expandPath(opthnd.optarg);
+        break;
+      case 'm':
+        options.evaluator = require.resolve(opthnd.optarg);
+        break;
+      case 't':
+        options.sandboxType = opthnd.optarg;
+        break;
+      case 'l':
+        options.sandboxLibexecDir = expandPath(opthnd.optarg);
+        break;
+      case 'J':
+        options.json = true;
+        break;
+      case 'f':
+        pidfile.write(expandPath(opthnd.optarg));
+        break;
+      case 'sandbox-definitions':
+        options.sandboxDefinitions = expandPath(opthnd.optarg);
+        break;
+      case 'T':
+      {
+        const dump = Object.keys(require(options.sandboxDefinitions));
+        console.log(options.json ? JSON.stringify(dump) : dump);
+        process.exit(0);
+        break;
+      }
+      case 'D':
+        options.dump = true;
+        break;
+      case 'F':
+        options.dumpFiles = true;
+        break;
+      case 'v':
+        options.verbose++;
+        break;
+      default:
+        throw new Error(`unhandled option '${option}'`);
+    }
+  }
+
+  options.evaluatorOptions = process.argv.slice(parser.optind()); /* after --, pass to evaluator as-is */
+
+  const sandboxSetupDefs = require(options.sandboxDefinitions)[options.sandboxType];
   if (!sandboxSetupDefs)
     panic(`Invalid sandbox type: ${options.sandboxType}`);
 
-  if (options.rEvaluator)
-    evaluator = require.resolve(options.rEvaluator);
-  else
-    evaluator = path.resolve(options.evaluator);
-
   let args = [];    /* Command-line arguments to pass to evaluator */
   let files = [];   /* Startup files passed to evaluator as supplementary arguments */
 
@@ -123,7 +164,7 @@ function main() {
     args = args.concat(['-p', options.port]);
 
   /* Decode log-level parameters and translate as appropriate for the type of evaluator in use */
-  if (options.debugLogLevel || options.debugLogLevel === 0)
+  if (options.logLevel || options.logLevel === 0)
   {
     switch(options.sandboxType)
     {
@@ -143,22 +184,22 @@ function main() {
           'notice':    5,
           'info':      6,
           'debug':     7,
-        }[options.debugLogLevel];
+        }[options.logLevel];
 
-        if (typeof level === 'undefined' && (!isNaN(Number(options.debugLogLevel))))
+        if (typeof level === 'undefined' && (!isNaN(Number(options.logLevel))))
         {
           console.warn('Warning: numeric log levels are deprecated');
-          level = Number(options.debugLogLevel);
+          level = Number(options.logLevel);
         }
 
         if (typeof level === 'undefined')
-          panic(`unknown debug log level ${options.debugLogLevell}`);
+          panic(`unknown debug log level ${options.logLevell}`);
 
         args = args.concat('-d', level);
         break;
       }
       case 'node':
-        process.env.DCP_DEBUG_EVALUATOR = options.debugLogLevel;
+        process.env.DCP_DEBUG_EVALUATOR = options.logLevel;
         break;
       default:
         panic(`debug log level not supported for ${options.sandboxType} evaluators`);
@@ -167,8 +208,8 @@ function main() {
   }
 
   /* Pass options after -- as evaluator-specific options */
-  if (options._)
-    args = args.concat(options._);
+  if (options.evaluatorOptions.length > 0)
+    args = args.concat(options.evaluatorOptions);
 
   /* Definitions in the JSON file are either in the sandboxLibexecDir, or pathed
    * via normal CommonJS (node_modules) rules.  The syntax for detecting which must
@@ -183,15 +224,16 @@ function main() {
 
   if (options.dump) {
     if (options.json) {
-      console.log(JSON.stringify({evaluator, files, args, options}));
+      console.log(JSON.stringify({ evaluator: options.evaluator, files, args, options }));
     } else {
-      console.log('Evaluator command:', [evaluator].concat(files).concat(args).join(' '));
+      console.log('Evaluator command:', [path.resolve(options.prefix, options.evaluator)].concat(files).concat(args).join(' '));
       console.log('Options:', options);
     }
     process.exit(0);
   }
 
-  if (options.dumpFiles) {
+  if (options.dumpFiles)
+  {
     let dump = files;
 
     if (options.json)
@@ -200,15 +242,6 @@ function main() {
     process.exit(0);
   }
 
-  if (options.dumpTypes) {
-    let dump = Object.keys(require(options.sandboxDefinitions));
-
-    if (options.json)
-      dump = JSON.stringify(dump);
-    console.log(dump);
-    process.exit(0);
-  }
-
   if (!options.stdio && options.sandboxType === 'node') {
     if (process.env.I_WANT_AN_INSECURE_DCP_WORKER !== 'badly') {
       console.warn('***\x07 WARNING: Node evaluator is not suitable for production work - ^C to abort');
@@ -225,8 +258,8 @@ function main() {
 
   if (!options.stdio) {
     console.log('Starting DCP Evaluator -- ');
-    console.log('. Evaluator program at:\t\t', evaluator);
-    console.log('. Listening on:      \t\t', options.port);
+    console.log('. Evaluator program at:\t\t',  options.evaluator);
+    console.log('. Listening on:      \t\t',    options.port);
     console.log('. Worker environment type:\t', options.sandboxType);
     console.log('. Worker environment:  \t');
     files.forEach(f => {
@@ -235,15 +268,12 @@ function main() {
     console.log('');
   }
 
-  if (options.pidFile)
-    require('../lib/pidfile').write(options.pidFile);
-
   if (options.verbose) {
     console.log('. cwd:', path.join(options.prefix));
-    console.log('. run:', [evaluator].concat(files).concat(args).map(a => `"${a}"`).join(' '));
+    console.log('. run:', [options.evaluator].concat(files).concat(args).map(a => `"${a}"`).join(' '));
   }
 
-  const child = require('child_process').spawn(evaluator, files.concat(args), {
+  const child = require('child_process').spawn(options.evaluator, files.concat(args), {
     cwd: path.join(options.prefix),
     windowsHide: true,
     stdio: ['inherit', 'inherit', 'inherit'],

package/bin/dcp-task-probe

@@ -102,7 +102,7 @@ Examples (bash syntax):
   - # dcp-task-probe --merge 'minimumWage={"CPU":1,"GPU":3}' -c
 Options:
   -h | --help                    show this help and exit
-  -C | --dumpConfig              dump worker configuration instead of requesting a task
+  -C | --showConfig [dot.path]   show derived worker configuration instead of requesting a task
   -v | --version                 increase verbosity (show job info)
   -H | --hostname=               set evaluator hostname (for capability probing)
   -p | --port=                   set evaluator port number (for capability probing)
@@ -272,13 +272,13 @@ function cpuCommaGpu(str)
 /**
  * Parse and process command-line options. Most options result in changes to the running dcpConfig;
  * other options are reflected in the returned object:
- *  - dumpConfig
+ *  - showConfig
  *  - workerId
  */
 async function processOptions(workerConfig)
 {
   const opts = { verbose: 0 };
-  const parser = new getopt.BasicParser('h(help)C(dumpConfig)S:(set)M:(merge)a:(allowedOrigins)'
+  const parser = new getopt.BasicParser('h(help)C(showConfig)S:(set)M:(merge)a:(allowedOrigins)'
                                         + 'c:(cores)u:(utilization)m:(maxSandboxes)w:(workerId)'
                                         + 'j:(job)g:(computeGroup)G(leaveGlobalGroup)'
                                         + 'H:(hostname)p:(port)r(replay)l(load)i(input)o(output)'
@@ -302,8 +302,18 @@ async function processOptions(workerConfig)
         break;
 
       case 'C':
-        opts.dumpConfig = true;
+      {
+        /* no direct support for optional arguments in posix-getopt; dig into internals */
+        const nextArg = parser.gop_argv[parser.gop_optind];
+        if (nextArg?.[0] === '-')
+          opts.showConfig === ''; /* defined but empty - match yargs behaviour */
+        else
+        {
+          opts.showConfig = nextArg;
+          parser.gop_optind++;
+        }
         break;
+      }
 
       case 'v':
         opts.verbose += 1;
@@ -338,7 +348,7 @@ async function processOptions(workerConfig)
         break;
 
       case 'G':
-        workerConfig.leavePublicGroup = true;
+        workerConfig.leaveGlobalGroup = true;
         break;
 
       case 'H':
@@ -462,7 +472,7 @@ function readWorkerSnapshot()
 
   if (!readWorkerSnapshot.cache)
   {
-    const serializer = dcpConfig.worker.requestSnapshot.endsWith('kvin') ? require('kvin').KVIN : JSON;
+    const serializer = dcpConfig.worker.requestSnapshot.endsWith('kvin') ? require('dcp/internal/kvin').KVIN : JSON;
     debug('dcp-task-probe')('loading snapshot', detailedView(dcpConfig.worker.requestSnapshot));
     readWorkerSnapshot.cache = serializer.parse(fs.readFileSync(dcpConfig.worker.requestSnapshot));
   }
@@ -609,12 +619,11 @@ async function main()
     await probeEvaluator(initialWorkerConfig);
   const dummyWorker = new (require('dcp/worker').DistributiveWorker)(initialWorkerConfig, function FakeSandboxConstructor(){});
 
-  if (opts.dumpConfig)
+  if (opts.hasOwnProperty('showConfig'))
   {
-    /* work around the cores getter/setter */
-    const dump = Object.assign({}, dummyWorker.config);
-    console.log(dump);
-    process.exit(0);
+    const val = opts.showConfig ? eval(`dummyWorker.config.${opts.showConfig}`) : dummyWorker.config;
+    console.log(JSON.stringify(val, null, 2));
+    process.exit();
   }
 
   const { workerId, response, workerConfig } = await requestTask(dummyWorker, opts);