dcp-worker 3.2.24 → 3.2.26

package/docs/CODEOWNERS

@@ -0,0 +1,36 @@
+# @file    CODEOWNERS - Defines who develops and maintains a feature, and own
+#          the resulting files or directories in the repository.
+#
+#          See https://docs.gitlab.com/ee/user/project/code_owners.html
+#
+# @author  Bryan Hoang <bryan@distributive.network>
+# @date    August 2022
+
+# Username mapping:
+# @wesgarland -> Wes Garland <wes@distributive.network>
+
+
+[]
+
+[Brandon]
+
+[Eddie]
+/package-lock.json @eroosenmaallen
+
+[Wes]
+/README.md @wesgarland
+/docs/ @wesgarland
+/.eslintrc.json @wesgarland
+/.tidelift @wesgarland
+/lib/ @wesgarland
+/LICENSE.md @wesgarland
+/.gitignore @wesgarland
+/etc/ @wesgarland
+/.gitlab-ci.yml @wesgarland
+/bin/ @wesgarland
+/etc/ @wesgarland
+/lib/ @wesgarland
+/bin/ @wesgarland
+
+[Wes or Eddie]
+/package.json @wesgarland @eroosenmaallen

package/docs/CODEOWNERS.template

@@ -0,0 +1,10 @@
+# @file    CODEOWNERS - Defines who develops and maintains a feature, and own
+#          the resulting files or directories in the repository.
+#
+#          See https://docs.gitlab.com/ee/user/project/code_owners.html
+#
+# @author  Bryan Hoang <bryan@distributive.network>
+# @date    August 2022
+
+# Username mapping:
+# @wesgarland -> Wes Garland <wes@distributive.network>

package/etc/dcp-worker-config.js

@@ -0,0 +1,65 @@
+/**
+ * @file        dcp-worker-config.js
+ *              Default configuration for the standalone DCP Worker package.
+ *              Copy this file before modifying, so that changes are preserved
+ *              during the upgrade cycle. Suggested locations:
+ *              - /etc/dcp/dcp-worker/dcp-config.js, or 
+ *              - ~/.dcp/dcp-worker/dcp-config.js.
+ *
+ *              Those files have a higher precedence than the configuration
+ *              that ships with the npm package; changes made in those files
+ *              will be merged into the running configuration, overriding the
+ *              defaults specified here.
+ *
+ *              Windows users can also affect these changes by adding entries
+ *              to the registry. This is the preferred method for enterprise
+ *              deployment.
+ *
+ * @author      Wes Garland
+ * @date        Feb 2021
+ */
+{
+  worker: {
+    trustComputeGroupOrigins: true,  /* Trust the scheduler to modify allowOrigins via Compute Group configuration */
+
+    /* Allow lists permitting supervisor network access beyond DCP messages to services */
+    allowOrigins: {
+      fetchWorkFunctions: [ dcpConfig.scheduler.location.origin ],
+      fetchArguments:     [ dcpConfig.scheduler.location.origin ],
+      fetchData:          [ dcpConfig.scheduler.location.origin ],
+      sendResults:        [ dcpConfig.scheduler.location.origin ],
+      any:                [],
+    },
+
+    /* Vector describing the lowest-value work this worker will accept. */
+    minimumWage: {
+      'CPU':  0, /* DCC per second of CPU time */
+      'GPU':  0, /* DCC per second of GPU time */
+      'in':   0, /* DCC per byte of inbound network traffic */
+      'out':  0, /* DCC per byte of outbound network traffic */
+    },
+
+    /* Extra Compute Groups this worker can participate in. Join credentials are supplied by
+     * Distributive and/or local IT staff at site-licensed locations.
+     */
+    computeGroups: [
+      // { joinKey: 'scott', joinSecret: 'tiger' },
+      // { joinKey: 'scott', joinHash: 'eh1-672937c2b944982e071185b888770f8b8ea67c11f56d545e403e0d513c609b87' },
+      // keystore('~/.dcp/scott'),
+    ],
+
+    jobAddresses: [],               /* If specified, restrict the worker to only these jobs */
+    paymentAddress: undefined,      /* Bank account where earned funds are transfered if not specified on command-line */
+  },
+
+  /* The evaluator is a secure environment for creating DCP Worker sandboxes, used when the Worker
+   * is running in Node.js. This configuration specifies where this worker's evaluator daemon is
+   * listening. Killing the evaluator stops all work from happening on this worker; the worker
+   * will run in the background waiting for it to re-launch when this happens. 
+   */
+  evaluator: {
+    listen:   new URL('dcpsaw://localhost:9000/'),
+  },
+
+  cookie: require('process').env.DCP_CONFIG_COOKIE, /* used to verify that configuration file was actually loaded */
+}

package/etc/dcp-worker-config.js.md5

@@ -0,0 +1,2 @@
+ee17a8c00be52fbefd8c642d312317e7  etc/dcp-worker-config.js
+### DO NOT MODIFY THIS FILE!!! ###

package/lib/pidfile.js

@@ -0,0 +1,96 @@
+/**
+ * @file        pidfile.js
+ *              Library code to record/remove pid files
+ *
+ * @author      Robert Mirandola, robert@distributive.network
+ * @author      Wes Garland, wes@distributive.network
+ * @date        April 2023
+ */
+'use strict';
+
+const fs   = require('fs');
+const path = require('path');
+
+/**
+ * Write a pid file for the current process. This file contains only the process' pid. Before the file
+ * is written, we to check to see if there is already a file there. If there is we warn, and -- if there
+ * is a process running with that pid -- we exit the process after briefly blocking the event loop.
+ *
+ * As the pid file is created, we register a cleanup which removes the file when the process exits. The
+ * cleanup depends on the process<dcpExit> event fired by dcp-client during process tear-down.
+ *
+ * @param {string} filename           the location of the pid file
+ */
+exports.write = async function pidfile$$write(filename)
+{
+  var fd;
+  
+  if (fs.existsSync(filename))
+  {
+    console.warn(`Warning: found pidfile at ${filename}`);
+    let oldPid = parseInt(fs.readFileSync(filename, 'utf8'), 10);
+
+    try
+    {
+      if (oldPid)
+        process.kill(oldPid, 0);
+    }
+    catch(error)
+    {
+      if (error.code === 'ESRCH') /* no such process */
+      {
+        oldPid = 0;
+        removePidFile();
+      }
+    }
+
+    if (oldPid)
+    {
+      console.error(`Process at PID ${oldPid} is still running; cannot start new process with pidfile ${filename}`);
+      require('dcp/utils').sleep(3); /* put the brakes on a respawn loop */
+      process.exit(1);
+    }
+  }
+
+  try
+  {
+    fd = fs.openSync(filename, 'wx');
+    fs.writeSync(fd, Buffer.from(process.pid + '\n'), 0);
+    process.on('dcpExit', removePidFile);   // Cleanup PID file on exit
+  }
+  catch (error)
+  {
+    console.warn(`Warning: Could not create pidfile at ${filename} (${error.code || error.message})`);
+    removePidFile();
+  }
+
+  function removePidFile()
+  {
+    try
+    {
+      fs.unlinkSync(filename);
+      if (typeof fd === 'number')
+        fs.closeSync(fd);
+    }
+    catch (error)
+    {
+      console.warn(`Warning: Could not remove pidfile at ${filename} (${error.code})`);
+    }
+  }
+}
+
+/**
+ * Generate a default pidfile name for the current process
+ *
+ * @param {string} basename     optional, name of current process
+ * @returns string which contains an absolute path
+ */
+exports.getDefaultPidFileName = function getDefaultPidFileName(basename)
+{
+  var defaultPidFileName = basename || path.basename(require.main.filename, '.js') + '.pid';
+
+  if (!path.isAbsolute(defaultPidFileName))
+    defaultPidFileName = path.resolve(__dirname, '..', 'run', defaultPidFileName);
+
+  return defaultPidFileName;
+}

package/lib/remote-console.js

@@ -8,7 +8,7 @@
  *              troubleshooting, as any attacker on the network will have the same OS-level
  *              priviledges as the DCP Service Worker.
  *
- *              To enable this feature, create a file in ../etc name enabled-debug-console. That
+ *              To enable this feature, create a file in ../etc named enable-debug-console. That
  *              file should contain a number which identities the port number this service will
  *              listen on.
  *
@@ -29,6 +29,9 @@ var dcpConfig;
 var mainEval;
 var ci;
 
+require('dcp-client'); /* plumb in modules from bundle even if library has not been initialized */
+const debugging = require('dcp/internal/debugging').scope('dcp-worker');
+
 function daemonEval()
 {
   try {
@@ -43,7 +46,7 @@ function daemonEval()
 }
 
 function callbackTelnet(port, client, registry) {
-  console.log(' ! telnetd - listening on port', port);
+  debugging() && console.debug(' ! telnetd - listening on port', port);
 }
 
 /**
@@ -55,42 +58,33 @@ exports.setMainEval = function removeConsole$$setMainEval()
   mainEval = arguments[0];
 }
 
-exports.init = function remoteConsole$$init(port, ...commands)
+exports.init = function remoteConsole$$init(...commands)
 {
   try
   {
-    let historyFilename;
-    
-    if (!port)
-    {
-      /* invoked before main */
-      const edcFilename = path.resolve(__dirname, '..', 'etc', 'enable-debug-console');
-      if (!fs.existsSync(edcFilename))
-        return;
+    /* invoked before main */
+    const edcFilename = path.resolve(__dirname, '..', 'etc', 'enable-debug-console');
+    if (!fs.existsSync(edcFilename))
+      return;
+    const edc = fs.readFileSync(edcFilename, 'ascii').trim();
+    if (edc === 'false')
+      return;
+    const port = parseInt(edc, 10);
+    if (!port && port !== 0)
+      throw new Error('invalid port: ' + edc);
+
+    console.warn('*** Enabling telnet daemon on port', port, '(security risk) ***');
+
+    ci = require('telnet-console').start({
+      port,
+      callbackTelnet,
+      eval: daemonEval,
+      histfile: edcFilename + '.history',
+    }, ...commands);
 
-      port = JSON.parse(fs.readFileSync(edcFilename, 'ascii').trim());
-      if (!port)
-        return;
-      historyFilename = edcFilename + '.history';
-    }
-    else
-    {
-      historyFilename = path.resolve(__dirname, '..', 'etc', `telnetd-repl-port-${port}.history`);
-    }
-    
-    if (port)
-    {
-      console.warn(`*** Enabling telnet daemon on port ${port} (security risk) ***`);
-      ci = require('telnet-console').start({
-        port,
-        callbackTelnet,
-        eval: daemonEval,
-        histfile: historyFilename,
-      }, ...commands);
-      exports.init = () => {
-        throw new Error('remote console has already been initialized');
-      };
-    }
+    exports.init = () => {
+      throw new Error('remote console has already been initialized');
+    };
   }
   catch(e)
   {

package/lib/startWorkerLogger.js

@@ -41,8 +41,6 @@ const workerEvents = {
   submit:         'onSubmit',
   submitError:    'onSubmitError',
   payment:        'onPayment',
-  error:          'onError',
-  warning:        'onWarning',
 }
 const supervisorEvents = {
   submittingResult: 'onSubmitStart',

package/lib/worker-loggers/common-types.js

@@ -5,8 +5,6 @@
  * @property {function} onFetchingSlices
  * @property {function} onFetchedSlices
  * @property {function} onFetchSlicesFailed
- * @property {function} onError
- * @property {function} onWarning
  * @property {SandboxCallback} sandbox$onSliceStart
  * @property {SandboxCallback} sandbox$onSliceProgress
  * @property {SandboxCallback} sandbox$onSliceFinish
@@ -23,4 +21,4 @@
  * @typedef {function} SandboxCallback
  * @param {Sandbox} sandbox
  * @param {object} workerData
- */
+ */

package/lib/worker-loggers/console.js

@@ -80,16 +80,6 @@ const consoleLogger = {
   onSubmitError(ev) {
     console.log(" ! Failed to submit results:", ev);
   },
-
-  onError(ev)
-  {
-    this.options.verbose && console.error(" ! Worker Error:", ev);
-  },
-
-  onWarning(ev)
-  {
-    this.options.verbose && console.warn(" ! Worker Warning:", ev);
-  }
 };
 
 Object.assign(exports, consoleLogger);

package/lib/worker-loggers/dashboard.js

@@ -35,8 +35,6 @@ const dashboardLogger = {
     const grid = new contrib.grid({rows: 3, cols: 5, screen: this.screen});
 
     const log = grid.set(0, 2, 2, 3, components.log, {
-      fg: 'green',
-      selectedFg: 'green',
       label: 'Worker Log',
       scrollable: true,
       alwaysScroll: true,
@@ -55,6 +53,7 @@ const dashboardLogger = {
       }
       log.log.apply(log, arguments);
     }
+    require('../remote-console').reintercept();
 
     this.sandboxes = grid.set(0, 0, 2, 2, components.sandboxes, {
       label: 'Sandboxes',
@@ -70,18 +69,27 @@ const dashboardLogger = {
     this.workerInfo = grid.set(2, 0, 1, 5, blessed.text);
     this.updateWorkerInfo();
 
-    setInterval(() => this.screen.render(), 1000);
+    setInterval(() => this.screen.render(), 1000).unref();
 
-    this.screen.key(['escape', 'C-c'], () => {
-      console.log('076: Exit requested...');
-      if (typeof options.exitGuard?.exit === 'function')
-        options.exitGuard.exit(0);
-      else
-        process.exit(0);
+    function raise(sig)
+    {
+      process.kill(process.pid, sig);
+    }
+
+    /* Apply key bindings which mimic canonical input mode */
+    this.screen.key(['C-c'], () => raise('SIGINT'));
+    this.screen.key(['C-z'], () => raise('SIGTSTP'));
+    this.screen.key(['C-\\'], () => raise('SIGQUIT'));
+
+    this.screen.key(['escape'], () => {
+      console.log('Stopping worker...');
+      worker.stop();
     });
   },
 
   updateWorkerInfo() {
+    const workerOptions = dcpConfig.worker;
+    
     this.workerInfo.setLabel(`Worker Status [${this.sliceFetchStatus}]`);
     this.workerInfo.setContent([
       chalk.green(` DCCs Earned: ${chalk.bold(this.totalDCCs.toFixed(7))}`),
@@ -90,6 +98,9 @@ const dashboardLogger = {
       `        Bank: ${chalk.yellow(dcpConfig.bank.location.href)}`,
       `Bank Account: ${chalk.yellow(this.supervisor.paymentAddress || 'Starting...')}`,
       `    Identity: ${chalk.yellow(this.supervisor.identityKeystore? this.supervisor.identityKeystore.address : 'Starting...')}`,
+      `        Jobs: ${workerOptions.jobAddresses?.length ? workerOptions.jobAddresses.join(', ') : '<any>'}`,
+      ` Priv Groups: ${Object.keys(workerOptions.computeGroups).length}`,
+      `   Pub Group: ${workerOptions.leavePublicGroup ? 'no' : 'yes'}`,
     ].join('\n'));
   },
 
@@ -118,7 +129,7 @@ const dashboardLogger = {
           sandboxData.progressData.progress = 0;
           this.sandboxes.update();
         }
-      }, 500);
+      }, 500).unref();
     } else {
       sandboxData.progressData.progress = ev.progress;
       sandboxData.progressData.indeterminate = false;
@@ -173,14 +184,6 @@ const dashboardLogger = {
     this.sliceFetchStatus = SLICE_FETCH_STATUS.NO_WORK;
     this.updateWorkerInfo();
   },
-
-  onError(error) {
-    this.options.verbose && console.error('Unexpected error from worker:', error);
-  },
-
-  onWarning(error) {
-    this.options.verbose >= 2 && console.warn('Unexpected warning from worker:', error);
-  },
 };
 
 Object.assign(exports, dashboardLogger);

package/npm-hooks/postpublish

@@ -0,0 +1,11 @@
+#! /bin/bash
+#
+# @file         post-publish
+#               Hook which tags the package in git with the package version
+#               as it gets published.
+# @author       Wes Garland, wes@distributive.network
+# @date         Nov 2022
+#
+cd `dirname "$0"`/..
+PACKAGE_VERSION=$(cat package.json | egrep '^\s*"version":' | head -1 | awk -F: '{ print $2 }' | sed 's/[\",]//g' | tr -d '[[:space:]]')
+git tag v$PACKAGE_VERSION && git push --tags

package/npm-hooks/prepack

@@ -0,0 +1,17 @@
+#! /bin/bash
+#
+# @file         prepack
+#               Hook which generates the config md5 on pack (or publish)
+# @author       Wes Garland, wes@distributive.network
+# @date         April 2023
+#
+set -e
+set -u
+set -o pipefail
+
+cd `dirname "$0"`/..
+
+(
+  md5sum etc/dcp-worker-config.js
+  echo "### DO NOT MODIFY THIS FILE!!! ###"
+) > etc/dcp-worker-config.js.md5

package/npm-hooks/prepublish

@@ -0,0 +1,42 @@
+#! /bin/bash
+#
+# @file         pre-publish
+#               Hook which prevents package publishing if the repository
+#               isn't in a nice, clean state.
+# @author       Wes Garland, wes@distributive.network
+# @date         Nov 2022
+#
+cd `dirname "$0"`/..
+
+yellow='\e[33m'
+normal='\e[0m'
+
+git ls-files --error-unmatch `find . -type f | egrep -v '^\./(node_modules|\.git)'` >/dev/null
+if [ "$?" != "0" ]; then
+  echo
+  echo -e "${yellow}pre-publish: abort due to files in package which aren't in repo${normal}" >/dev/stderr
+  echo
+  exit 1
+fi
+
+(git status --porcelain; echo DONE DONE) \
+| while read status filename
+  do
+    case "$status" in
+      "M")
+        echo "${filename} is not checked in"
+        EXITCODE=1
+      ;;
+      "DONE")
+         exit $EXITCODE
+      ;;
+      *)
+      ;;
+    esac
+  done
+if [ "$?" != "0" ]; then
+  echo
+  echo -e "${yellow}pre-publish: abort due to modified files${normal}" >/dev/stderr
+  echo
+  exit 1
+fi

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcp-worker",
-  "version": "3.2.24",
+  "version": "3.2.26",
   "description": "JavaScript portion of DCP Workers for Node.js",
   "main": "bin/dcp-worker",
   "keywords": [
@@ -29,14 +29,16 @@
   "scripts": {
     "start": "node bin/dcp-worker start",
     "start-evaluator": "node bin/dcp-evaluator-start",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "peter tests",
+    "x-prepack": "npm-hooks/prepack",
+    "postpublish": "npm-hooks/postpublish",
+    "prepublishOnly": "npm-hooks/prepublish"
   },
   "dependencies": {
     "blessed": "^0.1.81",
     "blessed-contrib": "^4.11.0",
-    "bravojs": "^1.0.16",
     "chalk": "^4.1.0",
-    "dcp-client": "^4.2.27",
+    "dcp-client": "^4.2.28",
     "semver": "^7.3.8"
   },
   "optionalDependencies": {

package/etc/dcp-config.js

@@ -1,51 +0,0 @@
-/**
- * @file        dcp-config.js
- *              Configuration values for the standalone Worker package.
- *              These values have a higher precedence than the scheduler-provided values,
- *              but can be overridden via the usual dcp-config semantics -- .dcp, /etc, 
- *              Windows Registry, etc.  See dcp-client library documentation for details.
- *              Specify DCP_CONFIG in the environment to use an alternate module.
- *
- * @author      Wes Garland
- * @date        Feb 2021
- */
-
-const dcpConfig =
-{
-  worker: { /******* Note: all properties passed to web; no filtering! ********/
-
-    trustComputeGroupOrigins: true, // trust the scheduler to give network access to origins on a per-compute-group basis
-
-    /* Allow lists permitting supervisor network access beyond DCP messages to services */
-    allowOrigins: {
-      any: [],
-      fetchWorkFunctions: [],
-      fetchArguments: [],
-      fetchData: [],
-      sendResults: [],
-    },
-
-    minimumWage: {
-      CPU:  0,
-      GPU:  0,
-      'in': 0,
-      out:  0,
-    },
-
-    computeGroups: {},              // integer-one-indexed; format is 1:{ joinKey,joinHash } or 2:{ joinKey, joinSecret }
-    jobAddresses: [],               // Specific job addresses the worker may work on. If not empty, worker will only work on those jobs.
-    maxWorkingSandboxes: 1,
-    paymentAddress: null,           // user must to specify
-    evaluatorOptions: {}            /** @todo: add default options here. Note: right now they will be a bit off until we get localexec's evaluator a bit less special cased. */
-  },
-  evaluator:
-  {
-    listen: new URL('dcpsaw://localhost:9000/'),
-    libDir: '../libexec/evaluator', // relative to prefix
-  },
-}
-
-if (!dcpConfig.evaluator.location)
-  dcpConfig.evaluator.location = new URL(dcpConfig.evaluator.listen.href);
-
-Object.assign(exports, dcpConfig);