dcp-worker 3.2.30-9 → 3.2.31

package/catalog-info.yaml

@@ -0,0 +1,21 @@
+# @file         catalog-info.yml
+# @author       Brandon Christie <brandon@distributive.network>
+# @date         Mar 2023
+#
+# @description  Lists all of the components and the paths to all
+#               the documentation that will be published to backstage.
+
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: dcp-worker
+  description: Documentation for the dcp-worker.
+  tags:
+    - documentation
+  annotations:
+    gitlab.com/project-id: '18110002'
+    backstage.io/techdocs-ref: dir:./docs
+spec:
+  type: website
+  lifecycle: production
+  owner: core

package/docs/CODEOWNERS

@@ -18,9 +18,11 @@
 /package-lock.json @eroosenmaallen
 
 [Wes]
+/npm-hooks/ @wesgarland
 /README.md @wesgarland
 /docs/ @wesgarland
 /.eslintrc.json @wesgarland
+/.npmrc @wesgarland
 /.tidelift @wesgarland
 /lib/ @wesgarland
 /LICENSE.md @wesgarland

package/docs/mkdocs.yml

@@ -0,0 +1,16 @@
+# @file         mkdocs.yml
+# @author       Brandon Christie <brandon@distributive.network>
+# @date         Aug 2023
+#
+# @description  Determines how the component of name: 'dcp-worker'
+#               is formatted on backstage.
+
+site_name: dcp-worker
+repo_url: https://gitlab.com/Distributed-Compute-Protocol/dcp-worker
+edit_uri: edit/develop/docs
+docs_dir: .
+nav:
+  - README.md
+plugins:
+  - techdocs-core
+  - same-dir

package/etc/dcp-worker-config.js

@@ -49,7 +49,7 @@
       // keystore('~/.dcp/scott'),
     ],
 
-    jobAddresses: [],               /* If specified, restrict the worker to only these jobs */
+    jobAddresses: false,               /* If specified, restrict the worker to only these jobs */
     paymentAddress: undefined,      /* Bank account where earned funds are transfered if not specified on command-line */
   },
 

package/etc/dcp-worker-config.js.md5

@@ -1,2 +1,2 @@
-8edf1c57bee521ed7f151a21b8f3f34d
+4d49057dd014344e4be5a266e9754824
 ### DO NOT MODIFY THIS FILE!!! ###

package/lib/blessed-components/index.js

@@ -1,3 +1,4 @@
+'use strict';
 
 Object.assign(exports, require('./log'));
 Object.assign(exports, require('./sandboxes'));

package/lib/blessed-components/log.js

@@ -7,6 +7,7 @@
  *  log component, but features text wrapping and behaves
  *  closer to the built-in console log methods.
  */
+'use strict';
 
 const { Box } = require('blessed');
 

package/lib/blessed-components/sandboxes.js

@@ -67,11 +67,7 @@ class Sandboxes extends Box {
   update(data=this.data) {
     this.data = data;
     for (let i = 0; i < this.data.length; i++) {
-      if (i < this.progressBars.length) {
-        this.updateProgressBar(i, this.data[i]);
-      } else {
-        this.createProgressBar();
-      }
+      this.updateProgressBar(i, this.data[i]);
     }
 
     if (this.data.length < this.progressBars.length) {
@@ -80,7 +76,15 @@ class Sandboxes extends Box {
       }
     }
 
-    this.setLabel(`${this.options.label} (${this.data.length})`);
+    this.setLabel(`${this.options.label} (${this.progressBars.length})`);
+  }
+
+  // Deletes last progress bar
+  deleteProgressBar() {
+    let i = this.progressBars.length - 1;
+    this.progressBars[i].label.destroy()
+    this.progressBars[i].progressBar.destroy()
+    this.progressBars.pop(i)
   }
 }
 

package/lib/check-scheduler-version.js

@@ -6,6 +6,7 @@
  * @author      Sam Cantor
  * @date        Nov 2020
  */
+'use strict';
 
 exports.check = function checkSchedulerVersion$$check(quiet)
 {

package/lib/dashboard-tui.js

@@ -0,0 +1,231 @@
+/**
+ *  @file       worker-loggers/dashboard.js
+ *  @author     Ryan Rossiter, ryan@kingsds.network
+ *  @date       April 2020
+ *  @author     Wes Garland, wes@distributive.network
+ *  @date       June 2023
+ *
+ *  This module uses the blessed library to create a monitoring dashboard for the worker.
+ *  A corresponding worker-logger, dashboard.js, knows how to log to this dashboard.
+ */
+'use strict';
+
+const dcpConfig  = require('dcp/dcp-config');
+const chalk      = require('chalk');
+const blessed    = require('blessed');
+const contrib    = require('blessed-contrib');
+const components = require('./blessed-components');
+const utils      = require('../lib/utils');
+
+const { replaceWorkerEvent, replaceSandboxEvent }  = require('./default-ui-events');
+
+const SLICE_FETCH_STATUS = {
+  IDLE: chalk.yellow('Idle'),
+  FETCHING: chalk.blue('Fetching Work...'),
+  WORKING: chalk.green('Working'),
+  NO_WORK: chalk.red('No Work Available'),
+}
+
+const usingDebugger = require('module')._cache.niim instanceof require('module').Module;
+const screenConf = {
+  input:  usingDebugger ? new (require('events').EventEmitter) : undefined,
+  output: usingDebugger ? new (require('events').EventEmitter) : undefined,
+};
+/** 
+ */
+exports.init = function dashboard$$init(worker, options)
+{
+  var sliceFetchStatus = SLICE_FETCH_STATUS.IDLE;
+  var totalDCCs = 0;
+  const screen = blessed.screen(screenConf);
+  const grid = new contrib.grid({rows: 3, cols: 5, screen});
+  const workerInfoPane = grid.set(2, 0, 1, 5, blessed.text);
+  
+  const logPane = grid.set(0, 2, 2, 3, components.log, {
+    label: 'Worker Log',
+    scrollable: true,
+    alwaysScroll: true,
+    mouse: true,
+    scrollbar: {
+      bg: 'blue',
+    },
+  });
+
+  const sandboxPane = grid.set(0, 0, 2, 2, components.sandboxes, {
+    label: 'Sandboxes',
+    defaultProgressBars: 0,
+    scrollable: true,
+    alwaysScroll: true,
+    mouse: true,
+    scrollbar: {
+      bg: 'blue',
+    },
+  });
+
+  const passwordBox = blessed.textbox({
+    parent: screen,
+    border: 'line',
+    top: 'center',
+    left: 'center',
+    width: '50%',
+    height: 'shrink',
+    padding: {
+      top: 1
+    },
+    censor: true,
+    inputOnFocus: true,
+    label: 'Password Prompt:',
+    hidden: true,
+  });
+
+  function askPassword(promptMessage)
+  {
+    return new Promise((resolve, reject) => {
+      passwordBox.focus();
+      passwordBox.show();
+      passwordBox.setLabel(promptMessage);
+
+      function passwordSubmitFn(value)
+      {
+        passwordBox.hide();
+        passwordBox.removeListener('submit', passwordSubmitFn);
+        passwordBox.setValue('');
+        resolve(value);
+      }
+
+      passwordBox.on('submit', passwordSubmitFn);
+      screen.render();
+    });
+  }
+
+  // override wallet.passphrasePrompt with password box
+  require('dcp/wallet').passphrasePrompt = (promptMessage) => {
+    return askPassword(promptMessage);
+  };
+
+  delete exports.init; /* singleton */
+
+  if (!usingDebugger)
+    exports.logPane = logPane; /* now dashboard log can find the pane */
+  setInterval(() => screen.render(), 50).unref(); /* 50ms = 20 fps */
+  updateWorkerInfo();
+  screen.render();
+  
+  /* Apply key bindings which mimic canonical input mode */
+  screen.key(['C-c'], ()    => raise('SIGINT'));
+  screen.key(['C-z'], ()    => raise('SIGTSTP'));
+  screen.key(['\u001c'], () => raise('SIGQUIT')); /* C-\ */
+  
+  screen.key(['escape'], () => {
+    console.log('Stopping worker...');
+    worker.stop();
+  });
+
+  function updateWorkerInfo()
+  {
+    const workerOptions = worker.workerOptions;
+    
+    workerInfoPane.setLabel(`Worker Status [${sliceFetchStatus}]`);
+    workerInfoPane.setContent([
+      chalk.green(` DCCs Earned: ${chalk.bold(totalDCCs.toFixed(3))}`),
+      '',
+      `   Scheduler: ${chalk.yellow(dcpConfig.scheduler.location.href)}`,
+      `        Bank: ${chalk.yellow(dcpConfig.bank.location.href)}`,
+      `Bank Account: ${chalk.yellow(worker.paymentAddress || 'Starting...')}`,
+      `    Identity: ${chalk.yellow(worker.identityKeystore? worker.identityKeystore.address : 'Starting...')}`,
+      `        Jobs: ${workerOptions.jobAddresses?.length ? workerOptions.jobAddresses.join(', ') : '<any>'}`,
+      ` Priv Groups: ${Object.keys(workerOptions.computeGroups).length}`,
+      `   Pub Group: ${workerOptions.leavePublicGroup ? 'no' : 'yes'}`,
+    ].join('\n'));
+  }
+
+  /* Override default event behaviour to work better with the Dashboard. */
+
+  /** XXXpfr @todo Is this correct?  Or should we init progressData inside 'slice' like we used to.  */
+  replaceSandboxEvent('ready', function dashboard$$job(sandbox, sandboxData, ev) {
+    sandboxData.progressData = {
+      indeterminate: true,
+      progress: 0,
+      label: sandbox?.public ? sandbox.public.name: '<no-label>',
+    };
+  });
+  
+  replaceSandboxEvent('slice', function dashboard$$slice(sandbox, sandboxData, ev) {
+    sandboxPane.data.push(sandboxData.progressData);
+    sandboxPane.update();
+  });
+
+  replaceSandboxEvent('progress', function dashboard$$progress(sandbox, sandboxData, ev) {
+    if (!ev)
+    {
+      sandboxData.progressData.progress = 100;
+      setTimeout(() => {
+        if (sandboxData.progressData.indeterminate)
+        {
+          sandboxData.progressData.progress = 0;
+          sandboxPane.update();
+        }
+      }, 500).unref();
+    }
+    else
+    {
+      sandboxData.progressData.progress = ev;
+      sandboxData.progressData.indeterminate = false;
+    }
+
+    sandboxPane.update();
+  });
+  
+  replaceSandboxEvent('sliceEnd', function dashboard$$sliceEnd(sandbox, sandboxData, ev) {
+    sandboxPane.data = sandboxPane.data.filter(d => d != sandboxData.progressData);
+    sandboxData.progressData.progress = 0;
+    sandboxPane.update();
+  });
+
+  replaceSandboxEvent('end', function dashboard$$end(sandbox, sandboxData, ev) {
+    sandboxPane.data = sandboxPane.data.filter(d => d != sandboxData.progressData);
+    sandboxPane.deleteProgressBar();
+    sandboxData.progressData.progress = 0;
+    sandboxPane.update();
+  });
+
+  replaceWorkerEvent('beforeFetch', function dashboard$$beforeFetch(ev) {
+    sliceFetchStatus = SLICE_FETCH_STATUS.FETCHING;
+    updateWorkerInfo();
+  });
+  
+  replaceWorkerEvent('fetch', function dashboard$$fetch(ev) {
+    sliceFetchStatus = SLICE_FETCH_STATUS.NO_WORK;
+    if (ev instanceof Error)
+      console.error('Error fetching slices:', ev);
+    else if ( !(utils.slicesFetched(ev) === 0 && sandboxPane.data.length === 0))
+      sliceFetchStatus = SLICE_FETCH_STATUS.WORKING;
+    updateWorkerInfo();
+  });
+
+  worker.on('end', () =>  { screen.destroy(); });
+
+  worker.on('sandbox', function dashboard$$sandbox(ev) {
+    sandboxPane.createProgressBar();
+    sandboxPane.update();
+  });
+
+  worker.on('payment', function dashboard$$payment(ev) {
+    const payment = parseFloat(ev);
+    
+    if (!isNaN(payment))
+      totalDCCs += payment;
+
+    sandboxPane.update();
+    updateWorkerInfo();
+  });
+};
+
+/**
+ * Send a signal to the caller
+ * @param {number|string} sig    the signal to raise
+ */
+function raise(sig)
+{
+  process.kill(process.pid, sig);
+}

package/lib/default-ui-events.js

@@ -0,0 +1,171 @@
+/**
+ *  @file       default-event.js
+ *              Default worker/sandbox events, providing default logging behaviours (event handlers)
+ *              for the dcp-worker.
+ *
+ *              - All event handlers use the *current* global console object to emit messages;
+ *                enhanced logging subsystems should intercept this object to achieve their desired
+ *                behaviours.
+ *
+ *              - All event handlers invoke functions which are properties of the eventHandlers return
+ *                value from the hook function. This means that alternate user interfaces can either
+ *                hook or intercept the properties of that object to modify the event handlers'
+ *                behaviour without actually removing/replacing the event handler on the instance of
+ *                Worker.
+ *
+ *              NOTE: This is just a convenience module. There is no requirement to use this module to
+ *                    hook worker events, this module mainly exists to make it easy for the
+ *                    dashboard-tui to replace event handlers with better ones, but it also makes it
+ *                    easier to sandbox events since we only need to register one event handler here
+ *                    to handle every sandbox.
+ *
+ *  @author     Ryan Rossiter, ryan@kingsds.network
+ *  @date       April 2020
+ *  @author     Wes Garland, wes@distributive.network
+ *  @date       June 2023
+ */
+'use strict';
+
+const utils = require('../lib/utils');
+
+const sandboxEventHandlers = {};
+const workerEventHandlers = {};
+
+/**
+ *   Sandbox   1:  Slice Started:   slice 1, 0x5b5214D48F0428669c4E: Simple Job
+ *   Sandbox   1:  Slice Completed: slice 1, 0x5b5214D48F0428669c4E: Simple Job: dt 114ms
+ */
+
+/**
+ * @param    worker    The instance of Worker to hook
+ * @param    options   cliArgs from worker
+ */
+exports.hook = function hookWorkerEvents$$hook(worker, options)
+{
+  const sliceMap = {};          // jobAddress --> ( sliceNumber, t0 )
+  const truncationLength = 22;  // Extra 2 for '0x'
+
+  delete exports.hook;
+
+  function makeSliceId (sandbox, sliceNumber)
+  {
+    if (!sandbox.jobAddress)
+      return '<no job>';
+    
+    const address = sandbox.jobAddress.slice(0, truncationLength);
+    const baseInfo = sandbox?.public ? `${address}: ${sandbox.public.name}` : address;
+
+    if (!sliceNumber)
+      sliceNumber = sandbox.sliceNumber;
+    return sliceNumber > 0 ? `slice ${sliceNumber}, ${baseInfo}` : baseInfo;
+  }
+
+  sandboxEventHandlers.ready = function sandboxReadyHandler(sandbox, sandboxData, ev) {
+    console.log(` . Sandbox ${sandboxData.shortId}:  Initialized`);
+  };
+
+  sandboxEventHandlers.slice = function sliceHandler(sandbox, sandboxData, ev) {
+    sliceMap[sandbox.id] = { slice: sandbox.sliceNumber, t0: Date.now() };
+    console.log(` . Sandbox ${sandboxData.shortId}:  Slice Started:   ${makeSliceId(sandbox)}`);
+  };
+
+  sandboxEventHandlers.progress = function progressHandler(sandbox, sandboxdData, ev) {
+    // Overridden in dashboard-tui.js
+  };
+
+  sandboxEventHandlers.sliceEnd = function sliceEndHandler(sandbox, sandboxData, ev) {
+    const sliceInfo = sliceMap[sandbox.id];
+    if (sliceInfo)
+      console.log(` * Sandbox ${sandboxData.shortId}:  Slice Completed: ${makeSliceId(sandbox, sliceInfo.slice)}: dt ${Date.now() - sliceInfo.t0}ms`);
+    else
+      console.log(` * Sandbox ${sandboxData.shortId}:  Slice Completed: ${makeSliceId(sandbox)}`);
+  };
+
+  sandboxEventHandlers.end = function endHandler(sandbox,sandboxData, ev) {
+    const sliceInfo = sliceMap[sandbox.id];
+    console.log(` * Sandbox ${sandboxData.shortId}:  Terminated:      ${makeSliceId(sandbox, sliceInfo?.slice)}`);
+    delete sliceMap[sandbox.id];
+  };
+
+  workerEventHandlers.payment = function paymentHandler(ev) {
+    const payment = parseFloat(ev);
+
+    if (isNaN(payment))
+      console.error(' ! Failed to parse payment:', payment);
+    else
+      console.log(` . Payment: ${payment.toFixed(3)} ⊇`);
+  };
+
+  workerEventHandlers.beforeFetch = function beforeFetchHandler() {
+    options.verbose && console.log(' * Fetching slices...');
+  };
+
+  workerEventHandlers.fetch = function fetchHandler(ev) {
+    if (ev instanceof Error)
+      console.error(' ! Failed to fetch slices:', ev);
+    else
+      options.verbose && console.log(' . Fetched', utils.slicesFetched(ev), 'slices');
+  };
+
+
+  workerEventHandlers.beforeResult = function beforeResultHandler() {
+    options.verbose >= 2 && console.log(' * Submitting results...');
+  };
+
+  workerEventHandlers.result = function resultHandler(ev) {
+    if (ev instanceof Error)
+      console.error(" ! Failed to submit results:", ev);
+    else
+      options.verbose >= 2 && console.log(' . Submitted');
+  };
+
+  /* Register the appropriate event handlers on the worker and on each sandbox. The handlers are
+   * registered in such a way that mutating the exports object to supply different handlers after
+   * registration will work.
+   *
+   * The handlers registered on each sandbox receive two extra arguments before the usual event
+   * arguments; these are the sandbox handle emitted by the Worker<sandbox> event and an object
+   * called `sandboxData` which is just arbitrary storage for the eventHandlers' use, eg for memos.
+   */
+  for (let eventName in workerEventHandlers)
+    worker.on(eventName, (...args) => workerEventHandlers[eventName](...args));
+
+  worker.on('sandbox', function newSandboxHandler(sandbox) {
+    const sandboxData = {
+      shortId: sandbox.id.toString(10).padStart(3)
+    };
+    for (let eventName in sandboxEventHandlers)
+      sandbox.on(eventName, (...args) => sandboxEventHandlers[eventName](sandbox, sandboxData, ...args));
+  });
+  
+  exports.sandboxEventHandlers = sandboxEventHandlers;
+  exports. workerEventHandlers =  workerEventHandlers;
+};
+
+/**
+ * Function to replace a worker event handler.
+ *
+ * @param {string}   eventName          name of the event to replace
+ * @param {function} eventHandler       new event handler
+ */
+exports.replaceWorkerEvent = function hookWorkerEvents$$replace(eventName, eventHandler)
+{
+  if (!workerEventHandlers.hasOwnProperty(eventName))
+    throw new Error('unknown worker event: ' + eventName + `(${Object.keys(workerEventHandlers).join(', ')})`);
+
+  workerEventHandlers[eventName] = eventHandler;
+}
+
+/**
+ * Function to replace a sandbox event handler.
+ *
+ * @param {string}   eventName          name of the event to replace
+ * @param {function} eventHandler       new event handler
+ */
+exports.replaceSandboxEvent = function hookSandboxEvents$$replace(eventName, eventHandler)
+{
+  if (!sandboxEventHandlers.hasOwnProperty(eventName))
+    throw new Error('unknown sandbox event: ' + eventName + `(${Object.keys(sandboxEventHandlers).join(', ')})`);
+
+  sandboxEventHandlers[eventName] = eventHandler;
+}

package/lib/pidfile.js

@@ -21,7 +21,7 @@ const path = require('path');
  *
  * @param {string} filename           the location of the pid file
  */
-exports.write = async function pidfile$$write(filename)
+exports.write = function pidfile$$write(filename)
 {
   var fd;
   

package/lib/remote-console.js

@@ -47,7 +47,7 @@ function daemonEval()
 
 function callbackTelnet(port, client, registry) {
   client.unref();
-  debugging() && console.debug(' ! telnetd - listening on port', port);
+  debugging() && console.notice(' ! telnetd - listening on port', port);
 }
 
 /**
@@ -76,8 +76,16 @@ exports.init = function remoteConsole$$init(...commands)
 
     console.warn('*** Enabling telnet daemon on port', port, '(security risk) ***');
 
+    if (port !== 0)
+      exports.port = port;
+    else
+    {
+      /* telnet-console library does not properly support port 0 so we mostly work-around here */
+      exports.port = Math.floor(1023 + (Math.random() * (63 * 1024)));
+    }
+
     ci = require('telnet-console').start({
-      port,
+      port: exports.port,
       callbackTelnet,
       eval: daemonEval,
       histfile: edcFilename + '.history',