dcp-worker 3.3.13 → 3.3.15

package/.gitlab-ci.yml

@@ -107,6 +107,28 @@ check:
         - node_modules/
       policy: pull
 
+test:
+  variables:
+    DCP_CLIENT_ALLOW_INSECURE_CONFIGURATION: true
+  stage: test
+  script:
+    - npx peter -v --junit test_report.xml tests/
+  cache:
+    - key:
+        prefix: node-modules
+        files:
+          - package-lock.json
+      paths:
+        - node_modules/
+      policy: pull
+  artifacts:
+    expire_in: 1 day
+    when: always
+    paths:
+      - test_report.xml
+    reports:
+      junit: test_report.xml
+
 publish_docs:
   stage: deploy
   needs:

package/bin/dcp-worker

@@ -94,9 +94,9 @@ function parseCliArgs()
         describe: 'Number of CPU and GPU cores to work with: the format is 7,1 (or just 7) for 7 CPU cores and 1 GPU',
         type: 'string',
       },
-      density: {
-        alias: 'd',
-        describe: 'default proportion of CPU,GPU to use when cores not specified',
+      utilization: {
+        alias: 'u',
+        describe: 'default proportion of CPU,GPU to utilize: the format is -u 0.8,0.75 for 80% CPU and 75% GPU utilization',
         type: 'string',
       },
       maxSandboxes: {
@@ -145,7 +145,6 @@ function parseCliArgs()
         describe: 'Restrict worker to a specific job (use N times for N jobs)',
         type: 'array',
       },
-
       join: {
         alias: 'g',
         hidden: true,
@@ -193,7 +192,8 @@ function parseCliArgs()
 
       showConfig: {
         hide: false,
-        describe: 'Show merged configuration node (eg worker)',
+        describe: 'Show merged configuration node (e.g., --showConfig worker)',
+        type: 'string',
       },
 
       logfile: {
@@ -255,18 +255,6 @@ function parseCliArgs()
   .wrap(process.stdout.columns || 80)
   .argv;
 
-  if (cliArgs.showConfig)
-  {
-    console.log(eval('dcpConfig.' + cliArgs.showConfig));
-    process.exit();
-  }
-
-  if (cliArgs.dumpConfig)
-  {
-    console.log(JSON.stringify(require('dcp/dcp-config'), null, 2));
-    processExit(0);
-  }
-
   return cliArgs;
 }
 
@@ -370,7 +358,7 @@ async function main()
    *
    * It is important to never disable leavePublicGroup as a side effect of any other operation, or
    * slight configuration errors could have large security impacts.
-   */
+z   */
   const dcpWorkerOptions = dcpConfig.worker;
   const forceOptions = {
     paymentAddress,
@@ -412,6 +400,8 @@ async function main()
 
   if (cliArgs.jobId)
   {
+    if (dcpWorkerOptions.jobAddresses === false || dcpWorkerOptions.jobAddresses === undefined)
+      dcpWorkerOptions.jobAddresses = [];
     dcpWorkerOptions.jobAddresses.push(...cliArgs.jobId);
     dcpWorkerOptions.priorityOnly = true;
   }
@@ -427,14 +417,26 @@ async function main()
   if (cliArgs.watchdogInterval)
     dcpWorkerOptions.watchdogInterval = cliArgs.watchdogInterval;
 
-  if (dcpConfig.bannedGPUs)
+  if (cliArgs.showConfig)
   {
-    for (let prop of dcpConfig.bannedGPUs)
+    console.log(eval('dcpConfig.' + cliArgs.showConfig));
+    process.exit();
+  }
+
+  if (cliArgs.dumpConfig)
+  {
+    console.log(JSON.stringify(require('dcp/dcp-config'), null, 2));
+    process.exit();
+  }
+
+  if (dcpConfig.worker?.bannedGPUs)
+  {
+    for (let prop in dcpConfig.worker.bannedGPUs)
     {
       if (!bannedGPUs[prop])
-        bannedGPUs[prop] = dcpConfig.bannedGPUs[prop];
+        bannedGPUs[prop] = dcpConfig.worker?.bannedGPUs[prop];
       else
-        bannedGPUs[prop] = new RegExp(`(${dcpConfig.bannedGPUs[prop].source})|(${bannedGPUs[prop].source})`);
+        bannedGPUs[prop] = new RegExp(`(${dcpConfig.worker?.bannedGPUs[prop].source})|(${bannedGPUs[prop].source})`);
     }
   }
 
@@ -449,6 +451,7 @@ async function main()
   nascentWorker.on('warning', (...payload) => console.warn (...payload));
   nascentWorker.on('stop', () => { console.log('Worker is stopping') });
   nascentWorker.on('end',  () => { logClosing('log', 'Worker has stopped') });
+  nascentWorker.on('job', job => console.log(` . Job: ${job.name} ${job.address.slice(0,8)} ${job.description || ''} ${job.link || ''}`));
 
   // Display clean diagnostic when not debugging and env var
   //   DCP_SUPERVISOR_DEBUG_DISPLAY_MAX_INFO isn't set.
@@ -521,10 +524,10 @@ async function main()
     bannerLog(' . Falling back on global group when preferred groups have no work');
   if (dcpWorkerOptions.leavePublicGroup)
     bannerLog(' . Leaving the global compute group');
-  if (dcpWorkerOptions.cores)
-    bannerLog(` . Configured Cores: ${dcpWorkerOptions.cores.cpu},${dcpWorkerOptions.cores.gpu}`);
-  if (typeof dcpWorkerOptions.maxSandboxes !== 'undefined')
-    bannerLog(` . Maximum Sandboxes: ${dcpWorkerOptions.maxSandboxes}`);
+  bannerLog(` . Configured Cores: { cpu: ${dcpWorkerOptions.cores.cpu}, gpu: ${dcpWorkerOptions.cores.gpu} }`);;
+  bannerLog(` . Utilization: { cpu: ${dcpWorkerOptions.utilization.cpu}, gpu: ${dcpWorkerOptions.utilization.gpu} }`);
+  bannerLog(` . Effective Cores: { cpu: ${dcpWorkerOptions.utilization.cpu * dcpWorkerOptions.cores.cpu}, gpu: ${dcpWorkerOptions.utilization.gpu * dcpWorkerOptions.cores.gpu} }`);
+  bannerLog(` . Maximum Sandboxes: ${dcpWorkerOptions.maxSandboxes}`);
   if (cliArgs.verbose)
     bannerLog(` + Verbosity level: ${cliArgs.verbose}`);
   if (telnetd.hasOwnProperty('port'))
@@ -599,8 +602,7 @@ async function main()
 }
 
 /**
- * Process the cores and maxSandboxes cli arguments.
- * Also processes density, but that isn't currently used and may be deprecated in the future.
+ * Process the cores, utilization and maxSandboxes cli arguments.
  *
  * cliArgs.cores is the core count of the hardware to use.
  * It can be specified with only the cpu or gpu component, or both.
@@ -608,35 +610,35 @@ async function main()
  * E.g. -c 2,1  => cores = { cpu: 2,         gpu: 1 }
  *      -c 10   => cores = { cpu: 10,        gpu: <default> }
  *      -c ,10  => cores = { cpu: <default>, gpu: 10 }
+ *      -u 0.80,0.75 => utilization = { cpu: 0.80, gpu: 0.75 }
  *      -m 5    => maxSandboxes = 5
- *      default maxSandboxes := Math.max( 1, Math.round((System RAM) / 2.5GB) );
  */
 function processCoresAndMaxSandboxes (dcpWorkerOptions, cliArgs)
 {
-  const DCPWorker = require('dcp/worker').Worker;
-  const defaultTargets = {
-    density: dcpWorkerOptions.defaultCoreDensity || { cpu: 0, gpu: 0 }, /* 0,0 => user specified corrupt conf */
-    cores: DCPWorker.defaultCores(dcpWorkerOptions),
-  };
+  if (typeof cliArgs['maxSandboxes'] !== 'undefined')
+    dcpWorkerOptions.maxSandboxes = Number(cliArgs['maxSandboxes']);
 
   const parseArg = (which) => {
-    if (!cliArgs[which])
-      dcpWorkerOptions[which] = defaultTargets[which];
-    else
+    if (cliArgs[which])
     {
+      dcpWorkerOptions[which] = {};
       const [cpu, gpu] = cliArgs[which].split(',');
-      dcpWorkerOptions[which] = { cpu: Number(cpu || defaultTargets[which].cpu),
-                                  gpu: Number(gpu || defaultTargets[which].gpu) };
+      if (cpu?.length > 0)
+        dcpWorkerOptions[which].cpu = Number(cpu);
+      if (gpu?.length > 0)
+        dcpWorkerOptions[which].gpu = Number(gpu);
     }
   };
 
-  parseArg('density');
+  parseArg('utilization');
   parseArg('cores');
 
-  if (typeof cliArgs['maxSandboxes'] !== 'undefined')
-    dcpWorkerOptions.maxSandboxes = Number(cliArgs['maxSandboxes']);
-  debugging() && console.debug(`dcp-worker: cores = { ${dcpWorkerOptions.cores.cpu}, ${dcpWorkerOptions.cores.gpu} }`);
-  debugging() && console.debug('dcp-worker: core density =', dcpWorkerOptions.density);
+  if (debugging())
+  {
+    console.debug(`dcp-worker: cores = { cpu: ${dcpWorkerOptions.cores.cpu}, gpu: ${dcpWorkerOptions.cores.gpu} }`);
+    console.debug('dcp-worker: utilization =', dcpWorkerOptions.utilization);
+    console.debug('dcp-worker: maxSandboxes =', dcpWorkerOptions.maxSandboxes);
+  }
 }
 
 /**

package/etc/dcp-worker-config.js

@@ -40,6 +40,9 @@
       'in':   0, /* DCC per megabyte of inbound network traffic */
       'out':  0, /* DCC per megabyte of outbound network traffic */
     },
+    bannedGPUs: {
+      description: /D3D.*version\s[0-2][0-9]/
+    },
 
     /* Extra Compute Groups this worker can participate in. Join credentials are supplied by
      * Distributive and/or local IT staff at site-licensed locations.

package/etc/dcp-worker-config.js.md5

@@ -1,2 +1,2 @@
-9911ec21ecad5d6fa18b1663715bed5b
+271fe64e9b77126815dbb9aff6a3412f
 ### DO NOT MODIFY THIS FILE!!! ###

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcp-worker",
-  "version": "3.3.13",
+  "version": "3.3.15",
   "description": "Node.js Worker for Distributive Compute Platform",
   "main": "bin/dcp-worker",
   "keywords": [
@@ -40,7 +40,7 @@
     "blessed": "^0.1.81",
     "blessed-contrib": "4.11.0",
     "chalk": "^4.1.0",
-    "dcp-client": "4.4.12",
+    "dcp-client": "^4.4.18",
     "kvin": "^1.2.7",
     "posix-getopt": "^1.2.1",
     "semver": "^7.3.8",
@@ -53,7 +53,8 @@
   "devDependencies": {
     "@distributive/eslint-config": "2.1.4",
     "@trunkio/launcher": "1.2.7",
-    "eslint": "8.56.0"
+    "eslint": "8.56.0",
+    "peter": "2.4.7"
   },
   "peerDependencies": {
     "node-eventlog": "https://gitpkg.now.sh/Distributive-Network/node-eventlog/package?dcp/0.0.1"

package/tests/worker-with-no-options.simple

@@ -0,0 +1,60 @@
+#! /usr/bin/env node
+
+/**
+ * @file        worker-with-no-options.simple
+ * 
+ * @author      Nazila Akhavan <nazila@distributive.network>
+ * @date        Aug 2024
+ * 
+ * peter-timeout: 60
+ */
+'use strict';
+
+const { tmpdir } =require('node:os');
+const { spawnSync } = require('node:child_process');
+const { mkdtempSync, rmSync, existsSync, writeFileSync } = require('node:fs');
+const { join, parse, basename } = require('node:path');
+const { strictEqual, deepEqual } = require('node:assert');
+
+const filename = parse(basename(__filename)).name;
+const tempd = mkdtempSync(join(tmpdir(), `${filename}-`));
+
+process.on('exit', () => { existsSync(tempd) && rmSync(tempd, { recursive: true }) });
+process.chdir(__dirname);
+
+async function main()
+{
+  const { Keystore } = require('dcp/wallet');
+  const ks = await new Keystore(null);
+  const id = await new Keystore(null);
+
+  writeFileSync(`${tempd}/default.keystore`, JSON.stringify(ks.toJSON()), { encoding: 'utf-8' });
+  writeFileSync(`${tempd}/id.keystore`, JSON.stringify(id.toJSON()), { encoding: 'utf-8' });
+  const res = spawnSync(
+    '../bin/dcp-worker', ['--dumpConfig'],
+    {
+      encoding: 'utf-8',
+      env: {
+        ...process.env,
+        DCP_RESOURCE_DIR: tempd
+      }
+    }
+  );
+  
+  const confObj = JSON.parse(res.stdout);
+  strictEqual(confObj.worker.paymentAddress, ks.address.valueOf(), `${confObj.worker.paymentAddress} payment address is not correct`);
+  strictEqual(confObj.worker.leavePublicGroup, undefined, `leaveComputeGroup is ${confObj.worker.leavePublicGroup}`);
+  deepEqual(confObj.worker.allowOrigins.any, [], `allowOrigins should be empty. It includes ${confObj.worker.allowOrigins.any}`);
+  deepEqual(confObj.worker.cores.cpu, undefined, `Mismatch in cores! cpu: ${confObj.worker.cores.cpu}`);
+  deepEqual(confObj.worker.cores.gpu, undefined, `Mismatch in cores! gpu: ${confObj.worker.cores.gpu}`);
+  deepEqual(confObj.worker.utilization, { cpu: 1, gpu: 0.75 }, `Mismatch in utilization! cpu: ${confObj.worker.utilization.cpu}, gpu: ${confObj.worker.utilization.gpu}`);
+  deepEqual(confObj.worker.computeGroups, [], `computeGroups should be an empty array it includes ${confObj.worker.computeGroups}`);
+  deepEqual(confObj.worker.maxSandboxes, undefined,  `Mismatch! maxSandboxes:${confObj.worker.maxSandboxes}`);
+  deepEqual(confObj.worker.jobAddresses, undefined, `jobAddresses should be false but it is ${confObj.worker.jobAddresses}`);
+
+  console.log('Done.');
+}
+
+require('dcp-client')
+  .init()
+  .then(main);

package/tests/worker-with-options.simple

@@ -0,0 +1,68 @@
+#! /usr/bin/env node
+
+/**
+ * @file        worker-with-options.simple
+ * 
+ * @author      Nazila Akhavan <nazila@distributive.network>
+ * @date        Aug 2024
+ * 
+ * peter-timeout: 60
+ */
+'use strict';
+
+const { tmpdir } =require('node:os');
+const { spawnSync } = require('node:child_process');
+const { mkdtempSync, rmSync, existsSync, writeFileSync } = require('node:fs');
+const { join, parse, basename } = require('node:path');
+const { strictEqual, deepEqual } = require('node:assert');
+
+const filename = parse(basename(__filename)).name;
+const tempd = mkdtempSync(join(tmpdir(), `${filename}-`));
+
+process.on('exit', () => { existsSync(tempd) && rmSync(tempd, { recursive: true }) });
+process.chdir(__dirname);
+
+async function main()
+{
+  const { Keystore } = require('dcp/wallet');
+  const ks = await new Keystore(null);
+  const id = await new Keystore(null);
+
+  writeFileSync(`${tempd}/default.keystore`, JSON.stringify(ks.toJSON()), { encoding: 'utf-8' });
+  writeFileSync(`${tempd}/id.keystore`, JSON.stringify(id.toJSON()), { encoding: 'utf-8' });
+  const res = spawnSync(
+    '../bin/dcp-worker', [
+      '--paymentAddress', '0x9fd0909Bd93f7bad915726C3AB499391384b8b4b',
+      '--leavePublicGroup',
+      '--allowedOrigins', 'test-url',
+      '--cores', '3,1',
+      '--utilization', '0.1,0.5',
+      '--maxSandboxes', '5',
+      '--job-id', '09fd0909Bd93f7bad915726C3AB4',
+      '--join', `joinKey1,joinHash1`, '-g', 'joinKey2,joinHash2',
+      '--dumpConfig' ],
+    {
+      encoding: 'utf-8',
+      env: {
+        ...process.env,
+        DCP_RESOURCE_DIR: tempd
+      }
+    }
+  );
+  
+  const confObj = JSON.parse(res.stdout);
+  strictEqual(confObj.worker.paymentAddress, '0x9fd0909Bd93f7bad915726C3AB499391384b8b4b', `${confObj.worker.paymentAddress} payment address is not corrrect`);
+  strictEqual(confObj.worker.leavePublicGroup, true, `leaveComputeGroup is ${confObj.worker.leavePublicGroup}`);
+  deepEqual(confObj.worker.allowOrigins.any.includes('test-url'), true, `allowOrigins only includes ${confObj.worker.allowOrigins.any}!`);
+  deepEqual(confObj.worker.cores, { cpu: 3, gpu: 1 }, `Mismatch in cores! cpu: ${confObj.worker.cores.cpu}, gpu: ${confObj.worker.cores.gpu}`);
+  deepEqual(confObj.worker.utilization, { cpu: 0.1, gpu: 0.5 }, `Mismatch in utilization! cpu: ${confObj.worker.utilization.cpu}, gpu: ${confObj.worker.utilization.gpu}`);
+  deepEqual(confObj.worker.computeGroups, [{ joinKey: 'joinKey1', joinSecret: 'joinHash1' }, { joinKey: 'joinKey2', joinSecret: 'joinHash2' }], 'Mismatch in computeGroups')
+  deepEqual(confObj.worker.maxSandboxes, 5, `Mismatch! maxSandboxes:${confObj.worker.maxSandboxes}`);
+  deepEqual(confObj.worker.jobAddresses, ['09fd0909Bd93f7bad915726C3AB4'], `Mismatch  in job addresses! jobAddresses:${confObj.worker.jobAddresses}`);
+
+  console.log('Done.');
+}
+
+require('dcp-client')
+  .init()
+  .then(main);