dcp-worker 4.0.0 → 4.1.0

package/bin/dcp-task-probe

@@ -9,14 +9,14 @@
  * - signature: ethereum signature object
  * - auth:
  *   - workerId: our worker id
- *   - authSlices: job address props -> array of number
+ *   - authSlices: job id props -> array of number
  *   - schedulerId: { address: scheduler's address }
- * - jobCommissions: job address props
+ * - jobCommissions: job id props
  *   - rate: number
  *   - bank account
  * - body:
- *   - newJobs: job address props
- *     - address: job address
+ *   - newJobs: job id props
+ *     - address: job id
  *     - metrics:
  *       - sliceCPUTime
  *       - sliceGPUTime
@@ -40,15 +40,14 @@
  *     - modulePath: array
  *     - dependencies: array of module identifiers
  *     - urlParameters:
- *       - job: job address
+ *       - job: job id
  *       - jobId: uuid
  *     - workerConsole: 0
  *     - worktime:
  *       - name
  *       - version
- *   - task: job address props -> array
- *     - jobAddress: address
- *     - uuid: uuid
+ *   - task: job id props -> array
+ *     - jobId: opaqueId
  *     - worker: worker opaque id
  *     - sliceNumber
  *     - cotaskCount
@@ -281,7 +280,7 @@ async function processOptions(workerConfig)
   const opts = { verbose: 0 };
   const parser = new getopt.BasicParser('h(help)C(dumpConfig)S:(set)M:(merge)a:(allowedOrigins)'
                                         + 'c:(cores)u:(utilization)m:(maxSandboxes)w:(workerId)'
-                                        + 'J:(jobAddress)j:(job)g:(computeGroup)G(leaveGlobalGroup)'
+                                        + 'j:(job)g:(computeGroup)G(leaveGlobalGroup)'
                                         + 'H:(hostname)p:(port)r(replay)l(load)i(input)o(output)'
                                         + 'v(verbose)',
                                         process.argv);
@@ -334,12 +333,8 @@ async function processOptions(workerConfig)
         workerConfig.maxSandboxes = Number(opthnd.optarg);
         break;
 
-      case 'J':
-        workerConfig.jobAddresses = (workerConfig.jobAddresses || []).concat(opthnd.optarg);
-        break;
-
-      case 'j': /** @todo integrate with upcoming scheduler changes to opaqueId /wg dec 2024 */
-        workerConfig.jobs = (workerConfig.jobs || []).concat(opthnd.optarg);
+      case 'j':
+        workerConfig.jobIds = (workerConfig.jobIds || []).concat(opthnd.optarg);
         break;
 
       case 'G':
@@ -474,24 +469,6 @@ function readWorkerSnapshot()
   return readWorkerSnapshot.cache;
 }
 
-/**
- * Generate non-global compute group secrets, hashed with the current task distributor connection's
- * dcpsid. Add in the public group if necessary.
- *
- * @todo unify with supervisor /wg dec2024
- */
-function makeComputeGroupsRequest(tdConn, dummyWorker)
-{
-  const options = Object.assign({}, dummyWorker.config);
-  const taskDistributor = { connection: tdConn };
-
-  dummyWorker.badSupervisorBackdoor.dcp4.generateWorkerComputeGroups.apply({ taskDistributor, options });
-  if (!dummyWorker.config.leavePublicGroup && !dummyWorker.config.leaveGlobalGroup)
-    options.computeGroups.unshift({ joinKey: 'public' });
-  return options.computeGroups;
-}
-
-
 /**
  * Request a task from the task distributor
  *
@@ -525,8 +502,7 @@ async function requestTask(dummyWorker, opts)
       jobQuanta:            [ 1 ]/* this.quanta.calculateJobQuanta() */,
       capabilities:         dummyWorker.config.capabilities,
       paymentAddress:       dummyWorker.config.paymentAddress,
-      jobAddresses:         dummyWorker.config.jobAddresses || [],
-      workerComputeGroups:  makeComputeGroupsRequest(tdConn, dummyWorker),
+      jobIds:               dummyWorker.config.jobIds || [],
       minimumWage:          dummyWorker.config.minimumWage,
       soteriaJobs:          [],
       overdueSlices:        [],
@@ -542,7 +518,16 @@ async function requestTask(dummyWorker, opts)
   debug('dcp-task-probe:request')('Task Request:', request);
 
   const rtStart = Date.now();
-  const response = await tdConn.request('requestTask', request);
+  const tdRequest = new tdConn.Request({operation: 'requestTask', data: request});
+  tdRequest.connection.on('beforeSend', function beforeSendHandler (request) {
+    /* need to delay hashing because connection might not be established yet, but we need the dcpsid */
+    if (request !== tdRequest)
+      return;
+    const hashedComputeGroups = require('dcp/compute-groups').generateTaskRequestWorkerComputeGroups(dummyWorker.config, request);
+    request.payload.data.workerComputeGroups = hashedComputeGroups;
+  });
+
+  const response = await tdRequest.send();
   const rtElapsed = Date.now() - rtStart;
   console.log('Task Request Time:   ', fmtMs(rtElapsed));
 
@@ -550,7 +535,6 @@ async function requestTask(dummyWorker, opts)
   if (!response.success)
     panic('request failed:', response.payload);
 
-  debugger;
   return { response, workerId: request.coreStats?.worker, workerConfig: dummyWorker.config };
 }
 

package/bin/dcp-worker

@@ -166,7 +166,7 @@ function parseCliArgs()
       showConfig: {
         hide: false,
         describe: 'Show worker config',
-        type: 'string',
+        type: 'boolean',
       },
 
       logfile: {
@@ -316,19 +316,20 @@ async function main()
         return isHash(b) ? { joinKey: a, joinHash: b } : { joinKey: a, joinSecret: b };
       })
       .filter((el) => el.joinKey); /* Filter out entries with no joinKey */
-
-    leafMergeInto(workerConfig.computeGroups, cliComputeGroups);
+    workerConfig.computeGroups = cliComputeGroups;
   }
 
   if (cliArgs.jobId)
   {
-    if (!workerConfig.jobAddresses)
-      workerConfig.jobAddresses = [];
-    workerConfig.jobAddresses.push(...cliArgs.jobId);
+    workerConfig.jobIds = [];
+    workerConfig.jobIds.push(...cliArgs.jobId);
   }
 
   if (cliArgs.allowedOrigins)
+  {
+    workerConfig.allowOrigins = { any: [] };
     workerConfig.allowOrigins.any.push(...cliArgs.allowedOrigins);
+  }
 
   if (cliArgs.watchdogInterval)
     workerConfig.watchdogInterval = cliArgs.watchdogInterval;
@@ -345,7 +346,7 @@ async function main()
 
   if (cliArgs.showConfig)
   {
-    console.log(eval('dcpConfig.' + cliArgs.showConfig)); // eslint-disable-line no-eval
+    console.log(JSON.stringify(worker.config, null, 2));
     process.exit();
   }
 
@@ -359,7 +360,7 @@ async function main()
   worker.on('warning', (...payload) => console.warn (...payload));
   worker.on('stop', () => { console.log('Worker is stopping') });
   worker.on('end',  () => { logClosing('log', 'Worker has stopped') });
-  worker.on('job', job => console.log(` . Job: ${job.name} ${job.address.slice(0,8)} ${job.description || ''} ${job.link || ''}`));
+  worker.on('job', job => console.log(` . Job: ${job.name} ${job.id.slice(0,8)} ${job.description || ''} ${job.link || ''}`));
 
   // Display clean diagnostic when not debugging and env var
   //   DCP_SUPERVISOR_DEBUG_DISPLAY_MAX_INFO isn't set.
@@ -425,8 +426,8 @@ async function main()
     return plural;
   }
 
-  if (worker.config.jobAddresses?.length > 0)
-    bannerLog(` * Processing only ${qty(worker.config.jobAddresses, 'job')} ` + worker.config.jobAddresses.join(', '));
+  if (worker.config.jobIds?.length > 0)
+    bannerLog(` * Processing only ${qty(worker.config.jobIds, 'job')} ` + worker.config.jobIds.join(', '));
   if (worker.config.computeGroups && Object.keys(worker.config.computeGroups).length > 0)
     bannerLog(` . Joining compute ${qty(worker.config.computeGroups, 'group')} ` + worker.config.computeGroups.map(el => el.joinKey).join(', '));
   if (worker.config.publicGroupFallback)
@@ -663,7 +664,7 @@ function sliceReport()
 
   report += ('Progress:') + '\n';
   worker.workingSandboxes.forEach(sb => {
-    const jobName = sb.job?.public?.name || `idek (${sb.jobAddress})`;
+    const jobName = sb.job?.public?.name || `idek (${sb.jobId})`;
     let el = Date.now() - sb.sliceStartTime;
     const t = el < 1000000
           ? toInterval(el)
@@ -677,7 +678,7 @@ function sliceReport()
           : 'ind';
     const stale = (el < 2000) ? '' : `(stale: ${toInterval(el)})`;
 
-    report += (` ${String(sb.id).padStart(4)}: ${sb.jobAddress} ${jobName.padEnd(34)} `+ `${t} ${pct} ${stale}`.padStart(13)) + '\n';
+    report += (` ${String(sb.id).padStart(4)}: ${sb.jobId} ${jobName.padEnd(34)} `+ `${t} ${pct} ${stale}`.padStart(13)) + '\n';
   });
 
   report += ('Slices:') + '\n';

package/lib/dashboard-tui.js

@@ -162,7 +162,7 @@ exports.init = function dashboard$$init(worker, options, bannerLogCumulative)
       `        Bank Account: ${chalk.yellow(worker.paymentAddress || 'Starting...')}`,
       `            Identity: ${chalk.yellow(worker.identityKeystore? worker.identityKeystore.address : 'Starting...')}`,
       `                 GPU: ${gpuInfo}`,
-      `                Jobs: ${worker.config.jobAddresses?.length ? worker.config.jobAddresses.join(', ') : '<any>'}`,
+      `                Jobs: ${worker.config.jobIds?.length ? worker.config.jobIds.join(', ') : '<any>'}`,
       `      Compute Groups: ${Object.keys(worker.config.computeGroups).length + (worker.config.leavePublicGroup ? 0 : 1)}`,
       `Global Compute Group: ${worker.config.leavePublicGroup ? 'no' : 'yes'}`,
       `           Worker Id: ${worker.id}`,
@@ -184,7 +184,7 @@ exports.init = function dashboard$$init(worker, options, bannerLogCumulative)
 
   replaceSandboxEventHandler('job', function dashboard$$sandboxJob(sandbox, sandboxData, job) {
     sandboxData.job = job;
-    sandboxData.label = job.name ? `${job.name} ${job.address.slice(0,8)}` : `<job ${job.address}>`;
+    sandboxData.label = job.name ? `${job.name} ${job.opaqueId.slice(0,8)}` : `<job ${job.opaqueId}>`;
     if (job.description)
       sandboxData.label += ': ' + job.description;
     sandboxPane.update();

package/lib/default-ui-events.js

@@ -44,18 +44,16 @@ function placeholder()
  */
 exports.hook = function defaultUiEvents$$hook(worker, options)
 {
-  const sliceMap = {};          // jobAddress --> ( sliceNumber, t0 )
-  const truncationLength = 22;  // Extra 2 for '0x'
+  const sliceMap = {};          // jobId --> ( sliceNumber, t0 )
 
   delete exports.hook;
 
   function makeSliceId (sandbox, sliceNumber)
   {
-    if (!sandbox.jobAddress)
+    if (!sandbox.jobId)
       return '<no job>';
     
-    const address = sandbox.jobAddress.slice(0, truncationLength);
-    const baseInfo = sandbox?.public ? `${address}: ${sandbox.public.name}` : address;
+    const baseInfo = sandbox?.public ? `${sandbox.jobId}: ${sandbox.public.name}` : sandbox.jobId;
 
     if (!sliceNumber)
       sliceNumber = sandbox.sliceNumber;
@@ -88,13 +86,13 @@ exports.hook = function defaultUiEvents$$hook(worker, options)
     delete sliceMap[sandbox.id];
   };
 
-  workerEventHandlers.payment = function paymentHandler(dcc, account, jobAddress) {
+  workerEventHandlers.payment = function paymentHandler(dcc, account, jobId) {
     const payment = parseFloat(dcc);
 
     if (isNaN(payment))
       console.error(' ! Failed to parse payment:', payment);
     else
-      console.log(` . Payment: ${payment.toFixed(3)} ⊇ ${jobAddress.slice(0, 8)}`);
+      console.log(` . Payment: ${payment.toFixed(3)} ⊇ ${jobId.slice(0, 8)}`);
   };
 
   workerEventHandlers.beforeFetch = function beforeFetchHandler() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcp-worker",
-  "version": "4.0.0",
+  "version": "4.1.0",
   "description": "Node.js Worker for Distributive Compute Platform",
   "main": "bin/dcp-worker",
   "keywords": [
@@ -39,7 +39,7 @@
     "blessed": "^0.1.81",
     "blessed-contrib": "4.11.0",
     "chalk": "^4.1.0",
-    "dcp-client": "^4.4.26",
+    "dcp-client": "^5.0.0",
     "kvin": "^1.2.7",
     "posix-getopt": "^1.2.1",
     "semver": "^7.3.8",

package/tests/worker-with-no-options.simple

@@ -31,7 +31,7 @@ async function main()
   writeFileSync(`${tempd}/default.keystore`, JSON.stringify(ks.toJSON()), { encoding: 'utf-8' });
   writeFileSync(`${tempd}/id.keystore`, JSON.stringify(id.toJSON()), { encoding: 'utf-8' });
   const res = spawnSync(
-    '../bin/dcp-worker', ['--dumpConfig'],
+    '../bin/dcp-worker', ['--showConfig'],
     {
       encoding: 'utf-8',
       env: {
@@ -42,15 +42,11 @@ async function main()
   );
   
   const confObj = JSON.parse(res.stdout);
-  strictEqual(confObj.worker.paymentAddress, ks.address.valueOf(), `${confObj.worker.paymentAddress} payment address is not correct`);
-  strictEqual(confObj.worker.leavePublicGroup, undefined, `leaveComputeGroup is ${confObj.worker.leavePublicGroup}`);
-  deepEqual(confObj.worker.allowOrigins.any, [], `allowOrigins should be empty. It includes ${confObj.worker.allowOrigins.any}`);
-  deepEqual(confObj.worker.cores.cpu, undefined, `Mismatch in cores! cpu: ${confObj.worker.cores.cpu}`);
-  deepEqual(confObj.worker.cores.gpu, undefined, `Mismatch in cores! gpu: ${confObj.worker.cores.gpu}`);
-  deepEqual(confObj.worker.utilization, { cpu: 1, gpu: 0.75 }, `Mismatch in utilization! cpu: ${confObj.worker.utilization.cpu}, gpu: ${confObj.worker.utilization.gpu}`);
-  deepEqual(confObj.worker.computeGroups, [], `computeGroups should be an empty array it includes ${confObj.worker.computeGroups}`);
-  deepEqual(confObj.worker.maxSandboxes, undefined,  `Mismatch! maxSandboxes:${confObj.worker.maxSandboxes}`);
-  deepEqual(confObj.worker.jobAddresses, undefined, `jobAddresses should be false but it is ${confObj.worker.jobAddresses}`);
+  strictEqual(confObj.paymentAddress, ks.address.valueOf(), `${confObj.paymentAddress} payment address is not correct`);
+  strictEqual(confObj.leavePublicGroup, undefined, `leaveComputeGroup is ${confObj.leavePublicGroup}`);
+  deepEqual(confObj.allowOrigins.any, [], `allowOrigins should be empty. It includes ${confObj.allowOrigins.any}`);
+  deepEqual(confObj.utilization, { cpu: 1, gpu: 0.75 }, `Mismatch in utilization! cpu: ${confObj.utilization.cpu}, gpu: ${confObj.utilization.gpu}`);
+  deepEqual(confObj.jobIds, undefined, `jobIds should be false but it is ${confObj.jobIds}`);
 
   console.log('Done.');
 }

package/tests/worker-with-options.simple

@@ -40,7 +40,7 @@ async function main()
       '--maxSandboxes', '5',
       '--job-id', '09fd0909Bd93f7bad915726C3AB4',
       '--join', `joinKey1,joinHash1`, '-g', 'joinKey2,joinHash2',
-      '--dumpConfig' ],
+      '--showConfig' ],
     {
       encoding: 'utf-8',
       env: {
@@ -51,14 +51,14 @@ async function main()
   );
   
   const confObj = JSON.parse(res.stdout);
-  strictEqual(confObj.worker.paymentAddress, '0x9fd0909Bd93f7bad915726C3AB499391384b8b4b', `${confObj.worker.paymentAddress} payment address is not corrrect`);
-  strictEqual(confObj.worker.leavePublicGroup, true, `leaveComputeGroup is ${confObj.worker.leavePublicGroup}`);
-  deepEqual(confObj.worker.allowOrigins.any.includes('test-url'), true, `allowOrigins only includes ${confObj.worker.allowOrigins.any}!`);
-  deepEqual(confObj.worker.cores, { cpu: 3, gpu: 1 }, `Mismatch in cores! cpu: ${confObj.worker.cores.cpu}, gpu: ${confObj.worker.cores.gpu}`);
-  deepEqual(confObj.worker.utilization, { cpu: 0.1, gpu: 0.5 }, `Mismatch in utilization! cpu: ${confObj.worker.utilization.cpu}, gpu: ${confObj.worker.utilization.gpu}`);
-  deepEqual(confObj.worker.computeGroups, [{ joinKey: 'joinKey1', joinSecret: 'joinHash1' }, { joinKey: 'joinKey2', joinSecret: 'joinHash2' }], 'Mismatch in computeGroups')
-  deepEqual(confObj.worker.maxSandboxes, 5, `Mismatch! maxSandboxes:${confObj.worker.maxSandboxes}`);
-  deepEqual(confObj.worker.jobAddresses, ['09fd0909Bd93f7bad915726C3AB4'], `Mismatch  in job addresses! jobAddresses:${confObj.worker.jobAddresses}`);
+  strictEqual(confObj.paymentAddress, '0x9fd0909Bd93f7bad915726C3AB499391384b8b4b', `${confObj.paymentAddress} payment address is not corrrect`);
+  strictEqual(confObj.leavePublicGroup, true, `leaveComputeGroup is ${confObj.leavePublicGroup}`);
+  deepEqual(confObj.allowOrigins.any.includes('test-url'), true, `allowOrigins only includes ${confObj.allowOrigins.any}!`);
+  deepEqual(confObj.cores, { cpu: 3, gpu: 1 }, `Mismatch in cores! cpu: ${confObj.cores.cpu}, gpu: ${confObj.cores.gpu}`);
+  deepEqual(confObj.utilization, { cpu: 0.1, gpu: 0.5 }, `Mismatch in utilization! cpu: ${confObj.utilization.cpu}, gpu: ${confObj.utilization.gpu}`);
+  deepEqual(confObj.computeGroups, [{ joinKey: 'joinKey1', joinSecret: 'joinHash1' }, { joinKey: 'joinKey2', joinSecret: 'joinHash2' }], 'Mismatch in computeGroups')
+  deepEqual(confObj.maxSandboxes, 5, `Mismatch! maxSandboxes:${confObj.maxSandboxes}`);
+  deepEqual(confObj.jobIds, ['09fd0909Bd93f7bad915726C3AB4'], `Mismatch  in job ids! jobIds:${confObj.jobIds}`);
 
   console.log('Done.');
 }