dcp-client 4.4.9-0 → 4.4.10-0

package/examples/nodejs/simple-worker.js

@@ -1,15 +1,127 @@
 #! /usr/bin/env node
+/**
+ * @file    simple-worker.js
+ *
+ *          Simple NodeJS application showing how to implement a simple DCP worker using the native evaluator.
+ *          Look at https://gitlab.com/Distributed-Compute-Protocol/dcp-native/-/releases to download the evaluator 
+ *          and start it using one of the dcp-evaluator-* scripts from the dcp-worker npm package.
+ *           
+ *          *********************************** NOTE 1 ***********************************
+ *          Your keystore should be placed in your home directory in .dcp/default.keystore.
+ *          When using the dcp-client API in NodeJS, this keystore will be used for communicating over DCP.
+ *  
+ *          *********************************** NOTE 2 ***********************************
+ *          Executing Job with DCP Worker
+ * 
+ *          Run the following commands in your terminal:
+ *          ```
+ *          npm add --global dcp-worker
+ *          dcp-worker --allowedOrigins http://localhost:<port number>
+ *          ```
+ * 
+ * @author  Wes Garland  <wes@distributive.network>
+ * @author  Kevin Yu     <kevin@distributive.network>
+ * @date    Aug 2019, April 2020, June 2024
+*/
 
-global.Promise = Promise = require("./promiseDebug").hook()
+'use strict';
 
-async function main() {
-debugger
-  const protocol = require('dcp/protocol')
-  const compute = require('dcp/compute')
-  const numberOfThreads = 4
+/**
+ * Setup event listeners for jobs
+ *
+ * @param {object} job - the job handle object
+ * @returns {void} 
+ */
+function addWorkerEventListeners(worker)
+{
+  // The start event fires after the worker is ready to fetch tasks, but
+  // before the first task has been fetched.
+  worker.on('start', () => {
+    console.log('Worker ready to fetch tasks');
+  });
 
-  compute.work(numberOfThreads)
+  // The fetch event is fired when the worker has fetched a task from the
+  // task distributor.
+  worker.on('fetch', (task) => {
+    if (task.slices)
+      console.log(`Worker has fetched ${Object.keys(task.slices).length} slices`);
+  });
+
+  // The sandbox event is emitted when a new sandbox is created. The event
+  // handler receives as its sole argument an EventEmitter which is used
+  // to emit sandbox events
+  worker.on('sandbox', (sandbox) => {
+    // The progress event is fired to indicate progress throughout a job this is triggered by the progress() call in the work function, however, quickly-repeating calls to progress() may be composited into a single event.
+    sandbox.on('progress', (event) => {
+      console.log('Sandbox progress', event);
+    });
+  });
+
+  // After a result is sent to the result-submitter for consideration
+  worker.on('payment', (event) => {
+    console.log('Work payment', event);
+  });
+
+  // Result event is fired immediately after the worker sends a result to
+  // the the result data sink
+  worker.on('result', (event) => {
+    console.log('Worker has sent a result back to the scheduler');
+  });
+
+  // Worker has stopped working and no new results will be sent to the
+  // result submitter or the result data sink and no more slices will be
+  // returned to the scheduler
+  worker.on('end', () => {
+    console.log('Worker has stopped working');
+  });
+
+  // Log errors
+  worker.on('error', console.error);
+}
+
+/**
+ * Main function to deploy a job with remote work function
+ *
+ * @returns {void} 
+ */
+async function main()
+{
+  const wallet = require('dcp/wallet');
+  const { Worker: DCPWorker } = require('dcp/worker');
+
+  // Bank account funds will deposit into
+  const paymentAddress = (await wallet.get()).address;
+
+  // Retrieve auth keystore object
+  const identity = await wallet.getId();
+
+  // Define stand-alone worker options
+  const sawOptions = {
+    hostname: dcpConfig.evaluator.location.hostname,
+    port: Number(dcpConfig.evaluator.location.port),
+  };
+
+  // Instantiate DCP worker
+  const worker = new DCPWorker(
+    identity,
+    {
+      cores: { cpu: 1 },
+      paymentAddress,
+      sandboxOptions: {
+        SandboxConstructor: require('../../lib/standaloneWorker').workerFactory(sawOptions),
+      },
+      allowOrigins: {
+        any: ['http://localhost:12345', 'http://localhost:12346'],
+      },
+    });
+
+  // Listen for work emitted events
+  addWorkerEventListeners(worker);
+
+
+  // Start the worker
+  await worker.start();
 }
 
-require('dcp-client').init().then(main).finally(() => setImmediate(process.exit))
+require('../../').init().then(main);
 

package/libexec/sandbox/bootstrap.js

@@ -145,7 +145,7 @@ self.wrapScriptLoading({ scriptName: 'bootstrap', finalScript: true }, function
     });
   }
 
-  function workerBootstrap$work$reject(reason = 'false') {
+  function workerBootstrap$work$reject(reason) {
     protectedStorage.workRejectReason = reason; // Memoize reason
     throw Symbol.for('workReject');
   }

package/libexec/sandbox/bravojs-env.js

@@ -359,11 +359,8 @@ prepPyodide`);
     }
 
     if (error === Symbol.for('workReject')) {
-      err['message'] = protectedStorage.workRejectReason;
-      err['name'] = 'EWORKREJECT';
-      err['stack'] = 'Slice was rejected in the sandbox by work.reject'
       reportTimes(metrics);
-      ring3PostMessage({ request: 'workError', error: err });
+      ring3PostMessage({ request: 'workReject', reason: protectedStorage.workRejectReason });
     }
     else
       ring3PostMessage({request: 'workError', error: err});

package/libexec/sandbox/lift-webgpu.js

@@ -224,13 +224,9 @@ self.wrapScriptLoading({ scriptName: 'lift-webgpu' }, function liftWebGPU$$fn(pr
     }
   }
 
-  // Want to use the original of these after all gpu functions are wrapped.
-  const underlyingGPUQueue = GPUQueue;
-  const underlyingGPUQueueProto = GPUQueue.prototype;
+  // Want to use the submit/onSubmittedWorkDone original functions for timing.
   const underlyingOnSubmittedWorkDone = GPUQueue.prototype.onSubmittedWorkDone;
   const underlyingSubmit = GPUQueue.prototype.submit;
-  const underlyingRequestDevice = GPUAdapter.prototype.requestDevice;
-  const underlyingDestroy = GPUDevice.prototype.destroy;
 
   // some of them will get re-wrapped, that's fine, we always refer to the original function
   const requiredWrappingGPUClasses = [
@@ -254,35 +250,13 @@ self.wrapScriptLoading({ scriptName: 'lift-webgpu' }, function liftWebGPU$$fn(pr
   requiredWrappingGPUClasses.forEach(liftWebGPUPrototype);
 
   let locked = false;
-  const gpuQueueRegistry = [];
+  const submittedDonePromises = [];
   protectedStorage.webGPU = {
     lock: () => { locked = true; },
     unlock: () => { locked = false; },
-    waitAllCommandToFinish: () => { return Promise.allSettled(gpuQueueRegistry.map((q) => q.onSubmittedWorkDone())); },
+    waitAllCommandToFinish: () => { return Promise.allSettled(submittedDonePromises); },
   };
 
-  // currently, the only queue exposed is the default queue
-  GPUAdapter.prototype.requestDevice = async function requestDevice(...args)
-  {
-    const device = await underlyingRequestDevice.call(this, ...args);
-    gpuQueueRegistry.push(device.queue);
-    return device;
-  }
-
-  /**
-   * Redefine GPUQueue constructor in order to add the queue to our webGPUQueueRegistry.
-   * Need to redefine the `GPUQueue` function for instanceof checks to work with the changed constructor,
-   * then add the original prototype to the new object. 
-   */
-  GPUQueue = function GPUQueue$$constructor(...args)
-  {
-    const queue = new underlyingGPUQueue(...args);
-    gpuQueueRegistry.push(queue);
-    return queue;
-  }
-  GPUQueue.prototype = underlyingGPUQueueProto;
-  GPUQueue.prototype.constructor = GPUQueue;
-
   // our submit keeps a global tracker of all submissions, so we can track the time of each submission 
   GPUQueue.prototype.submit = function submit(commandBuffers)
   {
@@ -291,19 +265,16 @@ self.wrapScriptLoading({ scriptName: 'lift-webgpu' }, function liftWebGPU$$fn(pr
     underlyingSubmit.call(this, commandBuffers);
 
     const submitTime = performance.now();
-    underlyingOnSubmittedWorkDone.call(this).then(() => {
+    const submitDonePromise = underlyingOnSubmittedWorkDone.call(this).then(() => {
+      const idx = submittedDonePromises.indexOf(submitDonePromise);
+      submittedDonePromises.splice(idx);
+
       const completedAt = performance.now();
       const duration = new TimeInterval();
       duration.overrideInterval(submitTime, completedAt);
       webGPUTimer.push(duration);
     });
+    submittedDonePromises.push(submitDonePromise);
   }
 
-  GPUDevice.prototype.destroy = function destroy()
-  {
-    const idx = gpuQueueRegistry.indexOf(this.queue);
-    gpuQueueRegistry.splice(idx);
-    underlyingDestroy.call(this);
-  }
 });
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcp-client",
-  "version": "4.4.9-0",
+  "version": "4.4.10-0",
   "description": "Core libraries for accessing DCP network",
   "keywords": [
     "dcp"

package/examples/nodejs/deploy-job.mjs

@@ -1,28 +0,0 @@
-// How to deploy a job using ES6 modules.
-
-import { init } from '../../index.js';
-
-const { compute } = await init();
-
-const inputSet = [1, 2, 3, 4];
-
-const workFunction = (input) => {
-  progress();
-  return input;
-};
-
-const job = compute.for(inputSet, workFunction);
-
-job.public.name = 'ESM example';
-
-job.on('accepted', ({ job: { id } }) => {
-  console.log('Job accepted with id', id);
-});
-
-job.on('result', ({ result }) => {
-  console.log('Received a result:', result);
-});
-
-const results = await job.exec();
-
-console.log(results);

package/examples/nodejs/events.js

@@ -1,67 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * @file        events.js
- *              Sample node application showing how to deploy a DCP job whilst receiving
- *              events describing the current state of the job, processing results
- *              as they are received, and so on.
- *
- *              Note: Your keystore should be placed in your home directory in .dcp/default.keystore.
- *              When using the dcp-client API in NodeJS, this keystore will be used for communicating over DCP.
- *
- * @author      Wes Garland, wes@kingsds.network
- * @date        Aug 2019, April 2020
- */
-
-const SCHEDULER_URL = new URL('https://scheduler.distributed.computer');
-
-/** Main program entry point */
-async function main() {
-  const compute = require('dcp/compute');
-  const wallet = require('dcp/wallet');
-  let startTime;
-
-  const job = compute.for(
-    ['red', 'green', 'yellow', 'blue', 'brown', 'orange', 'pink'],
-    (colour) => {
-      progress(0);
-      let sum = 0;
-      for (let i = 0; i < 10000000; i += 1) {
-        progress(i / 10000000);
-        sum += Math.random();
-      }
-      return { colour, sum };
-    },
-  );
-
-  job.on('accepted', () => {
-    console.log(` - Job accepted by scheduler, waiting for results`);
-    console.log(` - Job has id ${job.id}`);
-    startTime = Date.now();
-  });
-
-  job.on('readystatechange', (arg) => {
-    console.log(`new ready state: ${arg}`);
-  });
-
-  job.on('result', (ev) => {
-    console.log(
-      ` - Received result for slice ${ev.sliceNumber} at ${
-        Math.round((Date.now() - startTime) / 100) / 10
-      }s`,
-    );
-    console.log(` * Wow! ${ev.result.colour} is such a pretty colour!`);
-  });
-
-  job.public.name = 'events example, nodejs';
-
-  const ks = await wallet.get(); /* usually loads ~/.dcp/default.keystore */
-  job.setPaymentAccountKeystore(ks);
-  const results = await job.exec(compute.marketValue);
-  console.log('results=', Array.from(results));
-}
-
-/* Initialize DCP Client and run main() */
-require('../..')
-  .init(SCHEDULER_URL)
-  .then(main)

package/examples/nodejs/minimal.js

@@ -1,39 +0,0 @@
-#! /usr/bin/env node
-/** 
- * @file      minimal.html
- *
- *            Sample NodeJS application showing how to deploy a minimal DCP job.
- * 
- *            Note: Your keystore should be placed in your home directory in .dcp/default.keystore.
- *            When using the dcp-client API in NodeJS, this keystore will be used for communicating over DCP.
- * 
- * @author    Wes Garland, wes@kingsds.network
- * @date      Aug 2019, April 2020
- */ 
-
-const SCHEDULER_URL =  new URL('https://scheduler.distributed.computer') ;
-
-async function main() {
-  const compute = require('dcp/compute');
-
-  let job = compute.for(1, 10,
-    function(i) {
-      progress(0);
-      let sum = 0;
-      for (let i =0; i < 10000000; i++) {
-        sum += Math.random();
-        progress(i/10000000);
-      }
-      return i*3
-    }
-  )
-
-  job.on('result',function(ev) {
-    console.log('received result', ev.result);
-  }) 
-
-  job.public.name = 'minimal example, nodejs';                                
-  await job.exec(compute.marketValue);
-}
-
-require('dcp-client').init(SCHEDULER_URL, true).then(main);

package/examples/nodejs/remote-data/dataPattern.js

@@ -1,92 +0,0 @@
-#! /usr/bin/env node
-/*
- * @file   dataPattern.js - An example on using RemoteDataPattern class that works perfectly for
- *                          slices which their uri's has similar patterns. For more information please refer
- *                          to https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/scheduler/remote-storage.md#data-movement
- * 
- * There are two ways to stringify input data before sending on the wire
- * - JSON: It is a default method
- * - KVIN: if kvin.serialize(inputData)is being used we need to define the Content-Type `res.header("Content-Type", "application/x-kvin")`
- * 
- * Note that to allow workers fetch data from URLs, 
- *  - in the node worker: add `-a 'http://localhost:12345'` at the end of starting worker command.
- * 
- *  - in the localExec: 
- *    ```dcpConfig = require('dcp/dcp-config');
- *       dcpConfig.worker.allowOrigins.any.push('http://localhost:12345');
- *    ```
- *  - in the browser worker: in the console run `dcpConfig.worker.allowOrigins.any.push('http://localhost:12345')` also,
- *    you need to add following lines in the response function in slice.get
- *    ```res.header("Access-Control-Allow-Headers", "content-type");
- *       res.header("Access-Control-Allow-Origin", "*");
- *    ```
- * @author Nazila Akhavan <nazila@kingsds.network>
- * @date   Sep. 2021
- * 
- */
-
-const SCHEDULER_URL = new URL('https://scheduler.distributed.computer');
-const express = require('express');
- 
- /** Main program entry point */
-async function main() {
-  const compute = require('dcp/compute');
-  
-  const app = express();
-  const port1 = 12345;
-  app.get('/hello-world-1.json', (req, res) => {
-    let a = {a:1, b:2, c:3};
-    res.send(a);
-  })
-  app.get('/hello-world-2.json', (req, res) => {
-    let a = {x:1, y:2, z:3};
-    res.send(a);
-  })
-  app.listen(port1, () => {
-    console.log(`port ${port1} is ready!`)
-  })
-  
-  const { RemoteDataPattern } = require('dcp/compute');
-  let remoteData = new RemoteDataPattern('http://localhost:12345/hello-world-{slice}.json',2)
-  
-  let workerFunction = `async function(c){
-    let sum = 0;
-    for (let i = 0; i < 10000000; i += 1) {
-      progress(i / 10000000);
-      sum += Math.random();
-    }
-    return c;
-  }`
-
-  const job = compute.for(
-    remoteData,
-    workerFunction,
-  );
-
-  job.on('accepted', () => {
-    console.log(` - Job accepted by scheduler, waiting for results`);
-    console.log(` - Job has id ${job.id}`);
-  });
-
-  job.on('readystatechange', (arg) => {
-    console.log(`new ready state: ${arg}`);
-  });
-
-  job.on('result', (ev) => {
-    console.log(ev);
-  });
-
-  job.public.name = 'RemoteDataPattern-example';
-
-  const results = await job.exec(compute.marketValue);
-
-  console.log('results=', Array.from(results));
- }
- 
- /* Initialize DCP Client and run main() */
-require('../../..')
-  .init(SCHEDULER_URL)
-  .then(main)
-  .catch(console.error)
-  .finally(process.exit);
-  

package/examples/nodejs/remote-data/dataSet.js

@@ -1,98 +0,0 @@
-#! /usr/bin/env node
-/**
- * @file   dataSet.js - An example on using RemoteDataSet class
- *                      For more information please refer to https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/scheduler/remote-storage.md#data-movement
- * 
- * @author Nazila Akhavan <nazila@kingsds.network>
- * @date   Sep. 2021
- * 
- * 
- * There are two ways to stringify input data before sending on the wire
- * - JSON: It is a default method
- * - KVIN: if kvin.serialize(inputData)is being used we need to define the Content-Type `res.header("Content-Type", "application/x-kvin")`
- * 
- * Note that to allow workers fetch data from URLs, 
- *  - in the node worker: add `-a 'http://localhost:12345' 'http://localhost:12346'` at the end of starting worker command.
- * 
- *  - in the localExec: 
- *    ```dcpConfig = require('dcp/dcp-config');
- *       dcpConfig.worker.allowOrigins.any.push('http://localhost:12345', 'http://localhost:12346');
- *    ```
- *  - in the browser worker: in the console run `dcpConfig.worker.allowOrigins.any.push('http://localhost:12345', 'http://localhost:12346')` also,
- *    you need to add following lines in the response function in slice.get
- *    ```res.header("Access-Control-Allow-Headers", "content-type");
- *       res.header("Access-Control-Allow-Origin", "*");
- *    ``` 
- */
-
-const SCHEDULER_URL = new URL('https://scheduler.distributed.computer');
-const express = require('express');
- 
- /** Main program entry point */
-async function main() {
-  const compute = require('dcp/compute');
-  
-  const slice1 = express();
-  const port1 = 12345;
-  slice1.get('/hello.json', (req, res) => {
-    let a = "Hello world!";
-    res.send(a);
-  })
-  slice1.listen(port1, () => {
-    console.log(`port ${port1} is ready!`)
-  })
-
-  const slice2 = express();
-  const port2 = 12346;
-  slice2.get('/', (req, res) => {
-    let a = {x:1, y:2, z:3};
-    res.send(a);
-  })
-  slice2.listen(port2, () => {
-    console.log(`port ${port2} is ready!`)
-  })
-
-  const { RemoteDataSet } = require('dcp/compute');
-  let remoteDataSet = new RemoteDataSet([`http://localhost:12345/hello.json`, `http://localhost:12346`])
-  
-  let workerFunction = `async function(c){
-    let sum = 0;
-    for (let i = 0; i < 10000000; i += 1) {
-      progress(i / 10000000);
-      sum += Math.random();
-    }
-    return c;
-  }`
-
-  const job = compute.for(
-    remoteDataSet,
-    workerFunction,
-  );
-
-  job.on('accepted', () => {
-    console.log(` - Job accepted by scheduler, waiting for results`);
-    console.log(` - Job has id ${job.id}`);
-  });
-
-  job.on('readystatechange', (arg) => {
-    console.log(`new ready state: ${arg}`);
-  });
-
-  job.on('result', (ev) => {
-    console.log(ev);
-  });
-
-  job.public.name = 'RemoteDataSet-example';
- 
-  const results = await job.exec(compute.marketValue);
-
-  console.log('results=', Array.from(results));
- }
- 
- /* Initialize DCP Client and run main() */
-require('../../..')
-  .init(SCHEDULER_URL)
-  .then(main)
-  .catch(console.error)
-  .finally(process.exit);
-  

package/examples/nodejs/remote-data/dataURL.js

@@ -1,99 +0,0 @@
-#! /usr/bin/env node
-/**
- * @file   dataURL.js - An example on deploying a job using URL for input data
- * 
- * @author Nazila Akhavan <nazila@kingsds.network>
- * @date   Sep. 2021
- * 
- * 
- * There are two ways to stringify input data before sending on the wire
- * - JSON: It is a default method
- * - KVIN: if kvin.serialize(inputData)is being used we need to define the Content-Type `res.header("Content-Type", "application/x-kvin")`
- * 
- * Note that to allow workers fetch data from URLs, 
- *  - in the node worker: add `-a 'http://localhost:12345' 'http://localhost:12346'` at the end of starting worker command.
- * 
- *  - in the localExec: 
- *    ```dcpConfig = require('dcp/dcp-config');
- *       dcpConfig.worker.allowOrigins.any.push(['http://localhost:12345' 'http://localhost:12346']);
- *    ```
- *  - in the browser worker: in the console run `dcpConfig.worker.allowOrigins.any.push('http://localhost:12345', 'http://localhost:12346')` also,
- *    you need to add following lines in the response function in slice.get
- *    ```res.header("Access-Control-Allow-Headers", "content-type");
- *       res.header("Access-Control-Allow-Origin", "*");
- *    ``` 
- */
-
-const SCHEDULER_URL = new URL('https://scheduler.distributed.computer');
-const kvin = require('kvin');
-const express = require('express');
- 
- /** Main program entry point */
-async function main() {
-  const compute = require('dcp/compute');
-  
-  /* example of using the default */
-  const slice1 = express();
-  const port1 = 12345;
-  slice1.get('/', (req, res) => {
-    let a = {x:1, y:2};
-    res.send(a);
-  })
-  slice1.listen(port1, () => {
-    console.log(`port ${port1} is ready!`)
-  })
-
-  /* example of using the kvin */
-  const slice2 = express();
-  const port2 = 12346;
-  slice2.get('/', (req, res) => {
-    let a = {x:1, y:2};
-    res.header("Content-Type", "application/x-kvin");
-    res.send(kvin.serialize(a));
-  })
-  slice2.listen(port2, () => {
-    console.log(`port ${port2} is ready!`);
-  })
-
-  let dcp_inputDataArray_urls = [new URL(`http://localhost:${port1}/`) , new URL(`http://localhost:${port2}/`) ];
-  
-  let workerFunction = `async function(c){
-    let sum = 0;
-    for (let i = 0; i < 10000000; i += 1) {
-      progress(i / 10000000);
-      sum += Math.random();
-    }
-    return c;
-  }`
-
-  const job = compute.for(
-    dcp_inputDataArray_urls,
-    workerFunction,
-  );
-
-  job.on('accepted', () => {
-    console.log(` - Job accepted by scheduler, waiting for results`);
-    console.log(` - Job has id ${job.id}`);
-  });
-
-  job.on('readystatechange', (arg) => {
-    console.log(`new ready state: ${arg}`);
-  });
-
-  job.on('result', (ev) => {
-    console.log(ev);
-  });
-
-  job.public.name = 'URL-data-example';
-
-  const results = await job.exec(compute.marketValue);
-  console.log('results=', Array.from(results));
- }
- 
- /* Initialize DCP Client and run main() */
-require('../../..')
-  .init(SCHEDULER_URL)
-  .then(main)
-  .catch(console.error)
-  .finally(process.exit);
-