dcp-client 4.1.13 → 4.1.17

package/generated/sandbox-definitions.json

@@ -1 +1 @@
-{"browser":["deny-node","kvin/kvin.js","script-load-wrapper","primitive-timers","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"node":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","primitive-timers","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"native":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"webGpuNative":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","webgpu-worker-environment.js","bootstrap"],"nodeTesting":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","primitive-timers","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"],"testing":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"]}
+{"browser":["deny-node","kvin/kvin.js","script-load-wrapper","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"node":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"native":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"webGpuNative":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","webgpu-worker-environment.js","bootstrap"],"nodeTesting":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"],"testing":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"]}

package/index.js

@@ -29,6 +29,7 @@ const os      = require('os');
 const fs      = require('fs')
 const path    = require('path');
 const process = require('process');
+const kvin    = require('kvin');
 const moduleSystem = require('module');
 const { spawnSync } = require('child_process');
 const { createContext, runInContext } = require('vm');
@@ -690,7 +691,6 @@ exports.initSync = function dcpClient$$initSync() {
       finalBundleCode = exports.fetchSync(finalBundleURL);
     } catch(e) {
       console.error('Error downloading autoUpdate bundle from ' + finalBundleURL);
-      console.log(require('dcp/utils').justFetchPrettyError(e));
       throw e;
     }
   }
@@ -810,7 +810,7 @@ exports.createAggregateConfig = async function dcpClient$$createAggregateConfig(
   if (!aggrConfig.scheduler.configLocation &&
       aggrConfig.scheduler.configLocation !== false) {
     addConfigs(aggrConfig.scheduler, localConfig.scheduler, { 
-      configLocation: new URL(`${aggrConfig.scheduler.location}etc/dcp-config.js`)
+      configLocation: new URL(`${aggrConfig.scheduler.location}etc/dcp-config.kvin`)
     });
   }
 
@@ -822,9 +822,9 @@ exports.createAggregateConfig = async function dcpClient$$createAggregateConfig(
     try {
       debugging() && console.debug(` * Loading configuration from ${aggrConfig.scheduler.configLocation.href}`); 
       remoteConfigCode = await require('dcp/protocol').fetchSchedulerConfig(aggrConfig.scheduler.configLocation);
+      remoteConfigCode = kvin.deserialize(remoteConfigCode);
     } catch(e) {
       console.error('Error: dcp-client::init could not fetch scheduler configuration at', '' + aggrConfig.scheduler.configLocation);
-      console.log(require('dcp/utils').justFetchPrettyError(e));
       throw e;
     }
     if (remoteConfigCode.length === 0)

package/lib/standaloneWorker.js

@@ -93,6 +93,10 @@ function StandaloneWorker(options = {}) {
     socket.setNoDelay(dcpConfig.worker ? dcpConfig.worker.nagle : true);
     socket.connect(port, hostname);
     socket.on('connect', beginSession.bind(this));
+    socket.on('end', () => {
+      connected = false;
+      ee.emit('end')
+    })
     socket.on('error', function saWorker$$socket$errorHandler(e) {
       connecting = false;
       if (!dcpConfig.standaloneWorker.quiet)
@@ -279,7 +283,7 @@ function StandaloneWorker(options = {}) {
   /* Shutdown the stream(s) which are connected to the evaluator */
   shutdown = (e) => {
     if (connectTimer) {
-      clearTimeout(connectTimer);
+      clearBackoffInterval(connectTimer);
       connectTimer = true;
     }
 
@@ -309,6 +313,7 @@ function StandaloneWorker(options = {}) {
    *  type DIE:, which in turn eventuallys triggers shutdown.
    */
   this.terminate = function standaloneWorker$$Worker$terminate () {
+    debugging() && console.log(`312: saw.terminate on Evaluator ${this.serial}`);
     var wrappedMessage = this.serialize({ type: 'die' }) + '\n'
 
     try {
@@ -317,6 +322,11 @@ function StandaloneWorker(options = {}) {
       // Socket may have already been destroyed
     }
 
+    if (connectTimer) {
+      clearBackoffInterval(connectTimer);
+      connectTimer = true;
+    }
+
     /* If DIE: response doesn't arrive in a reasonable time -- clean up */
     dieTimer = setTimeout(shutdown, 7000);
   }

package/libexec/evaluator-node.js

@@ -84,29 +84,34 @@ exports.Evaluator = function Evaluator(inputStream, outputStream, files) {
     codeGeneration: { strings: true, wasm: true },
   });
 
-  for(const file of files) {
-    fd = fs.openSync(file, 'r');
-    if (mmap) {
-      fs.flockSync(fd, 'sh');
-      bootstrapCode = mmap.map(fs.fstatSync(fd).size, mmap.PROT_READ, mmap.MAP_SHARED, fd, 0, mmap.MADV_SEQUENTIAL).toString('utf8');
-    } else {
-      bootstrapCode = fs.readFileSync(fd, 'utf-8');
+  if(files) {
+    for(const file of files) {
+      fd = fs.openSync(file, 'r');
+      if (mmap) {
+        fs.flockSync(fd, 'sh');
+        bootstrapCode = mmap.map(fs.fstatSync(fd).size, mmap.PROT_READ, mmap.MAP_SHARED, fd, 0, mmap.MADV_SEQUENTIAL).toString('utf8');
+      } else {
+        bootstrapCode = fs.readFileSync(fd, 'utf-8');
+      }
+      fs.closeSync(fd);
+
+      vm.runInContext(bootstrapCode, this.sandboxGlobal, {
+        filename: path.basename(file),
+        lineOffset: 0,
+        columnOffset: 0,
+        contextName: 'Evaluator #' + this.id,
+        contextCodeGeneration: {
+          wasm: true,
+          strings: true
+        },
+        displayErrors: true,
+        timeout: 3600 * 1000,   /* gives us our own event loop; this is max time for one pass run-to-completion */
+        breakOnSigInt: true     /* also gives us our own event loop */
+      });
     }
-    fs.closeSync(fd);
-
-    vm.runInContext(bootstrapCode, this.sandboxGlobal, {
-      filename: path.basename(file),
-      lineOffset: 0,
-      columnOffset: 0,
-      contextName: 'Evaluator #' + this.id,
-      contextCodeGeneration: {
-        wasm: true,
-        strings: true
-      },
-      displayErrors: true,
-      timeout: 3600 * 1000,   /* gives us our own event loop; this is max time for one pass run-to-completion */
-      breakOnSigInt: true     /* also gives us our own event loop */
-    });
+  } else {
+    console.error('There are no files to run in the node evaluator --  this is ok in the presence of other errors.');
+    process.exit(1);
   }
 
   /* Pass any new data on the input stream to the onreadln()

package/libexec/sandbox/access-lists.js

@@ -84,6 +84,7 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, (ri
     'Set',
     'setInterval',
     'setTimeout',
+    'setImmediate',
     'sleep',
     'String',
     'Symbol',
@@ -560,11 +561,11 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, (ri
 
   addEventListener('message', async (event) => {
     try {
-      if (event.data.request === 'applyRequirements') {
+      if (event.request === 'applyRequirements') {
         // This event is fired when the worker is initialized with job requirements,
         // apply restrictions to the environment based on the requirements.
         // Assume the scheduler gave us a nicely-shaped req object.
-        const requirements = event.data.requirements;
+        const requirements = event.requirements;
         blockList.OffscreenCanvas = !requirements.environment.offscreenCanvas;
         applyAllAccessLists();
 

package/libexec/sandbox/bootstrap.js

@@ -22,11 +22,11 @@ self.wrapScriptLoading({ scriptName: 'bootstrap', finalScript: true }, (ring2Pos
   addEventListener('message', async (event) => {
     try {
       var indirectEval = eval // eslint-disable-line
-      if (event.data.request === 'eval') {
+      if (event.request === 'eval') {
         try {
-          let result = await indirectEval(event.data.data, event.data.filename)
+          let result = await indirectEval(event.data, event.filename)
           ring2PostMessage({
-            request: `evalResult::${event.data.msgId}`,
+            request: `evalResult::${event.msgId}`,
             data: result
           })
         } catch (error) {
@@ -42,7 +42,7 @@ self.wrapScriptLoading({ scriptName: 'bootstrap', finalScript: true }, (ring2Pos
             }
           })
         }
-      } else if (event.data.request === 'resetState') {
+      } else if (event.request === 'resetState') {
         // This event is fired when the web worker is about to be reused with another slice
         lastProgress = 0;
         postMessageSentTime = 0;

package/libexec/sandbox/bravojs-env.js

@@ -23,7 +23,7 @@ self.wrapScriptLoading({ scriptName: 'bravojs-env', ringTransition: true }, (rin
   }
   //Listens for postMessage from the sandbox
   addEventListener('message', async (event) => {
-    let message = event.data
+    let message = event
     let indirectEval = eval // eslint-disable-line
     switch (message.request) {
       case 'moduleGroup': /* Outside environment is sending us a module group */

package/libexec/sandbox/calculate-capabilities.js

@@ -173,7 +173,7 @@ self.wrapScriptLoading(
 
     addEventListener('message', async (event) => {
       try {
-        if (event.data.request === 'describe') {
+        if (event.request === 'describe') {
           const capabilities = await getCapabilities();
           ring2PostMessage({
             capabilities,

package/libexec/sandbox/event-loop-virtualization.js

@@ -5,87 +5,71 @@
  *  This gives DCP introspection capability to see how long a job
  *  should take, and how we can pay DCCs accordingly.
  * 
- *  The node, v8, and web worker evaluators have been modified to
- *  provide the following primitive functions onto the global object
- *  when this program is evaluated:
- *  - ontimer             will be invoked by the reactor when there are timers that
- *                        should need servicing based on the information provided to nextTimer().    
- *  - nextTimer()         sets when the next timer will be fired (in ms)
- *  - evalTimer           evaluates using either the actual setImmediate/setTimeout or uses 
- *                        Promise.resolve.then as a fallback
- *  
- *  Once this file has run, the following methods will be
- *  available on the global object for every evaluator:
- *  - setTimeout()        execute callback after minimum timeout time in ms
- *  - clearTimeout()      clear the timeout created by setTimeout
- *  - setInterval()       recurringly execute callback after minimum timeout time in ms
- *  - clearInterval()     clear the interval created by setInterval
- *  - queueMicrotask()    add a microtask to the microtask queue, bypassing 4ms timeout clamping 
+ *  All evaluators have their own implementation of the event loop at this
+ *  point, with corresponding timeout functions for their loop. This file will
+ *  create a wrapper for each of the timeouts, with a virtual event loop
+ *  to control code execution. 
  *
- *  @author     Parker Rowe, parker@kingsds.network
- *  @date       August 2020
+ *              Ryan Saweczko, ryansaweczko@kingsds.network
+ *  @date       January 2022
  * 
- *  @note       Unusual function scoping is done to eliminate spurious symbols
- *              from being accessible from the global object, to mitigate
- *              certain classes of security risks.  The global object here is
- *              the top of the scope chain (ie global object) for all code run
- *              by hosts in this environment.
  */
-/* globals self, ontimer, nextTimer, evalTimer */
+/* globals self */
+
 
 self.wrapScriptLoading({ scriptName: 'event-loop-virtualization' }, (ring0PostMessage) => {
-  let lastTimeStamp;
-  
-  if (typeof self.evalTimer === 'undefined'){
-    self.evalTimer = (func) => Promise.resolve().then(func);
-  }
-
-  (function privateScope(ontimer, nextTimer, evalTimer) {
-    const timers = [];
-    timers.serial = 0; /* If this isn't set, it becomes NaN */
-
-    function sortTimers() {
-      timers.sort(function (a, b) { return a.when - b.when; });
+
+  (function privateScope(realSetTimeout, realSetInterval, realSetImmediate, realClearTimeout, realClearInterval, realClearImmediate) {
+    let totalCPUTime = 0;
+    const events = [];
+    events.serial = 0;
+
+    function sortEvents() {
+      events.sort(function (a, b) { return a.when - b.when; });
     }
 
-    /* Fire any timers which are ready to run, being careful not to
-     * get into a recurring timer death loop without reactor mediation.
-     */
-    ontimer(function fireTimerCallbacks() {
-      let now = Date.now();
+    function serviceEvents()
+    {
+      serviceEvents.timeout = null;
+      serviceEvents.nextTimeout = null;
+      serviceEvents.servicing = true;
 
-      sortTimers();
-      for (let i = 0; i < timers.length; ++i) {
-        let timer = timers[i];
-        if (timer.when > now) {
-          break;
-        }
+      const startTime = performance.now();
+      let now = Date.now();
 
-        let measurementWrapper = async () => {
-          if (lastTimeStamp) {
-            ring0PostMessage({
-              request: 'measurement',
-              idle: performance.now() - lastTimeStamp
-            });
+      sortEvents();
+      if (events[0].when <= now)
+      {
+        const event = events.shift();
+        if (event.eventType === 'timer')
+        {
+          serviceEvents.executingTimeout = realSetTimeout(event.fn, 0, event.args);
+          if (event.recur)
+          {
+            event.when = Date.now() + event.recur;
+            events.push(event);
+            sortEvents();
           }
-
-          lastTimeStamp = performance.now();
-          await timer.fn();
-          lastTimeStamp = performance.now();
         }
-        Promise.resolve().then(measurementWrapper);
+        // Can add handles for events to the event loop as needed (ie messages)
+      }
 
-        if (timer.recur) {
-          timer.when = Date.now() + timer.recur;
-        } else {
-          timers.splice(i--, 1);
+      // Measure the time on the event loop after everything has executed
+      serviceEvents.measurerTimeout = realSetTimeout(endOfRealEventCycle,1);
+      function endOfRealEventCycle()
+      {
+        serviceEvents.servicing = false;
+        const endTime = performance.now();
+        totalCPUTime += endTime - startTime;
+
+        // Set timeout to rerun this function if there are events remaining that just can't be used yet
+        if (events.length > 0)
+        {
+          serviceEvents.nextTimeout = events[0].when
+          serviceEvents.timeout = realSetTimeout(serviceEvents, events[0].when - Date.now());
         }
       }
-      if (timers.length) {
-        sortTimers();
-        nextTimer(timers[0].when);
-      }
-    });
+    }
 
     /** Execute callback after at least timeout ms. 
      * 
@@ -94,7 +78,8 @@ self.wrapScriptLoading({ scriptName: 'event-loop-virtualization' }, (ring0PostMe
      *  @param    arg               array of arguments to be applied to the callback function
      *  @returns                    {object} A value which may be used as the timeoutId parameter of clearTimeout()
      */
-    self.setTimeout = function eventLoop$$Worker$setTimeout(callback, timeout, arg) {
+    setTimeout = function eventLoop$$Worker$setTimeout(callback, timeout, arg) {
+      timeout = timeout || 0;
       let timer, args;
       if (typeof callback === 'string') {
         let code = callback;
@@ -112,18 +97,30 @@ self.wrapScriptLoading({ scriptName: 'event-loop-virtualization' }, (ring0PostMe
         callback = () => fn.apply(fn, args);          // apply the arguments to the callback function
       }
 
-      timers.serial = +timers.serial + 1;
+      events.serial = +events.serial + 1;
       timer = {
+        eventType: 'timer',
         fn: callback,
         when: Date.now() + (+timeout || 0),
-        serial: timers.serial,
+        serial: events.serial,
         valueOf: function () { return this.serial; }
       }
-      timers.push(timer);
-      lastTimeStamp = performance.now();
-      if (timer.when <= timers[0].when) {
-        sortTimers();
-        nextTimer(timers[0].when);
+      events.push(timer);
+      sortEvents();
+      if (!serviceEvents.servicing)
+      {
+        if (!serviceEvents.nextTimeout)
+        {
+          realSetTimeout(serviceEvents, events[0].when - Date.now());
+        }
+        else
+        {
+          if (serviceEvents.nextTimeout > events[0].when)
+          {
+            realClearTimeout(serviceEvents.timeout);
+            realSetTimeout(serviceEvents, events[0].when - Date.now())
+          }
+        }
       }
       return timer;
     }
@@ -133,42 +130,38 @@ self.wrapScriptLoading({ scriptName: 'event-loop-virtualization' }, (ring0PostMe
      * 
      *  @param    timeoutId         {object} The value, returned from setTimeout(), identifying the timer.
      */
-    self.clearTimeout = function eventLoop$$Worker$clearTimeout(timeoutId) {
-      if (typeof timeoutId === "object") {
-        let i = timers.indexOf(timeoutId);
-        if (i != -1) {
-          timers.splice(i, 1);
-
-          /* if there is a timer at the top of the timers list, set that to be the nextTimer to fire
-           * otherwise, tell the event loop that there are no more timers to fire, and end the loop accordingly.
-           * this fixes a bug where you clear a timeout, but the program still waits that time before ending,
-           * despite never calling the callback function
-           * 
-           * for example:
-           * const timeout = setTimeout(() => console.log("hi"), 10000);
-           * clearTimeout(timeout); 
-           * 
-           * used to still wait 10 seconds before closing the program, despite never printing hi to the console
-           */
-          if (timers.length) {
-            nextTimer(timers[0].when);
-          }
-          else {
-            nextTimer(0);
+    clearTimeout = function eventLoop$$Worker$clearTimeout(timeoutId)
+    {
+      function checkService()
+      {
+        if (!serviceEvents.servicing)
+        {
+          if (events.length)
+          {
+            realClearTimeout(serviceEvents.timeout);
+            realSetTimeout(serviceEvents, events[0].when - Date.now())
           }
+          else
+            realClearTimeout(serviceEvents.timeout);
         }
-      } else if (typeof timeoutId === "number") { /* slow path - object has been reinterpreted in terms of valueOf() */
-        for (let i = 0; i < timers.length; i++) {
-          if (timers[i].serial === timeoutId) {
-            timers.splice(i, 1);
-
-            if (timers.length) {
-              nextTimer(timers[0].when);
-            }
-            else {
-              nextTimer(0);
-            }
-
+      }
+      if (typeof timeoutId === "object")
+      {
+        let i = events.indexOf(timeoutId);
+        if (i !== -1)
+          events.splice(i, 1);
+        if (i === 0)
+          checkService()
+      }
+      else if (typeof timeoutId === "number")
+      { /* slow path - object has been reinterpreted in terms of valueOf() */
+        for (let i = 0; i < events.length; i++)
+        {
+          if (events[i].serial === timeoutId)
+          {
+            events.splice(i, 1);
+            if (i === 0)
+              checkService()
             break;
           }
         }
@@ -182,41 +175,63 @@ self.wrapScriptLoading({ scriptName: 'event-loop-virtualization' }, (ring0PostMe
      *  @param    arg               array of arguments to be applied to the callback function
      *  @returns                    {object} A value which may be used as the intervalId paramter of clearInterval()
      */
-    self.setInterval = function eventLoop$$Worker$setInterval(callback, interval, arg) {
-      let timer = self.setTimeout(callback, +interval || 0, arg);
+    setInterval = function eventLoop$$Worker$setInterval(callback, interval, arg)
+    {
+      let timer = setTimeout(callback, +interval || 0, arg);
       timer.recur = interval;
       return timer;
     }
+    /** Execute callback after 0 ms, immediately when the event loop allows.
+     * 
+     *  @param    callback          {function} Callback function to fire after a minimum callback time
+     *  @param    arg               array of arguments to be applied to the callback function
+     *  @returns                    {object} A value which may be used as the intervalId paramter of clearImmediate()
+     */
+     setImmediate = function eventLoop$$Worker$setImmediate(callback, arg) {
+      let timer = setTimeout(callback, 0, arg);
+      return timer;
+    }
 
     /** Remove an interval timer from the list of pending interval timers, regardless of its current
      *  status. (Same as clearTimeout)
      *
      *  @param    intervalId         {object} The value, returned from setInterval(), identifying the timer.
      */
-    self.clearInterval = self.clearTimeout;
+    clearInterval = clearTimeout;
+    clearImmediate = clearTimeout
 
     /** queues a microtask to be executed at a safe time prior to control returning to the event loop
      * 
      *  @param    callback          {function} Callback function to fire
      */
-    self.queueMicrotask = function eventLoop$$Worker$queueMicrotask(callback) {
+    queueMicrotask = function eventLoop$$Worker$queueMicrotask(callback) {
       Promise.resolve().then(callback);
     }
 
     function clearAllTimers() {
-      timers.length = 0;
-      nextTimer(0);
-      lastTimeStamp = null;
+      events.length = 0;
+      realClearTimeout(serviceEvents.timeout);
+      realClearTimeout(serviceEvents.measurerTimeout);
+      realClearTimeout(serviceEvents.executingTimeout);
     }
 
     addEventListener('message', async (event) => {
       try {
-        if (event.data.request === 'clearTimers') {
+        if (event.request === 'clearTimers') {
           clearAllTimers();
           ring0PostMessage({
             request: 'clearTimersDone',
           });
         }
+        else if (event.request === 'resetAndGetCPUTime')
+        {
+          const cpuTime = totalCPUTime;
+          totalCPUTime = 0;
+          ring0PostMessage({
+            request: 'totalCPUTime',
+            CPU: cpuTime
+          })
+        }
       } catch (error) {
         ring0PostMessage({
           request: 'error',
@@ -228,10 +243,12 @@ self.wrapScriptLoading({ scriptName: 'event-loop-virtualization' }, (ring0PostMe
         });
       }
     });
-  })(self.ontimer, self.nextTimer, self.evalTimer);
-
-  ontimer = nextTimer = evalTimer = undefined;
-  delete self.ontimer;
-  delete self.nextTimer;
-  delete self.evalTimer;
+  })(self.setTimeout, self.setInterval, self.setImmediate, self.clearTimeout, self.clearInterval, self.clearImmediate);
+
+  self.setTimeout = setTimeout;
+  self.setInterval = setInterval;
+  self.setImmediate = setImmediate;
+  self.clearTimeout = clearTimeout;
+  self.clearInterval = clearInterval;
+  self.clearImmediate = clearImmediate;
 });