dcp-client 4.2.6 → 4.2.9

package/generated/sandbox-definitions.json

@@ -1 +1 @@
-{"browser":["deny-node","kvin/kvin.js","script-load-wrapper","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"node":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"native":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"webGpuNative":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","webgpu-worker-environment.js","bootstrap"],"nodeTesting":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"],"testing":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","event-loop-virtualization","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"]}
+{"browser":["deny-node","kvin/kvin.js","script-load-wrapper","wrap-event-listeners","event-loop-virtualization","gpu-timers","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"node":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","event-loop-virtualization","gpu-timers","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"native":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","event-loop-virtualization","webgpu-worker-environment","gpu-timers","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"webGpuNative":["deny-node","kvin/kvin.js","bootstrap"],"nodeTesting":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","event-loop-virtualization","gpu-timers","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"],"testing":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","event-loop-virtualization","webgpu-worker-environment","gpu-timers","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"]}

package/index.js

@@ -32,10 +32,11 @@ const process = require('process');
 const kvin    = require('kvin');
 const moduleSystem = require('module');
 const { spawnSync } = require('child_process');
-const { createContext, runInContext } = require('vm');
+const vm = require('vm');
 
 exports.debug = false;
 let initInvoked = false; /* flag to help us detect use of Compute API before init */
+let hadOriginalDcpConfig = globalThis.hasOwnProperty('dcpConfig'); /* flag to determine if the user set their own dcpConfig global variable before init */
 
 function debugging(what = 'dcp-client') {
   const debugSyms = []
@@ -107,6 +108,30 @@ const bundleSandbox = {
   },
 };
 
+function runSandboxedCode(sandbox, code, options)
+{
+  if (process.env.DCP_CLIENT_LEGACY_CONTEXT_SANDBOXING)
+  {
+    const script = new vm.Script(code, options);
+    return script.runInNewContext(vm.createContext(sandbox), options);
+  }
+
+  if (typeof runSandboxedCode.extraGlobalProps === 'undefined')
+    runSandboxedCode.extraGlobalProps = {};
+
+  for (let prop in sandbox)
+  {
+    if (!globalThis.hasOwnProperty(prop) || runSandboxedCode.extraGlobalProps.hasOwnProperty(prop))
+    {
+      globalThis[prop] = sandbox[prop];
+      runSandboxedCode.extraGlobalProps[prop] = true; /* Memoize global prop list mutation to allow re-mutation */
+    }
+  }
+  
+  const script = new vm.Script(code, options);
+  return script.runInThisContext(options);
+}
+
 /** Evaluate a file in a sandbox without polluting the global object.
  *  @param      filename        {string}    The name of the file to evaluate, relative to
  *  @param      sandbox         {object}    A sandbox object, used for injecting 'global' symbols as needed
@@ -114,7 +139,6 @@ const bundleSandbox = {
  */
 function evalScriptInSandbox(filename, sandbox, olFlag) {
   var code
-  var context = createContext(sandbox)
   try {
     code = fs.readFileSync(path.resolve(distDir, filename), 'utf-8')
     if (olFlag)
@@ -126,7 +150,8 @@ function evalScriptInSandbox(filename, sandbox, olFlag) {
     throw e
   }
 
-  return runInContext(code, context, filename, 0);
+
+  return runSandboxedCode(sandbox, code, { filename, lineNumber: 0 });
 }
 
 /** Evaluate code in a secure sandbox; in this case, the code is the configuration
@@ -134,18 +159,12 @@ function evalScriptInSandbox(filename, sandbox, olFlag) {
  *  during config file processing.
  *
  *  @param      code     {string}        The code to eval
- *  @param      context  {object}        An object that has been initialized as a context
- *                                       that will act like the context's global object
+ *  @param      sandbox  {object}        A sandbox object, used for injecting 'global' symbols as needed
  *  @param      filename {string}        The name of the file we're evaluating for stack-
  *                                       trace purposes.
  */
-function evalStringInSandbox(
-  code,
-  sandbox,
-  filename = '(dcp-client$$evalStringInSandbox)',
-) {
-  const context = createContext(sandbox);
-
+function evalStringInSandbox(code, sandbox, filename = '(dcp-client$$evalStringInSandbox)')
+{
   /**
    * Remove comments and then decide if this config file contains an IIFE. If
    * not, we need to wrap it as an IIFE to avoid "SyntaxError: Illegal return
@@ -167,7 +186,7 @@ function evalStringInSandbox(
    * printed.
    */
   try {
-    result = runInContext(code, context, {
+    result = runSandboxedCode(sandbox, code, {
       filename,
       lineOffset,
       /**
@@ -301,7 +320,7 @@ function addConfig (existing, neo) {
         existing[prop] = new (existing[prop].constructor)(neo[prop]);
       continue;
     }
-    if (typeof neo[prop] === 'object' && !Array.isArray(neo[prop]) && ['Function','Object'].includes(neo[prop].constructor.name)) {
+    if (typeof neo[prop] === 'object' && neo[prop] !== null && !Array.isArray(neo[prop]) && ['Function','Object'].includes(neo[prop].constructor.name)) {
       if (typeof existing[prop] === 'undefined') {
         existing[prop] = {}
       }
@@ -575,7 +594,7 @@ function initTail(aggrConfig, finalBundleCode, finalBundleURL) {
   require('dcp/dcp-url').patchup(aggrConfig);
 
   /* 3 */
-  if (global.dcpConfig) {
+  if (hadOriginalDcpConfig) {
     /* dcpConfig was defined before dcp-client was initialized: assume dev knows what he/she is doing */
     debugging() && console.debug('Dropping bundle dcp-config in favour of global dcpConfig')
     Object.assign(require('dcp/dcp-config'), global.dcpConfig);

package/libexec/evaluator-node.js

@@ -66,7 +66,7 @@ exports.Evaluator = function Evaluator(inputStream, outputStream, files) {
   /* Add properties to sandbox global scope */
   Object.assign(this.sandboxGlobal, {
     self: this.sandboxGlobal,
-    die: (exitCode) => { this.destroy(); process.exit(exitCode); },
+    die: () => { this.destroy(); },
     writeln: (string)  => { this.writeln(string) },
     onreadln: (handler) => { this.onreadlnHandler = handler },
     setTimeout,

package/libexec/sandbox/access-lists.js

@@ -23,6 +23,8 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'AsyncFunction',
     'Atomics',
     'BigInt',
+    'BigInt64Array',
+    'BigUint64Array',
     'Boolean',
     'Blob',
     'bravojs',
@@ -45,7 +47,6 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'Float32Array',
     'Float64Array',
     'Function',
-    'getWebGLTimer',
     'Headers',
     'Infinity',
     'Int16Array',
@@ -108,7 +109,6 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'WeakSet',
     'WebAssembly',
     'WebGL2RenderingContext',
-    'webGLOffset',
     'WebGLTexture',
     'WorkerGlobalScope',
     // Our own Webgpu symbols
@@ -123,18 +123,6 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'flushLastLog'
   ]);
 
-  // webGL timer getter to be overwritten in calculate-capabilities
-  self.getWebGLTimer = (function monkeypatchWebGL() {
-    let timer = 0;
-    function getTimer() {
-      return timer;
-    }
-    return getTimer;
-  })();
-
-  // Offset time for webGL when sandbox is re-used
-  self.webGLOffset = 0;
-
   // Origin time for performance polyfill
   const pt0 = new Date().getTime(); 
 

package/libexec/sandbox/bootstrap.js

@@ -131,15 +131,16 @@ self.wrapScriptLoading({ scriptName: 'bootstrap', finalScript: true }, function
     return true;
   }
 
-  function workerBootstrap$work$emit(eventName, value) {
-    if (typeof eventName !== 'string') {
-      throw new Error(`Event name passed to work.emit must be a string, not ${eventName}.`);
+  function workerBootstrap$work$emit(customEvent, value) {
+    if (typeof customEvent !== 'string') {
+      throw new Error(`Event name passed to work.emit must be a string, not ${customEvent}.`);
     }
 
     postMessage({
       request: 'emitEvent',
       payload: {
-        eventName,
+        eventName: 'custom',
+        customEvent,
         data: value,
       },
     });

package/libexec/sandbox/bravojs-env.js

@@ -184,12 +184,9 @@ self.wrapScriptLoading({ scriptName: 'bravojs-env', ringTransition: true }, func
   function reportRejectedGPUandTotal (t0) {
     try
     {
-      const webGLTimer = getWebGLTimer;
-      const offset = webGLOffset;
       const total = performance.now() - t0;
-
-      let webGL = webGLTimer() - offset;
-      self.webGLOffset = offset + webGL;
+      const webGL = protectedStorage.getAndResetWebGLTimer();
+      protectedStorage.subtractWebGLTimeFromCPUTime(webGL);
       ring3PostMessage({ request: 'measurement', total, webGL });
     }
     catch (error)
@@ -232,12 +229,9 @@ self.wrapScriptLoading({ scriptName: 'bravojs-env', ringTransition: true }, func
   {
     try
     {
-      const webGLTimer = getWebGLTimer;
-      const offset = webGLOffset;
-      const total = performance.now() - t0;
-
-      let webGL = webGLTimer() - offset;
-      self.webGLOffset = offset + webGL;
+      const total = performance.now() - t0 + 1; /* +1 to ensure we never have "0 second slices" */
+      const webGL = protectedStorage.getAndResetWebGLTimer();
+      protectedStorage.subtractWebGLTimeFromCPUTime(webGL); /* Because webGL is sync but doesn't use CPU */
       ring3PostMessage({ request: 'measurement', total, webGL });
       ring3PostMessage({ request: 'complete', result });
     }
@@ -277,6 +271,7 @@ self.wrapScriptLoading({ scriptName: 'bravojs-env', ringTransition: true }, func
      */
     try { await tryFlushMicroTaskQueue(); } catch(e) {};
     try { flushLastLog(); } catch(e) {};
+    try { protectedStorage.markCPUTimeAsDone(); } catch(e) {};
 
     if (rejection)
       errorCallback(rejection);
@@ -301,6 +296,6 @@ self.wrapScriptLoading({ scriptName: 'bravojs-env', ringTransition: true }, func
      * 1. shorten stack
      * 2. initialize the event loop measurement code
      */
-    setTimeout(() => runWorkFunction_inner(datum, (result) => reportResult(t0, result), (rejection) => reportError(t0, rejection)));
+    protectedStorage.setTimeout(() => runWorkFunction_inner(datum, (result) => reportResult(t0, result), (rejection) => reportError(t0, rejection)));
   }
 }); /* end of fn */

package/libexec/sandbox/calculate-capabilities.js

@@ -91,47 +91,12 @@ self.wrapScriptLoading({ scriptName: 'calculate-capabilities' }, function calcul
 
           // Destroy the WebGL context, from https://www.khronos.org/registry/webgl/extensions/WEBGL_lose_context/
           // "This is the recommended mechanism for applications to programmatically halt their use of the WebGL API."
-          gl.getExtension('WEBGL_lose_context').loseContext();
+          // gl.getExtension('WEBGL_lose_context').loseContext();
 
           bigTexture4096 = textureSize >= 4096;
           bigTexture8192 = textureSize >= 8192;
           bigTexture16384 = textureSize >= 16384;
           bigTexture32768 = textureSize >= 32768;
-
-          // Monkeypatch webGL *after* capability check to verify getContext exists AND not charge every job for GPU checking
-          getWebGLTimer = (function monkeypatchWebGL() {
-            let timer = 0;
-            function getTimer() {
-              return timer;
-            }
-            let oldGetContext = OffscreenCanvas.prototype.getContext;
-            OffscreenCanvas.prototype.getContext = function(type, options){
-              let context = oldGetContext(type, options);
-              for (let key of Object.getOwnPropertyNames(context.__proto__)){
-                if (typeof context[key] === 'function'){
-                  let func = context[key].bind(context);
-                  context[key] = (...args) => {
-                    let startTimer = performance.now();
-                    // Try using .then (assuming async) and falling back to synchronous
-                    try{
-                      ret = func(...args).then((ret)=>{
-                        let timeRes = performance.now() - startTimer;
-                        timer += timeRes;
-                        return ret;
-                      });
-                    }catch(err){
-                      ret = func(...args); 
-                      let timeRes = performance.now() - startTimer;
-                      timer += timeRes;
-                    }
-                    return ret;
-                  };
-                }
-              };  
-              return context;
-            };
-            return getTimer;
-          })();
         }catch(err){
           //it is possible on some machines, that offscreenCanvas is available but canvas.getContext('webgl' or 'webgl2') results in null
           //Any error in this using an extensions should likely result in specifications for that capability being set to false.

package/libexec/sandbox/event-loop-virtualization.js

@@ -32,6 +32,7 @@ self.wrapScriptLoading({ scriptName: 'event-loop-virtualization' }, function eve
       serviceEvents.timeout = null;
       serviceEvents.nextTimeout = null;
       serviceEvents.servicing = true;
+      serviceEvents.sliceIsFinished = false;
 
       const startTime = performance.now();
       let now = Date.now();
@@ -58,15 +59,30 @@ self.wrapScriptLoading({ scriptName: 'event-loop-virtualization' }, function eve
       function endOfRealEventCycle()
       {
         serviceEvents.servicing = false;
+        if (!serviceEvents.sliceIsFinished)
+        {
+          const endTime = performance.now();
+          totalCPUTime += endTime - startTime;
+  
+          // Set timeout to rerun this function if there are events remaining that just can't be used yet
+          if (events.length > 0)
+          {
+            serviceEvents.nextTimeout = events[0].when
+            serviceEvents.timeout = realSetTimeout(serviceEvents, events[0].when - Date.now());
+          }
+        }
+      }
+
+      protectedStorage.markCPUTimeAsDone = function markCPUTimeAsDone()
+      {
         const endTime = performance.now();
         totalCPUTime += endTime - startTime;
+        serviceEvents.sliceIsFinished = true;
+      }
 
-        // Set timeout to rerun this function if there are events remaining that just can't be used yet
-        if (events.length > 0)
-        {
-          serviceEvents.nextTimeout = events[0].when
-          serviceEvents.timeout = realSetTimeout(serviceEvents, events[0].when - Date.now());
-        }
+      protectedStorage.subtractWebGLTimeFromCPUTime = function subtractCPUTime(time)
+      {
+        totalCPUTime -= time;
       }
     }
 
@@ -124,6 +140,9 @@ self.wrapScriptLoading({ scriptName: 'event-loop-virtualization' }, function eve
       return timer;
     }
 
+    /** Ensure our trampoline setTimeout in bravojs-env will have the proper setTimeout, don't allow clients to see or overwrite to prevent measuring time */
+    protectedStorage.setTimeout = setTimeout;
+
     /** Remove a timeout from the list of pending timeouts, regardless of its current
      *  status.
      * 
@@ -215,6 +234,7 @@ self.wrapScriptLoading({ scriptName: 'event-loop-virtualization' }, function eve
       serviceEvents.timeout = null;
       serviceEvents.nextTimeout = null;
       serviceEvents.servicing = false;
+      serviceEvents.sliceIsFinished = false;
     }
 
     addEventListener('message', async (event) => {

package/libexec/sandbox/gpu-timers.js

@@ -0,0 +1,68 @@
+/**
+ *  @file       gpu-timers.js
+ *              Copyright (c) 2018, Kings Distributed Systems, Ltd.  All Rights Reserved. @todo: is this correct lol?
+ *
+ *              This file adds wrappers for webGL and webGPU functions so we can measure their GPU time
+ *
+ *  @author     Ryan Saweczko, ryansaweczko@kingsds.network
+ *  @date       July 2022
+ */
+
+/* global WebGPUWindow GPU */
+// @ts-nocheck
+
+self.wrapScriptLoading({ scriptName: 'gpu-timers' }, function gpuTimers$fn(protectedStorage, ring2PostMessage)
+{
+  /* Default if we don't have webGL */
+  protectedStorage.getAndResetWebGLTimer = function defaultTimer()
+  {
+    return 0;
+  }
+
+  if (self.OffscreenCanvas && new OffscreenCanvas(1,1))
+  {
+    let time = 0;
+    function getAndResetWebGLTimer()
+    {
+      const tmp = time;
+      time = 0;
+      return tmp;
+    }
+    protectedStorage.getAndResetWebGLTimer = getAndResetWebGLTimer;
+
+    /* Factory to wrap a function from a context with a timer */
+    function timeWebGLFactory(context, prop)
+    {
+      let originalFn = context[prop].bind(context);
+      
+      context[prop] = function wrappedWebGLFunction(...args)
+      {
+        var returnValue;
+        const start = performance.now();
+        try
+        {
+          returnValue =  originalFn(...args);
+          time += performance.now() - start;
+        }
+        catch(e)
+        {
+          time += performance.now() - start;
+          throw e;
+        }
+        return returnValue;
+      }
+    }
+  
+    /* Update all functions on the OffscreenCanvas getContext prototype to have timers */
+    let oldGetContext = OffscreenCanvas.prototype.getContext;
+    OffscreenCanvas.prototype.getContext = function(type, options)
+    {
+      let context = oldGetContext.bind(this)(type, options);
+      for (let key of Object.getOwnPropertyNames(context.__proto__))
+        if (typeof context[key] === 'function')
+          timeWebGLFactory(context, key);
+      return context;
+    };
+  }
+
+});

package/libexec/sandbox/sa-ww-simulation.js

@@ -135,6 +135,7 @@ try {
           serialize = newSerializer.serialize
           deserialize = newSerializer.deserialize
           outMsg = { type: 'nop', success: true }
+          send(outMsg);
           break
         case 'workerMessage':
           // if (inMsg.message.request === 'main') {
@@ -145,6 +146,7 @@ try {
           // }
           emitEvent('message', {data: inMsg.message})
           outMsg.success = true
+          send(outMsg)
           break
         case 'die':
           writeln('DIE: ' + Date())
@@ -157,7 +159,6 @@ try {
         outMsg.success = false
         outMsg.exception = { name: e.name, message: e.message, fileName: e.fileName, lineNumber: e.lineNumber, stack: e.stack }
         outMsg.e = e
-      } finally {
         send(outMsg)
       }
     }) /* receiveLine */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcp-client",
-  "version": "4.2.6",
+  "version": "4.2.9",
   "description": "Core libraries for accessing DCP network",
   "keywords": [
     "dcp"
@@ -52,8 +52,8 @@
   "devDependencies": {
     "@kingsds/eslint-config": "1.0.1",
     "eslint": "7.30.0",
-    "peter": "^2.2.0",
-    "express": "^4.17.1"
+    "express": "^4.17.1",
+    "peter": "^2.3.2"
   },
   "peerDependencies": {
     "dcp-worker": "^3.0.0"