dcp-client 4.3.5 → 4.3.7

package/generated/sandbox-definitions.json

@@ -1 +1 @@
-{"browser":["deny-node","kvin/kvin.js","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"node":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"native":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"nodeTesting":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"],"testing":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"]}
+{"browser":["deny-node","kvin/kvin.js","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","worktimes","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","pyodide-core","calculate-capabilities","bootstrap"],"node":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","worktimes","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","pyodide-core","calculate-capabilities","bootstrap"],"native":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","worktimes","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","pyodide-core","calculate-capabilities","bootstrap"],"nodeTesting":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","worktimes","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","pyodide-core","calculate-capabilities","bootstrap","testing.js"],"testing":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","worktimes","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","pyodide-core","calculate-capabilities","bootstrap","testing.js"]}

package/index.js

@@ -373,11 +373,17 @@ function magicView(node, seen)
  *
  * Returns false is the file simply does not exist.
  *
+ * Setting `DCP_CLIENT_ALLOW_INSECURE_CONFIGURATION` to a non-empty value disables
+ * the security check.
+ *
  * @param {string} fullPath    the full path to the file to check
  * @param {object} statBuf     [optional] existing stat buf for the file
  */
 function checkConfigFileSafePerms(fullPath, statBuf)
 {
+  if (process.env.DCP_CLIENT_ALLOW_INSECURE_CONFIGURATION)
+    return true;
+
   const fun = checkConfigFileSafePerms;
 
   if (!fs.existsSync(fullPath))

package/libexec/sandbox/access-lists.js

@@ -12,24 +12,17 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
   const ring1PostMessage = self.postMessage;
   const global = typeof globalThis === 'undefined' ? self : globalThis;
 
-  // aggregated from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects#Reflection
   const allowList = new Set([
-    '__proto__',
-    'addEventListener',
-    'applyAccesslist',
+    // global objects, aggregated from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects#Reflection
+    'AggregateError',
     'Array',
     'ArrayBuffer',
-    'AsyncFunction',
     'Atomics',
     'BigInt',
     'BigInt64Array',
     'BigUint64Array',
     'Boolean',
-    'Blob',
-    'bravojs',
-    'clearInterval',
-    'clearTimeout',
-    'console',
+
     'constructor',
     'DataView',
     'Date',
@@ -41,12 +34,10 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'escape',
     'eval',
     'EvalError',
-    'File',
-    'FileReader',
     'Float32Array',
     'Float64Array',
     'Function',
-    'Headers',
+    'globalThis',
     'Infinity',
     'Int16Array',
     'Int32Array',
@@ -58,7 +49,6 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'Math',
     'module',
     'NaN',
-    'navigator',
     'null',
     'Number',
     'Object',
@@ -67,26 +57,18 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'onmessage',
     'parseFloat',
     'parseInt',
-    'performance',
     'postMessage',
     'Promise',
     'propertyIsEnumerable',
     'Proxy',
-    'pt0',
     'RangeError',
     'ReferenceError',
     'Reflect',
     'RegExp',
-    'removeEventListener',
-    'requestAnimationFrame',
     'require',
     'Response',
-    'self',
     'Set',
-    'setInterval',
-    'setTimeout',
-    'setImmediate',
-    'sleep',
+    'SharedArrayBuffer',
     'String',
     'Symbol',
     'SyntaxError',
@@ -96,7 +78,6 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'toString',
     'TypeError',
     'URIError',
-    'URL',
     'Uint16Array',
     'Uint32Array',
     'Uint8Array',
@@ -106,11 +87,44 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'valueOf',
     'WeakMap',
     'WeakSet',
+    'WeakRef',
+    '__proto__',
+
+    // WorkerGlobalScope symbols, aggregated from https://developer.mozilla.org/en-US/docs/Web/API/WorkerGlobalScope
+    // 'caches',
+    //'crossOriginIsolated',
+    'crypto',
+    //'fonts',
+    // 'indexedDB',
+    'isSecureContext',
+    'location',
+    'navigator',
+    //'origin',
+    'performance',
+    // 'scheduler',
+    'self',
+    'console',
+    'atob',
+    'btoa',
+    'clearInterval',
+    'clearTimeout',
+    // 'fetch',
+    // 'importScripts',
+    'queueMicrotask',
+    'setInterval',
+    'setTimeout',
+    'structuredClone',
+    'reportError',
+    'WorkerGlobalScope',
+
+    // WebAssembly symbols
     'WebAssembly',
+
+    // WebGL symbols
     'WebGL2RenderingContext',
     'WebGLTexture',
-    'WorkerGlobalScope',
-    // All webGPU symbols are allowed
+
+    // WebGPU symbols
     'WebGPUWindow',
     'GPU',
     'GPUAdapter',
@@ -152,9 +166,15 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'GPUTextureView',
     'GPUUncapturedErrorEvent',
     'GPUValidationError',
-    // Our own symbols
+
+    // DCP symbols / chosen web API additions 
     'progress',
     'work',
+    'bravojs',
+    'setImmediate',
+    'Blob',
+    'addEventListener',
+    'removeEventListener',
   ]);
 
   // Origin time for performance polyfill
@@ -169,15 +189,15 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     // Assumption that if performance exists, performance.now must exist
     performance: typeof performance !== 'undefined' ? performance : { 
       now: ()=>{ 
-        res = new Date().getTime() - pt0;
+        const res = new Date().getTime() - pt0;
         return res;
       } 
     },
     importScripts: function () {
       throw new Error('importScripts is not supported on DCP');
     },
-    WorkerGlobalScope: typeof globalThis === 'undefined' ? self : globalThis,
     globalThis: typeof globalThis === 'undefined' ? self : globalThis,
+    WorkerGlobalScope: typeof globalThis === 'undefined' ? self : globalThis, 
     // For browsers/SA-workers that don't support btoa/atob, modified from https://github.com/MaxArt2501/base64-js/blob/master/base64.js
     btoa: function (string) {
       var b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
@@ -659,10 +679,9 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
    *
    * @param {object} obj - The object, which will have the allow list applied to its properties.
    * @param {Set} allowList - A set of properties to allow people to access.
-   * @param {Set} blockList - An object of property names mapping to booleans to indicate whether access is allowed or not.
-   * @param {Set} polyfills - An object of property names that have been polyfilled.
+   * @param {Object} blockList - An object of property names mapping to booleans to indicate whether access is allowed or not.
    */
-  function applyAccessLists(obj, allowList, blockList = {}, polyfills = {}) {
+  function applyAccessLists(obj, allowList, blockList = {}) {
     if (!obj) { return; }
     Object.getOwnPropertyNames(obj).forEach(function (prop) {
       if (Object.getOwnPropertyDescriptor(obj, prop).configurable) {
@@ -670,17 +689,13 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
           let isSet = false;
           let propValue;
           Object.defineProperty(obj, prop, {
-            get: function () {
-              if (isSet) {
+            get: function getProtectedProperty() {
+              if (isSet)
                 return propValue;
-              } else {
-                if (prop in polyfills) {
-                  return polyfills[prop];
-                }
+              else 
                 return undefined;
-              }
             },
-            set: function (value) {
+            set: function setProtectedProperty(value) {
               propValue = value;
               isSet = true;
             },
@@ -711,30 +726,20 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
   }
 
   /**
-   * Applies a list of polyfills to symbols not present in the global object. Will apply
-   * this list through the objects entire prototype chain
+   * Assigns each property from some polyfill object to another object, inserting or replacing for each property
    * 
-   * @param {Object} obj - The global object to add properties on
-   * @param {Set} polyfills - An object of property names to create/polyfill 
+   * @param {Object} obj - The object to add properties on
+   * @param {Object} polyfills - An object of properties to create/polyfill 
    */
-  function applyPolyfills(obj, polyfills = {}) {
+  function applyPolyfills(obj, polyfills) {
     // Apply symbols from polyfill object
     for (let prop in polyfills) {
-      let found = false;
-      for (let o = obj; o.__proto__ && (o.__proto__ !== Object); o = o.__proto__) {
-        if (o.hasOwnProperty(prop)) {
-          found = true;
-          break;
-        }
-      }
-      if (found) { continue; }
       let propValue = polyfills[prop];
       Object.defineProperty(obj, prop, {
-        get: function () {
+        get: function getPolyfill() {
           return propValue;
-
         },
-        set: function (value) {
+        set: function setPolyfill(value) {
           propValue = value;
         },
         configurable: false
@@ -748,43 +753,23 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
    * so that the blockList is accessible to modify w/o adding it to the allowList.
    */
   function applyAllAccessLists() {
-    // We need to apply the access lists to global, global.__proto__, and global.__proto__.__proto__,
-    // because there's networking-accessing functions inside global.__proto__.__proto__, like fetch.
-    //
-    // If we're in a robust environment (node, browser, WebWorker, basically anything but v8),
-    // then we have to climb the prototype chain and apply the allowList there, but we have to stop
-    // before we allow Object's properties
+    // We need to apply the access lists to global, and the entirety of global's prototype chain
+    // because there's networking-accessing functions inside the chain, like fetch.
 
     var global = typeof globalThis === 'undefined' ? self : globalThis;
-
-    // Ternary expression to avoid a ReferenceError on navigator
-    let _GPU       = ((typeof navigator !== 'undefined') && (typeof navigator.gpu !== 'undefined')) ? navigator.gpu : 
-      (typeof GPU !== 'undefined'? GPU : undefined);
-
-    for (let g = global; g.__proto__ && (g.__proto__ !== Object); g = g.__proto__) {
-      applyAccessLists(g, allowList, blockList, polyfills);
-    }
+    for (let g = global; Object.getPrototypeOf(g); g = Object.getPrototypeOf(g))
+      applyAccessLists(g, allowList, blockList);
 
     if (typeof navigator === 'undefined')
+      navigator = { userAgent: 'not a browser' };
+    else
     {
-      navigator = {
-        userAgent: 'not a browser',
-        gpu: _GPU,
+      const navPolyFill = {
+        userAgent:  navigator.userAgent || 'not a browser', 
       };
-    }
-    else if (!protectedStorage.createdNewNavigator)
-    {
-      // We also want to allowList certain parts of navigator, but not others.
-      const navAllowlist = new Set([
-        'userAgent',
-        'gpu',
-      ]);
-      let navPolyfill = {
-        userAgent: typeof navigator.userAgent !== 'undefined'? navigator.userAgent : 'not a browser',
-        gpu: _GPU,
-      };
-      applyAccessLists(navigator.__proto__, navAllowlist, {}, {}, navPolyfill);
-      applyPolyfills(navigator.__proto__, navPolyfill);
+      if (navigator.gpu)
+        navPolyFill.gpu = navigator.gpu;
+      navigator = navPolyFill;
     }
   }
 
@@ -835,6 +820,12 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     };
   }
 
+  function allowWorktimeSymbols(symbols)
+  {
+    for (let symbol of symbols)
+      allowList.add(symbol);
+  }
+
   addEventListener('message', async (event) => {
     try {
       if (event.request === 'applyRequirements') {
@@ -845,6 +836,10 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
         blockList.OffscreenCanvas = !requirements.environment.offscreenCanvas;
         blockList.WebGPUWindow = !requirements.environment.webgpu;
         blockList.GPU = !requirements.environment.webgpu;
+
+        if (event.worktime && protectedStorage.worktimeGlobals[event.worktime])
+          allowWorktimeSymbols(protectedStorage.worktimeGlobals[event.worktime]);
+
         applyAllAccessLists();
 
         ring1PostMessage({ request: 'applyRequirementsDone' });

package/libexec/sandbox/bravojs-env.js

@@ -74,7 +74,7 @@ self.wrapScriptLoading({ scriptName: 'bravojs-env', ringTransition: true }, func
           protectedStorage.sandboxConfig = message.sandboxConfig;
           Object.assign(self.work.job.public, message.job.public); /* override locale-specific defaults if specified */
           // Load bravojs' module.main with the work function
-          module.declare(message.job.dependencies || (message.job.requireModules /* deprecated */), function mainModule(require, exports, module) {
+          module.declare(message.job.dependencies || (message.job.requireModules /* deprecated */), async function mainModule(require, exports, module) {
             try {
               if (exports.hasOwnProperty('job'))
                 throw new Error("Tried to assign sandbox when it was already assigned"); /* Should be impossible - might happen if throw during assign? */
@@ -83,11 +83,22 @@ self.wrapScriptLoading({ scriptName: 'bravojs-env', ringTransition: true }, func
               message.job.requirePath.map(p => require.paths.push(p));
               message.job.modulePath.map(p => module.paths.push(p));
               exports.arguments = message.job.arguments;
-
-              if (message.job.useStrict)
-                exports.job = indirectEval(`"use strict"; (${message.job.workFunction})`);
-              else
-                exports.job = indirectEval(`(${message.job.workFunction})`);
+              exports.worktime = message.job.worktime;
+
+              switch (message.job.worktime.name)
+              {
+                case 'map-basic':
+                  if (message.job.useStrict)
+                    exports.job = indirectEval(`"use strict"; (${message.job.workFunction})`);
+                  else
+                    exports.job = indirectEval(`(${message.job.workFunction})`);
+                  break;
+                case 'pyodide':
+                  exports.job = await generatePyodideFunction(message.job);
+                  break;
+                default:
+                  throw new Error(`Unsupported worktime: ${message.job.worktime.name}`);
+              }
             } catch(e) {
               reportError(e);
               return;
@@ -125,6 +136,148 @@ self.wrapScriptLoading({ scriptName: 'bravojs-env', ringTransition: true }, func
     }
   })
 
+  /**
+   * Factory function which generates a "map-basic"-like workFunction
+   * out of a Pyodide Worktime job (Python code, files, env variables).
+   *
+   * It takes any "images" passed in the workFunction "arguments" and
+   * writes them to the in memory filesystem provided by Emscripten.
+   * It adds any environment variables specified in the workFunction
+   * "arguments" to the pseudo-"process" for use.
+   * It globally imports a dcp module with function "set_slice_handler"
+   * which takes a python function as input. The python function passed
+   * to that slice handler is invoked by the function which this
+   * factory function returns.
+   *
+   * @param {Object} job The job data associated with the message
+   * @returns {Function} function pyodideWorkFn(slice) -> result
+   */
+  async function generatePyodideFunction(job)
+  {
+    var pythonSliceHandler;
+
+    const pyodide = await pyodideInit();
+    const sys = pyodide.pyimport('sys');
+
+    const findImports = pyodide.runPython('import pyodide; pyodide.code.find_imports');
+    const findPythonModuleLoader = pyodide.runPython('import importlib; importlib.find_loader');
+
+    const parsedArguments = parsePyodideArguments(job.arguments);
+
+    // write images to file and set environment variables
+    const prepPyodide = pyodide.runPython(`
+import tarfile, io
+import os, sys
+
+def prepPyodide(args):
+  for image in args['images']:
+    image = bytes(image)
+    imageFile = io.BytesIO(image)
+    tar = tarfile.open(mode='r', fileobj=imageFile)
+    tar.extractall()
+
+  for item, value in args['environmentVariables'].items():
+    os.environ[item] = value
+
+  sys.argv.extend(args['sysArgv'])
+
+  return
+
+prepPyodide`);
+
+    prepPyodide(pyodide.toPy(parsedArguments));
+
+    // register the dcp Python module 
+    if (!sys.modules.get('dcp'))
+    {
+      const create_proxy = pyodide.runPython('import pyodide;pyodide.ffi.create_proxy');
+
+      pyodide.registerJsModule('dcp', {
+        set_slice_handler: function pyodide$$dcp$$setSliceHandler(func) {
+          pythonSliceHandler = create_proxy(func);
+        },
+        progress,
+      });
+    }
+    pyodide.runPython( 'import dcp' );
+
+    // attempt to import packages from the package manager (if they're not already loaded)
+    const workFunctionPythonImports = findImports(job.workFunction).toJs();
+    const packageManagerImports = workFunctionPythonImports.filter(x=>!findPythonModuleLoader(x));
+    if (packageManagerImports.length > 0)
+    {
+      await fetchAndInitPyodidePackages(packageManagerImports);
+      await pyodide.loadPackage(packageManagerImports);
+    }
+
+    return workFunctionWrapper;
+
+    /**
+     * Evaluates the Python WorkFunction string and then executes the slice
+     * handler Python function. If no call to `dcp.set_slice_handler` is passed
+     * or a non function is passed to it.
+     * This function specifically only takes one parameter since Pyodide Slice
+     * Handlers only accept one parameter.
+     */
+    async function workFunctionWrapper(datum)
+    {
+      const pyodide = await pyodideInit(); // returns the same promise when called multiple times
+
+      // load and execute the Python Workfunction, this populates the pythonSliceHandler variable
+      await pyodide.runPython(job.workFunction);
+
+      // failure to specify a slice handler is considered an error
+      if (!pythonSliceHandler)
+        throw new Error('ENOSLICEHANDLER: Must specify the slice handler using `dcp.set_slice_handler(fn)`');
+
+      // setting the slice handler to a non function or lambda is not supported
+      else if (typeof pythonSliceHandler !== 'function')
+        throw new Error('ENOSLICEHANDLER: Slice Handler must be a function');
+
+      const sliceHandlerResult = await pythonSliceHandler(datum);
+
+      // if it is a PyProxy, convert its value to JavaScript
+      if (sliceHandlerResult.toJs)
+        return sliceHandlerResult.toJs();
+
+      return sliceHandlerResult;
+    }
+
+    /*
+     * Refer to the "The Pyodide Worktime"."Work Function (JS)"."Arguments"."Commands"
+     * part of the DCP Worktimes spec.
+     */
+    function parsePyodideArguments(args)
+    {
+      var index = 1;
+      const numArgs = args[0];
+      const images = [];
+      const environmentVariables = {};
+      const sysArgv = args.slice(numArgs);
+
+      while (index < numArgs)
+      {
+        switch (args[index])
+        {
+          case 'gzImage':
+            const image = args[index+1];
+            images.push(image);
+            index+=2;
+            break;
+          case 'env':
+            const env = args[index+1].split(/=(.*)/s);
+            index+=2;
+            environmentVariables[env[0]] = env[1];
+            break;
+          default:
+            throw new Error(`Invalid argument ${args[index]}`);
+        }
+      }
+
+      return { sysArgv, images, environmentVariables };
+    }
+  }
+
   /** A module.declare suitable for running when processing modules arriving as part
   * of a  module group or other in-memory cache.
   */

package/libexec/sandbox/calculate-capabilities.js

@@ -48,9 +48,7 @@ self.wrapScriptLoading({ scriptName: 'calculate-capabilities' }, function calcul
         return arguments[0];
       });`
 
-      webgpu =
-        (typeof navigator !== 'undefined' &&
-          typeof navigator.gpu !== 'undefined');
+      webgpu = Boolean(globalThis.navigator?.gpu);
 
       if (webgpu) {
         try {
@@ -135,9 +133,11 @@ self.wrapScriptLoading({ scriptName: 'calculate-capabilities' }, function calcul
     addEventListener('message', async (event) => {
       try {
         if (event.request === 'describe') {
+          const worktimes = globalThis.worktimes;
           const capabilities = await getCapabilities();
           ring2PostMessage({
             capabilities,
+            worktimes,
             request: 'describe',
           });
         }