dcp-client 4.3.6 → 4.3.8

package/generated/sandbox-definitions.json

@@ -1 +1 @@
-{"browser":["deny-node","kvin/kvin.js","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"node":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"native":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap"],"nodeTesting":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"],"testing":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","calculate-capabilities","bootstrap","testing.js"]}
+{"browser":["deny-node","kvin/kvin.js","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","worktimes","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","pyodide-core","calculate-capabilities","bootstrap"],"node":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","worktimes","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","pyodide-core","calculate-capabilities","bootstrap"],"native":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","worktimes","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","pyodide-core","calculate-capabilities","bootstrap"],"nodeTesting":["kvin/kvin.js","sa-ww-simulation","script-load-wrapper","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","worktimes","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","pyodide-core","calculate-capabilities","bootstrap","testing.js"],"testing":["deny-node","kvin/kvin.js","sa-ww-simulation","script-load-wrapper","native-event-loop","wrap-event-listeners","timer-classes","event-loop-virtualization","unique-timing","worktimes","access-lists","bravojs-init","bravojs/bravo.js","bravojs-env","pyodide-core","calculate-capabilities","bootstrap","testing.js"]}

package/index.js

@@ -373,11 +373,17 @@ function magicView(node, seen)
  *
  * Returns false is the file simply does not exist.
  *
+ * Setting `DCP_CLIENT_ALLOW_INSECURE_CONFIGURATION` to a non-empty value disables
+ * the security check.
+ *
  * @param {string} fullPath    the full path to the file to check
  * @param {object} statBuf     [optional] existing stat buf for the file
  */
 function checkConfigFileSafePerms(fullPath, statBuf)
 {
+  if (process.env.DCP_CLIENT_ALLOW_INSECURE_CONFIGURATION)
+    return true;
+
   const fun = checkConfigFileSafePerms;
 
   if (!fs.existsSync(fullPath))

package/libexec/sandbox/access-lists.js

@@ -12,35 +12,17 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
   const ring1PostMessage = self.postMessage;
   const global = typeof globalThis === 'undefined' ? self : globalThis;
 
-  // aggregated from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects#Reflection
   const allowList = new Set([
-    '__proto__',
-    
-    // properties guarnteed by polyfills
-    'globalThis',
-    'location',
-    'performance',
-    'importScripts',
-    'WorkerGlobalScope',
-    'btoa',
-    'atob',
-    'Blob',
-
-    // General allowed symbols
-    'addEventListener',
-    'applyAccesslist',
+    // global objects, aggregated from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects#Reflection
+    'AggregateError',
     'Array',
     'ArrayBuffer',
-    'AsyncFunction',
     'Atomics',
     'BigInt',
     'BigInt64Array',
     'BigUint64Array',
     'Boolean',
-    'bravojs',
-    'clearInterval',
-    'clearTimeout',
-    'console',
+
     'constructor',
     'DataView',
     'Date',
@@ -52,12 +34,10 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'escape',
     'eval',
     'EvalError',
-    'File',
-    'FileReader',
     'Float32Array',
     'Float64Array',
     'Function',
-    'Headers',
+    'globalThis',
     'Infinity',
     'Int16Array',
     'Int32Array',
@@ -69,7 +49,6 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'Math',
     'module',
     'NaN',
-    'navigator',
     'null',
     'Number',
     'Object',
@@ -82,21 +61,14 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'Promise',
     'propertyIsEnumerable',
     'Proxy',
-    'pt0',
     'RangeError',
     'ReferenceError',
     'Reflect',
     'RegExp',
-    'removeEventListener',
-    'requestAnimationFrame',
     'require',
     'Response',
-    'self',
     'Set',
-    'setInterval',
-    'setTimeout',
-    'setImmediate',
-    'sleep',
+    'SharedArrayBuffer',
     'String',
     'Symbol',
     'SyntaxError',
@@ -106,7 +78,6 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'toString',
     'TypeError',
     'URIError',
-    'URL',
     'Uint16Array',
     'Uint32Array',
     'Uint8Array',
@@ -116,10 +87,44 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'valueOf',
     'WeakMap',
     'WeakSet',
+    'WeakRef',
+    '__proto__',
+
+    // WorkerGlobalScope symbols, aggregated from https://developer.mozilla.org/en-US/docs/Web/API/WorkerGlobalScope
+    // 'caches',
+    //'crossOriginIsolated',
+    'crypto',
+    //'fonts',
+    // 'indexedDB',
+    'isSecureContext',
+    'location',
+    'navigator',
+    //'origin',
+    'performance',
+    // 'scheduler',
+    'self',
+    'console',
+    'atob',
+    'btoa',
+    'clearInterval',
+    'clearTimeout',
+    // 'fetch',
+    // 'importScripts',
+    'queueMicrotask',
+    'setInterval',
+    'setTimeout',
+    'structuredClone',
+    'reportError',
+    'WorkerGlobalScope',
+
+    // WebAssembly symbols
     'WebAssembly',
+
+    // WebGL symbols
     'WebGL2RenderingContext',
     'WebGLTexture',
-    // All webGPU symbols are allowed
+
+    // WebGPU symbols
     'WebGPUWindow',
     'GPU',
     'GPUAdapter',
@@ -161,9 +166,15 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     'GPUTextureView',
     'GPUUncapturedErrorEvent',
     'GPUValidationError',
-    // Our own symbols
+
+    // DCP symbols / chosen web API additions 
     'progress',
     'work',
+    'bravojs',
+    'setImmediate',
+    'Blob',
+    'addEventListener',
+    'removeEventListener',
   ]);
 
   // Origin time for performance polyfill
@@ -178,15 +189,15 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     // Assumption that if performance exists, performance.now must exist
     performance: typeof performance !== 'undefined' ? performance : { 
       now: ()=>{ 
-        res = new Date().getTime() - pt0;
+        const res = new Date().getTime() - pt0;
         return res;
       } 
     },
     importScripts: function () {
       throw new Error('importScripts is not supported on DCP');
     },
-    WorkerGlobalScope: typeof globalThis === 'undefined' ? self : globalThis,
     globalThis: typeof globalThis === 'undefined' ? self : globalThis,
+    WorkerGlobalScope: typeof globalThis === 'undefined' ? self : globalThis, 
     // For browsers/SA-workers that don't support btoa/atob, modified from https://github.com/MaxArt2501/base64-js/blob/master/base64.js
     btoa: function (string) {
       var b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
@@ -673,7 +684,7 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
   function applyAccessLists(obj, allowList, blockList = {}) {
     if (!obj) { return; }
     Object.getOwnPropertyNames(obj).forEach(function (prop) {
-      if (Object.getOwnPropertyDescriptor(obj, prop).configurable) {
+      if (Object.getOwnPropertyDescriptor(obj, prop)?.configurable) {
         if (!allowList.has(prop)) {
           let isSet = false;
           let propValue;
@@ -809,6 +820,12 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
     };
   }
 
+  function allowWorktimeSymbols(symbols)
+  {
+    for (let symbol of symbols)
+      allowList.add(symbol);
+  }
+
   addEventListener('message', async (event) => {
     try {
       if (event.request === 'applyRequirements') {
@@ -819,6 +836,10 @@ self.wrapScriptLoading({ scriptName: 'access-lists', ringTransition: true }, fun
         blockList.OffscreenCanvas = !requirements.environment.offscreenCanvas;
         blockList.WebGPUWindow = !requirements.environment.webgpu;
         blockList.GPU = !requirements.environment.webgpu;
+
+        if (event.worktime && protectedStorage.worktimeGlobals[event.worktime])
+          allowWorktimeSymbols(protectedStorage.worktimeGlobals[event.worktime]);
+
         applyAllAccessLists();
 
         ring1PostMessage({ request: 'applyRequirementsDone' });

package/libexec/sandbox/bravojs-env.js

@@ -74,7 +74,7 @@ self.wrapScriptLoading({ scriptName: 'bravojs-env', ringTransition: true }, func
           protectedStorage.sandboxConfig = message.sandboxConfig;
           Object.assign(self.work.job.public, message.job.public); /* override locale-specific defaults if specified */
           // Load bravojs' module.main with the work function
-          module.declare(message.job.dependencies || (message.job.requireModules /* deprecated */), function mainModule(require, exports, module) {
+          module.declare(message.job.dependencies || (message.job.requireModules /* deprecated */), async function mainModule(require, exports, module) {
             try {
               if (exports.hasOwnProperty('job'))
                 throw new Error("Tried to assign sandbox when it was already assigned"); /* Should be impossible - might happen if throw during assign? */
@@ -83,11 +83,22 @@ self.wrapScriptLoading({ scriptName: 'bravojs-env', ringTransition: true }, func
               message.job.requirePath.map(p => require.paths.push(p));
               message.job.modulePath.map(p => module.paths.push(p));
               exports.arguments = message.job.arguments;
-
-              if (message.job.useStrict)
-                exports.job = indirectEval(`"use strict"; (${message.job.workFunction})`);
-              else
-                exports.job = indirectEval(`(${message.job.workFunction})`);
+              exports.worktime = message.job.worktime;
+
+              switch (message.job.worktime.name)
+              {
+                case 'map-basic':
+                  if (message.job.useStrict)
+                    exports.job = indirectEval(`"use strict"; (${message.job.workFunction})`);
+                  else
+                    exports.job = indirectEval(`(${message.job.workFunction})`);
+                  break;
+                case 'pyodide':
+                  exports.job = await generatePyodideFunction(message.job);
+                  break;
+                default:
+                  throw new Error(`Unsupported worktime: ${message.job.worktime.name}`);
+              }
             } catch(e) {
               reportError(e);
               return;
@@ -125,6 +136,148 @@ self.wrapScriptLoading({ scriptName: 'bravojs-env', ringTransition: true }, func
     }
   })
 
+  /**
+   * Factory function which generates a "map-basic"-like workFunction
+   * out of a Pyodide Worktime job (Python code, files, env variables).
+   *
+   * It takes any "images" passed in the workFunction "arguments" and
+   * writes them to the in memory filesystem provided by Emscripten.
+   * It adds any environment variables specified in the workFunction
+   * "arguments" to the pseudo-"process" for use.
+   * It globally imports a dcp module with function "set_slice_handler"
+   * which takes a python function as input. The python function passed
+   * to that slice handler is invoked by the function which this
+   * factory function returns.
+   *
+   * @param {Object} job The job data associated with the message
+   * @returns {Function} function pyodideWorkFn(slice) -> result
+   */
+  async function generatePyodideFunction(job)
+  {
+    var pythonSliceHandler;
+
+    const pyodide = await pyodideInit();
+    const sys = pyodide.pyimport('sys');
+
+    const findImports = pyodide.runPython('import pyodide; pyodide.code.find_imports');
+    const findPythonModuleLoader = pyodide.runPython('import importlib; importlib.find_loader');
+
+    const parsedArguments = parsePyodideArguments(job.arguments);
+
+    // write images to file and set environment variables
+    const prepPyodide = pyodide.runPython(`
+import tarfile, io
+import os, sys
+
+def prepPyodide(args):
+  for image in args['images']:
+    image = bytes(image)
+    imageFile = io.BytesIO(image)
+    tar = tarfile.open(mode='r', fileobj=imageFile)
+    tar.extractall()
+
+  for item, value in args['environmentVariables'].items():
+    os.environ[item] = value
+
+  sys.argv.extend(args['sysArgv'])
+
+  return
+
+prepPyodide`);
+
+    prepPyodide(pyodide.toPy(parsedArguments));
+
+    // register the dcp Python module 
+    if (!sys.modules.get('dcp'))
+    {
+      const create_proxy = pyodide.runPython('import pyodide;pyodide.ffi.create_proxy');
+
+      pyodide.registerJsModule('dcp', {
+        set_slice_handler: function pyodide$$dcp$$setSliceHandler(func) {
+          pythonSliceHandler = create_proxy(func);
+        },
+        progress,
+      });
+    }
+    pyodide.runPython( 'import dcp' );
+
+    // attempt to import packages from the package manager (if they're not already loaded)
+    const workFunctionPythonImports = findImports(job.workFunction).toJs();
+    const packageManagerImports = workFunctionPythonImports.filter(x=>!findPythonModuleLoader(x));
+    if (packageManagerImports.length > 0)
+    {
+      await fetchAndInitPyodidePackages(packageManagerImports);
+      await pyodide.loadPackage(packageManagerImports);
+    }
+
+    return workFunctionWrapper;
+
+    /**
+     * Evaluates the Python WorkFunction string and then executes the slice
+     * handler Python function. If no call to `dcp.set_slice_handler` is passed
+     * or a non function is passed to it.
+     * This function specifically only takes one parameter since Pyodide Slice
+     * Handlers only accept one parameter.
+     */
+    async function workFunctionWrapper(datum)
+    {
+      const pyodide = await pyodideInit(); // returns the same promise when called multiple times
+
+      // load and execute the Python Workfunction, this populates the pythonSliceHandler variable
+      await pyodide.runPython(job.workFunction);
+
+      // failure to specify a slice handler is considered an error
+      if (!pythonSliceHandler)
+        throw new Error('ENOSLICEHANDLER: Must specify the slice handler using `dcp.set_slice_handler(fn)`');
+
+      // setting the slice handler to a non function or lambda is not supported
+      else if (typeof pythonSliceHandler !== 'function')
+        throw new Error('ENOSLICEHANDLER: Slice Handler must be a function');
+
+      const sliceHandlerResult = await pythonSliceHandler(datum);
+
+      // if it is a PyProxy, convert its value to JavaScript
+      if (sliceHandlerResult.toJs)
+        return sliceHandlerResult.toJs();
+
+      return sliceHandlerResult;
+    }
+
+    /*
+     * Refer to the "The Pyodide Worktime"."Work Function (JS)"."Arguments"."Commands"
+     * part of the DCP Worktimes spec.
+     */
+    function parsePyodideArguments(args)
+    {
+      var index = 1;
+      const numArgs = args[0];
+      const images = [];
+      const environmentVariables = {};
+      const sysArgv = args.slice(numArgs);
+
+      while (index < numArgs)
+      {
+        switch (args[index])
+        {
+          case 'gzImage':
+            const image = args[index+1];
+            images.push(image);
+            index+=2;
+            break;
+          case 'env':
+            const env = args[index+1].split(/=(.*)/s);
+            index+=2;
+            environmentVariables[env[0]] = env[1];
+            break;
+          default:
+            throw new Error(`Invalid argument ${args[index]}`);
+        }
+      }
+
+      return { sysArgv, images, environmentVariables };
+    }
+  }
+
   /** A module.declare suitable for running when processing modules arriving as part
   * of a  module group or other in-memory cache.
   */

package/libexec/sandbox/calculate-capabilities.js

@@ -133,9 +133,11 @@ self.wrapScriptLoading({ scriptName: 'calculate-capabilities' }, function calcul
     addEventListener('message', async (event) => {
       try {
         if (event.request === 'describe') {
+          const worktimes = globalThis.worktimes;
           const capabilities = await getCapabilities();
           ring2PostMessage({
             capabilities,
+            worktimes,
             request: 'describe',
           });
         }