dcp-client 5.4.1 → 5.4.3

package/libexec/sandbox/pyodide-worktime.js

@@ -0,0 +1,197 @@
+/**
+ *  @file pyodide-worktime.js
+ *  This file sets up the hooks for the Pyodide worktime. It registers the processSlice
+ *  initSandbox hooks, which run slices and setup the sandbox respectively.
+ *
+ *  @author Wes Garland <wes@distributive.network>
+ *  @author Severn Lortie <severn@distributive.network>
+ *  @author Will Pringle <will@distributive.network>
+ *  @author Hamada Gasmallah <hamada@distributive.network>
+ *
+ *  @date February, 2024
+ *  @copyright Copyright (c) 2018-2024, Distributive, Ltd.  All Rights Reserved.
+ */
+'use-strict';
+/* global self, bravojs, addEventListener, postMessage */
+// @ts-nocheck
+
+self.wrapScriptLoading({ scriptName: 'pyodide-worktime'}, function pyodideWorktime$$fn(protectedStorage, ring1PostMessage, wrapPostMessage)
+{
+  let wrappedPythonSliceHandler;
+  /**
+   * Registers the worktime callbacks that allow the worktime controller (worktimes.js) to setup our environment and run the worktime
+   */
+  function registerWorktime()
+  {
+    protectedStorage.worktimes.registerWorktime({
+      name: 'pyodide',
+      version: '0.28.0',
+      processSlice,
+      initSandbox,
+    });
+  }
+
+  /**
+   * Function which generates a "map-basic"-like workFunction
+   * out of a Pyodide Worktime job (Python code, files, env variables).
+   *
+   * It takes any "images" passed in the workFunction "arguments" and
+   * writes them to the in memory filesystem provided by Emscripten.
+   * It adds any environment variables specified in the workFunction
+   * "arguments" to the pseudo-"process" for use.
+   * It globally imports a dcp module with function "set_slice_handler"
+   * which takes a python function as input. The python function passed
+   * to that slice handler is invoked by the function which this
+   * function creates.
+   *
+   * @param {Object} job The job being assigned to the sandbox
+   */
+  async function initSandbox(job)
+  {
+    var pythonSliceHandler;
+
+    const pyodide = await pyodideInit();
+    const sys = pyodide.pyimport('sys');
+
+    const findImports = pyodide.runPython('import pyodide; pyodide.code.find_imports');
+    const findPythonModuleLoader = pyodide.runPython('import importlib; importlib.util.find_spec');
+
+    const parsedArguments = parsePyodideArguments(job.arguments);
+
+    // write images to file and set environment variables
+    const prepPyodide = pyodide.runPython(`
+import tarfile, io
+import os, sys
+
+def prepPyodide(args):
+  for image in args['images']:
+    image = bytes(image)
+    imageFile = io.BytesIO(image)
+    tar = tarfile.open(mode='r', fileobj=imageFile)
+
+    # Don't overwrite directories which corrupts Pyodide's in memory filesystem
+    def safe_extract(tar, path="/"):
+      for member in tar.getmembers():
+          if member.isdir():
+              dir_path = os.path.join(path, member.name)
+              if not os.path.exists(dir_path):
+                  os.makedirs(dir_path)
+          else:
+              tar.extract(member, path)
+    safe_extract(tar)
+
+  for item, value in args['environmentVariables'].items():
+    os.environ[item] = value
+
+  sys.argv.extend(args['sysArgv'])
+
+  return
+
+prepPyodide`);
+
+    prepPyodide(pyodide.toPy(parsedArguments));
+
+    // register the dcp Python module 
+    if (!sys.modules.get('dcp'))
+    {
+      const create_proxy = pyodide.runPython('import pyodide;pyodide.ffi.create_proxy');
+
+      pyodide.registerJsModule('dcp', {
+        set_slice_handler: function pyodide$$dcp$$setSliceHandler(func) {
+          pythonSliceHandler = create_proxy(func);
+        },
+        progress,
+      });
+    }
+    pyodide.runPython( 'import dcp' );
+
+    // attempt to import packages from the package manager (if they're not already loaded)
+    const workFunctionPythonImports = findImports(job.workFunction).toJs();
+    const packageManagerImports = workFunctionPythonImports.filter(x=>!findPythonModuleLoader(x));
+    if (packageManagerImports.length > 0)
+    {
+      await fetchAndInitPyodidePackages(packageManagerImports);
+      await pyodide.loadPackage(packageManagerImports);
+    }
+
+    wrappedPythonSliceHandler = workFunctionWrapper;
+
+    /**
+     * Evaluates the Python WorkFunction string and then executes the slice
+     * handler Python function. If no call to `dcp.set_slice_handler` is passed
+     * or a non function is passed to it.
+     * This function specifically only takes one parameter since Pyodide Slice
+     * Handlers only accept one parameter.
+     */
+    async function workFunctionWrapper(datum)
+    {
+      const pyodide = await pyodideInit(); // returns the same promise when called multiple times
+
+      // load and execute the Python Workfunction, this populates the pythonSliceHandler variable
+      await pyodide.runPython(job.workFunction);
+
+      // failure to specify a slice handler is considered an error
+      if (!pythonSliceHandler)
+        throw new Error('ENOSLICEHANDLER: Must specify the slice handler using `dcp.set_slice_handler(fn)`');
+
+      // setting the slice handler to a non function or lambda is not supported
+      else if (typeof pythonSliceHandler !== 'function')
+        throw new Error('ENOSLICEHANDLER: Slice Handler must be a function');
+
+      const sliceHandlerResult = await pythonSliceHandler(pyodide.toPy(datum));
+
+      // if it is a PyProxy, convert its value to JavaScript
+      if (sliceHandlerResult?.toJs)
+        return sliceHandlerResult.toJs();
+
+      return sliceHandlerResult;
+    }
+
+    /*
+     * Refer to the "The Pyodide Worktime"."Work Function (JS)"."Arguments"."Commands"
+     * part of the DCP Worktimes spec.
+     */
+    function parsePyodideArguments(args)
+    {
+      var index = 1;
+      const numArgs = args[0];
+      const images = [];
+      const environmentVariables = {};
+      const sysArgv = args.slice(numArgs);
+
+      while (index < numArgs)
+      {
+        switch (args[index])
+        {
+          case 'gzImage':
+            const image = args[index+1];
+            images.push(image);
+            index+=2;
+            break;
+          case 'env':
+            const env = args[index+1].split(/=(.*)/s);
+            index+=2;
+            environmentVariables[env[0]] = env[1];
+            break;
+          default:
+            throw new Error(`Invalid argument ${args[index]}`);
+        }
+      }
+
+      return { sysArgv, images, environmentVariables };
+    }
+  }
+
+  /**
+   * The processSlice hook function which runs the work function for input slice data
+   * @param {*} data The slice data
+   * @returns {*} The slice result
+   */
+  async function processSlice(data)
+  {
+    const result = await wrappedPythonSliceHandler(data);
+    return result;
+  }
+
+  registerWorktime();
+}); /* end of fn */

package/libexec/sandbox/worktimes.js

@@ -1,66 +1,294 @@
 /**
- * @file      worktimes.js
- *            Specify available worktimes, allow registering custom worktimes
- *            The single source of authority for what Worktimes are available.
- * 
- * @author    Will Pringle     <will@distributive.network>
- *            Hamada Gasmallah <hamada@distributive.network>
- * @date      January 2024
+ * @file dcp-client/libexec/sandbox/worktimes.js
+ * Specify available worktimes, allow registering custom worktimes
+ * The single source of authority for what Worktimes are available.
+ *
+ * @author Wes Garland, wes@distributive.network
+ *         Severn Lortie, severn@distributive.network
+ *         Will Pringle, will@distributive.network
+ *         Hamada Gasmallah, hamada@distributive.network
+ * @date   January 2024
+ * @copyright Copyright (c) 2018-2024, Distributive, Ltd.  All Rights Reserved.
  */
-'use strict';
+'use-strict';
+/* global self, bravojs, addEventListener, postMessage */
+// @ts-nocheck
 
-function worktimes$$fn(protectedStorage, _ring2PostMessage)
-{
+self.wrapScriptLoading({ scriptName: 'worktimes' }, function worktimes$$fn(protectedStorage, ring1PostMessage, wrapPostMessage) {
+  // This file starts at ring 2, but transitions to ring 3 partway through it.
+  const ring2PostMessage = self.postMessage; 
+  let ring3PostMessage;
+  protectedStorage.worktimes = { registeredWorktimes: [] };
   // when preparing a worktime, add it's globals to this object.
   // only if the job assigned to the evaluator uses that worktime, they will
   // be added to the allow-list
   protectedStorage.worktimeGlobals = {};
 
-  const worktimes = [
-    { name: 'map-basic', versions: ['1.0.0'] },
-    { name: 'pyodide',   versions: ['0.28.0'] },
-  ];
+  /**
+   * Register a worktime with the evaluator.
+   * @param {object} worktimeInfo Information about the worktime
+   * @param {string} worktimeInfo.name The name of the worktime
+   * @param {string} worktimeInfo.version The semantic version (semver) number of the worktime
+   * @param {function} worktimeInfo.initSandbox A function which initializes the worktime. It is passed the job as its only argument
+   * @param {function} worktimeInfo.processSlice A function which runs a slice. Must take the form processSlice(data) --> result
+   */
+  function registerWorktime(worktimeInfo)
+  {
+    protectedStorage.worktimes.registeredWorktimes.push(worktimeInfo);
+  }
+  protectedStorage.worktimes.registerWorktime = registerWorktime;
 
-  function registerWorktime(name, version)
+  /**
+   * Returns a list of worktimes in the form [{name: string, version:string}]
+   * @returns {object[]} array of worktimeInfo objects
+   */
+  function getWorktimeList()
   {
-    const foundWorktime = globalThis.worktimes.find(wt => wt.name === name);
-    // if we found a worktime and the version isn't already added, add it
-    if (foundWorktime && !foundWorktime.versions.includes(version))
-      foundWorktime.versions.push(version);
-    // if this is a new worktime, add it
-    else if (!foundWorktime)
-      globalThis.worktimes.push({ name, versions: [version]});
+    const worktimeList = [];
+    for (const worktime of protectedStorage.worktimes.registeredWorktimes)
+      worktimeList.push({ name: worktime.name, version: worktime.version });
+    return worktimeList;
   }
+  protectedStorage.worktimes.getWorktimeList = getWorktimeList;
+  globalThis.getWorktimeList = getWorktimeList; // hook for worker-info.js, removed by access-list before sandboxes see it
 
-  // nodejs-like environment
-  if (typeof module?.exports === 'object')
-    exports.worktimes = worktimes;
-  else // inside the sandbox
+  /**
+   * Get a worktime given its name and version.
+   * @note The version parameter here is not a semver range. It must be a literal version, e.g. 0.23.1
+   * @param {string} name The name of the worktime
+   * @param {string} version The version of the worktime
+   * @returns {object} The worktime
+   */
+  function getWorktime(name, version)
   {
-    globalThis.worktimes        = worktimes;
-    globalThis.registerWorktime = registerWorktime;
+    return protectedStorage.worktimes.registeredWorktimes.find(wt => wt.name === name && wt.version === version);
   }
-}
-
-// nodejs-like environment
-if (typeof module?.exports === 'object')
-{
-  worktimes$$fn({});
-  // sort the worktime versions from lowest to greatest
-  const semver = require('semver');
-  for (const wt of exports.worktimes)
+  protectedStorage.worktimes.getWorktime = getWorktime;
+
+  //Listens for postMessage from the sandbox
+  addEventListener('message', async function worktimes$$sandboxPostMessageHandler(event) {
+    let message = event;
+    switch (message.request)
+    {
+      /* Sandbox assigned a specific job by supervisor */
+      case 'assign':
+      {
+        try
+        {
+          if (typeof module.main !== 'undefined')
+            throw new Error('Main module was provided before job assignment');
+
+          protectedStorage.sandboxConfig = message.sandboxConfig;
+          Object.assign(self.work.job.public, message.job.public); /* override locale-specific defaults if specified */
+          mainModuleFactoryFactory(message.job);
+        }
+        catch (error)
+        {
+          ring2PostMessage({request: 'reject', error});
+        }
+        break; /* end of assign */
+      }
+      /* Supervisor has asked us to execute the work function. message.data is input datum. */
+      case 'main':
+      {
+        try
+        {
+          await runWorkFunction(message.data);
+        }
+        catch (error)
+        {
+          ring3PostMessage({ request: 'sandboxError', error });
+        }
+        break;
+      }
+      default:
+        break;
+    }
+  })
+
+  /* Report metrics to sandbox/supervisor */
+  function reportTimes (metrics)
   {
-    wt.versions.sort((left, right) => {
-      if (semver.lt(left, right))
-        return 1;
-      else if (semver.eq(left, right))
-        return 0;
-      else
-        return -1;
-    });
+    const { total, webGL, webGPU, CPU } = metrics;
+    ring3PostMessage({ request: 'measurement', total, webGL, webGPU, CPU });
   }
-}
-// inside the sandbox
-else
-  self.wrapScriptLoading({ scriptName: 'worktimes' }, worktimes$$fn);
 
+  /* Report an error from the work function to the supervisor */
+  function reportError (error, metrics)
+  {
+    const err = new Error('initial state');
+
+    for (let prop of [ 'message', 'name', 'code', 'stack', 'lineNumber', 'columnNumber' ])
+    {
+      try
+      {
+        if (typeof error[prop] !== 'undefined')
+          err[prop] = error[prop];
+      }
+      catch(e){};
+    }
+
+    reportTimes(metrics); // Report metrics for both 'workReject' and 'workError'.
+    ring3PostMessage({ request: 'workError', error: err });
+  }
+
+  /**
+   * Report a result from work function and metrics to the supervisor.
+   * @param     result  the value that the work function returned promise resolved to
+   */
+  function reportResult (result, metrics)
+  {
+    try
+    {
+      reportTimes(metrics);
+      ring3PostMessage({ request: 'complete', result });
+    }
+    catch (error)
+    {
+      ring3PostMessage({ request: 'sandboxError', error });
+    }
+  }
+  
+  /**
+   * Actual mechanics for running a work function. ** This function will never reject **
+   *
+   * @param     successCallback         callback to invoke when the work function has finished running;
+   *                                    it receives as its argument the resolved promise returned from
+   *                                    the work function
+   * @param     errorCallback           callback to invoke when the work function rejects. It receives
+   *                                    as its argument the error that it rejected with.
+   * @returns   unused promise   
+   */
+  async function runWorkFunction_inner(datum, wallDuration, successCallback, errorCallback)
+  {
+    /** @typedef {import("./timer-classes.js").TimeInterval} TimeInterval */
+    var rejection = false;
+    var result;
+    let metrics;
+    protectedStorage.emitConsoleMessages = true;
+    protectedStorage.flushConsoleBuffer();
+    try
+    {
+      result = await module.main.worktime.processSlice(datum);
+    }
+    catch (error)
+    {
+      rejection = error;
+    }
+
+    try
+    {
+      // reset the device states and flush all pending tasks
+      protectedStorage.lockTimers(); // lock timers so no new timeouts will be run.
+      if (protectedStorage.webGPU)
+        protectedStorage.webGPU.lock();
+
+      // Let microtask queue finish before getting metrics. With all event-loop possibilities locked,
+      // only the microtask could trigger new code, so waiting for a setTimeout guarantees everything's done
+      await new Promise((r) => protectedStorage.realSetTimeout.call(globalThis, r, 1));
+
+      // flush any pending console events, especially in the case of a repeating message that hasn't been emitted yet 
+      // then, disable emission of any more console messages
+      try { protectedStorage.dispatchSameConsoleMessage(); } catch(e) {};
+      protectedStorage.emitConsoleMessages = false;
+
+      metrics = await protectedStorage.bigBrother.globalTrackers.getMetrics();
+
+      await protectedStorage.bigBrother.globalTrackers.reset();
+    }
+    catch (error)
+    {
+      ring3PostMessage({ request: 'sandboxError', error });
+    }
+    finally
+    {
+      // due to the nature of the micro task queue, await, our `reset()` cancels all the things that could cause new
+      // tasks, and we wait for all pending task to finish in `reset()`, we are guaranteed to have an empty task queue
+      // now. Hence it's ok to stop the wall clock measurement now
+      wallDuration.stop();
+
+      // safety: wallDuration is always stopped, `length` will not throw
+      metrics = { ...metrics, total: wallDuration.length };
+    }
+
+    if (rejection)
+      errorCallback(rejection, metrics);
+    else
+      successCallback(result, metrics);
+
+    /* CPU time measurement ends when this function's return value is resolved or rejected */
+  }
+
+  /**
+   * Run the work function, returning a promise that resolves once the function has finished
+   * executing.
+   *
+   * @param datam     an element of the input set
+   */
+  async function runWorkFunction(datum)
+  {
+    // reset the time used for feature detection
+    protectedStorage.bigBrother.globalTrackers.resetRecordedTime();
+    const wallDuration = new protectedStorage.TimeInterval();
+    protectedStorage.bigBrother.globalTrackers.wallDuration = wallDuration;
+
+    if (protectedStorage.webGPU)
+      protectedStorage.webGPU.unlock();
+    await protectedStorage.unlockTimers();
+
+    /* Use setTimeout trampoline to
+     * 1. shorten stack
+     * 2. initialize the event loop measurement code
+     */
+    protectedStorage.setTimeout(() => runWorkFunction_inner(datum, wallDuration, reportResult, reportError));
+  }
+  /**
+   * Factory function which returns the main module factory for use as the second argument in module.declare.
+   *
+   * @param {object} job     the job property of the assign message from the supervisor.
+   * @returns {function} mainModuleFactory
+   */
+  function mainModuleFactoryFactory(job)
+  {
+    module.declare(job.dependencies, mainModuleFactory);
+
+    /* mainModule: this is the function that is run first by BravoJS once the module system has been
+      * loaded. It functions as the main module in a CommonJS environment, and its job is to initialize
+      *  the work function for use. Once initialized, the work function is accessible as the main module's
+      * `job` export.  The work function is eventually invoked by a message from the supervisor which
+      * invokes runWorkFunction() above.
+      *
+      * This function is infallible; any exceptions or rejections caught are reported to the Supervisor.
+      */
+    async function mainModuleFactory(require, exports, module)
+    {
+      try
+      {
+        if (exports.hasOwnProperty('job'))
+          throw new Error("Tried to assign sandbox when it was already assigned"); /* Should be impossible - might happen if throw during assign? */
+        exports.worktime = false;
+        job.requirePath.map(p => require.paths.push(p));
+        job.modulePath.map(p => module.paths.push(p));
+
+        exports.worktime = protectedStorage.worktimes.getWorktime(job.worktime.name, job.worktime.version);
+        if (!exports.worktime)
+          throw new Error(`Unsupported worktime: ${job.worktime.name}`);
+        await exports.worktime.initSandbox(job);
+      }
+      catch(error)
+      {
+        reportError(error);
+        return;
+      }
+
+      ring2PostMessage({
+        request: 'assigned',
+        jobAddress: job.address,
+      });
+
+      // Now that the evaluator is assigned, wrap post message for ring 3
+      wrapPostMessage();
+      ring3PostMessage = self.postMessage;
+    } /* end of main module */
+  }
+}); /* end of worktimes$$wrapScriptLoading IIFE */

package/npm-hooks/prepublish

@@ -11,6 +11,28 @@ cd `dirname "$0"`/..
 yellow='\e[33m'
 normal='\e[0m'
 
+currentWorktimes=$(CONSOLE_RESULTS=true node build/generate-worktimes-json)
+recordedWorktimes=$(< ./generated/worktimes.json)
+if [[ "$currentWorktimes" != "$recordedWorktimes" ]]; then
+  echo
+  echo -e "Generated worktimes: ${recordedWorktimes}" > /dev/stderr
+  echo -e "Current worktimes:   ${currentWorktimes}" > /dev/stderr
+  echo -e "${yellow}pre-publish: abort due to generated worktimes not matching currently available worktimes${normal}" > /dev/stderr
+  echo
+  exit 1
+fi
+
+currentSandboxFiles=$(CONSOLE_RESULTS=true node build/generate-sandbox-definitions-json)
+recordedSandboxFiles=$(< ./generated/sandbox-definitions.json)
+if [[ "$currentSandboxFiles" != "$recordedSandboxFiles" ]]; then
+  echo
+  echo -e "Generated sandbox definitions: ${currentSandboxFiles}" > /dev/stderr
+  echo -e "Current sandbox definitions:   ${recordedSandboxFiles}" > /dev/stderr
+  echo -e "${yellow}pre-publish: abort due to generated sandbox definitions not matching generation file${normal}" > /dev/stderr
+  echo
+  exit 1
+fi
+
 git ls-files --error-unmatch `find . -type f | egrep -v '^\./(node_modules|\.git|\.dcp-build|build)'` >/dev/null
 if [ "$?" != "0" ]; then
   echo

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "dcp-client",
-  "version": "5.4.1",
+  "version": "5.4.3",
   "dcp": {
-    "version": "d91005494a226e4361d9ae1a2f739dc2dc45750a",
+    "version": "3853a8dc41e0c33886fc337534f5c21aaf62dc56",
     "repository": "git@gitlab.com:Distributed-Compute-Protocol/dcp.git"
   },
   "description": "Core libraries for accessing DCP network",