dcp-client 4.3.0-4 → 4.3.0-5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3967,7 +3967,7 @@ eval("// Copyright Joyent, Inc. and other Node contributors.\n//\n// Permission
3967
3967
  \**********************/
3968
3968
  /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
3969
3969
 
3970
- eval("/**\n * @file build.js\n * Provide build information for DCP rtlink code, the same way as the dcp/build\n * module is injected for dcp-client code. That is what src/common/dcp-build.js\n * was supposed to do. We collide symbols here with a dcp-client built-in, so\n * that the code works the same everywhere but a gives different results depending\n * on how it was linked.\n *\n * This file is expressly processed by the copy-stamp rule in the core install\n * map, so that the symbols in here get substituted with their 'real' values as\n * the product is installed. A side effect of this is that dev versions running\n * out of the source tree will have symbol names instead of symbol values here.\n *\n * @author Wes Garland, wes@distributive.network\n * @date Nov 2022\n */\n\nexports.version = '050c7461ed59f387f7f585e05615ff7ae0eef059';\nexports.branch = 'develop';\n\n/* When/how configure.sh ran */\nexports.config = __webpack_require__(/*! ../etc/local-config.json */ \"./etc/local-config.json\");\ndelete exports.config.install;\n\n/* When/how install.sh ran (not for dcp-client) */\nexports.install = {\n timestamp: new Date(Number('__TIMESTAMP__') * 1000),\n build: '__DCP_BUILD',\n};\n\n\n\n\n//# sourceURL=webpack://dcp/./src/build.js?");
3970
+ eval("/**\n * @file build.js\n * Provide build information for DCP rtlink code, the same way as the dcp/build\n * module is injected for dcp-client code. That is what src/common/dcp-build.js\n * was supposed to do. We collide symbols here with a dcp-client built-in, so\n * that the code works the same everywhere but a gives different results depending\n * on how it was linked.\n *\n * This file is expressly processed by the copy-stamp rule in the core install\n * map, so that the symbols in here get substituted with their 'real' values as\n * the product is installed. A side effect of this is that dev versions running\n * out of the source tree will have symbol names instead of symbol values here.\n *\n * @author Wes Garland, wes@distributive.network\n * @date Nov 2022\n */\n\nexports.version = '8a95bd250c03421b473f543c2e9c7a85516d622c';\nexports.branch = 'prod-20230922';\n\n/* When/how configure.sh ran */\nexports.config = __webpack_require__(/*! ../etc/local-config.json */ \"./etc/local-config.json\");\ndelete exports.config.install;\n\n/* When/how install.sh ran (not for dcp-client) */\nexports.install = {\n timestamp: new Date(Number('__TIMESTAMP__') * 1000),\n build: '__DCP_BUILD',\n};\n\n\n\n\n//# sourceURL=webpack://dcp/./src/build.js?");
3971
3971
 
3972
3972
  /***/ }),
3973
3973
 
@@ -4019,7 +4019,7 @@ eval("/**\n * @file dcp-assert.js\n * Simple assertion modul
4019
4019
  \*********************************/
4020
4020
  /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
4021
4021
 
4022
- eval("/* provided dependency */ var process = __webpack_require__(/*! ./node_modules/process/browser.js */ \"./node_modules/process/browser.js\");\n/**\n * @file dcp-build.js Return an object describing the current DCP build.\n * @author Ryan Rossiter <ryan@kingsds.network>\n * @author Wes Garland, wes@distributive.network\n * @date July 2020\n * @date Apr 2023\n */\n\nif ((__webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\").platform) === 'nodejs')\n{\n const { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\n const fs = requireNative('fs');\n const path = requireNative('path');\n\n try\n {\n const rtlink = requireNative('dcp-rtlink');\n const localConfig = false\n || process.env.DCP_LOCAL_CONFIG_JSON /* dcp-client bundler's temp dir */\n || path.join(rtlink.runningLocation, 'etc', 'local-config.json'); /* installed or src dir config */\n Object.assign(exports, JSON.parse(fs.readFileSync(localConfig), 'utf-8'));\n }\n catch(error)\n {\n if (error.code !== 'MODULE_NOT_FOUND')\n throw error;\n /* If we arrive here, we couldn't resolve dcp-rtlink => we're on dcp-client */\n Object.assign(exports, {\"version\":\"050c7461ed59f387f7f585e05615ff7ae0eef059\",\"branch\":\"develop\",\"dcpClient\":{\"version\":\"4.3.0-3\",\"resolved\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#83021d39982e2d0e6fd5b400e59a4f7501fb3e7e\",\"overridden\":false},\"built\":\"Mon Sep 18 2023 14:14:55 GMT-0400 (Eastern Daylight Saving Time)\",\"config\":{\"generated\":\"Mon 18 Sep 2023 02:14:54 PM EDT by erose on lorge\",\"build\":\"debug\"},\"webpack\":\"5.88.2\",\"node\":\"v18.17.1\"}.config);\n }\n}\nelse\n{\n /* For all non-node platforms, just assume that we're also on dcp-client */\n Object.assign(exports, {\"version\":\"050c7461ed59f387f7f585e05615ff7ae0eef059\",\"branch\":\"develop\",\"dcpClient\":{\"version\":\"4.3.0-3\",\"resolved\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#83021d39982e2d0e6fd5b400e59a4f7501fb3e7e\",\"overridden\":false},\"built\":\"Mon Sep 18 2023 14:14:55 GMT-0400 (Eastern Daylight Saving Time)\",\"config\":{\"generated\":\"Mon 18 Sep 2023 02:14:54 PM EDT by erose on lorge\",\"build\":\"debug\"},\"webpack\":\"5.88.2\",\"node\":\"v18.17.1\"}.config);\n}\n\nif (typeof exports.build === 'undefined') {\n throw new Error('Could not determine build type!');\n}\n\n\n//# sourceURL=webpack://dcp/./src/common/dcp-build.js?");
4022
+ eval("/* provided dependency */ var process = __webpack_require__(/*! ./node_modules/process/browser.js */ \"./node_modules/process/browser.js\");\n/**\n * @file dcp-build.js Return an object describing the current DCP build.\n * @author Ryan Rossiter <ryan@kingsds.network>\n * @author Wes Garland, wes@distributive.network\n * @date July 2020\n * @date Apr 2023\n */\n\nif ((__webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\").platform) === 'nodejs')\n{\n const { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\n const fs = requireNative('fs');\n const path = requireNative('path');\n\n try\n {\n const rtlink = requireNative('dcp-rtlink');\n const localConfig = false\n || process.env.DCP_LOCAL_CONFIG_JSON /* dcp-client bundler's temp dir */\n || path.join(rtlink.runningLocation, 'etc', 'local-config.json'); /* installed or src dir config */\n Object.assign(exports, JSON.parse(fs.readFileSync(localConfig), 'utf-8'));\n }\n catch(error)\n {\n if (error.code !== 'MODULE_NOT_FOUND')\n throw error;\n /* If we arrive here, we couldn't resolve dcp-rtlink => we're on dcp-client */\n Object.assign(exports, {\"version\":\"8a95bd250c03421b473f543c2e9c7a85516d622c\",\"branch\":\"prod-20230922\",\"dcpClient\":{\"version\":\"4.3.0\",\"resolved\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#2ba1f35c876d61c52ffec4f5c67f2c5c8449ec5c\",\"overridden\":false},\"built\":\"Tue Sep 26 2023 09:10:57 GMT-0400 (Eastern Daylight Saving Time)\",\"config\":{\"generated\":\"Tue 26 Sep 2023 09:10:56 AM EDT by erose on lorge\",\"build\":\"debug\"},\"webpack\":\"5.88.2\",\"node\":\"v18.17.1\"}.config);\n }\n}\nelse\n{\n /* For all non-node platforms, just assume that we're also on dcp-client */\n Object.assign(exports, {\"version\":\"8a95bd250c03421b473f543c2e9c7a85516d622c\",\"branch\":\"prod-20230922\",\"dcpClient\":{\"version\":\"4.3.0\",\"resolved\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#2ba1f35c876d61c52ffec4f5c67f2c5c8449ec5c\",\"overridden\":false},\"built\":\"Tue Sep 26 2023 09:10:57 GMT-0400 (Eastern Daylight Saving Time)\",\"config\":{\"generated\":\"Tue 26 Sep 2023 09:10:56 AM EDT by erose on lorge\",\"build\":\"debug\"},\"webpack\":\"5.88.2\",\"node\":\"v18.17.1\"}.config);\n}\n\nif (typeof exports.build === 'undefined') {\n throw new Error('Could not determine build type!');\n}\n\n\n//# sourceURL=webpack://dcp/./src/common/dcp-build.js?");
4023
4023
 
4024
4024
  /***/ }),
4025
4025
 
@@ -4293,7 +4293,7 @@ eval("/**\n * @file password.js\n * Modal providing a way to
4293
4293
  \**********************************************/
4294
4294
  /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
4295
4295
 
4296
- eval("/**\n * @file client-modal/utils.js\n * @author KC Erb\n * @date Mar 2020\n * \n * All shared functions among the modals.\n */\nconst { fetchRelative } = __webpack_require__(/*! ./fetch-relative */ \"./src/dcp-client/client-modal/fetch-relative.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nexports.OnCloseErrorCode = 'DCP_CM:CANCELX';\n\nif (DCP_ENV.isBrowserPlatform) {\n // Provide as export for the convenience of `utils.MicroModal` instead of a separate require.\n exports.MicroModal = __webpack_require__(/*! micromodal */ \"./node_modules/micromodal/dist/micromodal.es.js\")[\"default\"];\n}\n\n/**\n * Return a unique string, formatted as a GET parameter, that changes often enough to\n * always force the browser to fetch the latest version of our resource.\n *\n * @note Currently always returns the Date-based poison due to webpack. \n */\nfunction cachePoison() {\n if (true)\n return '?ucp=050c7461ed59f387f7f585e05615ff7ae0eef059'; /* installer token */\n return '?ucp=' + Date.now();\n}\n \n/* Detect load type - on webpack, load dynamic content relative to webpack bundle;\n * otherwise load relative to the current scheduler's configured portal.\n */\nexports.myScript = (typeof document !== 'undefined') && document.currentScript;\nexports.corsProxyHref = undefined;\nif (exports.myScript && exports.myScript === (__webpack_require__(/*! ./fetch-relative */ \"./src/dcp-client/client-modal/fetch-relative.js\").myScript)) {\n let url = new ((__webpack_require__(/*! dcp/common/dcp-url */ \"./src/common/dcp-url.js\").DcpURL))(exports.myScript.src);\n exports.corsProxyHref = url.resolve('../cors-proxy.html');\n}\n\n/**\n * Look for modal id and required ids on page based on config, if not found, provide from dcp-client.\n * The first id in the required array must be the id of the modal's form element.\n * @param {Object} modalConfig Modal configuration object\n * @param {string} modalConfig.id Id of parent modal element\n * @param {string[]} modalConfig.required Array of required ids in parent modal element\n * @param {string[]} [modalConfig.optional] Array of optional ids in parent modal element\n * @param {string} modalConfig.path Relative path to modal html in dcp-client\n * @returns {DOMElement[]} Array of modal elements on page [config.id, ...config.required]\n */\nexports.initModal = async function (modalConfig, onClose) {\n exports.corsProxyHref = exports.corsProxyHref || dcpConfig.portal.location.resolve('dcp-client/cors-proxy.html');\n\n // Call ensure modal on any eager-loaded modals.\n if (modalConfig.eagerLoad) {\n Promise.all(\n modalConfig.eagerLoad.map(config => ensureModal(config))\n )\n };\n\n const [elements, optionalElements] = await ensureModal(modalConfig);\n\n // Wire up form to prevent default, resolve on submission, reject+reset when closed (or call onClose when closed)\n const [modal, form] = elements;\n form.reset(); // ensure that form is fresh\n let formResolve, formReject;\n let formPromise = new Promise( function(res, rej) {\n formResolve = res;\n formReject = rej;\n });\n form.onsubmit = function (submitEvent) {\n submitEvent.preventDefault();\n modal.setAttribute(\"data-state\", \"submitted\");\n formResolve(submitEvent);\n }\n\n exports.MicroModal.show(modalConfig.id, { \n disableFocus: true, \n onClose: onClose || getDefaultOnClose(formReject)\n });\n return [elements, formPromise, optionalElements];\n};\n\n// Ensure all required modal elements are on page according to modalConfig\nasync function ensureModal(modalConfig) {\n let allRequiredIds = [modalConfig.id, ...modalConfig.required];\n let missing = allRequiredIds.filter( id => !document.getElementById(id) );\n if (missing.length > 0) {\n if (missing.length !== allRequiredIds.length)\n console.warn(`Some of the ids needed to replace the default DCP-modal were found, but not all. So the default DCP-Modal will be used. Missing ids are: [${missing}].`);\n let contents = await fetchRelative(exports.corsProxyHref, modalConfig.path + cachePoison());\n const container = document.createElement('div');\n container.innerHTML = contents;\n document.body.appendChild(container);\n }\n\n const elements = allRequiredIds.map(id => document.getElementById(id));\n const optionalElements = (modalConfig.optional || []).map(id => document.getElementById(id));\n return [elements, optionalElements];\n};\n\n// This onClose is called by MicroModal and thus has the modal passed to it.\nfunction getDefaultOnClose (formReject) {\n return (modal) => {\n modal.offsetLeft; // forces style recalc\n const origState = modal.dataset.state;\n // reset form including data-state\n modal.setAttribute(\"data-state\", \"new\");\n // reject if closed without submitting form.\n if (origState !== \"submitted\") {\n const err = new DCPError(\"Modal was closed but modal's form was not submitted.\", exports.OnCloseErrorCode);\n formReject(err);\n }\n }\n}\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/client-modal/utils.js?");
4296
+ eval("/**\n * @file client-modal/utils.js\n * @author KC Erb\n * @date Mar 2020\n * \n * All shared functions among the modals.\n */\nconst { fetchRelative } = __webpack_require__(/*! ./fetch-relative */ \"./src/dcp-client/client-modal/fetch-relative.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nexports.OnCloseErrorCode = 'DCP_CM:CANCELX';\n\nif (DCP_ENV.isBrowserPlatform) {\n // Provide as export for the convenience of `utils.MicroModal` instead of a separate require.\n exports.MicroModal = __webpack_require__(/*! micromodal */ \"./node_modules/micromodal/dist/micromodal.es.js\")[\"default\"];\n}\n\n/**\n * Return a unique string, formatted as a GET parameter, that changes often enough to\n * always force the browser to fetch the latest version of our resource.\n *\n * @note Currently always returns the Date-based poison due to webpack. \n */\nfunction cachePoison() {\n if (true)\n return '?ucp=8a95bd250c03421b473f543c2e9c7a85516d622c'; /* installer token */\n return '?ucp=' + Date.now();\n}\n \n/* Detect load type - on webpack, load dynamic content relative to webpack bundle;\n * otherwise load relative to the current scheduler's configured portal.\n */\nexports.myScript = (typeof document !== 'undefined') && document.currentScript;\nexports.corsProxyHref = undefined;\nif (exports.myScript && exports.myScript === (__webpack_require__(/*! ./fetch-relative */ \"./src/dcp-client/client-modal/fetch-relative.js\").myScript)) {\n let url = new ((__webpack_require__(/*! dcp/common/dcp-url */ \"./src/common/dcp-url.js\").DcpURL))(exports.myScript.src);\n exports.corsProxyHref = url.resolve('../cors-proxy.html');\n}\n\n/**\n * Look for modal id and required ids on page based on config, if not found, provide from dcp-client.\n * The first id in the required array must be the id of the modal's form element.\n * @param {Object} modalConfig Modal configuration object\n * @param {string} modalConfig.id Id of parent modal element\n * @param {string[]} modalConfig.required Array of required ids in parent modal element\n * @param {string[]} [modalConfig.optional] Array of optional ids in parent modal element\n * @param {string} modalConfig.path Relative path to modal html in dcp-client\n * @returns {DOMElement[]} Array of modal elements on page [config.id, ...config.required]\n */\nexports.initModal = async function (modalConfig, onClose) {\n exports.corsProxyHref = exports.corsProxyHref || dcpConfig.portal.location.resolve('dcp-client/cors-proxy.html');\n\n // Call ensure modal on any eager-loaded modals.\n if (modalConfig.eagerLoad) {\n Promise.all(\n modalConfig.eagerLoad.map(config => ensureModal(config))\n )\n };\n\n const [elements, optionalElements] = await ensureModal(modalConfig);\n\n // Wire up form to prevent default, resolve on submission, reject+reset when closed (or call onClose when closed)\n const [modal, form] = elements;\n form.reset(); // ensure that form is fresh\n let formResolve, formReject;\n let formPromise = new Promise( function(res, rej) {\n formResolve = res;\n formReject = rej;\n });\n form.onsubmit = function (submitEvent) {\n submitEvent.preventDefault();\n modal.setAttribute(\"data-state\", \"submitted\");\n formResolve(submitEvent);\n }\n\n exports.MicroModal.show(modalConfig.id, { \n disableFocus: true, \n onClose: onClose || getDefaultOnClose(formReject)\n });\n return [elements, formPromise, optionalElements];\n};\n\n// Ensure all required modal elements are on page according to modalConfig\nasync function ensureModal(modalConfig) {\n let allRequiredIds = [modalConfig.id, ...modalConfig.required];\n let missing = allRequiredIds.filter( id => !document.getElementById(id) );\n if (missing.length > 0) {\n if (missing.length !== allRequiredIds.length)\n console.warn(`Some of the ids needed to replace the default DCP-modal were found, but not all. So the default DCP-Modal will be used. Missing ids are: [${missing}].`);\n let contents = await fetchRelative(exports.corsProxyHref, modalConfig.path + cachePoison());\n const container = document.createElement('div');\n container.innerHTML = contents;\n document.body.appendChild(container);\n }\n\n const elements = allRequiredIds.map(id => document.getElementById(id));\n const optionalElements = (modalConfig.optional || []).map(id => document.getElementById(id));\n return [elements, optionalElements];\n};\n\n// This onClose is called by MicroModal and thus has the modal passed to it.\nfunction getDefaultOnClose (formReject) {\n return (modal) => {\n modal.offsetLeft; // forces style recalc\n const origState = modal.dataset.state;\n // reset form including data-state\n modal.setAttribute(\"data-state\", \"new\");\n // reject if closed without submitting form.\n if (origState !== \"submitted\") {\n const err = new DCPError(\"Modal was closed but modal's form was not submitted.\", exports.OnCloseErrorCode);\n formReject(err);\n }\n }\n}\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/client-modal/utils.js?");
4297
4297
 
4298
4298
  /***/ }),
4299
4299
 
@@ -4325,7 +4325,7 @@ eval("/**\n * @file Module that implements Compute API\n * @module dcp/comput
4325
4325
  /***/ ((module, exports, __webpack_require__) => {
4326
4326
 
4327
4327
  "use strict";
4328
- eval("/* module decorator */ module = __webpack_require__.nmd(module);\n/* provided dependency */ var process = __webpack_require__(/*! ./node_modules/process/browser.js */ \"./node_modules/process/browser.js\");\n/**\n * @file dcp-client-bundle-src.js\n * Top-level file which gets webpacked into the bundle consumed by dcp-client 2.5\n * @author Wes Garland, wes@kingsds.network\n * @date July 2019\n */\n\n{\n let thisScript = typeof document !== 'undefined' ? (typeof document.currentScript !== 'undefined' && document.currentScript) || document.getElementById('_dcp_client_bundle') : {}\n let realModuleDeclare\n\n if ( false || typeof module.declare === 'undefined') {\n realModuleDeclare = ( true) ? module.declare : 0\n if (false) {}\n module.declare = function moduleUnWrapper (deps, factory) {\n factory(null, module.exports, module)\n return module.exports\n }\n }\n\n let _debugging = () => false\n if (process.env.DCP_CONFIG_USE_DEPRECATED_FUTURE)\n dcpConfig.future = (__webpack_require__(/*! ../common/config-future.js */ \"./src/common/config-future.js\").futureFactory)(_debugging, dcpConfig);\n\n /* These modules are official API and must be part of DCP Client */\n let officialApi = {\n 'protocol': __webpack_require__(/*! ../protocol-v4 */ \"./src/protocol-v4/index.js\"),\n 'compute': (__webpack_require__(/*! ./compute */ \"./src/dcp-client/compute.js\").compute),\n 'worker': __webpack_require__(/*! ./worker */ \"./src/dcp-client/worker/index.js\"),\n 'wallet': __webpack_require__(/*! ./wallet */ \"./src/dcp-client/wallet/index.js\"),\n };\n\n /* Some of these modules are API-track. Some of them need to be published to be\n * available for top-level resolution by DCP internals. Those (mostly) should have\n * been written using relative module paths.....\n */\n let modules = Object.assign({\n 'dcp-build': {\"version\":\"050c7461ed59f387f7f585e05615ff7ae0eef059\",\"branch\":\"develop\",\"dcpClient\":{\"version\":\"4.3.0-3\",\"resolved\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#83021d39982e2d0e6fd5b400e59a4f7501fb3e7e\",\"overridden\":false},\"built\":\"Mon Sep 18 2023 14:14:55 GMT-0400 (Eastern Daylight Saving Time)\",\"config\":{\"generated\":\"Mon 18 Sep 2023 02:14:54 PM EDT by erose on lorge\",\"build\":\"debug\"},\"webpack\":\"5.88.2\",\"node\":\"v18.17.1\"},\n 'dcp-xhr': __webpack_require__(/*! ../common/dcp-xhr */ \"./src/common/dcp-xhr.js\"),\n 'dcp-env': __webpack_require__(/*! ../common/dcp-env */ \"./src/common/dcp-env.js\"),\n 'dcp-url': __webpack_require__(/*! ../common/dcp-url */ \"./src/common/dcp-url.js\"),\n 'cli': __webpack_require__(/*! ../common/cli */ \"./src/common/cli.js\"),\n 'dcp-timers': __webpack_require__(/*! ../common/dcp-timers */ \"./src/common/dcp-timers.js\"),\n 'dcp-dot-dir': __webpack_require__(/*! ../common/dcp-dot-dir */ \"./src/common/dcp-dot-dir.js\"),\n 'dcp-assert': __webpack_require__(/*! ../common/dcp-assert */ \"./src/common/dcp-assert.js\"),\n 'dcp-events': __webpack_require__(/*! ../common/dcp-events */ \"./src/common/dcp-events/index.js\"),\n 'utils': __webpack_require__(/*! ../utils */ \"./src/utils/index.js\"),\n 'debugging': __webpack_require__(/*! ../debugging */ \"./src/debugging.js\"),\n 'publish': __webpack_require__(/*! ../common/dcp-publish */ \"./src/common/dcp-publish.js\"),\n 'compute-groups': {\n ...__webpack_require__(/*! ./compute-groups */ \"./src/dcp-client/compute-groups/index.js\"),\n publicGroupOpaqueId: (__webpack_require__(/*! ../common/scheduler-constants */ \"./src/common/scheduler-constants.js\").computeGroups[\"public\"].opaqueId),\n },\n 'bank-util': __webpack_require__(/*! ./bank-util */ \"./src/dcp-client/bank-util.js\"),\n 'protocol-v4': __webpack_require__(/*! ../protocol-v4 */ \"./src/protocol-v4/index.js\"), /* deprecated */\n 'client-modal': __webpack_require__(/*! ./client-modal */ \"./src/dcp-client/client-modal/index.js\"),\n 'eth': __webpack_require__(/*! ./wallet/eth */ \"./src/dcp-client/wallet/eth.js\"),\n 'serialize': __webpack_require__(/*! ../utils/serialize */ \"./src/utils/serialize.js\"),\n 'kvin': __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\"),\n 'job': __webpack_require__(/*! ./job */ \"./src/dcp-client/job/index.js\"),\n 'range-object': __webpack_require__(/*! ./range-object */ \"./src/dcp-client/range-object.js\"),\n 'stats-ranges': __webpack_require__(/*! ./stats-ranges */ \"./src/dcp-client/stats-ranges.js\"),\n 'job-values': __webpack_require__(/*! ./job-values */ \"./src/dcp-client/job-values.js\"),\n 'signal-handler': __webpack_require__(/*! ../node-libs/signal-handler */ \"./src/node-libs/signal-handler.js\"),\n 'standard-objects': {}\n }, officialApi);\n\n /* Export the JS Standard Classes (etc) from the global object of the bundle evaluation context,\n * in case we have code somewhere that needs to use these for instanceof checks.\n */\n ;[ Object, Function, Boolean, Symbol,\n Error, EvalError, RangeError, ReferenceError, SyntaxError, TypeError, URIError,\n Number, Math, Date,\n String, RegExp,\n Array, Int8Array, Uint8Array, Uint8ClampedArray, Int16Array, Uint16Array, Int32Array, Uint32Array, Float32Array, Float64Array,\n Map, Set, WeakMap, WeakSet,\n ArrayBuffer, DataView, JSON,\n Promise, \n Reflect, Proxy, Intl, WebAssembly, __webpack_require__\n ].forEach(function (obj) {\n if (obj.name && (typeof obj === 'function' || typeof obj === 'object'))\n modules['standard-objects'][obj.name] = obj\n })\n\n /* Export the constructors used by object literals and boxing objects. Usually these are the same as\n * the standard objects, but not always -- evaluation environments like node vm contexts can have\n * different standard objects supplied via the sandboxing object than the engine uses internally.\n */\n modules['engine-constructors'] = {\n Object: ({}).constructor,\n Array: ([]).constructor,\n Number: (0).constructor,\n String: ('').constructor,\n Boolean: (false).constructor,\n Function: (()=>1).constructor,\n };\n\n if (typeof BigInt !== 'undefined')\n {\n modules['standard-objects']['BigInt'] === BigInt;\n modules['engine-constructors'].BigInt = (0n).constructor;\n }\n if (typeof BigInt64Array !== 'undefined')\n modules['standard-objects']['BigInt64Array'] === BigInt64Array;\n if (typeof BigInt64Array !== 'undefined')\n modules['standard-objects']['BigUint64Array'] === BigUint64Array;\n\n module.declare([], function(require, exports, module) {\n Object.assign(exports, modules)\n exports['dcp-config'] = dcpConfig\n exports['dcp-default-config'] = {\"_serializeVerId\":\"v8\",\"what\":{\"ctr\":0,\"ps\":{\"dcp\":{\"ctr\":0,\"ps\":{\"connectionOptions\":{\"ctr\":0,\"ps\":{\"default\":{\"ctr\":0,\"ps\":{\"connectTimeout\":60,\"disconnectTimeout\":900,\"lingerTimeout\":18000,\"identityUnlockTimeout\":300,\"batchWaitTime\":0.03,\"ttl\":{\"raw\":{\"min\":15,\"max\":600,\"default\":120}},\"transports\":{\"arr\":[\"socketio\"]},\"socketio\":{\"raw\":{}}}}}},\"validitySlopValue\":10,\"validityStampCachePurgeInterval\":60,\"maxConnectionTimeout\":300000}},\"worker\":{\"raw\":{}},\"evaluator\":{\"ctr\":0,\"ps\":{\"listen\":{\"ctr\":\"dcpUrl$$DcpURL\",\"ps\":{},\"arg\":\"http://localhost:9000/\"},\"location\":{\"ctr\":\"dcpUrl$$DcpURL\",\"ps\":{},\"arg\":\"http://localhost:9000/\"},\"friendLocation\":{\"seen\":6}}},\"supervisor\":{\"raw\":{\"dcp\":{\"connectionOptions\":{\"default\":{\"identityUnlockTimeout\":900}}},\"tuning\":{\"watchdogInterval\":7,\"minSandboxStartDelay\":0.1,\"maxSandboxStartDelay\":0.7,\"minSandboxSlack\":0.2,\"maxSandboxSlack\":0.5,\"maxSandboxSliceRetries\":1,\"cachedJobsThreshold\":0,\"prefetchInterval\":30,\"pruneFrequency\":30000,\"mustPruneMultiplier\":1.3,\"defaultDelayIncrement\":50,\"maxExtraSandboxes\":8,\"maxResultSubmissionRetries\":3,\"reservedSliceLifetime\":300000},\"repoMan\":{\"frequency\":60000,\"thresholdMultiplier\":2},\"sandbox\":{\"generalTimeout\":6000,\"sliceTimeout\":86400000,\"progressTimeout\":300000,\"progressReportInterval\":1200000,\"maxSandboxUse\":1000},\"pCores\":0}},\"standaloneWorker\":{\"undefined\":true},\"scheduler\":{\"ctr\":0,\"ps\":{\"location\":{\"ctr\":\"dcpUrl$$DcpURL\",\"ps\":{},\"arg\":\"https://scheduler.distributed.computer/\"},\"worker\":{\"ctr\":0,\"ps\":{\"types\":{\"arr\":[\"v4\"]},\"operations\":{\"raw\":\"1.0.0\"}}},\"compatibility\":{\"raw\":{\"minimum\":{\"dcp\":\"^5.0.0\",\"dcp-client\":\"^4.0.0\",\"dcp-worker\":\"^2.1.0\",\"operations\":{\"work\":\"^4.1.0\",\"compute\":\"^1.0.0\",\"bank\":\"^4.0.0\"}},\"exclusions\":{}}}}}}}};\n })\n if (realModuleDeclare)\n module.declare = realModuleDeclare\n\n bundleExports = thisScript.exports = exports; /* must be last expression evaluated! */\n}\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/index.js?");
4328
+ eval("/* module decorator */ module = __webpack_require__.nmd(module);\n/* provided dependency */ var process = __webpack_require__(/*! ./node_modules/process/browser.js */ \"./node_modules/process/browser.js\");\n/**\n * @file dcp-client-bundle-src.js\n * Top-level file which gets webpacked into the bundle consumed by dcp-client 2.5\n * @author Wes Garland, wes@kingsds.network\n * @date July 2019\n */\n\n{\n let thisScript = typeof document !== 'undefined' ? (typeof document.currentScript !== 'undefined' && document.currentScript) || document.getElementById('_dcp_client_bundle') : {}\n let realModuleDeclare\n\n if ( false || typeof module.declare === 'undefined') {\n realModuleDeclare = ( true) ? module.declare : 0\n if (false) {}\n module.declare = function moduleUnWrapper (deps, factory) {\n factory(null, module.exports, module)\n return module.exports\n }\n }\n\n let _debugging = () => false\n if (process.env.DCP_CONFIG_USE_DEPRECATED_FUTURE)\n dcpConfig.future = (__webpack_require__(/*! ../common/config-future.js */ \"./src/common/config-future.js\").futureFactory)(_debugging, dcpConfig);\n\n /* These modules are official API and must be part of DCP Client */\n let officialApi = {\n 'protocol': __webpack_require__(/*! ../protocol-v4 */ \"./src/protocol-v4/index.js\"),\n 'compute': (__webpack_require__(/*! ./compute */ \"./src/dcp-client/compute.js\").compute),\n 'worker': __webpack_require__(/*! ./worker */ \"./src/dcp-client/worker/index.js\"),\n 'wallet': __webpack_require__(/*! ./wallet */ \"./src/dcp-client/wallet/index.js\"),\n };\n\n /* Some of these modules are API-track. Some of them need to be published to be\n * available for top-level resolution by DCP internals. Those (mostly) should have\n * been written using relative module paths.....\n */\n let modules = Object.assign({\n 'dcp-build': {\"version\":\"8a95bd250c03421b473f543c2e9c7a85516d622c\",\"branch\":\"prod-20230922\",\"dcpClient\":{\"version\":\"4.3.0\",\"resolved\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#2ba1f35c876d61c52ffec4f5c67f2c5c8449ec5c\",\"overridden\":false},\"built\":\"Tue Sep 26 2023 09:10:57 GMT-0400 (Eastern Daylight Saving Time)\",\"config\":{\"generated\":\"Tue 26 Sep 2023 09:10:56 AM EDT by erose on lorge\",\"build\":\"debug\"},\"webpack\":\"5.88.2\",\"node\":\"v18.17.1\"},\n 'dcp-xhr': __webpack_require__(/*! ../common/dcp-xhr */ \"./src/common/dcp-xhr.js\"),\n 'dcp-env': __webpack_require__(/*! ../common/dcp-env */ \"./src/common/dcp-env.js\"),\n 'dcp-url': __webpack_require__(/*! ../common/dcp-url */ \"./src/common/dcp-url.js\"),\n 'cli': __webpack_require__(/*! ../common/cli */ \"./src/common/cli.js\"),\n 'dcp-timers': __webpack_require__(/*! ../common/dcp-timers */ \"./src/common/dcp-timers.js\"),\n 'dcp-dot-dir': __webpack_require__(/*! ../common/dcp-dot-dir */ \"./src/common/dcp-dot-dir.js\"),\n 'dcp-assert': __webpack_require__(/*! ../common/dcp-assert */ \"./src/common/dcp-assert.js\"),\n 'dcp-events': __webpack_require__(/*! ../common/dcp-events */ \"./src/common/dcp-events/index.js\"),\n 'utils': __webpack_require__(/*! ../utils */ \"./src/utils/index.js\"),\n 'debugging': __webpack_require__(/*! ../debugging */ \"./src/debugging.js\"),\n 'publish': __webpack_require__(/*! ../common/dcp-publish */ \"./src/common/dcp-publish.js\"),\n 'compute-groups': {\n ...__webpack_require__(/*! ./compute-groups */ \"./src/dcp-client/compute-groups/index.js\"),\n publicGroupOpaqueId: (__webpack_require__(/*! ../common/scheduler-constants */ \"./src/common/scheduler-constants.js\").computeGroups[\"public\"].opaqueId),\n },\n 'bank-util': __webpack_require__(/*! ./bank-util */ \"./src/dcp-client/bank-util.js\"),\n 'protocol-v4': __webpack_require__(/*! ../protocol-v4 */ \"./src/protocol-v4/index.js\"), /* deprecated */\n 'client-modal': __webpack_require__(/*! ./client-modal */ \"./src/dcp-client/client-modal/index.js\"),\n 'eth': __webpack_require__(/*! ./wallet/eth */ \"./src/dcp-client/wallet/eth.js\"),\n 'serialize': __webpack_require__(/*! ../utils/serialize */ \"./src/utils/serialize.js\"),\n 'kvin': __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\"),\n 'job': __webpack_require__(/*! ./job */ \"./src/dcp-client/job/index.js\"),\n 'range-object': __webpack_require__(/*! ./range-object */ \"./src/dcp-client/range-object.js\"),\n 'stats-ranges': __webpack_require__(/*! ./stats-ranges */ \"./src/dcp-client/stats-ranges.js\"),\n 'job-values': __webpack_require__(/*! ./job-values */ \"./src/dcp-client/job-values.js\"),\n 'signal-handler': __webpack_require__(/*! ../node-libs/signal-handler */ \"./src/node-libs/signal-handler.js\"),\n 'standard-objects': {}\n }, officialApi);\n\n /* Export the JS Standard Classes (etc) from the global object of the bundle evaluation context,\n * in case we have code somewhere that needs to use these for instanceof checks.\n */\n ;[ Object, Function, Boolean, Symbol,\n Error, EvalError, RangeError, ReferenceError, SyntaxError, TypeError, URIError,\n Number, Math, Date,\n String, RegExp,\n Array, Int8Array, Uint8Array, Uint8ClampedArray, Int16Array, Uint16Array, Int32Array, Uint32Array, Float32Array, Float64Array,\n Map, Set, WeakMap, WeakSet,\n ArrayBuffer, DataView, JSON,\n Promise, \n Reflect, Proxy, Intl, WebAssembly, __webpack_require__\n ].forEach(function (obj) {\n if (obj.name && (typeof obj === 'function' || typeof obj === 'object'))\n modules['standard-objects'][obj.name] = obj\n })\n\n /* Export the constructors used by object literals and boxing objects. Usually these are the same as\n * the standard objects, but not always -- evaluation environments like node vm contexts can have\n * different standard objects supplied via the sandboxing object than the engine uses internally.\n */\n modules['engine-constructors'] = {\n Object: ({}).constructor,\n Array: ([]).constructor,\n Number: (0).constructor,\n String: ('').constructor,\n Boolean: (false).constructor,\n Function: (()=>1).constructor,\n };\n\n if (typeof BigInt !== 'undefined')\n {\n modules['standard-objects']['BigInt'] === BigInt;\n modules['engine-constructors'].BigInt = (0n).constructor;\n }\n if (typeof BigInt64Array !== 'undefined')\n modules['standard-objects']['BigInt64Array'] === BigInt64Array;\n if (typeof BigInt64Array !== 'undefined')\n modules['standard-objects']['BigUint64Array'] === BigUint64Array;\n\n module.declare([], function(require, exports, module) {\n Object.assign(exports, modules)\n exports['dcp-config'] = dcpConfig\n exports['dcp-default-config'] = {\"_serializeVerId\":\"v8\",\"what\":{\"ctr\":0,\"ps\":{\"dcp\":{\"ctr\":0,\"ps\":{\"connectionOptions\":{\"ctr\":0,\"ps\":{\"default\":{\"ctr\":0,\"ps\":{\"connectTimeout\":60,\"disconnectTimeout\":900,\"lingerTimeout\":18000,\"identityUnlockTimeout\":300,\"batchWaitTime\":0.03,\"ttl\":{\"raw\":{\"min\":15,\"max\":600,\"default\":120}},\"transports\":{\"arr\":[\"socketio\"]},\"socketio\":{\"raw\":{}}}}}},\"validitySlopValue\":10,\"validityStampCachePurgeInterval\":60,\"maxConnectionTimeout\":300000}},\"worker\":{\"raw\":{}},\"evaluator\":{\"ctr\":0,\"ps\":{\"listen\":{\"ctr\":\"dcpUrl$$DcpURL\",\"ps\":{},\"arg\":\"http://localhost:9000/\"},\"location\":{\"ctr\":\"dcpUrl$$DcpURL\",\"ps\":{},\"arg\":\"http://localhost:9000/\"},\"friendLocation\":{\"seen\":6}}},\"supervisor\":{\"raw\":{\"dcp\":{\"connectionOptions\":{\"default\":{\"identityUnlockTimeout\":900}}},\"tuning\":{\"watchdogInterval\":7,\"minSandboxStartDelay\":0.1,\"maxSandboxStartDelay\":0.7,\"minSandboxSlack\":0.2,\"maxSandboxSlack\":0.5,\"maxSandboxSliceRetries\":1,\"cachedJobsThreshold\":0,\"prefetchInterval\":30,\"pruneFrequency\":30000,\"mustPruneMultiplier\":1.3,\"defaultDelayIncrement\":50,\"maxExtraSandboxes\":8,\"maxResultSubmissionRetries\":3,\"reservedSliceLifetime\":300000},\"repoMan\":{\"frequency\":60000,\"thresholdMultiplier\":2},\"sandbox\":{\"generalTimeout\":6000,\"sliceTimeout\":86400000,\"progressTimeout\":300000,\"progressReportInterval\":1200000,\"maxSandboxUse\":1000},\"pCores\":0}},\"standaloneWorker\":{\"undefined\":true},\"scheduler\":{\"ctr\":0,\"ps\":{\"location\":{\"ctr\":\"dcpUrl$$DcpURL\",\"ps\":{},\"arg\":\"https://scheduler.distributed.computer/\"},\"worker\":{\"ctr\":0,\"ps\":{\"types\":{\"arr\":[\"v4\"]},\"operations\":{\"raw\":\"1.0.0\"}}},\"compatibility\":{\"raw\":{\"minimum\":{\"dcp\":\"^5.0.0\",\"dcp-client\":\"^4.0.0\",\"dcp-worker\":\"^2.1.0\",\"operations\":{\"work\":\"^4.1.0\",\"compute\":\"^1.0.0\",\"bank\":\"^4.0.0\"}},\"exclusions\":{}}}}}}}};\n })\n if (realModuleDeclare)\n module.declare = realModuleDeclare\n\n bundleExports = thisScript.exports = exports; /* must be last expression evaluated! */\n}\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/index.js?");
4329
4329
 
4330
4330
  /***/ }),
4331
4331
 
@@ -4447,7 +4447,7 @@ eval("/* provided dependency */ var process = __webpack_require__(/*! ./node_mod
4447
4447
  \*************************************************/
4448
4448
  /***/ ((module, __unused_webpack_exports, __webpack_require__) => {
4449
4449
 
4450
- eval("/**\n * @file /src/schedmsg/schedmsg-web.js\n * @author Ryan Rossiter, ryan@kingsds.network\n * @date March 2020\n *\n * This is the SchedMsg implementation for commands that are browser-specific\n * or have browser-specific behaviour.\n */\n\nconst { SchedMsg } = __webpack_require__(/*! ./schedmsg */ \"./src/dcp-client/schedmsg/schedmsg.js\");\n\nclass SchedMsgWeb extends SchedMsg {\n constructor(worker) {\n super(worker);\n\n this.registerHandler('announce', this.onAnnouncement.bind(this));\n this.registerHandler('openPopup', this.onOpenPopup.bind(this));\n this.registerHandler('reload', this.onReload.bind(this));\n }\n\n onAnnouncement({ message }) {\n \n window.alert('DCP Worker Announcement: ' + message);\n\n }\n\n onOpenPopup({ href }) {\n window.open(href);\n }\n\n onReload() {\n const hash = window.location.hash;\n\n let newUrl = window.location.href.replace(/#.*/, '');\n newUrl += (newUrl.indexOf('?') === -1 ? '?' : '&');\n newUrl += 'dcp=050c7461ed59f387f7f585e05615ff7ae0eef059,' + Date.now() + hash;\n\n window.location.replace(newUrl);\n }\n}\n\nObject.assign(module.exports, {\n SchedMsgWeb\n});\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/schedmsg/schedmsg-web.js?");
4450
+ eval("/**\n * @file /src/schedmsg/schedmsg-web.js\n * @author Ryan Rossiter, ryan@kingsds.network\n * @date March 2020\n *\n * This is the SchedMsg implementation for commands that are browser-specific\n * or have browser-specific behaviour.\n */\n\nconst { SchedMsg } = __webpack_require__(/*! ./schedmsg */ \"./src/dcp-client/schedmsg/schedmsg.js\");\n\nclass SchedMsgWeb extends SchedMsg {\n constructor(worker) {\n super(worker);\n\n this.registerHandler('announce', this.onAnnouncement.bind(this));\n this.registerHandler('openPopup', this.onOpenPopup.bind(this));\n this.registerHandler('reload', this.onReload.bind(this));\n }\n\n onAnnouncement({ message }) {\n \n window.alert('DCP Worker Announcement: ' + message);\n\n }\n\n onOpenPopup({ href }) {\n window.open(href);\n }\n\n onReload() {\n const hash = window.location.hash;\n\n let newUrl = window.location.href.replace(/#.*/, '');\n newUrl += (newUrl.indexOf('?') === -1 ? '?' : '&');\n newUrl += 'dcp=8a95bd250c03421b473f543c2e9c7a85516d622c,' + Date.now() + hash;\n\n window.location.replace(newUrl);\n }\n}\n\nObject.assign(module.exports, {\n SchedMsgWeb\n});\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/schedmsg/schedmsg-web.js?");
4451
4451
 
4452
4452
  /***/ }),
4453
4453
 
@@ -4724,7 +4724,7 @@ eval("/**\n * @file dcp/src/dcp-client/worker/supervisor2/gpu_support.js\n *\n *
4724
4724
  /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
4725
4725
 
4726
4726
  "use strict";
4727
- eval("/**\n * @file dcp/src/dcp-client/worker/supervisor2/index.js\n * Code managing sandboxes, tasks, jobs, and slices within in a DCP Worker.\n * @author Wes Garland, wes@distributive.network\n * Paul, paul@distributive.network\n * @date Dec 2020\n * June 2022, Jan-April 2023\n * @module supervisor\n * @copyright Copyright (c) 2018-2023, Distributive Corp. All Rights Reserved\n */\n/*\n * initial ready reconnecting stopping stopped paused broken\n * |-- ctor ----------------------------------------------------------------------------------------------------------------->\n * |-- work ----------------------------------------------------------------------------------------------------------------->\n * |-- work --------------------------------------------------------------------------------------------------->\n * |-- work -------------------------------------------------------------------------------->\n * |-- work --------------------------------------------------------->\n * |-- work --------------------------------->\n * |-- Worker.pause --------------------------------------------------------------------------->\n * <-- Worker.unpause -------------------------------------------------------------------------|\n * |-- work ----->\n * |-- stopWork ---------------------------->\n * |-- postStopShutdown --->\n * |-- PM.connectTo --> (ProtocolManager)\n * <-- PM.connectTo --| (ProtocolManager)\n * |-- stopWork ------------------------------------------->\n * <-- work -----------------------------------------------------------------------|\n * <-- stopWork -----------------------------------------------------|\n */\n/* global dcpConfig */ // eslint-disable-line no-redeclare\n// @ts-check\n\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nconst constants = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst { assert } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\nconst dcp_timers = __webpack_require__(/*! dcp/common/dcp-timers */ \"./src/common/dcp-timers.js\");\nconst EventEmitter = __webpack_require__(/*! events */ \"./node_modules/events/events.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst { Address } = __webpack_require__(/*! dcp/dcp-client/wallet/eth */ \"./src/dcp-client/wallet/eth.js\");\nconst { Keystore } = __webpack_require__(/*! dcp/dcp-client/wallet/keystore */ \"./src/dcp-client/wallet/keystore.js\");\nconst RingBuffer = __webpack_require__(/*! dcp/utils/ringBuffer */ \"./src/utils/ringBuffer.js\");\nconst { Synchronizer } = __webpack_require__(/*! dcp/common/concurrency */ \"./src/common/concurrency.js\");\nconst { JobManager } = __webpack_require__(/*! ./job-manager */ \"./src/dcp-client/worker/supervisor2/job-manager.js\");\nconst { Sandbox, SandboxError } = __webpack_require__(/*! ./sandbox2 */ \"./src/dcp-client/worker/supervisor2/sandbox2.js\");\nconst { sliceStatus } = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst { OriginAccessManager } = __webpack_require__(/*! dcp/dcp-client/worker/origin-access-manager */ \"./src/dcp-client/worker/origin-access-manager.js\");\nconst { a$sleepMs, booley, toJobMap, encodeDataURI, stringify, nextEma } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\n\nconst { ModuleCache } = __webpack_require__(/*! ./module-cache */ \"./src/dcp-client/worker/supervisor2/module-cache.js\");\nconst { Promise_any } = __webpack_require__(/*! ./promise_any */ \"./src/dcp-client/worker/supervisor2/promise_any.js\");\nconst { ProtocolManager } = __webpack_require__(/*! ./protocol-manager */ \"./src/dcp-client/worker/supervisor2/protocol-manager.js\");\nconst { EvaluatorManager } = __webpack_require__(/*! ./evaluator-manager */ \"./src/dcp-client/worker/supervisor2/evaluator-manager.js\");\nconst { DelayManager } = __webpack_require__(/*! ./delay-manager */ \"./src/dcp-client/worker/supervisor2/delay-manager.js\");\nconst { Options } = __webpack_require__(/*! ./options */ \"./src/dcp-client/worker/supervisor2/options.js\");\nconst common = __webpack_require__(/*! ./common */ \"./src/dcp-client/worker/supervisor2/common.js\");\nconst { debugBuild, selectiveDebug, selectiveDebug2, minimalDiag, selectiveSupEx } = common;\nconst supShared = __webpack_require__(/*! ../SupShared */ \"./src/dcp-client/worker/SupShared.js\");\nconst { canScheduleGPU } = __webpack_require__(/*! ./gpu_support */ \"./src/dcp-client/worker/supervisor2/gpu_support.js\");\n\n/** @typedef {string} opaqueId */ // 22 character base64 string\n/** @typedef {import('./slice2').Slice} Slice */\n/** @typedef {import('dcp/utils/jsdoc-types').Auth} Auth */\n/** @typedef {import('dcp/utils/jsdoc-types').Body} Body */\n/** @typedef {import('./sandbox2').SandboxHandle} SandboxHandle */\n/** @typedef {import('dcp/utils/jsdoc-types').SliceObj} SliceObj */\n/** @typedef {import('dcp/dcp-client/worker/index').Worker} Worker */\n/** @typedef {import('dcp/utils/jsdoc-types').TDPayload} TDPayload */\n/** @typedef {import('dcp/utils/jsdoc-types').Signature} Signature */\n/** @typedef {import('dcp/utils/jsdoc-types').SliceMessage} SliceMessage */\n/** @typedef {import('dcp/dcp-client/wallet/keystore').Keystore} Keystore */\n/** @typedef {import('dcp/utils/jsdoc-types').SupervisorOptions} SupervisorOptions */\n/** @typedef {import('dcp/protocol-v4/connection/connection').Connection} Connection */\n\n//\n// Index to functionality -- search for '_Idx' to toggle through the index.\n//\n// 1) class Supervisor\n// 2) work, checkCapabilities\n// 3) safeEmit, workerEmit, jobEmit, error, warning, mungeError, jobDescriptor, setState\n// 4) returnAllSlices, postStopShutdown, abort, stopWork, purgeJob\n// 5) roundRobinSlices, makeJobSelectionCursor, handleSandboxWorkError, handleFailedSlice\n// 6) returnSlices, returnSlice, emitProgressReport\n// 7) jobQuanta, repoMan, predictLoad(viz., clairvoyance), waitUntilWorkIsReady, generateWorkerComputeGroups\n// 8) availableSandboxSpace, fetchTask, addTaskToWorkload, fetchFromTD, clearUnusedJobManagersAndModuleCache\n// 9) createSandbox, returnSandbox, hookUpSandboxListeners, pruneSandboxes\n// 10) recordResult, sendToResultSubmitter, sendResultToRemote\n// 11) handleWorkReject\n//\n\n// _Idx\n//\n// class Supervisor\n//\n\n/**\n * Supervisor constructor\n *\n * A supervisor manages the communication with the scheduler, manages sandboxes, and\n * decides which workload should be sent to which sandboxes when.\n *\n * Possible states: 'initial', 'ready', 'reconnecting', 'stopping', 'stopped', 'broken'\n * Start state:\n * - initial\n *\n * Intermediate states:\n * - ready\n * - reconnecting\n * - stopping\n *\n * Terminal states:\n * - stopped\n * - broken\n *\n * Valid transitions:\n * - initial -> ready -> reconnecting -> ready\n * - ready -> stopping -> stopped\n * - initial -> broken\n */\nclass Supervisor extends EventEmitter\n{\n /**\n * @constructor\n * @param {Worker} worker\n * @param {Keystore} identity\n * @param {SupervisorOptions} options\n */\n constructor (worker, identity, options)\n {\n super({ captureRejections: false });\n\n if (!(identity instanceof Keystore))\n throw new Error(`identity ${JSON.stringify(identity)} must be an instance of Keystore`);\n\n debugging('supervisor') && console.debug('Supervisor.options', options);\n assert(options === worker.workerOptions);\n \n /** @type {Worker} */\n this.worker = worker;\n /** @type {Keystore} */\n this.identityKeystore = identity;\n /** @type {Options} */\n this.options = new Options(options, worker);\n\n selectiveDebug() && console.debug('Supervisor: cores.cpu, cores.gpu, maxSandboxes', options.cores?.cpu, options.cores?.gpu, this.options.maxSandboxes);\n\n /** @type {ModuleCache} */\n this.moduleCache = new ModuleCache(this);\n\n // Manage delays and exponential backoff.\n this.delayManager = new DelayManager(this, this.options.defaultDelayIncrement);\n\n /* See https://distributive.atlassian.net/browse/DCP-3175 */\n /** @type {OriginAccessManager} */\n this.originManager = OriginAccessManager.construct(this.options.allowOrigins);\n\n /** @type {ProtocolManager} */\n this.dcp4 = new ProtocolManager(this);\n\n /** @type {common.DebuggingTools} */\n this.dbg = new common.DebuggingTools(this);\n\n // Turn on for max speed debugging.\n if (false)\n {}\n\n /** @type { Synchronizer } */\n this.state = new Synchronizer('initial', [ 'initial', 'ready', 'reconnecting', 'stopping', 'stopped', 'paused', 'broken']);\n /** @type {Object<string, JobManager>} */\n this.jobMap = {}; // jobAddress => jobManager\n\n /** @type {JobManager[]} */\n this.jobManagerInventory = common.InventoryArray('jobManagers');\n /** @type {Sandbox[]} */\n this.sandboxInventory = []; // All sandboxes that are being used by the job managers. Makes sure we don't lose sandboxes.\n /** @type {{ next: cbNext, push: cbPush }} */\n this.cursor = null;\n /** @type {number} */\n this.defaultQuanta = 1.0;\n\n /**\n * Evaluator down management.\n **/\n this.evaluator = new EvaluatorManager();\n\n // There are 2 kinds of barriers.\n // 1) fetchTaskBarrier is a barrier for the task fetching from task distributor path.\n // 2) roundRobinBarrier is a barrier for the slice execution path.\n /** @type {boolean} */\n this.fetchTaskBarrier = false;\n /** @type {boolean} */\n this.roundRobinBarrier = false;\n\n /** @type {object[]} */\n this.rejectedJobs = [];\n /**\n * An N-slot ring buffer of job addresses. Stores all jobs that have had no more than 1 slice run in the ring buffer.\n * Required for the implementation of discrete jobs\n * @type {RingBuffer}\n */\n this.ringBufferofJobs = new RingBuffer(200); // N = 200 should be more than enough.\n /**\n * When true we await waitUntilWorkIsReady until at least 1 job is ready with at least 1 ready slice.\n * waitUntilWorkIsReady\n * @type {boolean}\n */\n this.waitForWork = true;\n /**\n * Last repoMan time stamp.\n * @type {number}\n **/\n this.lastRepoMan = Date.now();\n /**\n * Last prune time stamp.\n * @type {number}\n **/\n this.lastPrune = Date.now();\n /**\n * General time stamp.\n * @type {number}\n **/\n this.lastTime = Date.now();\n /**\n * Fetch started time stamp.\n * @type {number}\n **/\n this.fetchTaskStarted = 0;\n /**\n * The capabilities of a random sandbox.\n * @todo XXXpfr Re-work this once fetchTask uses the capabilities of every sandbox to fetch slices.\n * @type {object}\n */\n this.capabilities = null;\n /**\n * EMA times series of CPUTime + GPUTime over all jobs.\n * Each EMA entry is computed right before calling fetchTask.\n * @type {number}\n */\n this.localTime = 0;\n /**\n * EMA times series of sliceCPUTime + sliceGPUTime over all jobs.\n * Each EMA entry is computed right before calling fetchTask.\n * @type {number}\n */\n this.globalTime = 0;\n /**\n * When this.sliceTiming is set to be true, it displays the timings of a every slice\n * slice['queueingDelta'] = timespan of when slice is passed to jobManager.runQueuedSlice until sandbox.work\n * slice['executionDelta'] = timespan of execution in sandbox\n * slice['resultDelta'] = timespan of when sandbox finishes executing until recordResult completes.\n * @type {boolean}\n */\n this.sliceTiming = false;\n\n try\n {\n // Start up the connections.\n this.dcp4.instantiateAllConnections();\n }\n catch(error)\n {\n this.error('Failed to set up DCP connections:', error);\n this.setState('initial', 'broken');\n this.worker.stop(true).finally(() => { throw error; });\n }\n }\n\n //\n // Compatibility layer between Sup1, Sup2 and the Sup interface exposed by Worker.\n //\n /**\n * Get all sandboxes.\n * @type {Sandbox[]}\n */\n get sandboxes () { return this.sandboxInventory.filter((sandbox) => !sandbox.isTerminated); }\n /**\n * Get all working sandboxes.\n * @type {Sandbox[]}\n */\n get workingSandboxes () { return this.sandboxInventory.filter((sandbox) => sandbox.isWorking); }\n /**\n * Get the number of working sandboxes.\n * @type {number}\n */\n get workingSandboxCount () { return this.workingSandboxes.length; }\n /**\n * Get all slices over all jobs..\n * @type {Slice[]}\n */\n get slices () {\n const slices = [];\n this.jobManagerInventory.forEach((jobManager) => { slices.push(...jobManager.sliceInventory); });\n return slices;\n }\n /**\n * Get all queued slices over all jobs..\n * @type {Slice[]}\n */\n get queuedSlices () {\n const slices = [];\n this.jobManagerInventory.forEach((jobManager) => { slices.push(...jobManager.queuedSlices); });\n return slices;\n }\n /**\n * Get all queued slices over all jobs..\n * @type {Slice[]}\n */\n get workingSlices () {\n const slices = [];\n this.jobManagerInventory.forEach((jobManager) => { slices.push(...jobManager.workingSlices); });\n return slices;\n }\n /** @type {opaqueId} */\n get workerId () { return this.options.workerId; }\n /** @type {opaqueId} */\n set workerId (id) { this.options.workerId = id; }\n get version() { return '2.0.0' }\n /**\n * @deprecated\n * @todo XXXpfr Rip out this sup2/sup1 special-casing when we finally kill sup1.\n * @type {boolean}\n */\n get isSupervisor1 () { return false; }\n /**\n * @deprecated\n * @todo XXXpfr Rip out this sup2/sup1 special-casing when we finally kill sup1.\n * @type {boolean}\n */\n get isSupervisor2 () { return true; }\n\n //\n // Miscellaneous properties.\n //\n\n /**\n * Dynamic maxWorkingCores.\n * The maximum number of cores that can be executing slices. Slices are scheduled\n * using density. E.g. suppose a job has GPUDensity is 0 and CPUDensity is 0.5,\n * then 2 slices of this job can be scheduled on a single core.\n * @type {number}\n */\n get maxWorkingCores () { return this.options.cores?.cpu; }\n /**\n * Dynamic maxWorkingGPUs.\n * The maximum number of GPUs that can be executing slices. Slices are scheduled\n * using density. E.g. suppose a job has GPUDensity is 0.5 and CPUDensity is 0.5,\n * then 2 slices of this job can be scheduled on a single GPU core and a single CPU core.\n * @type {number}\n */\n get maxWorkingGPUs () { return this.options.cores?.gpu; }\n /**\n * @deprecated\n * @todo XXXpfr Get rid of this after Sup1 dies.\n */\n get lastDcpsid () { return this.dcp4.lastDcpsid; }\n /**\n * @deprecated\n * @todo XXXpfr Get rid of this after Sup1 dies.\n */\n set lastDcpsid (dcpsid) { this.dcp4.lastDcpsid = dcpsid; }\n /**\n * Indicates whether supervisor is ready for business.\n * @type {boolean}\n */\n get isReady () { return this.worker.working && this.state.is('ready'); }\n /**\n * The # of sandboxes not being used.\n * @type {number}\n */\n get unusedSandboxCount () { return this.options.maxSandboxes - this.workingSliceCount; }\n /**\n * The unused amount of CPU density in the cores.\n * @type {number}\n */\n get unusedCoreSpace () { return this.maxWorkingCores - this.workingSliceDensity; }\n /**\n * The unused amount of GPU density in the cores.\n * Use Math.max(1, this.maxWorkingGPUs) so there's always enough room to schedule\n * a GPU slice when this.workingGPUDensity = 0. In RoundRobinSlices we use the accumulated\n * recent history ( canScheduleGPU(maxWorkingGPUs) ) to check whether the average recent\n * density is within this.maxWorkingGPUs.\n * @type {number}\n */\n get unusedGPUSpace () { return Math.max(1, this.maxWorkingGPUs) - this.workingGPUDensity; }\n /** @type {number} */\n get workingSliceDensity ()\n {\n let density = 0;\n for (const jobMan of this.jobManagerInventory)\n density += jobMan.workingSliceDensity;\n return density;\n }\n /** @type {number} */\n get workingGPUDensity ()\n {\n let density = 0;\n for (const jobMan of this.jobManagerInventory)\n density += jobMan.workingGPUDensity;\n return density;\n }\n /** @type {number} */\n get workingSliceCount ()\n {\n let count = 0;\n for (const jobMan of this.jobManagerInventory)\n count += jobMan.workingSliceCount;\n return count;\n }\n /**\n * Compute the estimated time to completion of all work.\n * The time is measured as if there were only a single slice running at a time.\n * workRemaining is the amount of time until completion.\n * @type {number}\n */\n get workRemaining ()\n {\n let workRemaining = 0;\n for (const jobMan of this.jobManagerInventory)\n workRemaining += jobMan.workRemaining;\n return workRemaining;\n }\n\n // _Idx\n //\n // work, checkCapabilities\n //\n\n /**\n * Set up sandboxes and interval timers, then start to search for work.\n * Called in Worker.start().\n * Initial entry point after Worker constructor.\n * We need to start searching for work here to allow starting and stopping a worker.\n */\n work ()\n {\n const abort = async (error) => {\n // May be in a stopping/stopped state, because dcp-worker was hit with ctrl-C.\n this.setState(['ready', 'stopping', 'stopped', 'reconnecting'], 'broken');\n await this.worker.stop(true);\n throw error;\n };\n /* Provide opportunity for calling code to hook ready/error events. */\n dcp_timers.setImmediate(() => {\n try\n {\n // [ 'initial', 'ready', 'reconnecting', 'stopping', 'stopped', 'paused', 'broken' ]\n if (this.state.isNot('initial'))\n {\n if (this.state.in(['ready', 'stopping', 'reconnecting']))\n {\n this.warning(`Supervisor.work was called when supervisor is already ${this.state.valueOf()}.`, 'Please either wait and try again or restart worker.');\n return;\n }\n else if (this.state.is('broken'))\n {\n this.warning(\"Cannot call Supervisor.work when supervisor is in a 'broken' state. Please restart worker.\");\n return;\n }\n this.state.set(['stopped', 'paused'], 'initial');\n }\n this.evaluator.initialize();\n this.dcp4.instantiateAllConnections();\n\n // Beacon interval timer.\n this.progressReportTimer = dcp_timers.setInterval(() => this.emitProgressReport(), this.options.progressReportInterval);\n // Watchdog: fetchTask-driven interval timer.\n this.watchdogTimer = dcp_timers.setInterval(() => this.fetchTask(), this.options.watchdogInterval);\n\n // Interval timers helps keep workers and localExec alive forever.\n this.progressReportTimer.unref();\n this.watchdogTimer.unref();\n\n if ( false || debugging('supervisor'))\n {\n this.sliceDebuggingTimer = setInterval(() => {\n this.jobManagerInventory.forEach((jobMan) => {\n const { unassigned, ready, reserved, working, workdone, complete, failed, finished } = jobMan.dumpSlices ('RRS', false, false);\n console.debug(`RRS(${jobMan.identifier},${this.unusedSandboxCount},${this.unusedCoreSpace},${this.workingSliceCount},${this.workingSliceDensity}): u/r/rsv/w/wd/c/f/fsh ${unassigned}/${ready}/${reserved}/${working}/${workdone}/${complete}/${failed}/${finished}`, jobMan.identifier, this.sandboxInventory.length);\n });\n }, 30 * 1000);\n if (this.sliceDebuggingTimer.unref)\n this.sliceDebuggingTimer.unref();\n }\n\n this.state.set('initial', 'ready');\n\n // Create 1 sandbox now to get the capabilities which are sent to Task Distributor by fetchTask.\n this.createSandbox()\n .then((sandbox) => {\n this.sandboxInventory.push(sandbox);\n debugging('supervisor') && console.debug('work() after createSandbox', this.sandboxInventory.length, sandbox.identifier, Date.now() - this.lastTime, this.options.watchdogInterval);\n this.fetchTask() // Don't wait for watchdog.\n .catch (async (error) => {\n this.error('work() failed when calling fetchTask', error);\n await abort(error);\n });\n })\n .catch(async (error) => {\n this.error('work() failed when calling createSandbox, exiting...', error);\n await abort(error);\n });\n }\n catch(error)\n {\n this.error('work() failed', error);\n if (this.state.is('initial')) this.state.set('initial', 'broken');\n else if (!this.state.is('broken')) this.setState('ready', 'broken');\n this.worker.stop(true).finally(() => { throw error; });\n }\n });\n }\n\n /** Construct capabilities when necessary. */\n checkCapabilities (sandbox)\n {\n /**\n * Assign the capabilities of one the sandboxes before fetching slices from the scheduler.\n * @todo XXXpfr Re-work this once fetchTask uses the capabilities of every sandbox to fetch slices.\n */\n this.capabilities = sandbox.capabilities;\n if (DCP_ENV.isBrowserPlatform && this.capabilities.browser)\n this.capabilities.browser.chrome = DCP_ENV.isBrowserChrome;\n\n debugging('supervisor') && console.debug('Supervisor.checkCapabilities computed', Date.now() - this.lastTime);\n }\n\n // _Idx\n //\n // safeEmit, workerEmit, jobEmit,\n // error, warning, mungeError, jobDescriptor, setState\n //\n\n /**\n * Safe event emitter.\n * @param {EventEmitter} emitter\n * @param {string} event\n * @param {...any} args\n */\n safeEmit(emitter, event, ...args)\n {\n try\n {\n emitter.emit(event, ...args);\n }\n catch (error)\n {\n this.error(`Event handler for event ${event} threw an exception`, error);\n }\n }\n\n /**\n * Safe event emitter on worker.\n * @param {string} event\n * @param {...any} args\n */\n workerEmit(event, ...args)\n {\n this.safeEmit(this.worker, event, ...args);\n }\n\n /**\n * Safe event emitter on slice.jobHandle.\n * @param {Slice} slice\n * @param {string} event\n * @param {...any} args\n */\n jobEmit(slice, event, ...args)\n {\n this.safeEmit(slice.jobHandle, event, ...args);\n }\n\n /**\n * Error feedback to user.\n * @param {string} message\n * @param {Array<Error>|Error|string} [coreError]\n * @param {string} [additionalInfo]\n * @param {boolean} [supressStack=false]\n */\n error (message, coreError, additionalInfo, supressStack = false)\n {\n const isString = (s) => { return (typeof s === 'string' || s instanceof String); };\n if (coreError instanceof AggregateError)\n coreError = coreError.errors;\n if (Array.isArray(coreError) && coreError.length > 0) // Emit error for every element of array.\n return coreError.flat().forEach((c_err) => this.error(message, c_err, additionalInfo));\n\n debugging('supervisor') && console.debug('Supervisor.error:', message, coreError, additionalInfo);\n if (!message)\n message = 'Supervisor.error called w/o valid message';\n if (additionalInfo)\n {\n if (typeof additionalInfo === 'object')\n // @ts-ignore\n additionalInfo = (additionalInfo instanceof Error) ? additionalInfo.message : JSON.stringify(additionalInfo);\n else if (typeof additionalInfo !== 'string')\n additionalInfo = String(additionalInfo);\n\n if (!isString(additionalInfo))\n additionalInfo = additionalInfo.toString();\n if (!coreError)\n coreError = '';\n else if (!isString(coreError))\n coreError = String(coreError);\n }\n\n let dcpError;\n if (additionalInfo)\n dcpError = new DCPError(message, coreError, additionalInfo, supressStack);\n else if (coreError && (coreError instanceof Error))\n dcpError = new DCPError(message, coreError, '', supressStack);\n else\n dcpError = new DCPError(message, '', '', supressStack);\n\n this.worker.emit('error', dcpError);\n }\n\n /**\n * Warning feedback to user.\n * @param {string[]} messages\n */\n warning (...messages)\n {\n debugging('supervisor') && console.debug('Supervisor.warning:', messages);\n if (messages.length < 1)\n messages = [ 'Supervisor.warning called w/o valid message(s)' ];\n messages.forEach((message) => this.worker.emit('warning', message));\n }\n\n /**\n * @deprecated\n * Create new object and copy the interesting properties from error.\n * Only show the stack for debug builds.\n * If timestamp isn't set, assign new Date().\n * @param {{ message }|string|object} error\n * @param {*} [errorCtor]\n * @returns {string|{ message }}\n */\n __mungeError (error, errorCtor)\n {\n if (typeof error === 'string')\n {\n const errorLines = error.split('\\n');\n return common.displayMaxDiagInfo() ? error : errorLines[0];\n }\n\n if (!error || typeof error !== 'object' || !('message' in error) || Array.isArray(error))\n return error;\n\n if (minimalDiag)\n return error.message;\n\n const errorObj = errorCtor ? new errorCtor(error.message) : { message: error.message };\n\n const props = common.displayMaxDiagInfo()\n ? [ 'type', 'process', 'name', 'origin', 'info', 'code', 'errorCode', 'operation', 'fileName', 'lineNumber', 'timestamp' ]\n : [ 'code', 'errorCode', 'fileName', 'lineNumber', 'timestamp' ]\n const predCopy = (prop) => {\n if (error[prop])\n errorObj[prop] = error[prop];\n };\n\n props.forEach((prop) => { predCopy(prop); });\n\n if (common.displayMaxDiagInfo())\n {\n predCopy('stack');\n if (errorObj['name'] === 'Error')\n delete errorObj['name'];\n }\n if (!errorObj['timestamp'])\n errorObj['timestamp'] = new Date();\n\n return errorObj;\n }\n\n /**\n * Get the job descriptor for the appropriate job manager,\n * which is the object value corresponding to jobAddress, in\n * the object returned by getJobsForTask in task-jobs.js.\n * @param {string} jobAddress\n * @returns {object}\n */\n jobDescriptor (jobAddress)\n {\n const jobManager = this.jobMap[jobAddress];\n if (!jobManager)\n throw new Error(`Cannot find the job descriptor corresponding to jobAddress ${jobAddress}`);\n return jobManager.jobMessage;\n }\n\n /**\n * Protect this.state when transitioning from currState -> nextState\n * It's dangerous to place this.state.set in a catch block with this.error or this.warning\n * because an uncaught exception will kill process before emitting the event-based diagnostic.\n * @param {string|string[]} currState\n * @param {string} nextState\n */\n setState(currState, nextState)\n {\n try { this.state.set(currState, nextState); }\n catch (e) { this.error('Supervisor.state.set error', e); }\n }\n\n // _Idx\n //\n // returnAllSlices, postStopShutdown, abort\n // stopWork, purgeJob\n //\n\n /** @returns {Promise<*>} */\n returnAllSlices ()\n {\n if (selectiveDebug())\n {\n const activeSlices = this.jobManagerInventory.map((jm) => jm.activeSlices).flat();\n if (activeSlices.length > 0)\n this.warning(`Returning active slices : ${stringify(activeSlices.map((slice) => slice.identifier), -1, 2)}`);\n }\n // The promises are all about returning the slices to the scheduler and there's no reason to await that.\n return Promise.all(this.jobManagerInventory.map((jm) => jm.destroy()));\n }\n\n /** @returns {Promise<*>} */\n postStopShutdown ()\n {\n for (const sandbox of this.sandboxInventory)\n sandbox.terminate(false);\n this.sandboxInventory = [];\n\n // There shouldn't be anything in the job managers, but just to be safe call returnAllSlices.\n // Clear jobManagerInventory, close all connections and set state to 'stopped'.\n return this.returnAllSlices()\n .finally(() => {\n // Re-enable is-screen-saver-active logic for the sandbox handle 'end' event handler.\n this.evaluator.pauseSandboxHandleEndHandler = false;\n this.jobManagerInventory = common.InventoryArray('jobManagers');\n return this.dcp4.closeConnections()\n .finally (() => {\n if (this.state.isNot('stopped'))\n this.setState('stopping', 'stopped');\n // This log message assume slices were returned to scheduler in a previous operation, which is the only current use case.\n // If we use this function in a different way in the future, update the log message.\n selectiveDebug() && console.debug(`Supervisor.postStopShutdown(${this.state}): terminated all sandboxes and returned all slices to scheduler...`);\n });\n });\n }\n\n /**\n * Stop the worker immediately and return all unfinished slices.\n * @returns {Promise<*>}\n */\n abort ()\n {\n return this.returnAllSlices()\n .finally (() => {\n return this.postStopShutdown();\n });\n }\n\n /**\n * Terminates sandboxes and returns slices.\n * Sets the working flag to false, call @this.work to start working again.\n *\n * If forceTerminate is true: Terminates all sandboxes and returns all slices.\n * If forceTerminate is false: Terminates non-working sandboxes and returns initial and ready slices.\n *\n * @param {boolean} [forceTerminate = true] - true if you want to stop the sandboxes from completing their current slice.\n * @returns {Promise<*>}\n */\n async stopWork (forceTerminate = true)\n {\n /** @returns {boolean} */\n const doNotWaitForWork = () => {\n return (this.evaluator.reallyDown || !this.sandboxInventory.filter(sbx => !sbx.isTerminated).length);\n }\n selectiveDebug() && console.debug(`Supervisor.stopWork(${forceTerminate}, ${this.state}): terminating sandboxes and returning slices to scheduler.`);\n\n // [ 'initial', 'ready', 'reconnecting', 'stopping', 'stopped', 'paused', 'broken']\n if (this.state.in(['stopping', 'stopped', 'reconnecting']))\n {\n this.warning(`Supervisor.stopWork was called when supervisor is in state ${this.state.valueOf()}.`, 'Please either wait and try again or restart worker.');\n return;\n }\n else if (this.state.is('initial'))\n {\n this.warning('Cannot call stopWork before worker has started. Please either wait and try again or restart worker.');\n return;\n }\n this.state.set(['ready', 'paused', 'broken'], 'stopping');\n\n this.dcp4.instantiateAllConnections();\n\n // Do not enter is-screen-saver-active logic in the sandbox handle 'end' event handler.\n this.evaluator.pauseSandboxHandleEndHandler = true;\n\n if (forceTerminate)\n return this.abort();\n else\n {\n const slicesToReturn = [];\n for (const jm of this.jobManagerInventory)\n slicesToReturn.push(...jm.queuedSlices);\n\n const reason = `stopWork returning all non-finished slices that are not working`;\n this.returnSlices(slicesToReturn, reason);\n\n for (let k = 0; k < 3; k++)\n {\n await new Promise((resolve) => {\n // Count the slices that have been working or close-to-working but haven't submitted results yet.\n let activeSliceCount = 0;\n for (const jm of this.jobManagerInventory)\n activeSliceCount += jm.activeSlices.length;\n // When no active slices we're done.\n if (activeSliceCount === 0)\n resolve();\n // When no work can be completed we return all slices and leave.\n if (doNotWaitForWork())\n {\n this.returnAllSlices();\n resolve();\n }\n selectiveDebug() && console.debug(`StopWork: waiting for ${activeSliceCount} working slices to finish`, k);\n // Resolve and finish stopWork once all sandboxes have finished submitting their results.\n this.worker.on('result', () => {\n selectiveDebug() && console.debug(`StopWork: result handler, activeSliceCount ${activeSliceCount-1}`);\n if (--activeSliceCount === 0)\n {\n this.warning('All sandboxes empty, stopping worker and closing all connections');\n resolve();\n }\n });\n this.on('evalDown', () => {\n this.warning('Evaluator is down.', 'Force return all slices to scheduler, stopping worker and closing all connections.');\n this.returnAllSlices();\n resolve();\n });\n });\n }\n\n for (const jm of this.jobManagerInventory)\n this.safeEmit(jm.jobHandle, 'flush');\n\n if (selectiveDebug())\n {\n console.debug(`stopWork(${this.state.valueOf()}): After waiting for working slices to finish: workingSbxes: ${this.workingSandboxCount}, totalSbxes: ${this.sandboxInventory.length}, jobs: ${this.jobManagerInventory.length}`);\n this.jobManagerInventory.forEach((jm) => {\n console.debug('stopWork job', jm.identifier);\n console.debug(jm.countSliceStr('stopWork'));\n });\n }\n }\n\n return this.postStopShutdown();\n }\n\n /**\n * Purge all traces of the job.\n * @param {JobManager} jobManager\n */\n purgeJob (jobManager)\n {\n selectiveDebug() && console.debug(`Supervisor.purgeJob ${jobManager.identifier}.`);\n // If the slice from a job never completes and the job address exists in the ringBufferofJobs,\n // then we remove it to allow for another slice (from the same job) to be obtained by fetchTask\n this.ringBufferofJobs.buf = this.ringBufferofJobs.filter(element => element !== jobManager.address);\n this.jobManagerInventory.delete(jobManager);\n this.moduleCache.removeJob(jobManager.address);\n this.dbg.cleanUpDeadJob(jobManager.address);\n jobManager.destroy();\n }\n\n // _Idx\n //\n // roundRobinSlices, makeJobSelectionCursor, handleSandboxWorkError, handleFailedSlice\n //\n\n /**\n * Round-robin through the job managers, picking 1 slice to run each time.\n * Try to have the same number of working sandboxes for each job.\n * Try to run a slice on every available sandbox.\n *\n * The basic idea behind the scheduling of slices in this implementation is to keep as\n * many slices from different jobs running as possible, so as to reduce the likelihood\n * of resource contention between sandboxes.\n *\n * Slices are scheduled based on the following ruleset:\n * 1) cursor = makeJobSelectionCursor(), then cursor.next() returns a slice chosen as follows.\n * 2) Let concurrency range from 1 to maxWorkingCores.\n * 3) For a given concurrency, let readyJobs be all jobs such that jobMan.workingSliceDensity < concurrency.\n * 4) Do an ascending sort of readyJobs wrt jobMan.emaTotalTime.\n * 5) Pick a slice from the longest job in readyJobs that doesn't have any executing slices.\n * 6) Alternately shift a slice from readySlices vs choose a slice from a random nearly finished job, and remove slice from readySlices.\n * 7) When there are no more almost finished jobs with slices, shift slices from readyJobs.\n * 8) Jobs which have slicePriority set by the task-distributor may have slices chosen ahead of the above algorithm.\n * 9) Jobs with a slicePriority closer to 1 are more likely to be chosen.\n * 10) After finishing concurrency at maxWorkingCores, cursor.next() returns null, so create a new cursor.\n * @returns {Promise<any>}\n */\n roundRobinSlices ()\n {\n //\n // Should we try to put all runSlice promises in an array and return Promise.all(runslice-promises) ?\n //\n try\n {\n /**\n * The amount of space available for the CPU-component of slices to run in sandboxes.\n * If space is 2.5 and there are 6 slices with density 0.4, and there are enough non-working usable\n * sandboxes, then all 6 slices will be scheduled to run.\n * @type {number}\n */\n const unusedCoreSpace = this.unusedCoreSpace;\n /**\n * The number of sandboxes not currently being used.\n * @type {number}\n */\n const unusedSandboxCount = this.unusedSandboxCount;\n /**\n * The amount of space available for the GPU-component of slices to run in sandboxes.\n * @type {number}\n */\n const unusedGPUSpace = this.unusedGPUSpace;\n if (unusedCoreSpace < common.doNotSchedule || this.roundRobinBarrier || unusedSandboxCount < 1)\n {\n selectiveDebug2() && console.debug('RRS: bail early space/barrier/unusedSlots', unusedCoreSpace, this.roundRobinBarrier, unusedSandboxCount);\n return;\n }\n // roundRobinBarrier is a barrier for the slice execution path.\n this.roundRobinBarrier = true;\n if (this.evaluator.down && this.evaluator.createSandboxRefCount > 0)\n return;\n selectiveDebug2() && console.debug('BarrierState:RRS:', this.fetchTaskBarrier, this.roundRobinBarrier);\n\n if (selectiveDebug2())\n {\n this.jobManagerInventory.forEach((jobMan) => {\n const { unassigned, ready, reserved, working, workdone, complete, failed, finished } = jobMan.dumpSlices ('RRS', false, false);\n console.debug(`RRS(${jobMan.identifier},${unusedSandboxCount},${unusedCoreSpace}): u/r/rsv/w/wd/c/f/fsh ${unassigned}/${ready}/${reserved}/${working}/${workdone}/${complete}/${failed}/${finished}`, jobMan.identifier, this.sandboxInventory.length);\n });\n }\n\n if ( false || selectiveDebug())\n {\n let totalReady = 0, totalReadyDensity = 0;\n for (const jobMan of this.jobManagerInventory)\n {\n const currentReady = jobMan.readySlices.length;\n const currentReadyDensity = jobMan.readySlices.length * jobMan.estimateDensity;\n totalReady += currentReady;\n totalReadyDensity += currentReadyDensity;\n console.debug(`RRS: job ${jobMan.identifier}, density ${jobMan.estimateDensity}, readySlices ${currentReady}, readyDensity ${currentReadyDensity}`);\n }\n console.debug(`RRS: space ${unusedCoreSpace}, unusedSandboxCount ${unusedSandboxCount}, totalReady ${totalReady}, totalReadyDensity ${totalReadyDensity}`);\n }\n\n /** @type {Slice[]} */\n const slices = [];\n /** @type {number} */\n let density = 0;\n /** @type {number} */\n let gpuDensity = 0;\n\n const isSpaceAvailable = (density) => {\n const result = density < unusedCoreSpace && slices.length < unusedSandboxCount;\n selectiveDebug2() && console.debug('RRS: isSpaceAvailable', density < unusedCoreSpace, slices.length < unusedSandboxCount);\n return result;\n }\n\n // When the cursor is almost done and RRS tries to schedule slices,\n // it makes sense to recreate the cursor once to ensure enough slices can be pulled from cursor.\n let recreateCursorCount = 0;\n\n while (isSpaceAvailable(density + common.schedulingSlop))\n {\n // Get existing cursor or create new one.\n if (!this.cursor)\n this.cursor = this.makeJobSelectionCursor();\n\n // Get the next slice, then check to see whether it can be used.\n const slice = this.cursor.next();\n if (!slice)\n {\n if (/*!okToSchedule ||*/ ++recreateCursorCount > 1)\n {\n this.cursor = null;\n break;\n }\n // Start a new cursor.\n this.cursor = this.makeJobSelectionCursor();\n continue;\n }\n let okToSchedule = true\n const job = slice.jobManager;\n density += job.estimateDensity;\n if (job.useGPU)\n {\n okToSchedule = canScheduleGPU(this.maxWorkingGPUs);\n if (okToSchedule)\n {\n gpuDensity += job.estimateGPUDensity;\n okToSchedule = (gpuDensity <= unusedGPUSpace);\n selectiveDebug2() && console.debug(`RRS: GPU scheduling(${okToSchedule},${this.workingSliceCount},${density.toFixed(7)},${unusedCoreSpace.toFixed(7)}): gpuDensity/gpuSpace ${gpuDensity.toFixed(7)}/${unusedGPUSpace.toFixed(7)}, jobGPUDensity/jobCPUDensity ${job.estimateGPUDensity.toFixed(7)}/${job.estimateDensity.toFixed(7)}`);\n }\n }\n if (okToSchedule && density <= unusedCoreSpace + common.schedulingSlop) // Ok, if it's only over by a little bit.\n slices.push(slice);\n else\n {\n slice.unReserve();\n density -= job.estimateDensity;\n if (job.useGPU)\n gpuDensity -= job.estimateGPUDensity;\n else\n this.cursor.push(slice); // If useGPU, then skip pulling a slice from job\n break;\n }\n selectiveDebug2() && console.debug('RRS: density/space/numSlices/unusedSlots/jobDensity', density, unusedCoreSpace, slices.length, unusedSandboxCount, job.estimateDensity);\n }\n\n selectiveSupEx() && density > 0 && console.debug(`roundRobinSlices(${this.workingSliceCount},${this.workingSliceDensity}): Found density ${density.toFixed(7)}/${unusedCoreSpace} with ${slices.length} slices:`, slices.map((slice) => slice.identifier), this.jobManagerInventory.map((jm) => `${jm.identifier}:${jm.estimateDensity.toFixed(7)}:${jm.emaSliceTime.toFixed(0)}`));\n\n // Execute the slices.\n if (slices.length > 0)\n {\n const lastSlice = slices.pop();\n for (const slice of slices)\n slice.jobManager.runSlice(slice);\n return lastSlice.jobManager.runSlice(lastSlice);\n }\n }\n finally\n {\n this.roundRobinBarrier = false;\n }\n }\n\n /**\n * @private\n * @callback cbNext\n * @returns {Slice}\n */\n /**\n * @private\n * @callback cbPush\n * @param {Slice} slice\n */\n\n /**\n * Factory function which instantiates a JobSelectionCursor. A JobSelectionCursor\n * steps the order that job slices should be selected for execution in the supervisor,\n * given the current state of the supervisor and the availability of jobs when the\n * inventory was snapshot. The entire slice scheduling algorithm is represented by\n * this cursor.\n *\n * The basic idea behind the scheduling of slices in this implementation is to keep as\n * many slices from different jobs running as possible, so as to reduce the likelihood\n * of resource contention between sandboxes.\n *\n * Slices are scheduled based on the following ruleset:\n * 1) cursor = makeJobSelectionCursor(), then cursor.next() returns a slice chosen as follows.\n * 2) Let concurrency range from 1 to maxWorkingCores.\n * 3) For a given concurrency, let readyJobs be all jobs such that jobMan.workingSliceDensity < concurrency.\n * 4) Do an ascending sort of readyJobs wrt jobMan.emaTotalTime.\n * 5) Pick a slice from the longest job in readyJobs that doesn't have any executing slices.\n * 6) Alternately shift a slice from readySlices vs choose a slice from a random nearly finished job, and remove slice from readySlices.\n * 7) When there are no more almost finished jobs with slices, shift slices from readyJobs.\n * 8) Jobs which have slicePriority set by the task-distributor may have slices chosen ahead of the above algorithm.\n * 9) Jobs with a slicePriority closer to 1 are more likely to be chosen.\n * 10) After finishing concurrency at maxWorkingCores, cursor.next() returns null, so create a new cursor.\n *\n * A custom selection of jobs can be passed in via the argument jobManagers.\n *\n * @param {JobManager[]} [jobManagers]\n * @returns {{ next: cbNext, push: cbPush }}\n */\n makeJobSelectionCursor (jobManagers)\n {\n /* Variables in this scope function as state information for next() */\n /** @type {JobManager[]} */\n var candidateJobs; // The jobs available with slices ready to execute.\n /** @type {JobManager[]} */\n var readyJobs; // The jobs from which slices are selected for a given concurrency level.\n /** @type {JobManager[]} */\n var preferedJobs = []; // Those jobs in readyJobs with a slicePreference property.\n /** @type {JobManager[]} */\n var lowDensityJobs = []; // Jobs with density <= 0.6, will be scheduled again.\n /** @type {Slice[]} */\n var pendingSlices = [];\n /**\n * Upper bound of the sum of the working slices densities allowed for a given job.\n * type {number}\n **/\n var concurrency = 0;\n /** type {number} */\n var jobIdx = 0;\n /** @type {boolean} */\n var lowDensityPass = false;\n\n const that = this;\n if (!jobManagers)\n jobManagers = this.jobManagerInventory;\n\n const jobStateStr = (jobs) => {\n return jobs.map((jm) => `${jm.identifier} : ${jm.readySlices.length} : ${jm.workingSliceDensity} : ${Math.round(jm.emaTotalTime)}`);\n }\n const jobState = (hdr, jobs) => { console.debug(hdr, jobStateStr(jobs)); }\n\n /**\n * Populate readyJobs with jobs which are ready and have at least one slice which is ready,\n * and whose # of working slice density is less than concurrency. A reserved slice has a\n * finite lifetime and if exceeded, transition it back to ready.\n * @param {JobManager[]} jobManagers\n * @param {number} concurrency\n */\n function filterJobsAndCheckOldReservedSlices (jobManagers, concurrency) // eslint-disable-line no-shadow\n {\n candidateJobs = [], readyJobs = [];\n const fiveMinutesAgo = Date.now() - that.options.reservedSliceLifetime;\n for (const jobMan of jobManagers)\n {\n if (!jobMan.ready) continue;\n let readyCount = 0;\n for (const slice of jobMan.sliceInventory)\n {\n if (slice.isReady) readyCount++;\n else if (slice.isReserved && fiveMinutesAgo > slice.startTime)\n {\n slice.unReserve();\n readyCount++;\n }\n }\n if (readyCount > 0)\n {\n candidateJobs.push(jobMan);\n if (jobMan.workingSliceDensity < concurrency) readyJobs.push(jobMan);\n }\n }\n }\n\n function seed (concurrency) // eslint-disable-line no-shadow\n {\n /* Reset. */\n jobIdx = 0;\n\n /* Populate readyJobs with jobs which are ready and have at least one slice which is ready,\n and whose # of working slice density is less than concurrency. */\n filterJobsAndCheckOldReservedSlices(jobManagers, concurrency);\n // candidateJobs = jobManagers.filter((jobMan) => jobMan.readySlices.length > 0);\n // readyJobs = candidateJobs.filter((jobMan) => jobMan.workingSliceDensity < concurrency);\n\n if (!lowDensityPass && lowDensityJobs.length === 0)\n lowDensityJobs = jobManagers.filter((jm) => jm.estimateDensity > 0 && jm.estimateDensity <= 0.6);\n\n if (readyJobs.length > 1)\n {\n /* Asc sort by shortest average slice completion time. */\n const shortestSliceJobs = readyJobs.sort((a, b) => Math.round(a.emaTotalTime) - Math.round(b.emaTotalTime));\n const almostDoneIndices = shortestSliceJobs.filter((jm) => jm.almostDone).map((_, idx) => idx);\n readyJobs = [];\n\n /* Find longest job that isn't working. */\n for (let k = shortestSliceJobs.length - 1; k >= 0; k--)\n {\n const jobMan = shortestSliceJobs[k];\n if (jobMan.isNotWorking)\n {\n readyJobs.push(jobMan);\n shortestSliceJobs.splice(k, 1);\n break;\n }\n }\n\n /* Alternate the next shortest slice with a random almost done job. */\n if (almostDoneIndices.length > 0)\n {\n while (shortestSliceJobs.length > 0)\n {\n readyJobs.push(shortestSliceJobs.shift());\n if (almostDoneIndices.length < 1)\n break;\n else\n {\n const almostDoneIdx = almostDoneIndices[Math.floor(Math.random() * almostDoneIndices.length)];\n readyJobs.push(shortestSliceJobs[almostDoneIdx]);\n shortestSliceJobs.splice(almostDoneIdx, 1);\n }\n }\n }\n if (shortestSliceJobs.length > 0)\n readyJobs.push(...shortestSliceJobs);\n }\n /* Populate preferedJobs with jobs from readyJobs which also have a slicePreference set. */\n preferedJobs = candidateJobs.filter((jm) => jm.hasOwnProperty('slicePreference'));\n selectiveDebug2() && jobState(`makeJobSelectionCursor:seed(${concurrency}): readyJobs:`, readyJobs);\n }\n\n /**\n * Each invocation of next() identifies one slice to run, or returns false if none can run.\n * @returns {Slice}\n */\n function next ()\n {\n if (pendingSlices.length > 0)\n {\n const slice = pendingSlices.pop();\n return slice.markAsReserved();\n }\n\n if (concurrency === 0)\n seed(++concurrency);\n\n selectiveDebug2() && console.debug(`makeJobSelectionCursor(cc/idx/ready/working):next(${concurrency},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): maxWorkingCores ${that.maxWorkingCores}: begin`);\n while (true)\n {\n if (jobIdx >= readyJobs.length)\n {\n if (++concurrency > that.maxWorkingCores)\n break;\n seed(concurrency);\n }\n\n if (readyJobs.length < 1)\n {\n if (candidateJobs.length < 1)\n break;\n continue; /* No ready jobs at current concurrency level. */\n }\n\n selectiveDebug2() && console.debug(`makeJobSelectionCursor:next(${concurrency},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): before loop`);\n\n /* Schedule a prefered job slice based on random chance. */\n if (preferedJobs.length > 0)\n {\n let prioRan = Math.random();\n let list = preferedJobs.filter((jm) => jm['slicePreference'] >= prioRan);\n\n if (list.length > 0)\n {\n const jobMan = list[list.length * Math.random()];\n const slice = jobMan.reserveOneSlice();\n if (slice)\n return slice;\n }\n }\n\n /* Schedule a slice from next job; jobs are in increasing order of estimated run time. */\n while (jobIdx < readyJobs.length)\n {\n const jobMan = readyJobs[jobIdx];\n const slice = jobMan.reserveOneSlice();\n if ( false || selectiveDebug2())\n {\n slice && console.debug(`makeJobSelectionCursor:next(${concurrency},${lowDensityPass},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): found slice(slice:ready:working)`, `${slice.identifier} : ${slice.jobManager.readySlices.length} : ${slice.jobManager.workingSliceDensity}`);\n !slice && console.debug(`makeJobSelectionCursor:next(${concurrency},${lowDensityPass},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): no slices ready for job`, jobMan.identifier);\n }\n jobIdx++;\n if (slice)\n return slice;\n }\n\n /*\n * We did not schedule a slice with current seed. We need to re-seed to look for newly-available work\n * and sandboxes, ratcheting up the concurrency (max # of each job running) until we find something.\n */\n selectiveDebug2() && console.debug(`makeJobSelectionCursor:next(${concurrency},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): after loop`);\n }\n if (!lowDensityPass && lowDensityJobs.length > 0)\n {\n jobManagers = lowDensityJobs;\n concurrency = 0;\n lowDensityPass = true;\n return next();\n }\n selectiveDebug2() && console.debug(`makeJobSelectionCursor:next(${concurrency},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): null`, lowDensityPass);\n return null; /* Did not find any more work that fits. */\n }\n function push (slice) { pendingSlices.push(slice); }\n\n return { next, push };\n }\n\n\n /**\n * Handle sandbox.work(...) errors.\n * @todo The orginal code from 2019 did not terminate sandbox when not SandboxError and Sandbox code didn't already terminate. Do we want to try that?\n * The old 2019 sandbox code terminated upon error in start, assign, resetState, describe, applyRequirements and work.\n * So maybe that 2019 terminate yoga was a bunch of hooie.\n * @param {Sandbox} sandbox\n * @param {Slice} slice\n * @param {Error} error\n * @returns {string}\n */\n handleSandboxWorkError (sandbox, slice, error)\n {\n if (debugBuild && !(slice.isWorking || slice.isWorkDone)) // Sanity. Exception should never fire.\n throw new Error(`handleSandboxWorkError: slice ${slice.identifier} must be WORKING.`);\n\n /** @type {boolean} */\n const isSandboxError = error instanceof SandboxError;\n /** @type {string} */\n let reason;\n const jobAddress = common.truncateAddress(slice.jobAddress);\n\n if (isSandboxError)\n reason = error['errorCode']\n else\n {\n // This error was unrelated to the work being done.\n reason = 'Slice has failed to complete execution';\n if (!error)\n error = new Error(`Slice ${slice.sliceNumber} in state ${slice.state} of job ${jobAddress} failed to complete execution`);\n }\n selectiveDebug() && console.debug('handleSandboxWorkError', slice.identifier, error);\n\n let errorString, onlyDisplayErrorString = true;\n if (error.name === 'EWORKREJECT')\n {\n reason = 'EWORKREJECT'; // The status.js processing does not have a case for 'EWORKREJECT'.\n errorString = !slice.hasBeenRejected\n ? `Slice rejected work: ${error.message}.`\n : `Slice rejected work twice; terminate job: ${error.message}.`\n error.stack = 'Sandbox was terminated by work.reject()';\n this.handleWorkReject(slice, error);\n }\n else\n {\n if (!this.evaluator.down)\n {\n /** Do we to be more selective when we retry a slice? */\n if (/*!isSandboxError ||*/ slice['useRetryLogic'])\n {\n slice['sandboxErrorCount'] = ( slice['sandboxErrorCount'] ?? 0) + 1;\n if (slice['sandboxErrorCount'] <= this.options.maxSliceRetries)\n slice.resetState(); // Try to reuse the slice.\n }\n }\n if (!slice.isReady)\n {\n selectiveDebug() && console.debug(`handleSandboxWorkError: returning slice ${slice.identifier}`);\n this.returnSlice(slice, reason)\n .finally (() => {\n this.handleFailedSlice(slice, error)\n });\n }\n\n switch (reason)\n {\n case 'ENOPROGRESS':\n errorString = 'No progress error in sandbox.';\n break;\n case 'ESLICETOOSLOW':\n errorString = 'Slice too slow error in sandbox.';\n break;\n case 'EPERM_ORIGIN':\n errorString = `Could not fetch data; origin not allowed: ${error.message}.`;\n break;\n case 'EFETCH':\n errorString = `Could not fetch data: ${error.message}.`;\n break;\n case 'EUNCAUGHT':\n onlyDisplayErrorString = false;\n errorString = `Uncaught error in sandbox: ${error.message}.`;\n break;\n default:\n onlyDisplayErrorString = false;\n errorString = `Slice failed in sandbox: ${error.message}.`;\n break;\n }\n }\n\n // Always terminate sandbox.\n this.returnSandbox(sandbox);\n\n // Always display max info under debug builds, otherwise maximal error.\n // messages are displayed to the worker, only if both worker and client agree.\n const displayMaxInfo = slice.jobManager.displayMaxDiagInfo;\n\n const errorObject = {\n jobAddress,\n sliceNumber: slice.sliceNumber,\n sandbox: sandbox.id,\n jobName: sandbox.public ? sandbox.public.name : 'unnamed',\n };\n\n if (!displayMaxInfo && onlyDisplayErrorString)\n this.error(errorString, '', '', true);\n else\n {\n Object.entries(errorObject).forEach(([k,v]) => (errorString += `\\n ${k}: ${v}`));\n this.error(errorString, error, '', true);\n }\n\n return reason;\n }\n\n /**\n * Slice has thrown error during execution:\n * Mark slice as failed, compensate when job is dicrete, emit events.\n * @param {Slice} slice\n * @param {Error} error\n */\n handleFailedSlice (slice, error)\n {\n debugging('supervisor') && console.debug(`handleFailedSlice: ${slice.identifier}`, error);\n slice.collectResult(error, false /*success*/);\n\n // If the slice from a job never completes and the job address exists in the ringBufferofJobs,\n // then we remove it to allow for another slice (from the same job) to be obtained by fetchTask\n this.ringBufferofJobs.buf = this.ringBufferofJobs.filter(element => element !== slice.jobAddress);\n\n this.workerEmit( 'result', error);\n this.jobEmit(slice, 'result', error);\n }\n\n // _Idx\n //\n // returnSlices, returnSlice, emitProgressReport\n //\n\n /**\n * Bulk-return multiple slices, possibly for assorted jobs.\n * Returns slices to the scheduler to be redistributed.\n * Called in the sandbox terminate handler and purgeAllWork(jobAddress)\n * and stopWork(forceTerminate).\n *\n * @param {Slice[]} slices - The slice candidates to check if they can be returned to the scheduler.\n * @param {string} reason - Reason for the return: 'ENOPROGRESS', 'EUNCAUGHT', 'ESLICETOOSLOW', 'EPERM_ORIGIN', 'EFETCH', 'unknown'.\n * @returns {Promise<*>} - Response from the scheduler.\n */\n returnSlices (slices, reason)\n {\n /** @param {Slice[]} verifiedSlices */\n const compressPayload = (verifiedSlices) => {\n assert(verifiedSlices?.length > 0);\n if (verifiedSlices.length > 1)\n return {\n worker: this.workerId,\n slices: common.constructReturnSliceBuckets(verifiedSlices, reason),\n };\n return verifiedSlices[0].getReturnMessagePayload(this.workerId, reason);\n }\n\n if (!slices || !slices.length)\n return Promise.resolve();\n\n debugging('supervisor') && console.debug(`Supervisor.returnSlices(${this.state}): Returning slices`, slices.map(slice => slice.identifier));\n\n // Only return those slices which still exist in their respective jobManagers sliceInventory .\n const verifiedSlices = slices.filter((slice) => slice.jobManager.removeSlice(slice));\n if (verifiedSlices.length > 0)\n {\n selectiveSupEx() && console.debug('Supervisor.returnSlices: Returning slices', verifiedSlices.map(slice => slice.identifier));\n return this.dcp4.sliceReturn(compressPayload(verifiedSlices), slices, reason);\n }\n return Promise.resolve();\n }\n\n /**\n * Takes a slice and returns it to the scheduler to be redistributed.\n * Usually called when an exception is thrown by sandbox.work(...) .\n * Or when the supervisor tells it to forcibly stop working.\n *\n * @param {Slice} slice - The slice to return to the scheduler.\n * @param {string} reason - Reason for the return: ''ENOPROGRESS', 'EUNCAUGHT', 'ESLICETOOSLOW', 'EPERM_ORIGIN', 'EFETCH', 'unknown'.\n * @returns {Promise<*>} - Response from the scheduler.\n */\n returnSlice (slice, reason) { return this.returnSlices([ slice ], reason); }\n\n /**\n * Send beacon to status.js for 'progress' and sliceStatus.scheduled.\n *\n * Run in an interval created in the ctor.\n * @returns {void|Promise<*>}\n */\n emitProgressReport ()\n {\n const readySlices = [], workingSlices = [];\n this.jobManagerInventory.forEach((jobManager) => {\n readySlices.push(...jobManager.readySlices);\n workingSlices.push(...jobManager.workingSlices);\n });\n /** @type {SliceObj[]} */\n const slices = common.constructSliceBuckets( readySlices, sliceStatus.scheduled );\n common.constructSliceBuckets( workingSlices, 'progress', slices );\n\n debugging('supervisor') && console.debug('emitProgressReport:', stringify(slices));\n\n if (slices.length > 0)\n {\n const payload = { worker: this.workerId, slices };\n return this.dcp4.safeRSStatus(payload, 'Failed to emit progress report');\n }\n }\n\n // _Idx\n //\n // jobQuanta, repoMan, predictLoad(viz., clairvoyance), waitUntilWorkIsReady, generateWorkerComputeGroups\n //\n\n /**\n * For a given job, the scheduler stores an EMA approximation of slice completion time.\n * However, each worker also tracks the same information and the ratio of local-info to\n * scheduler-info (viz., global-info) is returned by this.jobQuanta so we can tell the\n * task distributor how much work to return from fetchTask so that the work actually takes\n * 5 minutes to complete when using all the worker sandboxes.\n * @returns {Object<string, number>}\n */\n jobQuanta ()\n {\n //\n // Prevent wild swings of this.defaultQuanta, which is roughly the ratio of\n // local_worker_slice_time / jobPerfData_measured_slice_time.\n // We limit this ratio to be between 1/8th and 8.\n const minQuanta = 0.125, maxQuanta = 8.0;\n //\n // Because there will be slgiht differences between local_worker_slice_time and\n // jobPerfData_measured_slice_time even when the worker is the only worker hooked\n // up to the DCP scheduler, we prove a little rounding. The rounding is 1/32 buckets.\n const discreteIncrement = 0.03125, discreteIncrementInverse = 32;\n\n /** @type {Object<string, number>} */\n const quanta = { 0: 1 };\n let averageLocalTime = 0, averageGlobalTime = 0;\n for (const jobMan of this.jobManagerInventory)\n {\n if (jobMan.emaSliceTime > 0 && jobMan.globalTime > 0)\n {\n quanta[jobMan.address] = jobMan.emaSliceTime;\n /** XXXpfr @todo Should we be using TotalTime here? */\n averageLocalTime += jobMan.emaSliceTime;\n averageGlobalTime += jobMan.globalTime;\n selectiveDebug2() && console.debug('jobQuanta: job state', this.dbg.sliceSandboxStr, `l-density/g-density, ${jobMan.estimateDensity}/${jobMan.metrics?.sliceCPUDensity}`, `local/global, ${jobMan.emaSliceTime}/${jobMan.globalTime}`);\n }\n }\n\n if (averageLocalTime && averageGlobalTime)\n {\n /** @todo XXXpfr Add 1 stddev? */\n // alpha=0.1 gives an effective period of 19\n const alpha = 0.1;\n this.localTime = nextEma(this.localTime, averageLocalTime, alpha);\n this.globalTime = nextEma(this.globalTime, averageGlobalTime, alpha);\n this.defaultQuanta = this.localTime / this.globalTime;\n\n // Discretize by discreteIncrement increments.\n this.defaultQuanta = (this.defaultQuanta > 1\n ? Math.floor(discreteIncrementInverse * this.defaultQuanta)\n : Math.ceil(discreteIncrementInverse * this.defaultQuanta)) * discreteIncrement;\n\n // Enforce reasonable cap and floor to keep things from getting too crazy.\n this.defaultQuanta = Math.min(Math.max(this.defaultQuanta, minQuanta), maxQuanta);\n\n // Fake jobAddress '0' to represent unknown jobs.\n quanta['0'] = this.defaultQuanta;\n }\n else\n this.defaultQuanta = 1.0;\n\n selectiveDebug() && console.debug(`jobQuanta: defaultQuanta ${quanta['0']}, this.localTime ${this.localTime}/${averageLocalTime}, this.globalTime ${this.globalTime}/${averageGlobalTime}, quanta:`, quanta);\n if (common.debugQuanta())\n {\n console.debug('localRawData:', this.dbg.localRawData);\n console.debug('localData:', this.dbg.localData);\n console.debug('globalData:', this.dbg.globalData);\n }\n return quanta;\n }\n\n /**\n * @todo XXXpfr Should we not schedule long slices to a worker with too low defaultQuanta?\n *\n * When the estimated time to completion of all work is more than\n * repoManMultiplier * targetTaskDuration * this.maxWorkingCores,\n * return slices until the excess is removed.\n * Be fair. Round-robin over all jobs until excess is eliminated.\n * Kill the long jobs 1st.\n */\n repoMan()\n {\n const threshold = this.options.repoManMultiplier * this.options.targetTaskDuration * this.maxWorkingCores;\n const workRemaining = this.workRemaining;\n let excess = workRemaining - threshold;\n selectiveDebug() && console.debug(`repoMan: excess ${excess}, workerRemaining ${workRemaining}, threshold ${threshold}`);\n if (excess > 0)\n {\n const slices = [];\n /** @param {JobManager[]} jmi */\n const returnFrom = (jmi) => {\n while (true)\n {\n const _excess = excess;\n for (const jobMan of jmi)\n {\n const _readySlices = jobMan.readySlices;\n if (_readySlices.length > 0)\n {\n const slice = _readySlices[0];\n slice.repoMan(); // Mark as FINISHED\n slices.push(slice);\n excess -= jobMan.adjSliceTime;\n if (excess <= 0)\n break;\n }\n }\n if (_excess === excess || excess <= 0)\n break;\n }\n }\n // Be fair. Round-robin over all jobs until excess is eliminated.\n // Except the long jobs are killed 1st.\n const longJobs = this.jobManagerInventory.filter((jobMan) => jobMan.emaSliceTime >= this.options.targetTaskDuration);\n if (longJobs.length > 0)\n returnFrom(longJobs);\n if (excess > 0)\n returnFrom(this.jobManagerInventory);\n selectiveDebug() && (slices.length > 0) && console.debug(`repoMan: excess ${excess}, workerRemaining ${workRemaining}, threshold ${threshold}, returned-slice-count ${slices.length}`);\n this.returnSlices(slices, 'repoMan');\n }\n }\n\n /**\n * Predict the total reduction in density of working sandboxes timeSpanMs from now.\n * This function is called right before fetchTask, in order to calculate how much space is available.\n * @returns {{ queued: Slice[], working: number }}\n */\n predictLoad()\n {\n const timeSpanMs = this.options.prefetchInterval\n const queued = [];\n let working = 0;\n for (const jobMan of this.jobManagerInventory)\n {\n const { queued: jmQueued, working: jmWorking } = jobMan.predictLoad(timeSpanMs);\n queued.push(...jmQueued);\n // Optimize to short-circuit when queued > 1, because we won't call fetchWork in that case.\n if (queued.length > 1)\n break;\n working += jmWorking;\n }\n selectiveDebug() && console.debug(`Supervisor.predictLoad: queued ${queued.length}/${this.queuedSlices.length}, working ${working}/${this.workingSlices.length}`)\n return { queued, working };\n }\n\n /**\n * On the first call to fetchTask\n * or when the last call to fetchTask found nothing,\n * or when there are no ready slices,\n * wait until at least 1 job is ready with at least 1 ready slice.\n * @param {Array<Promise<any>>} jobManagerPromises\n * @returns {Promise<any>}\n */\n waitUntilWorkIsReady (jobManagerPromises)\n {\n if (this.waitForWork)\n {\n debugging('supervisor') && console.debug(`waitUntilWorkIsReady: promise count ${jobManagerPromises?.length}`);\n this.waitForWork = false;\n // Promise.any is supported in Node 15, Chrome 85, Edge 85, Firefox 79, Safari 14, Opera 71.\n // It was implemented in node and browsers in 2nd half of 2020, so there's a good chance many\n // customers will not have browsers that support it. And currently (Jan. 2023) DCP uses node 14.\n return Promise_any(jobManagerPromises);\n }\n // Flush microtask queue\n return a$sleepMs(0);\n }\n\n /**\n * Generate the workerComputeGroups property of the requestTask message.\n *\n * Concatenate the compute groups object from dcpConfig with the list of compute groups\n * from the supervisor, and remove the public group if accidentally present. Finally,\n * we transform joinSecrets/joinHash into joinHashHash for secure transmission.\n *\n * @note computeGroup objects with joinSecrets are mutated to record their hashes. This\n * affects the supervisor options and dcpConfig. Re-adding a joinSecret property\n * to one of these will cause the hash to be recomputed.\n */\n generateWorkerComputeGroups ()\n {\n return supShared.generateWorkerComputeGroups(this, this.dcp4.taskDistributor);\n }\n\n // _Idx\n //\n // availableSandboxSpace, fetchTask, addTaskToWorkload, fetchFromTD, clearUnusedJobManagersAndModuleCache\n //\n\n /**\n * Returns the number of unused sandbox slots to fill -- sent to fetchTask.\n * @param {Slice[]} queued\n * @param {number} working\n * @returns {number}\n */\n availableSandboxSpace (queued, working)\n {\n // If we find more than 1 queued slices, bail early.\n if (queued.length > 1)\n return 0; // We have more than 1 ready slices, no need to fetch.\n\n let longSliceCount = 0;\n if (queued.length < 1)\n this.waitForWork = true; // There are no ready slices.\n else if (queued[0].isLong)\n longSliceCount = 1;\n\n // There are almost no ready slices (there may be 0 or 1), fetch a full task.\n // The task is full, in the sense that it will contain slices whose\n // aggregate execution time is roughly this.maxWorkingCores * 5-minutes.\n // However, there can only be this.maxWorkingCores # of long slices on a worker,\n // Thus we need to know whether the last slice in this.readySlices() is long or not.\n // (A long slice has estimated execution time >= 5-minutes or is an estimation slice.)\n\n const numCores = this.maxWorkingCores - working - longSliceCount;\n selectiveDebug2() && console.debug('availableSandboxSpace', numCores, working, longSliceCount);\n return numCores;\n }\n\n /**\n * Ask the scheduler (task distributor) for work (Rq).\n * @param {object[]} [jobs=[]]\n * @returns {Promise<*>}\n */\n async fetchTask (jobs = [])\n {\n if (!this.isReady)\n return;\n\n const now = Date.now();\n const { queued, working } = this.predictLoad();\n const unusedFutureCoreSpace = this.maxWorkingCores - working;\n if (unusedFutureCoreSpace < common.doNotSchedule)\n {\n debugging('supervisor') && console.debug('fetchTask: There are no unused sandbox slots.', now - this.lastTime);\n return;\n }\n \n // Record fetch start time.\n this.fetchTaskStarted = now;\n\n // We check for pruning about every 25 seconds, or when must prune level is reached.\n if (this.sandboxInventory.length > this.options.mustPruneSandboxLevel\n || now > this.lastPrune + this.options.pruneFrequency)\n {\n this.lastPrune = now;\n this.pruneSandboxes();\n }\n\n // Every 60 seconds check to see if the estimated time to completion of all work is more than\n // repoManMultiplier * this.targetTaskDuration() * this.maxWorkingCores,\n // and then return slices until the excess is removed.\n // Be fair. Round-robin over all jobs until excess is eliminated. Kill the long jobs 1st.\n if (now > this.lastRepoMan + this.options.repoManFrequency)\n {\n this.lastRepoMan = now;\n this.repoMan();\n }\n\n // There are 2 barriers wrt fetchTask,\n // 1) fetchTaskBarrier is a barrier for the task fetching from task distributor path.\n // 2) roundRobinBarrier is a barrier for the slice execution path.\n\n try\n {\n const cpuSpaceToFill = this.availableSandboxSpace(queued, working);\n selectiveDebug2() && console.debug('Supervisor.fetchTask', cpuSpaceToFill, queued.length, working);\n if (cpuSpaceToFill < 1)\n {\n debugging('supervisor') && console.debug('Supervisor.fetchTask: Sufficient slices exist, so start executing.', now - this.lastTime, cpuSpaceToFill, queued.length, working);\n return this.roundRobinSlices();\n }\n selectiveDebug2() && console.debug('fetchTask begin q/w/slots/space/future-space', queued.length, working, this.unusedSandboxCount, this.unusedCoreSpace, unusedFutureCoreSpace);\n\n if (this.fetchTaskBarrier)\n return;\n // fetchTaskBarrier is a barrier for the task fetching from task distributor path.\n this.fetchTaskBarrier = true;\n\n /* @todo XXXpfr Think about how to do targetLoad.longSlices better.\n * Ideas:\n * 1) While branchy Javascript is CPU bound, that doesn't mean hyperthreading isn't useful.\n * When a branch is mispredicted the whole CPU instruction pipeline is flushed, which is\n * a huge perf hit and while waiting to fill the pipeline again, a hyperthread can get\n * a whole bunch of work done.\n * 2) Setting the Sup2.cores.cpu to #lCores is probably too much, but I've had great success with\n * Sup2.cores.cpu = dcpConfig.supervisor.tuning.coreRatio.cpu * #lCores\n * Which is very close to optimal throughput of work done.\n * 3) When the scheduler is 1/2 very long slices, the short slices will tend to get starved.\n * The config property dcpConfig.scheduler.preventSliceStarvation when set to true (default false)\n * will always leave one vCore open for short slices in every worker. In the future, I want the\n * scheduler to detect short slice starvation and dynamical turn preventSliceStarvation on until\n * short slice starvation is alleviated and then turn it back off.\n * 4) maxSandboxes is currently set to\n * factor * Sup2.cores.cpu\n * where 1.2 <= factor <= 1.5 depending upon how many lCores a worker has. Where I assume that a\n * machine with a large number of lCores has sufficient memory to handle a bigger factor.\n * The factor boundaries can be adjusted in dcpConfig.supervisor, but I intend to also allow them\n * to be overridden at the dcpConfig.worker level, so if somebody has a 32 lCore machine with\n * 8GB of RAM they can adjust factor to be closer to 1. Ideally we could adjust the factor\n * boundaries at the job/CG level.\n */\n\n const request = {\n supervisor: this.version,\n numCores: cpuSpaceToFill, /** @deprecated This is for legacy schedulers. */\n numGPUs: this.maxWorkingGPUs, /** @deprecated This is for legacy schedulers. */\n targetLoad: { cpu: cpuSpaceToFill, gpu: this.maxWorkingGPUs, longSlices: Math.floor(cpuSpaceToFill) },\n coreStats: this.options.getStatisticsCPU(),\n jobQuanta: this.jobQuanta(),\n capabilities: this.capabilities,\n paymentAddress: this.options.paymentAddress,\n jobAddresses: jobs.concat(this.options.jobAddresses || []), // When set, only fetches slices for these jobs.\n workerComputeGroups: this.generateWorkerComputeGroups(),\n minimumWage: this.options.minimumWage,\n loadedJobs: this.jobManagerInventory.map(jobMan => jobMan.address),\n readyJobs: this.jobManagerInventory.filter(jobMan => jobMan.ready).map(jobMan => jobMan.address),\n previouslyWorkedJobs: this.ringBufferofJobs.buf, // Only discrete jobs.\n rejectedJobs: this.rejectedJobs,\n };\n // Workers should be part of the public compute group by default.\n if (!booley(this.options.leavePublicGroup))\n request.workerComputeGroups.push(constants.computeGroups.public);\n\n debugging('supervisor') && console.debug('fetchTask is calling fetchFromTD', Date.now() - this.lastTime);\n\n // Call Task Distributor and handle response with this.addTaskToWorkload.\n return this.fetchFromTD(request, (response) => this.addTaskToWorkload(request, response));\n }\n catch (error)\n {\n this.fetchTaskBarrier = false;\n this.error('Supervisor.fetchTask failed!', error);\n }\n }\n\n /**\n * Callback for fetchFromTD.\n * @param {object} request\n * @param {object} response\n */\n async addTaskToWorkload (request, response)\n {\n const constructFetchHandle = (size, jobs, slices) => {\n return { \n fetchStart: this.fetchTaskStarted,\n fetchEnd: Date.now(),\n fetchSize: size,\n jobs,\n slices,\n };\n };\n\n try\n {\n /** @type {TDPayload} */\n const payload = response.payload;\n if (!response.success)\n {\n debugging() && console.debug('Task fetch failure; request=', request);\n debugging() && console.debug('Task fetch failure; response=', payload);\n this.error(`Unable to request task from scheduler; will try again on a new connection: payload ${stringify(payload)}`);\n return;\n }\n\n if (!payload.body?.newJobs) // No slices found.\n {\n // Reset first fetch logic.\n this.waitForWork = true;\n /**\n * The 'fetch' event fires when the stask distributor found no work.\n * @link https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/worker/worker-events.md\n * @event Supervisor#fetch\n */\n this.workerEmit('fetch', constructFetchHandle(0, {}, {}));\n // There may be an extra slice to process.\n debugging('supervisor') && console.debug('Task distributor found no slices...');\n return this.roundRobinSlices();\n }\n\n /** @todo XXXpfr At this poin the line #'s are short by 42 -- figure out why. */\n\n /*\n * payload: { TDPayload }\n * TDPayload: { owner: Address, signature: Signature, auth: Auth, body: Body };\n * Auth: { workerId: string, authSlices: Object<string, SliceMessage[]>, schedulerId: { address: Address }, jobCommissions: Object<string, { rate: number, account: number }> }\n * Body: { newJobs: Object<string, object>, task: Object<string, SliceMessage[]>, computeGroupJobs: Object<string, string[]>, computeGroupOrigins: Object<string, Object<string, string[]>>, schedulerConfig: {{ targetTaskDuration: number }} }\n *\n * NOTE: authorizationMessage has type AuthMessage\n */\n\n const { body, ...authorizationMessage } = payload;\n const { newJobs, task, schedulerConfig } = body;\n const newJobKeys = Object.keys(newJobs);\n const jobCount = newJobKeys.length;\n\n let jobSliceMap = task;\n if (jobSliceMap.length) /** @deprecated Task came from legacy scheduler */\n // @ts-ignore\n jobSliceMap = toJobMap(task, sliceMsg => sliceMsg);\n\n if (schedulerConfig) // Otherwise the default is 300 seconds.\n this.options.targetTaskDuration = schedulerConfig.targetTaskDuration;\n\n /*\n * Ensure all jobs received from the scheduler (task distributor) are:\n * 1. If we have specified specific jobs the worker may work on, the received jobs are in the specified job list\n * 2. If we are in localExec, at most 1 unique job type was received (since localExec workers are designated for only one job)\n * If the received jobs are not within these parameters, stop the worker since the scheduler cannot be trusted at that point.\n */\n if (request.jobAddresses?.length > 0 && !newJobKeys.every((ele) => request.jobAddresses.includes(ele)))\n {\n // \"fetchTask:\" because that should make sense to somebody that doesn't know the internals of Supervisor.\n this.error(\"fetchTask: Worker received slices it shouldn't have; rejecting the work and stopping.\");\n this.stopWork(true);\n return;\n }\n\n // Clear out job managers w/o any queued slices,\n // and remove corresponding job references from module cache.\n // When a cached module no longer has any job references it is removed from the cache.\n this.clearUnusedJobManagersAndModuleCache(newJobs);\n\n /** @todo XXXpfr Figure out how not to construct this every time. */\n this.jobMap = {};\n this.jobManagerInventory.forEach(jobManager => {\n this.jobMap[jobManager.address] = jobManager;\n });\n\n selectiveDebug2() && console.debug(`addTaskToWorkload(${Date.now() - this.lastTime}): newJobs ${common.truncateAddress(newJobKeys)}, jobSliceMap ${common.compressJobMap(jobSliceMap, (s) => s.sliceNumber)}`);\n\n let sliceCount = 0;\n /** @type {Array<Promise<*>>} */\n const jobManagerPromises = [], jobs = {}, slices = {};\n // Populate the job managers with slices, creating new job managers when necessary.\n // Set up discrete job ring buffer.\n for (const [jobAddress, jobMessage] of Object.entries(newJobs))\n {\n /** @type {JobManager} */\n let jobManager;\n const sliceMessages = jobSliceMap[jobAddress];\n sliceCount += sliceMessages.length;\n\n if (this.jobMap.hasOwnProperty(jobAddress))\n {\n jobManager = this.jobMap[jobAddress];\n jobManager.update(jobMessage, sliceMessages, authorizationMessage);\n }\n else\n {\n // Add the slice messages to the job manager ctor, so that slice construction is after job manager is ready.\n jobManager = new JobManager(this, jobMessage, sliceMessages, authorizationMessage);\n this.jobMap[jobAddress] = jobManager;\n this.jobManagerInventory.push(jobManager);\n\n // Populate the ring buffer based on job's discrete property.\n if (jobMessage.requirements.discrete && this.ringBufferofJobs.find(address => address === jobAddress) === undefined)\n this.ringBufferofJobs.push(jobAddress);\n }\n jobs[jobAddress] = jobManager.jobHandle;\n slices[jobAddress] = task[jobAddress].length;\n\n jobManagerPromises.push(jobManager.jobPromise);\n }\n\n const payloadLength = kvin.stringify(payload).length; /** @TODO - fix per DCP-3750 */\n /**\n * The 'fetch' event fires when the supervisor has found work from the task distributor.\n * @link https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/worker/worker-events.md\n * @event Supervisor#fetch\n */\n this.workerEmit('fetch', constructFetchHandle(payloadLength, jobs, slices));\n\n const compressTask = () => { return common.compressJobMap(authorizationMessage.auth.authSlices); }\n selectiveSupEx() && console.debug(`addTaskToWorkload(${Date.now() - this.lastTime}): task: ${sliceCount}/${request.targetLoad.cpu}/${this.maxWorkingCores}, jobs: ${jobCount}, authSlices: ${compressTask()}, conversion:`, request.jobQuanta);\n\n // On the first call to fetchTask,\n // or when the last call to fetchTask found nothing,\n // or when there are no ready slices,\n // wait until at least 1 job with 1 slice is ready.\n await this.waitUntilWorkIsReady(jobManagerPromises);\n\n debugging('supervisor') && console.debug('addTaskToWorkload: Before calling roundRobinSlices; job states', this.jobManagerInventory.map((jm) => jm.identifier));\n\n // Start working on the new slices.\n return dcp_timers.setImmediate(() => this.roundRobinSlices());\n }\n catch (error)\n {\n this.workerEmit('fetch', error);\n this.error('Supervisor.fetchTask failed!', error);\n }\n finally\n {\n this.fetchTaskBarrier = false;\n }\n }\n\n /**\n * @private\n * @callback cbAddTaskToWorkload\n * @param {Response} response\n * @returns {Promise<void>}\n */\n\n /**\n * Call to fetch new slices from task distributor.\n * @param {*} request\n * @param {cbAddTaskToWorkload} addTaskToWorkload\n * @returns {Promise<any>}\n */\n async fetchFromTD (request, addTaskToWorkload)\n {\n selectiveDebug2() && console.debug('fetchFromTD begin; BarrierState:', this.fetchTaskBarrier, this.roundRobinBarrier);\n // Fetch a new task if we have insufficient slices queued, then start workers\n if (!this.fetchTaskBarrier)\n throw new Error('fetchTaskBarrier must be set when entering fetchFromTD.');\n\n this.dcp4.instantiateAllConnections();\n\n let fetchTimeout = dcp_timers.setTimeout(() => {\n this.fetchTaskBarrier = false;\n this.warning('Fetch exceeded timeout, will reconnect at next watchdog interval');\n this.dcp4.resetConnection('taskDistributor').catch(error => {\n this.error('Failed to close task-distributor connection', error);\n });\n this.dcp4.resetConnection('resultSubmitter').catch(error => {\n this.error('Failed to close result-submitter connection', error);\n });\n this.dcp4.instantiateAllConnections();\n }, 3 * 60 * 1000); // Max out at 3 minutes to fetch.\n // Allow workers and localExec to exit.\n fetchTimeout.unref();\n\n const finalize = () => {\n this.fetchTaskBarrier = false;\n if (fetchTimeout)\n dcp_timers.clearTimeout(fetchTimeout);\n fetchTimeout = null;\n }\n\n // Ensure result submitter and task distributor connections before fetching tasks.\n try\n {\n await Promise.all([\n this.dcp4.taskDistributor.keepalive(),\n this.dcp4.resultSubmitter.keepalive(),\n ]);\n }\n catch (error)\n {\n selectiveDebug() && console.debug('fetchTaskFromTD: Keep slices failed', error);\n this.warning('Failed to connect to result submitter, refusing to fetch slices.', 'Will try again at next fetch cycle.');\n this.dcp4.resetConnection('taskDistributor').catch(e => {\n this.error('Failed to close task-distributor connection', e);\n });\n this.dcp4.resetConnection('resultSubmitter').catch(e => {\n this.error('Failed to close result-submitter connection', e);\n });\n return finalize();\n }\n\n if (!this.dcp4.taskDistributor)\n {\n const msg = 'Unable to request task from scheduler; no connection to task distributor';\n this.warning(msg);\n this.workerEmit('fetch', new Error(msg));\n return finalize();\n }\n \n // The 'beforeFetch' event allows the user to cancel the requestTask request.\n let canceled = false;\n /**\n * The 'beforeFetch' event fires before the request is sent to requestTask in task distributor.\n * @link https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/worker/worker-events.md\n * @event Supervisor#beforeFetch\n */\n this.workerEmit('beforeFetch', () => { canceled = true; })\n selectiveDebug() && canceled && console.debug('User canceled the fetch task.');\n if (canceled)\n return finalize()\n\n return this.dcp4.taskDistributor.request('requestTask', request)\n .then((response) => {\n addTaskToWorkload(response);\n // Success! Restore this.dcp4.taskDistributor delay to retryMinSleepMs (currently 32ms.)\n // Is there a better way to reset than explicit calls?\n this.delayManager.resetEBO('taskDistributor');\n return response;\n })\n .catch((error) => {\n this.workerEmit('fetch', error);\n this.error('Unable to request task from scheduler. Will try again on a new connection.', error);\n this.dcp4.resetConnection('taskDistributor');\n })\n .finally(() => {\n return finalize();\n });\n }\n\n /**\n * Remove all unreferenced jobs in this.jobManagerInventory and this.moduleCache.\n * Since job-managers are inserted into this.jobManagerInventory with a push, the job managers at the beginning are oldest.\n * Only delete #deleteCount of the oldest job-managers:\n * let deleteCount = this.jobManagerInventory.length - cachedJobsThreshold;\n * Edit cachedJobsThreshold to adjust the cache cleanup threshold.\n * @param {Object<string, number[]>} newJobMap - Jobs that should not be removed from this.jobManagerInventory and this.moduleCache.\n */\n clearUnusedJobManagersAndModuleCache (newJobMap)\n {\n const emptyJobs = [];\n for (const jobMan of this.jobManagerInventory) // Grab oldest 1st\n {\n if (!newJobMap[jobMan.address])\n {\n let isEmpty = true;\n for (const slice of jobMan.sliceInventory)\n if (slice.isQueuedOrActive)\n {\n isEmpty = false;\n break;\n }\n if (isEmpty)\n {\n // Walk through whole list to purge empty jobs with no assigned sandboxes to save.\n if (jobMan.assignedSandboxes.length < 1)\n this.purgeJob(jobMan);\n else\n emptyJobs.push(jobMan)\n }\n }\n }\n let deleteCount = this.jobManagerInventory.length - this.options.cachedJobsThreshold;\n if (deleteCount > 0)\n {\n selectiveDebug() && console.debug(`Supervisor.clearUnusedJobManagersAndModuleCache: deleteCount ${deleteCount}/${this.jobManagerInventory.length}/${this.options.cachedJobsThreshold}.`);\n for (const jobMan of emptyJobs) // Grab oldest 1st\n {\n this.purgeJob(jobMan);\n if (--deleteCount < 1)\n break;\n }\n }\n }\n\n // _Idx\n //\n // createSandbox, returnSandbox, hookUpSandboxListeners, pruneSandboxes\n //\n\n /**\n * Automatically handle when the evaluator is down.\n *\n * With the screensaver worker, when the screensaver goes down, so does the evaluator.\n * And when the screensaver starts running again, so does the evaluator. The evaluator\n * may be stopped and started again with sa worker running, and have good behavior.\n * However, browser workers cannot have their evaluators stopped without also stopping\n * the worker (otherwise file-a-bug...)\n *\n * @param {boolean} [throwError=false]\n * @returns {Promise<Sandbox>}\n */\n async createSandbox (throwError = false)\n {\n selectiveDebug2() && console.debug('createSandbox', this.sandboxInventory.length, Date.now() - this.lastTime);\n // See if there are any READY_FOR_ASSIGN sandboxes (viz., sandbox.isReadyForAssign is true.)\n // If the evaluator just came back up (while worker is still running) there should not be any non-assigned sandboxes.\n // We're only considering sa worker (e.g. screensaver worker), because browser workers cannot stop the\n // evaluator w/o stopping the worker (I think -- if not true, file-a-bug.)\n if (this.sandboxInventory.length > 0 && this.sandboxInventory[0].isReadyForAssign)\n {\n selectiveDebug2() && console.debug(`Supervisor.createSandbox: Found ready-for-assign sandbox ${this.sandboxInventory[0].identifier}`);\n return this.sandboxInventory.shift();\n }\n\n // If the evaluator cannot start (e.g. if the evalServer is not running),\n // then the while loop will keep retrying until the evalServer comes online.\n try\n {\n this.evaluator.createSandboxRefCount++;\n\n let retry = 0;\n while (true)\n {\n let sandbox;\n try\n {\n sandbox = new Sandbox(this, { ...this.options.sandboxOptions });\n selectiveDebug2() && console.debug(`Supervisor.createSandbox(${sandbox.id}): Calling sandbox.start: ${this.evaluator.createSandboxRefCount}, eval-down ${this.evaluator.down}`);\n this.hookUpSandboxListeners(sandbox);\n await sandbox.start();\n if (!this.capabilities)\n this.checkCapabilities(sandbox);\n if (this.evaluator.reallyDown)\n {\n this.evaluator.reallyDown = false;\n selectiveDebug() && console.debug('Supervisor.createSandbox: Evaluator is up again.', this.evaluator.createSandboxRefCount);\n this.jobManagerInventory.forEach((jobManager) => jobManager.resetSlices('createSandbox'));\n }\n return sandbox;\n }\n catch (error)\n {\n if (throwError)\n throw error;\n selectiveDebug() && console.debug(`Supervisor.createSandbox: Failed to start sandbox ${sandbox.identifier}`, this.evaluator.createSandboxRefCount, this.evaluator.down, error.message);\n if (error.code === 'ENOWORKER')\n throw new DCPError(\"Cannot use localExec without dcp-worker installed. Use the command 'npm install dcp-worker' to install the neccessary modules.\", 'ENOWORKER');\n\n if (throwError)\n throw error;\n\n // The evaluator may be down or shutting down, keep retrying.\n if ((retry % 60) === 0)\n this.warning('Failed to start a sandbox; will keep retrying; screensaver worker or evaluator may be down...');\n await a$sleepMs(1000 * Math.min(5, ++retry));\n }\n }\n }\n finally\n {\n this.evaluator.createSandboxRefCount--;\n }\n }\n\n /**\n * Remove sandbox from inventory and terminate.\n * @param {Sandbox} sandbox\n */\n returnSandbox (sandbox)\n {\n // If sandbox is not in this.sandboxInventory then sandbox must already be terminated\n // <==> this.sandboxInventory.includes(sandbox) || sandbox.isTerminated().\n selectiveDebug2() && console.debug(`returnSandbox: ${sandbox.identifier}`);\n if (common.removeElement(this.sandboxInventory, sandbox))\n sandbox.terminate(false);\n else\n {\n // If sandbox is not in this.sandboxInventory then sandbox must already be terminated\n if (common.displayMaxDiagInfo() && !sandbox.isTerminated) // Design assumption.\n throw new Error(`returnSandbox: Sandbox ${sandbox.identifier} has already been removed.`);\n }\n }\n\n /**\n * For a given sandbox, hook up all the Sandbox listeners.\n * @param {Sandbox} sandbox\n */\n hookUpSandboxListeners (sandbox)\n {\n sandbox.addListener('start', () => {\n if (!sandbox.slice) return;\n const payload = sandbox.slice.getMessagePayload(this.workerId, 'begin');\n return this.dcp4.safeRSStatus(payload, `Failed to send 'begin' status for slice ${sandbox.slice.identifier}`);\n });\n\n const that = this;\n // Sandbox error handler.\n sandbox.on('sandboxError', function Supervisor$sandboxError(error) {\n selectiveDebug() && console.debug(`Sandbox ${sandbox.identifier} sandboxError-handler; error while executing work function`, error);\n const slice = sandbox.slice;\n if (!slice?.isWorking) // Sanity -- warning should never fire.\n this.warning(`handleSandboxError: slice ${slice?.identifier} must be WORKING.`);\n if (slice)\n slice['useRetryLogic'] = true;\n that.returnSandbox(sandbox);\n });\n\n // Sandbox complete handler.\n // When any sandbox completes, go through the Supervisor.fetchTask protocol.\n sandbox.addListener('complete', () => {\n // Try not to call fetchTask unless there's something there.\n selectiveDebug2() && console.debug('Sandbox complete listener', this.fetchTaskBarrier, this.roundRobinBarrier, this.unusedSandboxCount, Date.now() - this.lastTime);\n if (!this.fetchTaskBarrier)\n this.fetchTask();\n });\n\n // If the sandbox terminated and we are not shutting down, then we should return all work which is\n // currently not being computed if all sandboxes are dead and the attempt to create a new one fails.\n sandbox.sandboxHandle.on('end', async () => {\n if (this.sandboxInventory.length > 0 && !this.evaluator.pauseSandboxHandleEndHandler)\n {\n selectiveDebug() && console.debug(`hookUpSandboxListeners: Sandbox \"${sandbox.identifier}\" terminated handler`, this.sandboxInventory.length, Date.now() - this.lastTime);\n\n // Does there exist a non-terminated sandbox?\n let allSandboxesTerminated = true;\n for (const sbx of this.sandboxInventory)\n if (!sbx.isTerminated)\n {\n allSandboxesTerminated = false;\n break;\n }\n\n if (allSandboxesTerminated && !this.evaluator.downInterlock)\n {\n //\n // When we get here, all sandboxes have been terminated.\n //\n this.evaluator.downInterlock = true;\n selectiveDebug() && console.debug('hookUpSandboxListeners: Try to create 1 sandbox in the sandbox-terminated-handler...', sandbox.identifier);\n await this.createSandbox(true /*throwError*/)\n .then((sbx) => {\n this.evaluator.reallyDown = false;\n // This is the only place where non-assigned sandboxes are added to this.sandboxInventory.\n this.sandboxInventory.unshift(sbx);\n selectiveDebug() && console.debug('Sandbox terminate handler was able to create new sandbox', sandbox.identifier);\n })\n .catch(() => {\n //\n // Since all sandboxes have been terminated, if we cannot create a new sandbox,\n // that probably means we're on a screensaver worker and the screensaver is down.\n // Try to submit results for completed slices, but return all other non-finished\n // slices to the scheduler -- after a brief delay.\n //\n selectiveDebug() && console.debug('Sandbox terminate handler cannot create new sandbox; evaluator is down', sandbox.identifier);\n this.evaluator.reallyDown = true;\n this.emit('evalDown');\n const delay = 60; // seconds\n this.jobManagerInventory.forEach((jm) => jm.evaluatorDownCleanup(delay));\n this.warning('Stopping all work.', 'Screensaver worker or evaluator may be down.');\n })\n .finally(() => {\n this.sandboxInventory = this.sandboxInventory.filter(sbx => !sbx.isTerminated);\n this.evaluator.shuttingDown = false;\n this.evaluator.downInterlock = false;\n });\n }\n }\n });\n }\n\n /**\n * Terminate extra sandboxes over the limit.\n * @todo XXXpfr Prioritize keeping expensive to assign sandboxes.\n */\n pruneSandboxes ()\n {\n this.sandboxInventory = this.sandboxInventory.filter((sandbox) => !sandbox.isTerminated);\n let pruneCount = this.sandboxInventory.length - this.options.maxSandboxes;\n if (pruneCount <= 0)\n return;\n\n selectiveDebug() && console.debug(`Supervisor.pruneSandboxes START: pruneCount ${pruneCount}/${this.sandboxInventory.length}/${this.options.maxSandboxes}.`, this.dbg.dumpSandboxState());\n selectiveDebug2() && console.debug(this.sandboxInventory.map((sbx) => sbx.identifier));\n\n // Prune ready-for-assign sandboxes first.\n while (pruneCount > 0)\n {\n if (this.sandboxInventory[0].isReadyForAssign)\n {\n const startedSandbox = this.sandboxInventory.shift();\n startedSandbox.terminate(false);\n pruneCount--;\n }\n else\n break;\n }\n\n // Don't purge jobs here: can accidentally purge a job that TD just fetched (XXXpfr)\n\n /**\n * Do we really want to do a bunch of work to keep empty job assigned sandboxes around?\n * When in a private compute group, there will be fewer jobs and it's likely\n * that a given job will be seen again.\n * @todo XXXpfr Prioritize keeping expensive to assign sandboxes.\n */\n const liveJobs = [], emptyJobs = [];\n let maxAssignedSandboxCount = 0;\n for (const jobMan of this.jobManagerInventory)\n {\n let isAlive = false;\n for (const slice of jobMan.sliceInventory)\n if (slice.isQueuedOrActive)\n {\n isAlive = true;\n break;\n }\n if (isAlive)\n liveJobs.push(jobMan);\n else\n {\n const _assignedSandboxes = jobMan.assignedSandboxes;\n if (maxAssignedSandboxCount < _assignedSandboxes.length)\n maxAssignedSandboxCount = _assignedSandboxes.length;\n emptyJobs.push(jobMan);\n }\n }\n\n if (emptyJobs.length > 0)\n {\n // Prune the sandboxes from all jobs with no current work.\n // Try to keep approximately the same # of assigned sandboxes per job.\n for (let k = maxAssignedSandboxCount; k >= 0; k--)\n {\n for (const jobMan of emptyJobs)\n {\n const _assignedSandboxes = jobMan.assignedSandboxes;\n if (_assignedSandboxes.length > k)\n {\n debugging('supervisor') && console.debug(`pruneSandboxes(empty): sandbox${_assignedSandboxes[0].id}`, Date.now() - this.lastTime);\n // Terminate and remove from this.sandboxInventory.\n this.returnSandbox(_assignedSandboxes[0]);\n if (--pruneCount < 1)\n {\n debugging('supervisor') && console.debug(`Supervisor.pruneSandboxes FINISH: ${pruneCount}/${this.sandboxInventory.length}/${this.options.maxSandboxes}.`);\n return;\n }\n }\n }\n }\n }\n\n // Round-robin prune 1 extra assigned sandbox from each non-empty jobmanager.\n while (pruneCount > 0)\n {\n const _pruneCount = pruneCount;\n for (const jobMan of liveJobs)\n {\n const _assignedSandboxes = jobMan.assignedSandboxes;\n if (_assignedSandboxes.length > 0)\n {\n debugging('supervisor') && console.debug(`pruneSandboxes(non-empty): sandbox${_assignedSandboxes[0].id}`, Date.now() - this.lastTime);\n // Terminate and remove from this.sandboxInventory.\n this.returnSandbox(_assignedSandboxes[0]);\n if (--pruneCount < 1)\n {\n debugging('supervisor') && console.debug(`Supervisor.pruneSandboxes FINISH: ${pruneCount}/${this.sandboxInventory.length}/${this.options.maxSandboxes}.`);\n return;\n }\n }\n }\n if (_pruneCount === pruneCount) // Nothing left to prune.\n break;\n }\n\n debugging('supervisor') && console.debug(`Supervisor.pruneSandboxes FINISH: incomplete-prune ${pruneCount}/${this.sandboxInventory.length}/${this.options.maxSandboxes}.`);\n }\n\n // _Idx\n //\n // recordResult, sendToResultSubmitter, sendResultToRemote\n //\n\n /**\n * Submits the slice results to the result-submitter service.\n * Then remove the slice from the its job manager.\n *\n * @param {Slice} slice - The slice to submit.\n * @param {Sandbox} sandbox - The sandbox handle associated to the slice.\n * @returns {Promise<any>}\n */\n recordResult (slice, sandbox)\n {\n // It is possible for slice.result to be undefined when there are upstream errors.\n if (!slice.result)\n throw new Error(`Slice ${slice.identifier} completed work, but there is no result. This is ok when there are upstream errors.`);\n if (!slice.isComplete)\n throw new Error(`Cannot record result for slice ${slice.identifier} that has not completed execution successfully.`);\n if (!slice.timeReport)\n throw new Error(`Invalid time report for slice ${slice.identifier} in recordResult`);\n if (!slice.dataReport)\n throw new Error(`Invalid data report for slice ${slice.identifier} in recordResult`);\n\n const metrics = slice.jobManager.updateStatistics(slice, sandbox);\n selectiveDebug() && console.debug(`Supervisor: recording result for slice ${slice.identifier} with metrics`, this.dbg.justCPU(metrics));\n\n /** @see result-submitter::result for full message details */\n const payloadData = {\n slice: slice.sliceNumber,\n job: slice.jobAddress,\n worker: this.workerId,\n paymentAddress: this.options.paymentAddress,\n metrics,\n authorizationMessage: slice.authorizationMessage,\n };\n\n let canceled = false;\n const resultUrl = (slice.resultStorageType !== 'pattern') ? slice.resultStorageDetails : false;\n this.workerEmit( 'beforeResult', () => { canceled = true; }, resultUrl);\n this.jobEmit(slice, 'beforeResult', () => { canceled = true; }, resultUrl);\n selectiveDebug && canceled && console.debug(`User canceled the result submission operation for slice ${slice.identifier}.`);\n if (canceled)\n return this.returnSlice(slice, 'Canceled via beforeResult event');\n\n if (slice.resultStorageType === 'pattern')\n return this.sendResultToRemote(slice)\n .then((response) => {\n return this.sendToResultSubmitter(slice, sandbox.sandboxHandle, payloadData, response);\n });\n\n return this.sendToResultSubmitter(slice, sandbox.sandboxHandle, payloadData, encodeDataURI(slice.result.result));\n }\n\n /**\n * Send result to result submitter.\n * @param {Slice} slice\n * @param {SandboxHandle} sandboxHandle\n * @param {*} payloadData\n * @param {string} [result]\n * @returns {Promise<any>}\n */\n async sendToResultSubmitter (slice, sandboxHandle, payloadData, result)\n {\n // When handleRSError is hit, { slice, payload } is added to the queue this.dcp4.submitResultsQueueMap[slice.key] .\n // For a given slice, the queue is retried independent of other slices that failed to submit.\n // When a given slice hits the retry limit (6 retries) the slice is returned to scheduler.\n const handleRSError = (error, slice, payloadData) => { // eslint-disable-line no-shadow\n const msg = `Failed to submit results to scheduler for slice ${slice.identifier}`;\n if (!error) error = new Error(msg);\n this.error(msg, error);\n\n slice['retrySubmitResults'] = (slice['retrySubmitResults'] ?? 0) + 1;\n if (slice['retrySubmitResults'] > this.options.maxResultSubmissionRetries)\n {\n this.handleFailedSlice(slice, error);\n throw new Error(`Failed to submit results 6 times for slice ${slice.identifier}`);\n }\n\n // For a given slice, there's never more than one element in the corresponding queue.\n this.dcp4.submitResultsQueueMap[slice.key] = [ { slice, sandboxHandle, payloadData } ];\n return this.dcp4.resetConnection('resultSubmitter');\n }\n\n try\n {\n debugging('supervisor') && console.debug('Supervisor.recordResult: payloadData', result.slice(0, 256), slice.identifier);\n if (result)\n payloadData.result = result;\n\n await this.delayManager.nextDelay('recordResult', 2);\n //->console.log('recordResult', slice.identifier, this.evaluator.down, Date.now() - this.lastTime); // SAVE\n\n return this.dcp4.resultSubmitter.request('result', payloadData)\n .then((resp) => {\n const payload = resp.payload;\n if (!resp.success)\n {\n if (payload)\n {\n selectiveDebug() && console.debug('resultSubmitter.send failed', payload);\n throw new DCPError(`Call to result submitter failed when recording results for ${slice.identifier}.`, payload);\n }\n if (debugBuild)\n {\n selectiveDebug() && console.debug('resultSubmitter.send failed with no payload', slice.identifier);\n // Look inside\n for (const [ key, value ] of Object.entries(resp)) {\n if (key !== 'connection')\n console.debug(`${key}:`, value);\n }\n }\n throw new Error(`Call to result submitter failed when recording results for ${slice.identifier}.`);\n }\n\n debugging('supervisor') && console.debug('Successfully submitted results', slice.identifier);\n\n // Success! Restore this['resultSubmitter'] delay to retryMinSleepMs (currently 32ms.)\n // Is there a better way to reset than explicit calls?\n this.delayManager.resetEBO('resultSubmitter');\n\n common.debugQuanta() && this.dbg.addGlobal(slice, payload.metrics);\n slice.jobManager.update({ metrics: payload.metrics }); // Update metrics\n\n // Emit the 3 'payment' events.\n const paymentAddress = payloadData.paymentAddress.toString();\n this.workerEmit( 'payment', payload.slicePaymentAmount, paymentAddress, slice.jobAddress, slice.sliceNumber);\n this.jobEmit(slice, 'payment', payload.slicePaymentAmount, paymentAddress, slice.sliceNumber);\n this.safeEmit(sandboxHandle, 'payment', payload.slicePaymentAmount, paymentAddress);\n\n const payloadLength = kvin.stringify(payloadData).length; /** @TODO - fix per DCP-3750 */\n const resultUrl = (slice.resultStorageType !== 'pattern') ? slice.resultStorageDetails : false;\n this.workerEmit( 'result', resultUrl, payloadLength);\n this.jobEmit(slice, 'result', resultUrl, payloadLength);\n\n slice.markAsFinished();\n\n // Remove the slice from the job manager.\n slice.jobManager.removeSlice(slice);\n\n if (this.sliceTiming)\n {\n slice['resultDelta'] = Date.now() - slice['resultDelta'];\n console.debug(`recordResult(${slice['queueingDelta']}, ${slice['executionDelta']}, ${slice['resultDelta']}): Completed slice ${slice.identifier}.`, Date.now() - this.lastTime);\n }\n if (false)\n {}\n\n return resp;\n })\n .catch ((error) => {\n handleRSError (error, slice, payloadData);\n });\n }\n catch (error)\n {\n handleRSError (error, slice, payloadData);\n }\n }\n\n /**\n * Send a work function's result to a server that speaks our DCP Remote Data Server protocol.\n * E.g. https://gitlab.com/Distributed-Compute-Protocol/dcp-rds\n *\n * @param {Slice} slice - Slice object whose result we are sending.\n * @returns {Promise<string>}\n * @throws When HTTP status not in the 2xx range.\n */\n sendResultToRemote (slice)\n {\n return supShared.sendResultToRemote(this, slice);\n }\n\n // _Idx\n //\n // handleWorkReject\n //\n\n /**\n * Handles reassigning or returning a slice that rejected.\n *\n * If error.message === 'false' and slice.hasBeenRejected is false, reschedule the slice.\n * Set the slice.hasBeenRejected to be true.\n *\n * If error.message !== 'false' or slice.hasBeenRejected is true (i.e. has been rejected once already)\n * zthen return all slices from the job to the scheduler and terminate all sandboxes with that jobAddress.\n *\n * @param {Slice} slice\n * @param {Error} error\n */\n handleWorkReject (slice, error)\n {\n debugging() && console.debug('handleWorkReject', error.message, slice.hasBeenRejected, slice.identifier);\n\n const jobManager = slice.jobManager;\n jobManager.rejectedJobReasons.push(error.message); // memoize reasons\n\n // First time rejecting without a reason; try rescheduling the slice.\n if (error.message === 'false' && !slice.hasBeenRejected)\n {\n // Mark slice as rejected.\n slice.hasBeenRejected = true;\n // Reset slice state to allow re-execution.\n slice.resetState();\n }\n else\n {\n // Slice has been rejected twice, so add to array of rejected jobs.\n const rejectedJob = {\n address: slice.jobAddress,\n reasons: jobManager.rejectedJobReasons,\n };\n this.rejectedJobs.push(rejectedJob);\n // Broadcast failure.\n this.workerEmit( 'result', error);\n this.jobEmit(slice, 'result', error);\n // Purge the job.\n this.purgeJob(jobManager);\n // Tell everyone all about it, when allowed.\n if (jobManager.displayMaxDiagInfo)\n {\n const suffixMsg = 'All slices and sandboxes with the same jobAddress returned to the scheduler or terminated.';\n if (slice.hasBeenRejected)\n this.warning(`work.reject: The slice ${slice.identifier} was rejected twice.`, suffixMsg);\n else\n this.warning(`work.reject: The slice ${slice.identifier} was rejected with reason: ${error.message}.`, suffixMsg);\n }\n }\n }\n\n}\nexports.Supervisor = Supervisor;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/supervisor2/index.js?");
4727
+ eval("/**\n * @file dcp/src/dcp-client/worker/supervisor2/index.js\n * Code managing sandboxes, tasks, jobs, and slices within in a DCP Worker.\n * @author Wes Garland, wes@distributive.network\n * Paul, paul@distributive.network\n * @date Dec 2020\n * June 2022, Jan-April 2023\n * @module supervisor\n * @copyright Copyright (c) 2018-2023, Distributive Corp. All Rights Reserved\n */\n/*\n * initial ready reconnecting stopping stopped paused broken\n * |-- ctor ----------------------------------------------------------------------------------------------------------------->\n * |-- work ----------------------------------------------------------------------------------------------------------------->\n * |-- work --------------------------------------------------------------------------------------------------->\n * |-- work -------------------------------------------------------------------------------->\n * |-- work --------------------------------------------------------->\n * |-- work --------------------------------->\n * |-- Worker.pause --------------------------------------------------------------------------->\n * <-- Worker.unpause -------------------------------------------------------------------------|\n * |-- work ----->\n * |-- stopWork ---------------------------->\n * |-- postStopShutdown --->\n * |-- PM.connectTo --> (ProtocolManager)\n * <-- PM.connectTo --| (ProtocolManager)\n * |-- stopWork ------------------------------------------->\n * <-- work -----------------------------------------------------------------------|\n * <-- stopWork -----------------------------------------------------|\n */\n/* global dcpConfig */ // eslint-disable-line no-redeclare\n// @ts-check\n\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nconst constants = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst { assert } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\nconst dcp_timers = __webpack_require__(/*! dcp/common/dcp-timers */ \"./src/common/dcp-timers.js\");\nconst EventEmitter = __webpack_require__(/*! events */ \"./node_modules/events/events.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst { Address } = __webpack_require__(/*! dcp/dcp-client/wallet/eth */ \"./src/dcp-client/wallet/eth.js\");\nconst { Keystore } = __webpack_require__(/*! dcp/dcp-client/wallet/keystore */ \"./src/dcp-client/wallet/keystore.js\");\nconst RingBuffer = __webpack_require__(/*! dcp/utils/ringBuffer */ \"./src/utils/ringBuffer.js\");\nconst { Synchronizer } = __webpack_require__(/*! dcp/common/concurrency */ \"./src/common/concurrency.js\");\nconst { JobManager } = __webpack_require__(/*! ./job-manager */ \"./src/dcp-client/worker/supervisor2/job-manager.js\");\nconst { Sandbox, SandboxError } = __webpack_require__(/*! ./sandbox2 */ \"./src/dcp-client/worker/supervisor2/sandbox2.js\");\nconst { sliceStatus } = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst { OriginAccessManager } = __webpack_require__(/*! dcp/dcp-client/worker/origin-access-manager */ \"./src/dcp-client/worker/origin-access-manager.js\");\nconst { a$sleepMs, booley, toJobMap, encodeDataURI, stringify, nextEma } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\n\nconst { ModuleCache } = __webpack_require__(/*! ./module-cache */ \"./src/dcp-client/worker/supervisor2/module-cache.js\");\nconst { Promise_any } = __webpack_require__(/*! ./promise_any */ \"./src/dcp-client/worker/supervisor2/promise_any.js\");\nconst { ProtocolManager } = __webpack_require__(/*! ./protocol-manager */ \"./src/dcp-client/worker/supervisor2/protocol-manager.js\");\nconst { EvaluatorManager } = __webpack_require__(/*! ./evaluator-manager */ \"./src/dcp-client/worker/supervisor2/evaluator-manager.js\");\nconst { DelayManager } = __webpack_require__(/*! ./delay-manager */ \"./src/dcp-client/worker/supervisor2/delay-manager.js\");\nconst { Options } = __webpack_require__(/*! ./options */ \"./src/dcp-client/worker/supervisor2/options.js\");\nconst common = __webpack_require__(/*! ./common */ \"./src/dcp-client/worker/supervisor2/common.js\");\nconst { debugBuild, selectiveDebug, selectiveDebug2, minimalDiag, selectiveSupEx } = common;\nconst supShared = __webpack_require__(/*! ../SupShared */ \"./src/dcp-client/worker/SupShared.js\");\nconst { canScheduleGPU } = __webpack_require__(/*! ./gpu_support */ \"./src/dcp-client/worker/supervisor2/gpu_support.js\");\n\n/** @typedef {string} opaqueId */ // 22 character base64 string\n/** @typedef {import('./slice2').Slice} Slice */\n/** @typedef {import('dcp/utils/jsdoc-types').Auth} Auth */\n/** @typedef {import('dcp/utils/jsdoc-types').Body} Body */\n/** @typedef {import('./sandbox2').SandboxHandle} SandboxHandle */\n/** @typedef {import('dcp/utils/jsdoc-types').SliceObj} SliceObj */\n/** @typedef {import('dcp/dcp-client/worker/index').Worker} Worker */\n/** @typedef {import('dcp/utils/jsdoc-types').TDPayload} TDPayload */\n/** @typedef {import('dcp/utils/jsdoc-types').Signature} Signature */\n/** @typedef {import('dcp/utils/jsdoc-types').SliceMessage} SliceMessage */\n/** @typedef {import('dcp/dcp-client/wallet/keystore').Keystore} Keystore */\n/** @typedef {import('dcp/utils/jsdoc-types').SupervisorOptions} SupervisorOptions */\n/** @typedef {import('dcp/protocol-v4/connection/connection').Connection} Connection */\n\n//\n// Index to functionality -- search for '_Idx' to toggle through the index.\n//\n// 1) class Supervisor\n// 2) work, checkCapabilities\n// 3) safeEmit, workerEmit, jobEmit, error, warning, mungeError, jobDescriptor, setState\n// 4) returnAllSlices, postStopShutdown, abort, stopWork, purgeJob\n// 5) roundRobinSlices, makeJobSelectionCursor, handleSandboxWorkError, handleFailedSlice\n// 6) returnSlices, returnSlice, emitProgressReport\n// 7) jobQuanta, repoMan, predictLoad(viz., clairvoyance), waitUntilWorkIsReady, generateWorkerComputeGroups\n// 8) availableSandboxSpace, fetchTask, addTaskToWorkload, fetchFromTD, clearUnusedJobManagersAndModuleCache\n// 9) createSandbox, returnSandbox, hookUpSandboxListeners, pruneSandboxes\n// 10) recordResult, sendToResultSubmitter, sendResultToRemote\n// 11) handleWorkReject\n//\n\n// _Idx\n//\n// class Supervisor\n//\n\n/**\n * Supervisor constructor\n *\n * A supervisor manages the communication with the scheduler, manages sandboxes, and\n * decides which workload should be sent to which sandboxes when.\n *\n * Possible states: 'initial', 'ready', 'reconnecting', 'stopping', 'stopped', 'broken'\n * Start state:\n * - initial\n *\n * Intermediate states:\n * - ready\n * - reconnecting\n * - stopping\n *\n * Terminal states:\n * - stopped\n * - broken\n *\n * Valid transitions:\n * - initial -> ready -> reconnecting -> ready\n * - ready -> stopping -> stopped\n * - initial -> broken\n */\nclass Supervisor extends EventEmitter\n{\n /**\n * @constructor\n * @param {Worker} worker\n * @param {Keystore} identity\n * @param {SupervisorOptions} options\n */\n constructor (worker, identity, options)\n {\n super({ captureRejections: false });\n\n if (!(identity instanceof Keystore))\n throw new Error(`identity ${JSON.stringify(identity)} must be an instance of Keystore`);\n\n debugging('supervisor') && console.debug('Supervisor.options', options);\n assert(options === worker.workerOptions);\n \n /** @type {Worker} */\n this.worker = worker;\n /** @type {Keystore} */\n this.identityKeystore = identity;\n /** @type {Options} */\n this.options = new Options(options, worker);\n\n selectiveDebug() && console.debug('Supervisor: cores.cpu, cores.gpu, maxSandboxes', options.cores?.cpu, options.cores?.gpu, this.options.maxSandboxes);\n\n /** @type {ModuleCache} */\n this.moduleCache = new ModuleCache(this);\n\n // Manage delays and exponential backoff.\n this.delayManager = new DelayManager(this, this.options.defaultDelayIncrement);\n\n /* See https://distributive.atlassian.net/browse/DCP-3175 */\n /** @type {OriginAccessManager} */\n this.originManager = OriginAccessManager.construct(this.options.allowOrigins);\n\n /** @type {ProtocolManager} */\n this.dcp4 = new ProtocolManager(this);\n\n /** @type {common.DebuggingTools} */\n this.dbg = new common.DebuggingTools(this);\n\n // Turn on for max speed debugging.\n if (false)\n {}\n\n /** @type { Synchronizer } */\n this.state = new Synchronizer('initial', [ 'initial', 'ready', 'reconnecting', 'stopping', 'stopped', 'paused', 'broken']);\n /** @type {Object<string, JobManager>} */\n this.jobMap = {}; // jobAddress => jobManager\n\n /** @type {JobManager[]} */\n this.jobManagerInventory = common.InventoryArray('jobManagers');\n /** @type {Sandbox[]} */\n this.sandboxInventory = []; // All sandboxes that are being used by the job managers. Makes sure we don't lose sandboxes.\n /** @type {{ next: cbNext, push: cbPush }} */\n this.cursor = null;\n /** @type {number} */\n this.defaultQuanta = 1.0;\n\n /**\n * Evaluator down management.\n **/\n this.evaluator = new EvaluatorManager();\n\n // There are 2 kinds of barriers.\n // 1) fetchTaskBarrier is a barrier for the task fetching from task distributor path.\n // 2) roundRobinBarrier is a barrier for the slice execution path.\n /** @type {boolean} */\n this.fetchTaskBarrier = false;\n /** @type {boolean} */\n this.roundRobinBarrier = false;\n\n /** @type {object[]} */\n this.rejectedJobs = [];\n /**\n * An N-slot ring buffer of job addresses. Stores all jobs that have had no more than 1 slice run in the ring buffer.\n * Required for the implementation of discrete jobs\n * @type {RingBuffer}\n */\n this.ringBufferofJobs = new RingBuffer(200); // N = 200 should be more than enough.\n /**\n * When true we await waitUntilWorkIsReady until at least 1 job is ready with at least 1 ready slice.\n * waitUntilWorkIsReady\n * @type {boolean}\n */\n this.waitForWork = true;\n /**\n * Last repoMan time stamp.\n * @type {number}\n **/\n this.lastRepoMan = Date.now();\n /**\n * Last prune time stamp.\n * @type {number}\n **/\n this.lastPrune = Date.now();\n /**\n * General time stamp.\n * @type {number}\n **/\n this.lastTime = Date.now();\n /**\n * Fetch started time stamp.\n * @type {number}\n **/\n this.fetchTaskStarted = 0;\n /**\n * The capabilities of a random sandbox.\n * @todo XXXpfr Re-work this once fetchTask uses the capabilities of every sandbox to fetch slices.\n * @type {object}\n */\n this.capabilities = null;\n /**\n * EMA times series of CPUTime + GPUTime over all jobs.\n * Each EMA entry is computed right before calling fetchTask.\n * @type {number}\n */\n this.localTime = 0;\n /**\n * EMA times series of sliceCPUTime + sliceGPUTime over all jobs.\n * Each EMA entry is computed right before calling fetchTask.\n * @type {number}\n */\n this.globalTime = 0;\n /**\n * When this.sliceTiming is set to be true, it displays the timings of a every slice\n * slice['queueingDelta'] = timespan of when slice is passed to jobManager.runQueuedSlice until sandbox.work\n * slice['executionDelta'] = timespan of execution in sandbox\n * slice['resultDelta'] = timespan of when sandbox finishes executing until recordResult completes.\n * @type {boolean}\n */\n this.sliceTiming = false;\n\n try\n {\n // Start up the connections.\n this.dcp4.instantiateAllConnections();\n }\n catch(error)\n {\n this.error('Failed to set up DCP connections:', error);\n this.setState('initial', 'broken');\n this.worker.stop(true).finally(() => { throw error; });\n }\n }\n\n //\n // Compatibility layer between Sup1, Sup2 and the Sup interface exposed by Worker.\n //\n /**\n * Get all sandboxes.\n * @type {Sandbox[]}\n */\n get sandboxes () { return this.sandboxInventory.filter((sandbox) => !sandbox.isTerminated); }\n /**\n * Get all working sandboxes.\n * @type {Sandbox[]}\n */\n get workingSandboxes () { return this.sandboxInventory.filter((sandbox) => sandbox.isWorking); }\n /**\n * Get the number of working sandboxes.\n * @type {number}\n */\n get workingSandboxCount () { return this.workingSandboxes.length; }\n /**\n * Get all slices over all jobs..\n * @type {Slice[]}\n */\n get slices () {\n const slices = [];\n this.jobManagerInventory.forEach((jobManager) => { slices.push(...jobManager.sliceInventory); });\n return slices;\n }\n /**\n * Get all queued slices over all jobs..\n * @type {Slice[]}\n */\n get queuedSlices () {\n const slices = [];\n this.jobManagerInventory.forEach((jobManager) => { slices.push(...jobManager.queuedSlices); });\n return slices;\n }\n /**\n * Get all queued slices over all jobs..\n * @type {Slice[]}\n */\n get workingSlices () {\n const slices = [];\n this.jobManagerInventory.forEach((jobManager) => { slices.push(...jobManager.workingSlices); });\n return slices;\n }\n /** @type {opaqueId} */\n get workerId () { return this.options.workerId; }\n /** @type {opaqueId} */\n set workerId (id) { this.options.workerId = id; }\n get version() { return '2.0.0' }\n /**\n * @deprecated\n * @todo XXXpfr Rip out this sup2/sup1 special-casing when we finally kill sup1.\n * @type {boolean}\n */\n get isSupervisor1 () { return false; }\n /**\n * @deprecated\n * @todo XXXpfr Rip out this sup2/sup1 special-casing when we finally kill sup1.\n * @type {boolean}\n */\n get isSupervisor2 () { return true; }\n\n //\n // Miscellaneous properties.\n //\n\n /**\n * Dynamic maxWorkingCores.\n * The maximum number of cores that can be executing slices. Slices are scheduled\n * using density. E.g. suppose a job has GPUDensity is 0 and CPUDensity is 0.5,\n * then 2 slices of this job can be scheduled on a single core.\n * @type {number}\n */\n get maxWorkingCores () { return this.options.cores?.cpu; }\n /**\n * Dynamic maxWorkingGPUs.\n * The maximum number of GPUs that can be executing slices. Slices are scheduled\n * using density. E.g. suppose a job has GPUDensity is 0.5 and CPUDensity is 0.5,\n * then 2 slices of this job can be scheduled on a single GPU core and a single CPU core.\n * @type {number}\n */\n get maxWorkingGPUs () { return this.options.cores?.gpu; }\n /**\n * @deprecated\n * @todo XXXpfr Get rid of this after Sup1 dies.\n */\n get lastDcpsid () { return this.dcp4.lastDcpsid; }\n /**\n * @deprecated\n * @todo XXXpfr Get rid of this after Sup1 dies.\n */\n set lastDcpsid (dcpsid) { this.dcp4.lastDcpsid = dcpsid; }\n /**\n * Indicates whether supervisor is ready for business.\n * @type {boolean}\n */\n get isReady () { return this.worker.working && this.state.is('ready'); }\n /**\n * The # of sandboxes not being used.\n * @type {number}\n */\n get unusedSandboxCount () { return this.options.maxSandboxes - this.workingSliceCount; }\n /**\n * The unused amount of CPU density in the cores.\n * @type {number}\n */\n get unusedCoreSpace () { return this.maxWorkingCores - this.workingSliceDensity; }\n /**\n * The unused amount of GPU density in the cores.\n * Use Math.max(1, this.maxWorkingGPUs) so there's always enough room to schedule\n * a GPU slice when this.workingGPUDensity = 0. In RoundRobinSlices we use the accumulated\n * recent history ( canScheduleGPU(maxWorkingGPUs) ) to check whether the average recent\n * density is within this.maxWorkingGPUs.\n * @type {number}\n */\n get unusedGPUSpace () { return Math.max(1, this.maxWorkingGPUs) - this.workingGPUDensity; }\n /** @type {number} */\n get workingSliceDensity ()\n {\n let density = 0;\n for (const jobMan of this.jobManagerInventory)\n density += jobMan.workingSliceDensity;\n return density;\n }\n /** @type {number} */\n get workingGPUDensity ()\n {\n let density = 0;\n for (const jobMan of this.jobManagerInventory)\n density += jobMan.workingGPUDensity;\n return density;\n }\n /** @type {number} */\n get workingSliceCount ()\n {\n let count = 0;\n for (const jobMan of this.jobManagerInventory)\n count += jobMan.workingSliceCount;\n return count;\n }\n /**\n * Compute the estimated time to completion of all work.\n * The time is measured as if there were only a single slice running at a time.\n * workRemaining is the amount of time until completion.\n * @type {number}\n */\n get workRemaining ()\n {\n let workRemaining = 0;\n for (const jobMan of this.jobManagerInventory)\n workRemaining += jobMan.workRemaining;\n return workRemaining;\n }\n\n // _Idx\n //\n // work, checkCapabilities\n //\n\n /**\n * Set up sandboxes and interval timers, then start to search for work.\n * Called in Worker.start().\n * Initial entry point after Worker constructor.\n * We need to start searching for work here to allow starting and stopping a worker.\n */\n work ()\n {\n const abort = async (error) => {\n // May be in a stopping/stopped state, because dcp-worker was hit with ctrl-C.\n this.setState(['ready', 'stopping', 'stopped', 'reconnecting'], 'broken');\n await this.worker.stop(true);\n throw error;\n };\n /* Provide opportunity for calling code to hook ready/error events. */\n dcp_timers.setImmediate(() => {\n try\n {\n // [ 'initial', 'ready', 'reconnecting', 'stopping', 'stopped', 'paused', 'broken' ]\n if (this.state.isNot('initial'))\n {\n if (this.state.in(['ready', 'stopping', 'reconnecting']))\n {\n this.warning(`Supervisor.work was called when supervisor is already ${this.state.valueOf()}.`, 'Please either wait and try again or restart worker.');\n return;\n }\n else if (this.state.is('broken'))\n {\n this.warning(\"Cannot call Supervisor.work when supervisor is in a 'broken' state. Please restart worker.\");\n return;\n }\n this.state.set(['stopped', 'paused'], 'initial');\n }\n this.evaluator.initialize();\n this.dcp4.instantiateAllConnections();\n\n // Beacon interval timer.\n this.progressReportTimer = dcp_timers.setInterval(() => this.emitProgressReport(), this.options.progressReportInterval);\n // Watchdog: fetchTask-driven interval timer.\n this.watchdogTimer = dcp_timers.setInterval(() => this.fetchTask(), this.options.watchdogInterval);\n\n // Interval timers helps keep workers and localExec alive forever.\n this.progressReportTimer.unref();\n this.watchdogTimer.unref();\n\n if ( false || debugging('supervisor'))\n {\n this.sliceDebuggingTimer = setInterval(() => {\n this.jobManagerInventory.forEach((jobMan) => {\n const { unassigned, ready, reserved, working, workdone, complete, failed, finished } = jobMan.dumpSlices ('RRS', false, false);\n console.debug(`RRS(${jobMan.identifier},${this.unusedSandboxCount},${this.unusedCoreSpace},${this.workingSliceCount},${this.workingSliceDensity}): u/r/rsv/w/wd/c/f/fsh ${unassigned}/${ready}/${reserved}/${working}/${workdone}/${complete}/${failed}/${finished}`, jobMan.identifier, this.sandboxInventory.length);\n });\n }, 30 * 1000);\n if (this.sliceDebuggingTimer.unref)\n this.sliceDebuggingTimer.unref();\n }\n\n this.state.set('initial', 'ready');\n\n // Create 1 sandbox now to get the capabilities which are sent to Task Distributor by fetchTask.\n this.createSandbox()\n .then((sandbox) => {\n this.sandboxInventory.push(sandbox);\n debugging('supervisor') && console.debug('work() after createSandbox', this.sandboxInventory.length, sandbox.identifier, Date.now() - this.lastTime, this.options.watchdogInterval);\n this.fetchTask() // Don't wait for watchdog.\n .catch (async (error) => {\n this.error('work() failed when calling fetchTask', error);\n await abort(error);\n });\n })\n .catch(async (error) => {\n this.error('work() failed when calling createSandbox, exiting...', error);\n await abort(error);\n });\n }\n catch(error)\n {\n this.error('work() failed', error);\n if (this.state.is('initial')) this.state.set('initial', 'broken');\n else if (!this.state.is('broken')) this.setState('ready', 'broken');\n this.worker.stop(true).finally(() => { throw error; });\n }\n });\n }\n\n /** Construct capabilities when necessary. */\n checkCapabilities (sandbox)\n {\n /**\n * Assign the capabilities of one the sandboxes before fetching slices from the scheduler.\n * @todo XXXpfr Re-work this once fetchTask uses the capabilities of every sandbox to fetch slices.\n */\n this.capabilities = sandbox.capabilities;\n if (DCP_ENV.isBrowserPlatform && this.capabilities.browser)\n this.capabilities.browser.chrome = DCP_ENV.isBrowserChrome;\n\n debugging('supervisor') && console.debug('Supervisor.checkCapabilities computed', Date.now() - this.lastTime);\n }\n\n // _Idx\n //\n // safeEmit, workerEmit, jobEmit,\n // error, warning, mungeError, jobDescriptor, setState\n //\n\n /**\n * Safe event emitter.\n * @param {EventEmitter} emitter\n * @param {string} event\n * @param {...any} args\n */\n safeEmit(emitter, event, ...args)\n {\n try\n {\n emitter.emit(event, ...args);\n }\n catch (error)\n {\n this.error(`Event handler for event ${event} threw an exception`, error);\n }\n }\n\n /**\n * Safe event emitter on worker.\n * @param {string} event\n * @param {...any} args\n */\n workerEmit(event, ...args)\n {\n this.safeEmit(this.worker, event, ...args);\n }\n\n /**\n * Safe event emitter on slice.jobHandle.\n * @param {Slice} slice\n * @param {string} event\n * @param {...any} args\n */\n jobEmit(slice, event, ...args)\n {\n this.safeEmit(slice.jobHandle, event, ...args);\n }\n\n /**\n * Error feedback to user.\n * @param {string} message\n * @param {Array<Error>|Error|string} [coreError]\n * @param {string} [additionalInfo]\n * @param {boolean} [supressStack=false]\n */\n error (message, coreError, additionalInfo, supressStack = false)\n {\n const isString = (s) => { return (typeof s === 'string' || s instanceof String); };\n if (coreError instanceof AggregateError)\n coreError = coreError.errors;\n if (Array.isArray(coreError) && coreError.length > 0) // Emit error for every element of array.\n return coreError.flat().forEach((c_err) => this.error(message, c_err, additionalInfo));\n\n debugging('supervisor') && console.debug('Supervisor.error:', message, coreError, additionalInfo);\n if (!message)\n message = 'Supervisor.error called w/o valid message';\n if (additionalInfo)\n {\n if (typeof additionalInfo === 'object')\n // @ts-ignore\n additionalInfo = (additionalInfo instanceof Error) ? additionalInfo.message : JSON.stringify(additionalInfo);\n else if (typeof additionalInfo !== 'string')\n additionalInfo = String(additionalInfo);\n\n if (!isString(additionalInfo))\n additionalInfo = additionalInfo.toString();\n if (!coreError)\n coreError = '';\n else if (!isString(coreError))\n coreError = String(coreError);\n }\n\n let dcpError;\n if (additionalInfo)\n dcpError = new DCPError(message, coreError, additionalInfo, supressStack);\n else if (coreError && (coreError instanceof Error))\n dcpError = new DCPError(message, coreError, '', supressStack);\n else\n dcpError = new DCPError(message, '', '', supressStack);\n\n this.worker.emit('error', dcpError);\n }\n\n /**\n * Warning feedback to user.\n * @param {string[]} messages\n */\n warning (...messages)\n {\n debugging('supervisor') && console.debug('Supervisor.warning:', messages);\n if (messages.length < 1)\n messages = [ 'Supervisor.warning called w/o valid message(s)' ];\n messages.forEach((message) => this.worker.emit('warning', message));\n }\n\n /**\n * @deprecated\n * Create new object and copy the interesting properties from error.\n * Only show the stack for debug builds.\n * If timestamp isn't set, assign new Date().\n * @param {{ message }|string|object} error\n * @param {*} [errorCtor]\n * @returns {string|{ message }}\n */\n __mungeError (error, errorCtor)\n {\n if (typeof error === 'string')\n {\n const errorLines = error.split('\\n');\n return common.displayMaxDiagInfo() ? error : errorLines[0];\n }\n\n if (!error || typeof error !== 'object' || !('message' in error) || Array.isArray(error))\n return error;\n\n if (minimalDiag)\n return error.message;\n\n const errorObj = errorCtor ? new errorCtor(error.message) : { message: error.message };\n\n const props = common.displayMaxDiagInfo()\n ? [ 'type', 'process', 'name', 'origin', 'info', 'code', 'errorCode', 'operation', 'fileName', 'lineNumber', 'timestamp' ]\n : [ 'code', 'errorCode', 'fileName', 'lineNumber', 'timestamp' ]\n const predCopy = (prop) => {\n if (error[prop])\n errorObj[prop] = error[prop];\n };\n\n props.forEach((prop) => { predCopy(prop); });\n\n if (common.displayMaxDiagInfo())\n {\n predCopy('stack');\n if (errorObj['name'] === 'Error')\n delete errorObj['name'];\n }\n if (!errorObj['timestamp'])\n errorObj['timestamp'] = new Date();\n\n return errorObj;\n }\n\n /**\n * Get the job descriptor for the appropriate job manager,\n * which is the object value corresponding to jobAddress, in\n * the object returned by getJobsForTask in task-jobs.js.\n * @param {string} jobAddress\n * @returns {object}\n */\n jobDescriptor (jobAddress)\n {\n const jobManager = this.jobMap[jobAddress];\n if (!jobManager)\n throw new Error(`Cannot find the job descriptor corresponding to jobAddress ${jobAddress}`);\n return jobManager.jobMessage;\n }\n\n /**\n * Protect this.state when transitioning from currState -> nextState\n * It's dangerous to place this.state.set in a catch block with this.error or this.warning\n * because an uncaught exception will kill process before emitting the event-based diagnostic.\n * @param {string|string[]} currState\n * @param {string} nextState\n */\n setState(currState, nextState)\n {\n try { this.state.set(currState, nextState); }\n catch (e) { this.error('Supervisor.state.set error', e); }\n }\n\n // _Idx\n //\n // returnAllSlices, postStopShutdown, abort\n // stopWork, purgeJob\n //\n\n /** @returns {Promise<*>} */\n returnAllSlices ()\n {\n if (selectiveDebug())\n {\n const activeSlices = this.jobManagerInventory.map((jm) => jm.activeSlices).flat();\n if (activeSlices.length > 0)\n this.warning(`Returning active slices : ${stringify(activeSlices.map((slice) => slice.identifier), -1, 2)}`);\n }\n // The promises are all about returning the slices to the scheduler and there's no reason to await that.\n return Promise.all(this.jobManagerInventory.map((jm) => jm.destroy()));\n }\n\n /** @returns {Promise<*>} */\n postStopShutdown ()\n {\n for (const sandbox of this.sandboxInventory)\n sandbox.terminate(false);\n this.sandboxInventory = [];\n\n // There shouldn't be anything in the job managers, but just to be safe call returnAllSlices.\n // Clear jobManagerInventory, close all connections and set state to 'stopped'.\n return this.returnAllSlices()\n .finally(() => {\n // Re-enable is-screen-saver-active logic for the sandbox handle 'end' event handler.\n this.evaluator.pauseSandboxHandleEndHandler = false;\n this.jobManagerInventory = common.InventoryArray('jobManagers');\n return this.dcp4.closeConnections()\n .finally (() => {\n if (this.state.isNot('stopped'))\n this.setState('stopping', 'stopped');\n // This log message assume slices were returned to scheduler in a previous operation, which is the only current use case.\n // If we use this function in a different way in the future, update the log message.\n selectiveDebug() && console.debug(`Supervisor.postStopShutdown(${this.state}): terminated all sandboxes and returned all slices to scheduler...`);\n });\n });\n }\n\n /**\n * Stop the worker immediately and return all unfinished slices.\n * @returns {Promise<*>}\n */\n abort ()\n {\n return this.returnAllSlices()\n .finally (() => {\n return this.postStopShutdown();\n });\n }\n\n /**\n * Terminates sandboxes and returns slices.\n * Sets the working flag to false, call @this.work to start working again.\n *\n * If forceTerminate is true: Terminates all sandboxes and returns all slices.\n * If forceTerminate is false: Terminates non-working sandboxes and returns initial and ready slices.\n *\n * @param {boolean} [forceTerminate = true] - true if you want to stop the sandboxes from completing their current slice.\n * @returns {Promise<*>}\n */\n async stopWork (forceTerminate = true)\n {\n /** @returns {boolean} */\n const doNotWaitForWork = () => {\n return (this.evaluator.reallyDown || !this.sandboxInventory.filter(sbx => !sbx.isTerminated).length);\n }\n selectiveDebug() && console.debug(`Supervisor.stopWork(${forceTerminate}, ${this.state}): terminating sandboxes and returning slices to scheduler.`);\n\n // [ 'initial', 'ready', 'reconnecting', 'stopping', 'stopped', 'paused', 'broken']\n if (this.state.in(['stopping', 'stopped', 'reconnecting']))\n {\n this.warning(`Supervisor.stopWork was called when supervisor is in state ${this.state.valueOf()}.`, 'Please either wait and try again or restart worker.');\n return;\n }\n else if (this.state.is('initial'))\n {\n this.warning('Cannot call stopWork before worker has started. Please either wait and try again or restart worker.');\n return;\n }\n this.state.set(['ready', 'paused', 'broken'], 'stopping');\n\n this.dcp4.instantiateAllConnections();\n\n // Do not enter is-screen-saver-active logic in the sandbox handle 'end' event handler.\n this.evaluator.pauseSandboxHandleEndHandler = true;\n\n if (forceTerminate)\n return this.abort();\n else\n {\n const slicesToReturn = [];\n for (const jm of this.jobManagerInventory)\n slicesToReturn.push(...jm.queuedSlices);\n\n const reason = `stopWork returning all non-finished slices that are not working`;\n this.returnSlices(slicesToReturn, reason);\n\n for (let k = 0; k < 3; k++)\n {\n await new Promise((resolve) => {\n // Count the slices that have been working or close-to-working but haven't submitted results yet.\n let activeSliceCount = 0;\n for (const jm of this.jobManagerInventory)\n activeSliceCount += jm.activeSlices.length;\n // When no active slices we're done.\n if (activeSliceCount === 0)\n resolve();\n // When no work can be completed we return all slices and leave.\n if (doNotWaitForWork())\n {\n this.returnAllSlices();\n resolve();\n }\n selectiveDebug() && console.debug(`StopWork: waiting for ${activeSliceCount} working slices to finish`, k);\n // Resolve and finish stopWork once all sandboxes have finished submitting their results.\n this.worker.on('result', () => {\n selectiveDebug() && console.debug(`StopWork: result handler, activeSliceCount ${activeSliceCount-1}`);\n if (--activeSliceCount === 0)\n {\n this.warning('All sandboxes empty, stopping worker and closing all connections');\n resolve();\n }\n });\n this.on('evalDown', () => {\n this.warning('Evaluator is down.', 'Force return all slices to scheduler, stopping worker and closing all connections.');\n this.returnAllSlices();\n resolve();\n });\n });\n }\n\n for (const jm of this.jobManagerInventory)\n this.safeEmit(jm.jobHandle, 'flush');\n\n if (selectiveDebug())\n {\n console.debug(`stopWork(${this.state.valueOf()}): After waiting for working slices to finish: workingSbxes: ${this.workingSandboxCount}, totalSbxes: ${this.sandboxInventory.length}, jobs: ${this.jobManagerInventory.length}`);\n this.jobManagerInventory.forEach((jm) => {\n console.debug('stopWork job', jm.identifier);\n console.debug(jm.countSliceStr('stopWork'));\n });\n }\n }\n\n return this.postStopShutdown();\n }\n\n /**\n * Purge all traces of the job.\n * @param {JobManager} jobManager\n */\n purgeJob (jobManager)\n {\n selectiveDebug() && console.debug(`Supervisor.purgeJob ${jobManager.identifier}.`);\n // If the slice from a job never completes and the job address exists in the ringBufferofJobs,\n // then we remove it to allow for another slice (from the same job) to be obtained by fetchTask\n this.ringBufferofJobs.buf = this.ringBufferofJobs.filter(element => element !== jobManager.address);\n this.jobManagerInventory.delete(jobManager);\n this.moduleCache.removeJob(jobManager.address);\n this.dbg.cleanUpDeadJob(jobManager.address);\n jobManager.destroy();\n }\n\n // _Idx\n //\n // roundRobinSlices, makeJobSelectionCursor, handleSandboxWorkError, handleFailedSlice\n //\n\n /**\n * Round-robin through the job managers, picking 1 slice to run each time.\n * Try to have the same number of working sandboxes for each job.\n * Try to run a slice on every available sandbox.\n *\n * The basic idea behind the scheduling of slices in this implementation is to keep as\n * many slices from different jobs running as possible, so as to reduce the likelihood\n * of resource contention between sandboxes.\n *\n * Slices are scheduled based on the following ruleset:\n * 1) cursor = makeJobSelectionCursor(), then cursor.next() returns a slice chosen as follows.\n * 2) Let concurrency range from 1 to maxWorkingCores.\n * 3) For a given concurrency, let readyJobs be all jobs such that jobMan.workingSliceDensity < concurrency.\n * 4) Do an ascending sort of readyJobs wrt jobMan.emaTotalTime.\n * 5) Pick a slice from the longest job in readyJobs that doesn't have any executing slices.\n * 6) Alternately shift a slice from readySlices vs choose a slice from a random nearly finished job, and remove slice from readySlices.\n * 7) When there are no more almost finished jobs with slices, shift slices from readyJobs.\n * 8) Jobs which have slicePriority set by the task-distributor may have slices chosen ahead of the above algorithm.\n * 9) Jobs with a slicePriority closer to 1 are more likely to be chosen.\n * 10) After finishing concurrency at maxWorkingCores, cursor.next() returns null, so create a new cursor.\n * @returns {Promise<any>}\n */\n roundRobinSlices ()\n {\n //\n // Should we try to put all runSlice promises in an array and return Promise.all(runslice-promises) ?\n //\n try\n {\n /**\n * The amount of space available for the CPU-component of slices to run in sandboxes.\n * If space is 2.5 and there are 6 slices with density 0.4, and there are enough non-working usable\n * sandboxes, then all 6 slices will be scheduled to run.\n * @type {number}\n */\n const unusedCoreSpace = this.unusedCoreSpace;\n /**\n * The number of sandboxes not currently being used.\n * @type {number}\n */\n const unusedSandboxCount = this.unusedSandboxCount;\n /**\n * The amount of space available for the GPU-component of slices to run in sandboxes.\n * @type {number}\n */\n const unusedGPUSpace = this.unusedGPUSpace;\n if (unusedCoreSpace < common.doNotSchedule || this.roundRobinBarrier || unusedSandboxCount < 1)\n {\n selectiveDebug2() && console.debug('RRS: bail early space/barrier/unusedSlots', unusedCoreSpace, this.roundRobinBarrier, unusedSandboxCount);\n return;\n }\n // roundRobinBarrier is a barrier for the slice execution path.\n this.roundRobinBarrier = true;\n if (this.evaluator.down && this.evaluator.createSandboxRefCount > 0)\n return;\n selectiveDebug2() && console.debug('BarrierState:RRS:', this.fetchTaskBarrier, this.roundRobinBarrier);\n\n if (selectiveDebug2())\n {\n this.jobManagerInventory.forEach((jobMan) => {\n const { unassigned, ready, reserved, working, workdone, complete, failed, finished } = jobMan.dumpSlices ('RRS', false, false);\n console.debug(`RRS(${jobMan.identifier},${unusedSandboxCount},${unusedCoreSpace}): u/r/rsv/w/wd/c/f/fsh ${unassigned}/${ready}/${reserved}/${working}/${workdone}/${complete}/${failed}/${finished}`, jobMan.identifier, this.sandboxInventory.length);\n });\n }\n\n if ( false || selectiveDebug())\n {\n let totalReady = 0, totalReadyDensity = 0;\n for (const jobMan of this.jobManagerInventory)\n {\n const currentReady = jobMan.readySlices.length;\n const currentReadyDensity = jobMan.readySlices.length * jobMan.estimateDensity;\n totalReady += currentReady;\n totalReadyDensity += currentReadyDensity;\n console.debug(`RRS: job ${jobMan.identifier}, density ${jobMan.estimateDensity}, readySlices ${currentReady}, readyDensity ${currentReadyDensity}`);\n }\n console.debug(`RRS: space ${unusedCoreSpace}, unusedSandboxCount ${unusedSandboxCount}, totalReady ${totalReady}, totalReadyDensity ${totalReadyDensity}`);\n }\n\n /** @type {Slice[]} */\n const slices = [];\n /** @type {number} */\n let density = 0;\n /** @type {number} */\n let gpuDensity = 0;\n\n const isSpaceAvailable = (density) => {\n const result = density < unusedCoreSpace && slices.length < unusedSandboxCount;\n selectiveDebug2() && console.debug('RRS: isSpaceAvailable', density < unusedCoreSpace, slices.length < unusedSandboxCount);\n return result;\n }\n\n // When the cursor is almost done and RRS tries to schedule slices,\n // it makes sense to recreate the cursor once to ensure enough slices can be pulled from cursor.\n let recreateCursorCount = 0;\n\n while (isSpaceAvailable(density + common.schedulingSlop))\n {\n // Get existing cursor or create new one.\n if (!this.cursor)\n this.cursor = this.makeJobSelectionCursor();\n\n // Get the next slice, then check to see whether it can be used.\n const slice = this.cursor.next();\n if (!slice)\n {\n if (/*!okToSchedule ||*/ ++recreateCursorCount > 1)\n {\n this.cursor = null;\n break;\n }\n // Start a new cursor.\n this.cursor = this.makeJobSelectionCursor();\n continue;\n }\n let okToSchedule = true\n const job = slice.jobManager;\n density += job.estimateDensity;\n if (job.useGPU)\n {\n okToSchedule = canScheduleGPU(this.maxWorkingGPUs);\n if (okToSchedule)\n {\n gpuDensity += job.estimateGPUDensity;\n okToSchedule = (gpuDensity <= unusedGPUSpace);\n selectiveDebug2() && console.debug(`RRS: GPU scheduling(${okToSchedule},${this.workingSliceCount},${density.toFixed(7)},${unusedCoreSpace.toFixed(7)}): gpuDensity/gpuSpace ${gpuDensity.toFixed(7)}/${unusedGPUSpace.toFixed(7)}, jobGPUDensity/jobCPUDensity ${job.estimateGPUDensity.toFixed(7)}/${job.estimateDensity.toFixed(7)}`);\n }\n }\n if (okToSchedule && density <= unusedCoreSpace + common.schedulingSlop) // Ok, if it's only over by a little bit.\n slices.push(slice);\n else\n {\n slice.unReserve();\n density -= job.estimateDensity;\n if (job.useGPU)\n gpuDensity -= job.estimateGPUDensity;\n else\n this.cursor.push(slice); // If useGPU, then skip pulling a slice from job\n break;\n }\n selectiveDebug2() && console.debug('RRS: density/space/numSlices/unusedSlots/jobDensity', density, unusedCoreSpace, slices.length, unusedSandboxCount, job.estimateDensity);\n }\n\n selectiveSupEx() && density > 0 && console.debug(`roundRobinSlices(${this.workingSliceCount},${this.workingSliceDensity}): Found density ${density.toFixed(7)}/${unusedCoreSpace} with ${slices.length} slices:`, slices.map((slice) => slice.identifier), this.jobManagerInventory.map((jm) => `${jm.identifier}:${jm.estimateDensity.toFixed(7)}:${jm.emaSliceTime.toFixed(0)}`));\n\n // Execute the slices.\n if (slices.length > 0)\n {\n const lastSlice = slices.pop();\n for (const slice of slices)\n slice.jobManager.runSlice(slice);\n return lastSlice.jobManager.runSlice(lastSlice);\n }\n }\n finally\n {\n this.roundRobinBarrier = false;\n }\n }\n\n /**\n * @private\n * @callback cbNext\n * @returns {Slice}\n */\n /**\n * @private\n * @callback cbPush\n * @param {Slice} slice\n */\n\n /**\n * Factory function which instantiates a JobSelectionCursor. A JobSelectionCursor\n * steps the order that job slices should be selected for execution in the supervisor,\n * given the current state of the supervisor and the availability of jobs when the\n * inventory was snapshot. The entire slice scheduling algorithm is represented by\n * this cursor.\n *\n * The basic idea behind the scheduling of slices in this implementation is to keep as\n * many slices from different jobs running as possible, so as to reduce the likelihood\n * of resource contention between sandboxes.\n *\n * Slices are scheduled based on the following ruleset:\n * 1) cursor = makeJobSelectionCursor(), then cursor.next() returns a slice chosen as follows.\n * 2) Let concurrency range from 1 to maxWorkingCores.\n * 3) For a given concurrency, let readyJobs be all jobs such that jobMan.workingSliceDensity < concurrency.\n * 4) Do an ascending sort of readyJobs wrt jobMan.emaTotalTime.\n * 5) Pick a slice from the longest job in readyJobs that doesn't have any executing slices.\n * 6) Alternately shift a slice from readySlices vs choose a slice from a random nearly finished job, and remove slice from readySlices.\n * 7) When there are no more almost finished jobs with slices, shift slices from readyJobs.\n * 8) Jobs which have slicePriority set by the task-distributor may have slices chosen ahead of the above algorithm.\n * 9) Jobs with a slicePriority closer to 1 are more likely to be chosen.\n * 10) After finishing concurrency at maxWorkingCores, cursor.next() returns null, so create a new cursor.\n *\n * A custom selection of jobs can be passed in via the argument jobManagers.\n *\n * @param {JobManager[]} [jobManagers]\n * @returns {{ next: cbNext, push: cbPush }}\n */\n makeJobSelectionCursor (jobManagers)\n {\n /* Variables in this scope function as state information for next() */\n /** @type {JobManager[]} */\n var candidateJobs; // The jobs available with slices ready to execute.\n /** @type {JobManager[]} */\n var readyJobs; // The jobs from which slices are selected for a given concurrency level.\n /** @type {JobManager[]} */\n var preferedJobs = []; // Those jobs in readyJobs with a slicePreference property.\n /** @type {JobManager[]} */\n var lowDensityJobs = []; // Jobs with density <= 0.6, will be scheduled again.\n /** @type {Slice[]} */\n var pendingSlices = [];\n /**\n * Upper bound of the sum of the working slices densities allowed for a given job.\n * type {number}\n **/\n var concurrency = 0;\n /** type {number} */\n var jobIdx = 0;\n /** @type {boolean} */\n var lowDensityPass = false;\n\n const that = this;\n if (!jobManagers)\n jobManagers = this.jobManagerInventory;\n\n const jobStateStr = (jobs) => {\n return jobs.map((jm) => `${jm.identifier} : ${jm.readySlices.length} : ${jm.workingSliceDensity} : ${Math.round(jm.emaTotalTime)}`);\n }\n const jobState = (hdr, jobs) => { console.debug(hdr, jobStateStr(jobs)); }\n\n /**\n * Populate readyJobs with jobs which are ready and have at least one slice which is ready,\n * and whose # of working slice density is less than concurrency. A reserved slice has a\n * finite lifetime and if exceeded, transition it back to ready.\n * @param {JobManager[]} jobManagers\n * @param {number} concurrency\n */\n function filterJobsAndCheckOldReservedSlices (jobManagers, concurrency) // eslint-disable-line no-shadow\n {\n candidateJobs = [], readyJobs = [];\n const fiveMinutesAgo = Date.now() - that.options.reservedSliceLifetime;\n for (const jobMan of jobManagers)\n {\n if (!jobMan.ready) continue;\n let readyCount = 0;\n for (const slice of jobMan.sliceInventory)\n {\n if (slice.isReady) readyCount++;\n else if (slice.isReserved && fiveMinutesAgo > slice.startTime)\n {\n slice.unReserve();\n readyCount++;\n }\n }\n if (readyCount > 0)\n {\n candidateJobs.push(jobMan);\n if (jobMan.workingSliceDensity < concurrency) readyJobs.push(jobMan);\n }\n }\n }\n\n function seed (concurrency) // eslint-disable-line no-shadow\n {\n /* Reset. */\n jobIdx = 0;\n\n /* Populate readyJobs with jobs which are ready and have at least one slice which is ready,\n and whose # of working slice density is less than concurrency. */\n filterJobsAndCheckOldReservedSlices(jobManagers, concurrency);\n // candidateJobs = jobManagers.filter((jobMan) => jobMan.readySlices.length > 0);\n // readyJobs = candidateJobs.filter((jobMan) => jobMan.workingSliceDensity < concurrency);\n\n if (!lowDensityPass && lowDensityJobs.length === 0)\n lowDensityJobs = jobManagers.filter((jm) => jm.estimateDensity > 0 && jm.estimateDensity <= 0.6);\n\n if (readyJobs.length > 1)\n {\n /* Asc sort by shortest average slice completion time. */\n const shortestSliceJobs = readyJobs.sort((a, b) => Math.round(a.emaTotalTime) - Math.round(b.emaTotalTime));\n const almostDoneIndices = shortestSliceJobs.filter((jm) => jm.almostDone).map((_, idx) => idx);\n readyJobs = [];\n\n /* Find longest job that isn't working. */\n for (let k = shortestSliceJobs.length - 1; k >= 0; k--)\n {\n const jobMan = shortestSliceJobs[k];\n if (jobMan.isNotWorking)\n {\n readyJobs.push(jobMan);\n shortestSliceJobs.splice(k, 1);\n break;\n }\n }\n\n /* Alternate the next shortest slice with a random almost done job. */\n if (almostDoneIndices.length > 0)\n {\n while (shortestSliceJobs.length > 0)\n {\n readyJobs.push(shortestSliceJobs.shift());\n if (almostDoneIndices.length < 1)\n break;\n else\n {\n const almostDoneIdx = almostDoneIndices[Math.floor(Math.random() * almostDoneIndices.length)];\n readyJobs.push(shortestSliceJobs[almostDoneIdx]);\n shortestSliceJobs.splice(almostDoneIdx, 1);\n }\n }\n }\n if (shortestSliceJobs.length > 0)\n readyJobs.push(...shortestSliceJobs);\n }\n /* Populate preferedJobs with jobs from readyJobs which also have a slicePreference set. */\n preferedJobs = candidateJobs.filter((jm) => jm.hasOwnProperty('slicePreference'));\n selectiveDebug2() && jobState(`makeJobSelectionCursor:seed(${concurrency}): readyJobs:`, readyJobs);\n }\n\n /**\n * Each invocation of next() identifies one slice to run, or returns false if none can run.\n * @returns {Slice}\n */\n function next ()\n {\n if (pendingSlices.length > 0)\n {\n const slice = pendingSlices.pop();\n return slice.markAsReserved();\n }\n\n if (concurrency === 0)\n seed(++concurrency);\n\n selectiveDebug2() && console.debug(`makeJobSelectionCursor(cc/idx/ready/working):next(${concurrency},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): maxWorkingCores ${that.maxWorkingCores}: begin`);\n while (true)\n {\n if (jobIdx >= readyJobs.length)\n {\n if (++concurrency > that.maxWorkingCores)\n break;\n seed(concurrency);\n }\n\n if (readyJobs.length < 1)\n {\n if (candidateJobs.length < 1)\n break;\n continue; /* No ready jobs at current concurrency level. */\n }\n\n selectiveDebug2() && console.debug(`makeJobSelectionCursor:next(${concurrency},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): before loop`);\n\n /* Schedule a prefered job slice based on random chance. */\n if (preferedJobs.length > 0)\n {\n let prioRan = Math.random();\n let list = preferedJobs.filter((jm) => jm['slicePreference'] >= prioRan);\n\n if (list.length > 0)\n {\n const jobMan = list[list.length * Math.random()];\n const slice = jobMan.reserveOneSlice();\n if (slice)\n return slice;\n }\n }\n\n /* Schedule a slice from next job; jobs are in increasing order of estimated run time. */\n while (jobIdx < readyJobs.length)\n {\n const jobMan = readyJobs[jobIdx];\n const slice = jobMan.reserveOneSlice();\n if ( false || selectiveDebug2())\n {\n slice && console.debug(`makeJobSelectionCursor:next(${concurrency},${lowDensityPass},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): found slice(slice:ready:working)`, `${slice.identifier} : ${slice.jobManager.readySlices.length} : ${slice.jobManager.workingSliceDensity}`);\n !slice && console.debug(`makeJobSelectionCursor:next(${concurrency},${lowDensityPass},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): no slices ready for job`, jobMan.identifier);\n }\n jobIdx++;\n if (slice)\n return slice;\n }\n\n /*\n * We did not schedule a slice with current seed. We need to re-seed to look for newly-available work\n * and sandboxes, ratcheting up the concurrency (max # of each job running) until we find something.\n */\n selectiveDebug2() && console.debug(`makeJobSelectionCursor:next(${concurrency},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): after loop`);\n }\n if (!lowDensityPass && lowDensityJobs.length > 0)\n {\n jobManagers = lowDensityJobs;\n concurrency = 0;\n lowDensityPass = true;\n return next();\n }\n selectiveDebug2() && console.debug(`makeJobSelectionCursor:next(${concurrency},${jobIdx},${readyJobs.length},${that.workingSliceDensity}): null`, lowDensityPass);\n return null; /* Did not find any more work that fits. */\n }\n function push (slice) { pendingSlices.push(slice); }\n\n return { next, push };\n }\n\n\n /**\n * Handle sandbox.work(...) errors.\n * @todo The orginal code from 2019 did not terminate sandbox when not SandboxError and Sandbox code didn't already terminate. Do we want to try that?\n * The old 2019 sandbox code terminated upon error in start, assign, resetState, describe, applyRequirements and work.\n * So maybe that 2019 terminate yoga was a bunch of hooie.\n * @param {Sandbox} sandbox\n * @param {Slice} slice\n * @param {Error} error\n * @returns {string}\n */\n handleSandboxWorkError (sandbox, slice, error)\n {\n if (debugBuild && !(slice.isWorking || slice.isWorkDone)) // Sanity. Exception should never fire.\n throw new Error(`handleSandboxWorkError: slice ${slice.identifier} must be WORKING.`);\n\n /** @type {boolean} */\n const isSandboxError = error instanceof SandboxError;\n /** @type {string} */\n let reason;\n const jobAddress = common.truncateAddress(slice.jobAddress);\n\n if (isSandboxError)\n reason = error['errorCode']\n else\n {\n // This error was unrelated to the work being done.\n reason = 'Slice has failed to complete execution';\n if (!error)\n error = new Error(`Slice ${slice.sliceNumber} in state ${slice.state} of job ${jobAddress} failed to complete execution`);\n }\n selectiveDebug() && console.debug('handleSandboxWorkError', slice.identifier, error);\n\n let errorString, onlyDisplayErrorString = true;\n if (error.name === 'EWORKREJECT')\n {\n reason = 'EWORKREJECT'; // The status.js processing does not have a case for 'EWORKREJECT'.\n errorString = !slice.hasBeenRejected\n ? `Slice rejected work: ${error.message}.`\n : `Slice rejected work twice; terminate job: ${error.message}.`\n error.stack = 'Sandbox was terminated by work.reject()';\n this.handleWorkReject(slice, error);\n }\n else\n {\n if (!this.evaluator.down)\n {\n /** Do we to be more selective when we retry a slice? */\n if (/*!isSandboxError ||*/ slice['useRetryLogic'])\n {\n slice['sandboxErrorCount'] = ( slice['sandboxErrorCount'] ?? 0) + 1;\n if (slice['sandboxErrorCount'] <= this.options.maxSliceRetries)\n slice.resetState(); // Try to reuse the slice.\n }\n }\n if (!slice.isReady)\n {\n selectiveDebug() && console.debug(`handleSandboxWorkError: returning slice ${slice.identifier}`);\n this.returnSlice(slice, reason)\n .finally (() => {\n this.handleFailedSlice(slice, error)\n });\n }\n\n switch (reason)\n {\n case 'ENOPROGRESS':\n errorString = 'No progress error in sandbox.';\n break;\n case 'ESLICETOOSLOW':\n errorString = 'Slice too slow error in sandbox.';\n break;\n case 'EPERM_ORIGIN':\n errorString = `Could not fetch data; origin not allowed: ${error.message}.`;\n break;\n case 'EFETCH':\n errorString = `Could not fetch data: ${error.message}.`;\n break;\n case 'EUNCAUGHT':\n onlyDisplayErrorString = false;\n errorString = `Uncaught error in sandbox: ${error.message}.`;\n break;\n default:\n onlyDisplayErrorString = false;\n errorString = `Slice failed in sandbox: ${error.message}.`;\n break;\n }\n }\n\n // Always terminate sandbox.\n this.returnSandbox(sandbox);\n\n // Always display max info under debug builds, otherwise maximal error.\n // messages are displayed to the worker, only if both worker and client agree.\n const displayMaxInfo = slice.jobManager.displayMaxDiagInfo;\n\n const errorObject = {\n jobAddress,\n sliceNumber: slice.sliceNumber,\n sandbox: sandbox.id,\n jobName: sandbox.public ? sandbox.public.name : 'unnamed',\n };\n\n if (!displayMaxInfo && onlyDisplayErrorString)\n this.error(errorString, '', '', true);\n else\n {\n Object.entries(errorObject).forEach(([k,v]) => (errorString += `\\n ${k}: ${v}`));\n this.error(errorString, error, '', true);\n }\n\n return reason;\n }\n\n /**\n * Slice has thrown error during execution:\n * Mark slice as failed, compensate when job is dicrete, emit events.\n * @param {Slice} slice\n * @param {Error} error\n */\n handleFailedSlice (slice, error)\n {\n debugging('supervisor') && console.debug(`handleFailedSlice: ${slice.identifier}`, error);\n slice.collectResult(error, false /*success*/);\n\n // If the slice from a job never completes and the job address exists in the ringBufferofJobs,\n // then we remove it to allow for another slice (from the same job) to be obtained by fetchTask\n this.ringBufferofJobs.buf = this.ringBufferofJobs.filter(element => element !== slice.jobAddress);\n\n this.workerEmit( 'result', error);\n this.jobEmit(slice, 'result', error);\n }\n\n // _Idx\n //\n // returnSlices, returnSlice, emitProgressReport\n //\n\n /**\n * Bulk-return multiple slices, possibly for assorted jobs.\n * Returns slices to the scheduler to be redistributed.\n * Called in the sandbox terminate handler and purgeAllWork(jobAddress)\n * and stopWork(forceTerminate).\n *\n * @param {Slice[]} slices - The slice candidates to check if they can be returned to the scheduler.\n * @param {string} reason - Reason for the return: 'ENOPROGRESS', 'EUNCAUGHT', 'ESLICETOOSLOW', 'EPERM_ORIGIN', 'EFETCH', 'unknown'.\n * @returns {Promise<*>} - Response from the scheduler.\n */\n returnSlices (slices, reason)\n {\n /** @param {Slice[]} verifiedSlices */\n const compressPayload = (verifiedSlices) => {\n assert(verifiedSlices?.length > 0);\n if (verifiedSlices.length > 1)\n return {\n worker: this.workerId,\n slices: common.constructReturnSliceBuckets(verifiedSlices, reason),\n };\n return verifiedSlices[0].getReturnMessagePayload(this.workerId, reason);\n }\n\n if (!slices || !slices.length)\n return Promise.resolve();\n\n debugging('supervisor') && console.debug(`Supervisor.returnSlices(${this.state}): Returning slices`, slices.map(slice => slice.identifier));\n\n // Only return those slices which still exist in their respective jobManagers sliceInventory .\n const verifiedSlices = slices.filter((slice) => slice.jobManager.removeSlice(slice));\n if (verifiedSlices.length > 0)\n {\n selectiveSupEx() && console.debug('Supervisor.returnSlices: Returning slices', verifiedSlices.map(slice => slice.identifier));\n return this.dcp4.sliceReturn(compressPayload(verifiedSlices), slices, reason);\n }\n return Promise.resolve();\n }\n\n /**\n * Takes a slice and returns it to the scheduler to be redistributed.\n * Usually called when an exception is thrown by sandbox.work(...) .\n * Or when the supervisor tells it to forcibly stop working.\n *\n * @param {Slice} slice - The slice to return to the scheduler.\n * @param {string} reason - Reason for the return: ''ENOPROGRESS', 'EUNCAUGHT', 'ESLICETOOSLOW', 'EPERM_ORIGIN', 'EFETCH', 'unknown'.\n * @returns {Promise<*>} - Response from the scheduler.\n */\n returnSlice (slice, reason) { return this.returnSlices([ slice ], reason); }\n\n /**\n * Send beacon to status.js for 'progress' and sliceStatus.scheduled.\n *\n * Run in an interval created in the ctor.\n * @returns {void|Promise<*>}\n */\n emitProgressReport ()\n {\n const readySlices = [], workingSlices = [];\n this.jobManagerInventory.forEach((jobManager) => {\n readySlices.push(...jobManager.readySlices);\n workingSlices.push(...jobManager.workingSlices);\n });\n /** @type {SliceObj[]} */\n const slices = common.constructSliceBuckets( readySlices, sliceStatus.scheduled );\n common.constructSliceBuckets( workingSlices, 'progress', slices );\n\n debugging('supervisor') && console.debug('emitProgressReport:', stringify(slices));\n\n if (slices.length > 0)\n {\n const payload = { worker: this.workerId, slices };\n return this.dcp4.safeRSStatus(payload, 'Failed to emit progress report');\n }\n }\n\n // _Idx\n //\n // jobQuanta, repoMan, predictLoad(viz., clairvoyance), waitUntilWorkIsReady, generateWorkerComputeGroups\n //\n\n /**\n * For a given job, the scheduler stores an EMA approximation of slice completion time.\n * However, each worker also tracks the same information and the ratio of local-info to\n * scheduler-info (viz., global-info) is returned by this.jobQuanta so we can tell the\n * task distributor how much work to return from fetchTask so that the work actually takes\n * 5 minutes to complete when using all the worker sandboxes.\n * @returns {Object<string, number>}\n */\n jobQuanta ()\n {\n //\n // Prevent wild swings of this.defaultQuanta, which is roughly the ratio of\n // local_worker_slice_time / jobPerfData_measured_slice_time.\n // We limit this ratio to be between 1/8th and 8.\n const minQuanta = 0.125, maxQuanta = 8.0;\n //\n // Because there will be slgiht differences between local_worker_slice_time and\n // jobPerfData_measured_slice_time even when the worker is the only worker hooked\n // up to the DCP scheduler, we prove a little rounding. The rounding is 1/32 buckets.\n const discreteIncrement = 0.03125, discreteIncrementInverse = 32;\n\n /** @type {Object<string, number>} */\n const quanta = { 0: 1 };\n let averageLocalTime = 0, averageGlobalTime = 0;\n for (const jobMan of this.jobManagerInventory)\n {\n if (jobMan.emaSliceTime > 0 && jobMan.globalTime > 0)\n {\n quanta[jobMan.address] = jobMan.emaSliceTime;\n /** XXXpfr @todo Should we be using TotalTime here? */\n averageLocalTime += jobMan.emaSliceTime;\n averageGlobalTime += jobMan.globalTime;\n selectiveDebug2() && console.debug('jobQuanta: job state', this.dbg.sliceSandboxStr, `l-density/g-density, ${jobMan.estimateDensity}/${jobMan.metrics?.sliceCPUDensity}`, `local/global, ${jobMan.emaSliceTime}/${jobMan.globalTime}`);\n }\n }\n\n if (averageLocalTime && averageGlobalTime)\n {\n /** @todo XXXpfr Add 1 stddev? */\n // alpha=0.1 gives an effective period of 19\n const alpha = 0.1;\n this.localTime = nextEma(this.localTime, averageLocalTime, alpha);\n this.globalTime = nextEma(this.globalTime, averageGlobalTime, alpha);\n this.defaultQuanta = this.localTime / this.globalTime;\n\n // Discretize by discreteIncrement increments.\n this.defaultQuanta = (this.defaultQuanta > 1\n ? Math.floor(discreteIncrementInverse * this.defaultQuanta)\n : Math.ceil(discreteIncrementInverse * this.defaultQuanta)) * discreteIncrement;\n\n // Enforce reasonable cap and floor to keep things from getting too crazy.\n this.defaultQuanta = Math.min(Math.max(this.defaultQuanta, minQuanta), maxQuanta);\n\n // Fake jobAddress '0' to represent unknown jobs.\n quanta['0'] = this.defaultQuanta;\n }\n else\n this.defaultQuanta = 1.0;\n\n selectiveDebug() && console.debug(`jobQuanta: defaultQuanta ${quanta['0']}, this.localTime ${this.localTime}/${averageLocalTime}, this.globalTime ${this.globalTime}/${averageGlobalTime}, quanta:`, quanta);\n if (common.debugQuanta())\n {\n console.debug('localRawData:', this.dbg.localRawData);\n console.debug('localData:', this.dbg.localData);\n console.debug('globalData:', this.dbg.globalData);\n }\n return quanta;\n }\n\n /**\n * @todo XXXpfr Should we not schedule long slices to a worker with too low defaultQuanta?\n *\n * When the estimated time to completion of all work is more than\n * repoManMultiplier * targetTaskDuration * this.maxWorkingCores,\n * return slices until the excess is removed.\n * Be fair. Round-robin over all jobs until excess is eliminated.\n * Kill the long jobs 1st.\n */\n repoMan()\n {\n const threshold = this.options.repoManMultiplier * this.options.targetTaskDuration * this.maxWorkingCores;\n const workRemaining = this.workRemaining;\n let excess = workRemaining - threshold;\n selectiveDebug() && console.debug(`repoMan: excess ${excess}, workerRemaining ${workRemaining}, threshold ${threshold}`);\n if (excess > 0)\n {\n const slices = [];\n /** @param {JobManager[]} jmi */\n const returnFrom = (jmi) => {\n while (true)\n {\n const _excess = excess;\n for (const jobMan of jmi)\n {\n const _readySlices = jobMan.readySlices;\n if (_readySlices.length > 0)\n {\n const slice = _readySlices[0];\n slice.repoMan(); // Mark as FINISHED\n slices.push(slice);\n excess -= jobMan.adjSliceTime;\n if (excess <= 0)\n break;\n }\n }\n if (_excess === excess || excess <= 0)\n break;\n }\n }\n // Be fair. Round-robin over all jobs until excess is eliminated.\n // Except the long jobs are killed 1st.\n const longJobs = this.jobManagerInventory.filter((jobMan) => jobMan.emaSliceTime >= this.options.targetTaskDuration);\n if (longJobs.length > 0)\n returnFrom(longJobs);\n if (excess > 0)\n returnFrom(this.jobManagerInventory);\n selectiveDebug() && (slices.length > 0) && console.debug(`repoMan: excess ${excess}, workerRemaining ${workRemaining}, threshold ${threshold}, returned-slice-count ${slices.length}`);\n this.returnSlices(slices, 'repoMan');\n }\n }\n\n /**\n * Predict the total reduction in density of working sandboxes timeSpanMs from now.\n * This function is called right before fetchTask, in order to calculate how much space is available.\n * @returns {{ queued: Slice[], working: number }}\n */\n predictLoad()\n {\n const timeSpanMs = this.options.prefetchInterval\n const queued = [];\n let working = 0;\n for (const jobMan of this.jobManagerInventory)\n {\n const { queued: jmQueued, working: jmWorking } = jobMan.predictLoad(timeSpanMs);\n queued.push(...jmQueued);\n // Optimize to short-circuit when queued > 1, because we won't call fetchWork in that case.\n if (queued.length > 1)\n break;\n working += jmWorking;\n }\n selectiveDebug() && console.debug(`Supervisor.predictLoad: queued ${queued.length}/${this.queuedSlices.length}, working ${working}/${this.workingSlices.length}`)\n return { queued, working };\n }\n\n /**\n * On the first call to fetchTask\n * or when the last call to fetchTask found nothing,\n * or when there are no ready slices,\n * wait until at least 1 job is ready with at least 1 ready slice.\n * @param {Array<Promise<any>>} jobManagerPromises\n * @returns {Promise<any>}\n */\n waitUntilWorkIsReady (jobManagerPromises)\n {\n if (this.waitForWork)\n {\n debugging('supervisor') && console.debug(`waitUntilWorkIsReady: promise count ${jobManagerPromises?.length}`);\n this.waitForWork = false;\n // Promise.any is supported in Node 15, Chrome 85, Edge 85, Firefox 79, Safari 14, Opera 71.\n // It was implemented in node and browsers in 2nd half of 2020, so there's a good chance many\n // customers will not have browsers that support it. And currently (Jan. 2023) DCP uses node 14.\n return Promise_any(jobManagerPromises);\n }\n // Flush microtask queue\n return a$sleepMs(0);\n }\n\n /**\n * Generate the workerComputeGroups property of the requestTask message.\n *\n * Concatenate the compute groups object from dcpConfig with the list of compute groups\n * from the supervisor, and remove the public group if accidentally present. Finally,\n * we transform joinSecrets/joinHash into joinHashHash for secure transmission.\n *\n * @note computeGroup objects with joinSecrets are mutated to record their hashes. This\n * affects the supervisor options and dcpConfig. Re-adding a joinSecret property\n * to one of these will cause the hash to be recomputed.\n */\n generateWorkerComputeGroups ()\n {\n return supShared.generateWorkerComputeGroups(this, this.dcp4.taskDistributor);\n }\n\n // _Idx\n //\n // availableSandboxSpace, fetchTask, addTaskToWorkload, fetchFromTD, clearUnusedJobManagersAndModuleCache\n //\n\n /**\n * Returns the number of unused sandbox slots to fill -- sent to fetchTask.\n * @param {Slice[]} queued\n * @param {number} working\n * @returns {number}\n */\n availableSandboxSpace (queued, working)\n {\n // If we find more than 1 queued slices, bail early.\n if (queued.length > 1)\n return 0; // We have more than 1 ready slices, no need to fetch.\n\n let longSliceCount = 0;\n if (queued.length < 1)\n this.waitForWork = true; // There are no ready slices.\n else if (queued[0].isLong)\n longSliceCount = 1;\n\n // There are almost no ready slices (there may be 0 or 1), fetch a full task.\n // The task is full, in the sense that it will contain slices whose\n // aggregate execution time is roughly this.maxWorkingCores * 5-minutes.\n // However, there can only be this.maxWorkingCores # of long slices on a worker,\n // Thus we need to know whether the last slice in this.readySlices() is long or not.\n // (A long slice has estimated execution time >= 5-minutes or is an estimation slice.)\n\n const numCores = this.maxWorkingCores - working - longSliceCount;\n selectiveDebug2() && console.debug('availableSandboxSpace', numCores, working, longSliceCount);\n return numCores;\n }\n\n /**\n * Ask the scheduler (task distributor) for work (Rq).\n * @param {object[]} [jobs=[]]\n * @returns {Promise<*>}\n */\n async fetchTask (jobs = [])\n {\n if (!this.isReady)\n return;\n\n const now = Date.now();\n const { queued, working } = this.predictLoad();\n const unusedFutureCoreSpace = this.maxWorkingCores - working;\n if (unusedFutureCoreSpace < common.doNotSchedule)\n {\n debugging('supervisor') && console.debug('fetchTask: There are no unused sandbox slots.', now - this.lastTime);\n return;\n }\n \n // Record fetch start time.\n this.fetchTaskStarted = now;\n\n // We check for pruning about every 25 seconds, or when must prune level is reached.\n if (this.sandboxInventory.length > this.options.mustPruneSandboxLevel\n || now > this.lastPrune + this.options.pruneFrequency)\n {\n this.lastPrune = now;\n this.pruneSandboxes();\n }\n\n // Every 60 seconds check to see if the estimated time to completion of all work is more than\n // repoManMultiplier * this.targetTaskDuration() * this.maxWorkingCores,\n // and then return slices until the excess is removed.\n // Be fair. Round-robin over all jobs until excess is eliminated. Kill the long jobs 1st.\n if (now > this.lastRepoMan + this.options.repoManFrequency)\n {\n this.lastRepoMan = now;\n this.repoMan();\n }\n\n // There are 2 barriers wrt fetchTask,\n // 1) fetchTaskBarrier is a barrier for the task fetching from task distributor path.\n // 2) roundRobinBarrier is a barrier for the slice execution path.\n\n try\n {\n const cpuSpaceToFill = this.availableSandboxSpace(queued, working);\n selectiveDebug2() && console.debug('Supervisor.fetchTask', cpuSpaceToFill, queued.length, working);\n if (cpuSpaceToFill < 1)\n {\n debugging('supervisor') && console.debug('Supervisor.fetchTask: Sufficient slices exist, so start executing.', now - this.lastTime, cpuSpaceToFill, queued.length, working);\n return this.roundRobinSlices();\n }\n selectiveDebug2() && console.debug('fetchTask begin q/w/slots/space/future-space', queued.length, working, this.unusedSandboxCount, this.unusedCoreSpace, unusedFutureCoreSpace);\n\n if (this.fetchTaskBarrier)\n return;\n // fetchTaskBarrier is a barrier for the task fetching from task distributor path.\n this.fetchTaskBarrier = true;\n\n /* @todo XXXpfr Think about how to do targetLoad.longSlices better.\n * Ideas:\n * 1) While branchy Javascript is CPU bound, that doesn't mean hyperthreading isn't useful.\n * When a branch is mispredicted the whole CPU instruction pipeline is flushed, which is\n * a huge perf hit and while waiting to fill the pipeline again, a hyperthread can get\n * a whole bunch of work done.\n * 2) Setting the Sup2.cores.cpu to #lCores is probably too much, but I've had great success with\n * Sup2.cores.cpu = dcpConfig.supervisor.tuning.coreRatio.cpu * #lCores\n * Which is very close to optimal throughput of work done.\n * 3) When the scheduler is 1/2 very long slices, the short slices will tend to get starved.\n * The config property dcpConfig.scheduler.preventSliceStarvation when set to true (default false)\n * will always leave one vCore open for short slices in every worker. In the future, I want the\n * scheduler to detect short slice starvation and dynamical turn preventSliceStarvation on until\n * short slice starvation is alleviated and then turn it back off.\n * 4) maxSandboxes is currently set to\n * factor * Sup2.cores.cpu\n * where 1.2 <= factor <= 1.5 depending upon how many lCores a worker has. Where I assume that a\n * machine with a large number of lCores has sufficient memory to handle a bigger factor.\n * The factor boundaries can be adjusted in dcpConfig.supervisor, but I intend to also allow them\n * to be overridden at the dcpConfig.worker level, so if somebody has a 32 lCore machine with\n * 8GB of RAM they can adjust factor to be closer to 1. Ideally we could adjust the factor\n * boundaries at the job/CG level.\n */\n\n const request = {\n supervisor: this.version,\n numCores: cpuSpaceToFill, /** @deprecated This is for legacy schedulers. */\n numGPUs: this.maxWorkingGPUs, /** @deprecated This is for legacy schedulers. */\n targetLoad: { cpu: cpuSpaceToFill, gpu: this.maxWorkingGPUs, longSlices: Math.floor(cpuSpaceToFill) },\n coreStats: this.options.getStatisticsCPU(),\n jobQuanta: this.jobQuanta(),\n capabilities: this.capabilities,\n paymentAddress: this.options.paymentAddress,\n jobAddresses: jobs.concat(this.options.jobAddresses || []), // When set, only fetches slices for these jobs.\n workerComputeGroups: this.generateWorkerComputeGroups(),\n minimumWage: this.options.minimumWage,\n loadedJobs: this.jobManagerInventory.map(jobMan => jobMan.address),\n readyJobs: this.jobManagerInventory.filter(jobMan => jobMan.ready).map(jobMan => jobMan.address),\n previouslyWorkedJobs: this.ringBufferofJobs.buf, // Only discrete jobs.\n rejectedJobs: this.rejectedJobs,\n };\n // Workers should be part of the public compute group by default.\n if (!booley(this.options.leavePublicGroup))\n request.workerComputeGroups.push(constants.computeGroups.public);\n\n debugging('supervisor') && console.debug('fetchTask is calling fetchFromTD', Date.now() - this.lastTime);\n\n // Call Task Distributor and handle response with this.addTaskToWorkload.\n return this.fetchFromTD(request, (response) => this.addTaskToWorkload(request, response));\n }\n catch (error)\n {\n this.fetchTaskBarrier = false;\n this.error('Supervisor.fetchTask failed!', error);\n }\n }\n\n /**\n * Callback for fetchFromTD.\n * @param {object} request\n * @param {object} response\n */\n async addTaskToWorkload (request, response)\n {\n const constructFetchHandle = (size, jobs, slices) => {\n return { \n fetchStart: this.fetchTaskStarted,\n fetchEnd: Date.now(),\n fetchSize: size,\n jobs,\n slices,\n };\n };\n\n try\n {\n /** @type {TDPayload} */\n const payload = response.payload;\n if (!response.success)\n {\n debugging() && console.debug('Task fetch failure; request=', request);\n debugging() && console.debug('Task fetch failure; response=', payload);\n this.error(`Unable to request task from scheduler; will try again on a new connection: payload ${stringify(payload)}`);\n return;\n }\n\n if (!payload.body?.newJobs) // No slices found.\n {\n // Reset first fetch logic.\n this.waitForWork = true;\n /**\n * The 'fetch' event fires when the stask distributor found no work.\n * @link https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/worker/worker-events.md\n * @event Supervisor#fetch\n */\n this.workerEmit('fetch', constructFetchHandle(0, {}, {}));\n // There may be an extra slice to process.\n debugging('supervisor') && console.debug('Task distributor found no slices...');\n return this.roundRobinSlices();\n }\n\n /** @todo XXXpfr At this poin the line #'s are short by 42 -- figure out why. */\n\n /*\n * payload: { TDPayload }\n * TDPayload: { owner: Address, signature: Signature, auth: Auth, body: Body };\n * Auth: { workerId: string, authSlices: Object<string, SliceMessage[]>, schedulerId: { address: Address }, jobCommissions: Object<string, { rate: number, account: number }> }\n * Body: { newJobs: Object<string, object>, task: Object<string, SliceMessage[]>, computeGroupJobs: Object<string, string[]>, computeGroupOrigins: Object<string, Object<string, string[]>>, schedulerConfig: {{ targetTaskDuration: number }} }\n *\n * NOTE: authorizationMessage has type AuthMessage\n */\n\n const { body, ...authorizationMessage } = payload;\n const { newJobs, task, schedulerConfig } = body;\n const newJobKeys = Object.keys(newJobs);\n const jobCount = newJobKeys.length;\n\n let jobSliceMap = task;\n if (jobSliceMap.length) /** @deprecated Task came from legacy scheduler */\n // @ts-ignore\n jobSliceMap = toJobMap(task, sliceMsg => sliceMsg);\n\n if (schedulerConfig) // Otherwise the default is 300 seconds.\n this.options.targetTaskDuration = schedulerConfig.targetTaskDuration;\n\n /*\n * Ensure all jobs received from the scheduler (task distributor) are:\n * 1. If we have specified specific jobs the worker may work on, the received jobs are in the specified job list\n * 2. If we are in localExec, at most 1 unique job type was received (since localExec workers are designated for only one job)\n * If the received jobs are not within these parameters, stop the worker since the scheduler cannot be trusted at that point.\n */\n if (request.jobAddresses?.length > 0 && !newJobKeys.every((ele) => request.jobAddresses.includes(ele)))\n {\n // \"fetchTask:\" because that should make sense to somebody that doesn't know the internals of Supervisor.\n this.error(\"fetchTask: Worker received slices it shouldn't have; rejecting the work and stopping.\");\n this.stopWork(true);\n return;\n }\n\n // Clear out job managers w/o any queued slices,\n // and remove corresponding job references from module cache.\n // When a cached module no longer has any job references it is removed from the cache.\n this.clearUnusedJobManagersAndModuleCache(newJobs);\n\n /** @todo XXXpfr Figure out how not to construct this every time. */\n this.jobMap = {};\n this.jobManagerInventory.forEach(jobManager => {\n this.jobMap[jobManager.address] = jobManager;\n });\n\n selectiveDebug2() && console.debug(`addTaskToWorkload(${Date.now() - this.lastTime}): newJobs ${common.truncateAddress(newJobKeys)}, jobSliceMap ${common.compressJobMap(jobSliceMap, (s) => s.sliceNumber)}`);\n\n let sliceCount = 0;\n /** @type {Array<Promise<*>>} */\n const jobManagerPromises = [], jobs = {}, slices = {};\n // Populate the job managers with slices, creating new job managers when necessary.\n // Set up discrete job ring buffer.\n for (const [jobAddress, jobMessage] of Object.entries(newJobs))\n {\n /** @type {JobManager} */\n let jobManager;\n const sliceMessages = jobSliceMap[jobAddress];\n sliceCount += sliceMessages.length;\n\n if (this.jobMap.hasOwnProperty(jobAddress))\n {\n jobManager = this.jobMap[jobAddress];\n jobManager.update(jobMessage, sliceMessages, authorizationMessage);\n }\n else\n {\n // Add the slice messages to the job manager ctor, so that slice construction is after job manager is ready.\n jobManager = new JobManager(this, jobMessage, sliceMessages, authorizationMessage);\n this.jobMap[jobAddress] = jobManager;\n //this.jobManagerInventory.push(jobManager); DO NOT PUT IN INVENTORY UNTIL jobManager.ready .\n\n // Populate the ring buffer based on job's discrete property.\n if (jobMessage.requirements.discrete && this.ringBufferofJobs.find(address => address === jobAddress) === undefined)\n this.ringBufferofJobs.push(jobAddress);\n }\n jobs[jobAddress] = jobManager.jobHandle;\n slices[jobAddress] = task[jobAddress].length;\n\n jobManagerPromises.push(jobManager.jobPromise);\n }\n\n const payloadLength = kvin.stringify(payload).length; /** @TODO - fix per DCP-3750 */\n /**\n * The 'fetch' event fires when the supervisor has found work from the task distributor.\n * @link https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/worker/worker-events.md\n * @event Supervisor#fetch\n */\n this.workerEmit('fetch', constructFetchHandle(payloadLength, jobs, slices));\n\n const compressTask = () => { return common.compressJobMap(authorizationMessage.auth.authSlices); }\n selectiveSupEx() && console.debug(`addTaskToWorkload(${Date.now() - this.lastTime}): task: ${sliceCount}/${request.targetLoad.cpu}/${this.maxWorkingCores}, jobs: ${jobCount}, authSlices: ${compressTask()}, conversion:`, request.jobQuanta);\n\n // On the first call to fetchTask,\n // or when the last call to fetchTask found nothing,\n // or when there are no ready slices,\n // wait until at least 1 job with 1 slice is ready.\n await this.waitUntilWorkIsReady(jobManagerPromises);\n\n debugging('supervisor') && console.debug('addTaskToWorkload: Before calling roundRobinSlices; job states', this.jobManagerInventory.map((jm) => jm.identifier));\n\n // Start working on the new slices.\n return dcp_timers.setImmediate(() => this.roundRobinSlices());\n }\n catch (error)\n {\n this.workerEmit('fetch', error);\n this.error('Supervisor.fetchTask failed!', error);\n }\n finally\n {\n this.fetchTaskBarrier = false;\n }\n }\n\n /**\n * @private\n * @callback cbAddTaskToWorkload\n * @param {Response} response\n * @returns {Promise<void>}\n */\n\n /**\n * Call to fetch new slices from task distributor.\n * @param {*} request\n * @param {cbAddTaskToWorkload} addTaskToWorkload\n * @returns {Promise<any>}\n */\n async fetchFromTD (request, addTaskToWorkload)\n {\n selectiveDebug2() && console.debug('fetchFromTD begin; BarrierState:', this.fetchTaskBarrier, this.roundRobinBarrier);\n // Fetch a new task if we have insufficient slices queued, then start workers\n if (!this.fetchTaskBarrier)\n throw new Error('fetchTaskBarrier must be set when entering fetchFromTD.');\n\n this.dcp4.instantiateAllConnections();\n\n let fetchTimeout = dcp_timers.setTimeout(() => {\n this.fetchTaskBarrier = false;\n this.warning('Fetch exceeded timeout, will reconnect at next watchdog interval');\n this.dcp4.resetConnection('taskDistributor').catch(error => {\n this.error('Failed to close task-distributor connection', error);\n });\n this.dcp4.resetConnection('resultSubmitter').catch(error => {\n this.error('Failed to close result-submitter connection', error);\n });\n this.dcp4.instantiateAllConnections();\n }, 3 * 60 * 1000); // Max out at 3 minutes to fetch.\n // Allow workers and localExec to exit.\n fetchTimeout.unref();\n\n const finalize = () => {\n this.fetchTaskBarrier = false;\n if (fetchTimeout)\n dcp_timers.clearTimeout(fetchTimeout);\n fetchTimeout = null;\n }\n\n // Ensure result submitter and task distributor connections before fetching tasks.\n try\n {\n await Promise.all([\n this.dcp4.taskDistributor.keepalive(),\n this.dcp4.resultSubmitter.keepalive(),\n ]);\n }\n catch (error)\n {\n selectiveDebug() && console.debug('fetchTaskFromTD: Keep slices failed', error);\n this.warning('Failed to connect to result submitter, refusing to fetch slices.', 'Will try again at next fetch cycle.');\n this.dcp4.resetConnection('taskDistributor').catch(e => {\n this.error('Failed to close task-distributor connection', e);\n });\n this.dcp4.resetConnection('resultSubmitter').catch(e => {\n this.error('Failed to close result-submitter connection', e);\n });\n return finalize();\n }\n\n if (!this.dcp4.taskDistributor)\n {\n const msg = 'Unable to request task from scheduler; no connection to task distributor';\n this.warning(msg);\n this.workerEmit('fetch', new Error(msg));\n return finalize();\n }\n \n // The 'beforeFetch' event allows the user to cancel the requestTask request.\n let canceled = false;\n /**\n * The 'beforeFetch' event fires before the request is sent to requestTask in task distributor.\n * @link https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/worker/worker-events.md\n * @event Supervisor#beforeFetch\n */\n this.workerEmit('beforeFetch', () => { canceled = true; })\n selectiveDebug() && canceled && console.debug('User canceled the fetch task.');\n if (canceled)\n return finalize()\n\n return this.dcp4.taskDistributor.request('requestTask', request)\n .then((response) => {\n addTaskToWorkload(response);\n // Success! Restore this.dcp4.taskDistributor delay to retryMinSleepMs (currently 32ms.)\n // Is there a better way to reset than explicit calls?\n this.delayManager.resetEBO('taskDistributor');\n return response;\n })\n .catch((error) => {\n this.workerEmit('fetch', error);\n this.error('Unable to request task from scheduler. Will try again on a new connection.', error);\n this.dcp4.resetConnection('taskDistributor');\n })\n .finally(() => {\n return finalize();\n });\n }\n\n /**\n * Remove all unreferenced jobs in this.jobManagerInventory and this.moduleCache.\n * Since job-managers are inserted into this.jobManagerInventory with a push, the job managers at the beginning are oldest.\n * Only delete #deleteCount of the oldest job-managers:\n * let deleteCount = this.jobManagerInventory.length - cachedJobsThreshold;\n * Edit cachedJobsThreshold to adjust the cache cleanup threshold.\n * @param {Object<string, number[]>} newJobMap - Jobs that should not be removed from this.jobManagerInventory and this.moduleCache.\n */\n clearUnusedJobManagersAndModuleCache (newJobMap)\n {\n const emptyJobs = [];\n for (const jobMan of this.jobManagerInventory) // Grab oldest 1st\n {\n if (!newJobMap[jobMan.address])\n {\n let isEmpty = true;\n for (const slice of jobMan.sliceInventory)\n if (slice.isQueuedOrActive)\n {\n isEmpty = false;\n break;\n }\n if (isEmpty)\n {\n // Walk through whole list to purge empty jobs with no assigned sandboxes to save.\n if (jobMan.assignedSandboxes.length < 1)\n this.purgeJob(jobMan);\n else\n emptyJobs.push(jobMan)\n }\n }\n }\n let deleteCount = this.jobManagerInventory.length - this.options.cachedJobsThreshold;\n if (deleteCount > 0)\n {\n selectiveDebug() && console.debug(`Supervisor.clearUnusedJobManagersAndModuleCache: deleteCount ${deleteCount}/${this.jobManagerInventory.length}/${this.options.cachedJobsThreshold}.`);\n for (const jobMan of emptyJobs) // Grab oldest 1st\n {\n this.purgeJob(jobMan);\n if (--deleteCount < 1)\n break;\n }\n }\n }\n\n // _Idx\n //\n // createSandbox, returnSandbox, hookUpSandboxListeners, pruneSandboxes\n //\n\n /**\n * Automatically handle when the evaluator is down.\n *\n * With the screensaver worker, when the screensaver goes down, so does the evaluator.\n * And when the screensaver starts running again, so does the evaluator. The evaluator\n * may be stopped and started again with sa worker running, and have good behavior.\n * However, browser workers cannot have their evaluators stopped without also stopping\n * the worker (otherwise file-a-bug...)\n *\n * @param {boolean} [throwError=false]\n * @returns {Promise<Sandbox>}\n */\n async createSandbox (throwError = false)\n {\n selectiveDebug2() && console.debug('createSandbox', this.sandboxInventory.length, Date.now() - this.lastTime);\n // See if there are any READY_FOR_ASSIGN sandboxes (viz., sandbox.isReadyForAssign is true.)\n // If the evaluator just came back up (while worker is still running) there should not be any non-assigned sandboxes.\n // We're only considering sa worker (e.g. screensaver worker), because browser workers cannot stop the\n // evaluator w/o stopping the worker (I think -- if not true, file-a-bug.)\n if (this.sandboxInventory.length > 0 && this.sandboxInventory[0].isReadyForAssign)\n {\n selectiveDebug2() && console.debug(`Supervisor.createSandbox: Found ready-for-assign sandbox ${this.sandboxInventory[0].identifier}`);\n return this.sandboxInventory.shift();\n }\n\n // If the evaluator cannot start (e.g. if the evalServer is not running),\n // then the while loop will keep retrying until the evalServer comes online.\n try\n {\n this.evaluator.createSandboxRefCount++;\n\n let retry = 0;\n while (true)\n {\n let sandbox;\n try\n {\n sandbox = new Sandbox(this, { ...this.options.sandboxOptions });\n selectiveDebug2() && console.debug(`Supervisor.createSandbox(${sandbox.id}): Calling sandbox.start: ${this.evaluator.createSandboxRefCount}, eval-down ${this.evaluator.down}`);\n this.hookUpSandboxListeners(sandbox);\n await sandbox.start();\n if (!this.capabilities)\n this.checkCapabilities(sandbox);\n if (this.evaluator.reallyDown)\n {\n this.evaluator.reallyDown = false;\n selectiveDebug() && console.debug('Supervisor.createSandbox: Evaluator is up again.', this.evaluator.createSandboxRefCount);\n this.jobManagerInventory.forEach((jobManager) => jobManager.resetSlices('createSandbox'));\n }\n return sandbox;\n }\n catch (error)\n {\n if (throwError)\n throw error;\n selectiveDebug() && console.debug(`Supervisor.createSandbox: Failed to start sandbox ${sandbox.identifier}`, this.evaluator.createSandboxRefCount, this.evaluator.down, error.message);\n if (error.code === 'ENOWORKER')\n throw new DCPError(\"Cannot use localExec without dcp-worker installed. Use the command 'npm install dcp-worker' to install the neccessary modules.\", 'ENOWORKER');\n\n if (throwError)\n throw error;\n\n // The evaluator may be down or shutting down, keep retrying.\n if ((retry % 60) === 0)\n this.warning('Failed to start a sandbox; will keep retrying; screensaver worker or evaluator may be down...');\n await a$sleepMs(1000 * Math.min(5, ++retry));\n }\n }\n }\n finally\n {\n this.evaluator.createSandboxRefCount--;\n }\n }\n\n /**\n * Remove sandbox from inventory and terminate.\n * @param {Sandbox} sandbox\n */\n returnSandbox (sandbox)\n {\n // If sandbox is not in this.sandboxInventory then sandbox must already be terminated\n // <==> this.sandboxInventory.includes(sandbox) || sandbox.isTerminated().\n selectiveDebug2() && console.debug(`returnSandbox: ${sandbox.identifier}`);\n if (common.removeElement(this.sandboxInventory, sandbox))\n sandbox.terminate(false);\n else\n {\n // If sandbox is not in this.sandboxInventory then sandbox must already be terminated\n if (common.displayMaxDiagInfo() && !sandbox.isTerminated) // Design assumption.\n throw new Error(`returnSandbox: Sandbox ${sandbox.identifier} has already been removed.`);\n }\n }\n\n /**\n * For a given sandbox, hook up all the Sandbox listeners.\n * @param {Sandbox} sandbox\n */\n hookUpSandboxListeners (sandbox)\n {\n sandbox.addListener('start', () => {\n if (!sandbox.slice) return;\n const payload = sandbox.slice.getMessagePayload(this.workerId, 'begin');\n return this.dcp4.safeRSStatus(payload, `Failed to send 'begin' status for slice ${sandbox.slice.identifier}`);\n });\n\n const that = this;\n // Sandbox error handler.\n sandbox.on('sandboxError', function Supervisor$sandboxError(error) {\n selectiveDebug() && console.debug(`Sandbox ${sandbox.identifier} sandboxError-handler; error while executing work function`, error);\n const slice = sandbox.slice;\n if (!slice?.isWorking) // Sanity -- warning should never fire.\n this.warning(`handleSandboxError: slice ${slice?.identifier} must be WORKING.`);\n if (slice)\n slice['useRetryLogic'] = true;\n that.returnSandbox(sandbox);\n });\n\n // Sandbox complete handler.\n // When any sandbox completes, go through the Supervisor.fetchTask protocol.\n sandbox.addListener('complete', () => {\n // Try not to call fetchTask unless there's something there.\n selectiveDebug2() && console.debug('Sandbox complete listener', this.fetchTaskBarrier, this.roundRobinBarrier, this.unusedSandboxCount, Date.now() - this.lastTime);\n if (!this.fetchTaskBarrier)\n this.fetchTask();\n });\n\n // If the sandbox terminated and we are not shutting down, then we should return all work which is\n // currently not being computed if all sandboxes are dead and the attempt to create a new one fails.\n sandbox.sandboxHandle.on('end', async () => {\n if (this.sandboxInventory.length > 0 && !this.evaluator.pauseSandboxHandleEndHandler)\n {\n selectiveDebug() && console.debug(`hookUpSandboxListeners: Sandbox \"${sandbox.identifier}\" terminated handler`, this.sandboxInventory.length, Date.now() - this.lastTime);\n\n // Does there exist a non-terminated sandbox?\n let allSandboxesTerminated = true;\n for (const sbx of this.sandboxInventory)\n if (!sbx.isTerminated)\n {\n allSandboxesTerminated = false;\n break;\n }\n\n if (allSandboxesTerminated && !this.evaluator.downInterlock)\n {\n //\n // When we get here, all sandboxes have been terminated.\n //\n this.evaluator.downInterlock = true;\n selectiveDebug() && console.debug('hookUpSandboxListeners: Try to create 1 sandbox in the sandbox-terminated-handler...', sandbox.identifier);\n await this.createSandbox(true /*throwError*/)\n .then((sbx) => {\n this.evaluator.reallyDown = false;\n // This is the only place where non-assigned sandboxes are added to this.sandboxInventory.\n this.sandboxInventory.unshift(sbx);\n selectiveDebug() && console.debug('Sandbox terminate handler was able to create new sandbox', sandbox.identifier);\n })\n .catch(() => {\n //\n // Since all sandboxes have been terminated, if we cannot create a new sandbox,\n // that probably means we're on a screensaver worker and the screensaver is down.\n // Try to submit results for completed slices, but return all other non-finished\n // slices to the scheduler -- after a brief delay.\n //\n selectiveDebug() && console.debug('Sandbox terminate handler cannot create new sandbox; evaluator is down', sandbox.identifier);\n this.evaluator.reallyDown = true;\n this.emit('evalDown');\n const delay = 60; // seconds\n this.jobManagerInventory.forEach((jm) => jm.evaluatorDownCleanup(delay));\n this.warning('Stopping all work.', 'Screensaver worker or evaluator may be down.');\n })\n .finally(() => {\n this.sandboxInventory = this.sandboxInventory.filter(sbx => !sbx.isTerminated);\n this.evaluator.shuttingDown = false;\n this.evaluator.downInterlock = false;\n });\n }\n }\n });\n }\n\n /**\n * Terminate extra sandboxes over the limit.\n * @todo XXXpfr Prioritize keeping expensive to assign sandboxes.\n */\n pruneSandboxes ()\n {\n this.sandboxInventory = this.sandboxInventory.filter((sandbox) => !sandbox.isTerminated);\n let pruneCount = this.sandboxInventory.length - this.options.maxSandboxes;\n if (pruneCount <= 0)\n return;\n\n selectiveDebug() && console.debug(`Supervisor.pruneSandboxes START: pruneCount ${pruneCount}/${this.sandboxInventory.length}/${this.options.maxSandboxes}.`, this.dbg.dumpSandboxState());\n selectiveDebug2() && console.debug(this.sandboxInventory.map((sbx) => sbx.identifier));\n\n // Prune ready-for-assign sandboxes first.\n while (pruneCount > 0)\n {\n if (this.sandboxInventory[0].isReadyForAssign)\n {\n const startedSandbox = this.sandboxInventory.shift();\n startedSandbox.terminate(false);\n pruneCount--;\n }\n else\n break;\n }\n\n // Don't purge jobs here: can accidentally purge a job that TD just fetched (XXXpfr)\n\n /**\n * Do we really want to do a bunch of work to keep empty job assigned sandboxes around?\n * When in a private compute group, there will be fewer jobs and it's likely\n * that a given job will be seen again.\n * @todo XXXpfr Prioritize keeping expensive to assign sandboxes.\n */\n const liveJobs = [], emptyJobs = [];\n let maxAssignedSandboxCount = 0;\n for (const jobMan of this.jobManagerInventory)\n {\n let isAlive = false;\n for (const slice of jobMan.sliceInventory)\n if (slice.isQueuedOrActive)\n {\n isAlive = true;\n break;\n }\n if (isAlive)\n liveJobs.push(jobMan);\n else\n {\n const _assignedSandboxes = jobMan.assignedSandboxes;\n if (maxAssignedSandboxCount < _assignedSandboxes.length)\n maxAssignedSandboxCount = _assignedSandboxes.length;\n emptyJobs.push(jobMan);\n }\n }\n\n if (emptyJobs.length > 0)\n {\n // Prune the sandboxes from all jobs with no current work.\n // Try to keep approximately the same # of assigned sandboxes per job.\n for (let k = maxAssignedSandboxCount; k >= 0; k--)\n {\n for (const jobMan of emptyJobs)\n {\n const _assignedSandboxes = jobMan.assignedSandboxes;\n if (_assignedSandboxes.length > k)\n {\n debugging('supervisor') && console.debug(`pruneSandboxes(empty): sandbox${_assignedSandboxes[0].id}`, Date.now() - this.lastTime);\n // Terminate and remove from this.sandboxInventory.\n this.returnSandbox(_assignedSandboxes[0]);\n if (--pruneCount < 1)\n {\n debugging('supervisor') && console.debug(`Supervisor.pruneSandboxes FINISH: ${pruneCount}/${this.sandboxInventory.length}/${this.options.maxSandboxes}.`);\n return;\n }\n }\n }\n }\n }\n\n // Round-robin prune 1 extra assigned sandbox from each non-empty jobmanager.\n while (pruneCount > 0)\n {\n const _pruneCount = pruneCount;\n for (const jobMan of liveJobs)\n {\n const _assignedSandboxes = jobMan.assignedSandboxes;\n if (_assignedSandboxes.length > 0)\n {\n debugging('supervisor') && console.debug(`pruneSandboxes(non-empty): sandbox${_assignedSandboxes[0].id}`, Date.now() - this.lastTime);\n // Terminate and remove from this.sandboxInventory.\n this.returnSandbox(_assignedSandboxes[0]);\n if (--pruneCount < 1)\n {\n debugging('supervisor') && console.debug(`Supervisor.pruneSandboxes FINISH: ${pruneCount}/${this.sandboxInventory.length}/${this.options.maxSandboxes}.`);\n return;\n }\n }\n }\n if (_pruneCount === pruneCount) // Nothing left to prune.\n break;\n }\n\n debugging('supervisor') && console.debug(`Supervisor.pruneSandboxes FINISH: incomplete-prune ${pruneCount}/${this.sandboxInventory.length}/${this.options.maxSandboxes}.`);\n }\n\n // _Idx\n //\n // recordResult, sendToResultSubmitter, sendResultToRemote\n //\n\n /**\n * Submits the slice results to the result-submitter service.\n * Then remove the slice from the its job manager.\n *\n * @param {Slice} slice - The slice to submit.\n * @param {Sandbox} sandbox - The sandbox handle associated to the slice.\n * @returns {Promise<any>}\n */\n recordResult (slice, sandbox)\n {\n // It is possible for slice.result to be undefined when there are upstream errors.\n if (!slice.result)\n throw new Error(`Slice ${slice.identifier} completed work, but there is no result. This is ok when there are upstream errors.`);\n if (!slice.isComplete)\n throw new Error(`Cannot record result for slice ${slice.identifier} that has not completed execution successfully.`);\n if (!slice.timeReport)\n throw new Error(`Invalid time report for slice ${slice.identifier} in recordResult`);\n if (!slice.dataReport)\n throw new Error(`Invalid data report for slice ${slice.identifier} in recordResult`);\n\n const metrics = slice.jobManager.updateStatistics(slice, sandbox);\n selectiveDebug() && console.debug(`Supervisor: recording result for slice ${slice.identifier} with metrics`, this.dbg.justCPU(metrics));\n\n /** @see result-submitter::result for full message details */\n const payloadData = {\n slice: slice.sliceNumber,\n job: slice.jobAddress,\n worker: this.workerId,\n paymentAddress: this.options.paymentAddress,\n metrics,\n authorizationMessage: slice.authorizationMessage,\n };\n\n let canceled = false;\n const resultUrl = (slice.resultStorageType !== 'pattern') ? slice.resultStorageDetails : false;\n this.workerEmit( 'beforeResult', () => { canceled = true; }, resultUrl);\n this.jobEmit(slice, 'beforeResult', () => { canceled = true; }, resultUrl);\n selectiveDebug && canceled && console.debug(`User canceled the result submission operation for slice ${slice.identifier}.`);\n if (canceled)\n return this.returnSlice(slice, 'Canceled via beforeResult event');\n\n if (slice.resultStorageType === 'pattern')\n return this.sendResultToRemote(slice)\n .then((response) => {\n return this.sendToResultSubmitter(slice, sandbox.sandboxHandle, payloadData, response);\n });\n\n return this.sendToResultSubmitter(slice, sandbox.sandboxHandle, payloadData, encodeDataURI(slice.result.result));\n }\n\n /**\n * Send result to result submitter.\n * @param {Slice} slice\n * @param {SandboxHandle} sandboxHandle\n * @param {*} payloadData\n * @param {string} [result]\n * @returns {Promise<any>}\n */\n async sendToResultSubmitter (slice, sandboxHandle, payloadData, result)\n {\n // When handleRSError is hit, { slice, payload } is added to the queue this.dcp4.submitResultsQueueMap[slice.key] .\n // For a given slice, the queue is retried independent of other slices that failed to submit.\n // When a given slice hits the retry limit (6 retries) the slice is returned to scheduler.\n const handleRSError = (error, slice, payloadData) => { // eslint-disable-line no-shadow\n const msg = `Failed to submit results to scheduler for slice ${slice.identifier}`;\n if (!error) error = new Error(msg);\n this.error(msg, error);\n\n slice['retrySubmitResults'] = (slice['retrySubmitResults'] ?? 0) + 1;\n if (slice['retrySubmitResults'] > this.options.maxResultSubmissionRetries)\n {\n this.handleFailedSlice(slice, error);\n throw new Error(`Failed to submit results 6 times for slice ${slice.identifier}`);\n }\n\n // For a given slice, there's never more than one element in the corresponding queue.\n this.dcp4.submitResultsQueueMap[slice.key] = [ { slice, sandboxHandle, payloadData } ];\n return this.dcp4.resetConnection('resultSubmitter');\n }\n\n try\n {\n debugging('supervisor') && console.debug('Supervisor.recordResult: payloadData', result.slice(0, 256), slice.identifier);\n if (result)\n payloadData.result = result;\n\n await this.delayManager.nextDelay('recordResult', 2);\n //->console.log('recordResult', slice.identifier, this.evaluator.down, Date.now() - this.lastTime); // SAVE\n\n return this.dcp4.resultSubmitter.request('result', payloadData)\n .then((resp) => {\n const payload = resp.payload;\n if (!resp.success)\n {\n if (payload)\n {\n selectiveDebug() && console.debug('resultSubmitter.send failed', payload);\n throw new DCPError(`Call to result submitter failed when recording results for ${slice.identifier}.`, payload);\n }\n if (debugBuild)\n {\n selectiveDebug() && console.debug('resultSubmitter.send failed with no payload', slice.identifier);\n // Look inside\n for (const [ key, value ] of Object.entries(resp)) {\n if (key !== 'connection')\n console.debug(`${key}:`, value);\n }\n }\n throw new Error(`Call to result submitter failed when recording results for ${slice.identifier}.`);\n }\n\n debugging('supervisor') && console.debug('Successfully submitted results', slice.identifier);\n\n // Success! Restore this['resultSubmitter'] delay to retryMinSleepMs (currently 32ms.)\n // Is there a better way to reset than explicit calls?\n this.delayManager.resetEBO('resultSubmitter');\n\n common.debugQuanta() && this.dbg.addGlobal(slice, payload.metrics);\n slice.jobManager.update({ metrics: payload.metrics }); // Update metrics\n\n // Emit the 3 'payment' events.\n const paymentAddress = payloadData.paymentAddress.toString();\n this.workerEmit( 'payment', payload.slicePaymentAmount, paymentAddress, slice.jobAddress, slice.sliceNumber);\n this.jobEmit(slice, 'payment', payload.slicePaymentAmount, paymentAddress, slice.sliceNumber);\n this.safeEmit(sandboxHandle, 'payment', payload.slicePaymentAmount, paymentAddress);\n\n const payloadLength = kvin.stringify(payloadData).length; /** @TODO - fix per DCP-3750 */\n const resultUrl = (slice.resultStorageType !== 'pattern') ? slice.resultStorageDetails : false;\n this.workerEmit( 'result', resultUrl, payloadLength);\n this.jobEmit(slice, 'result', resultUrl, payloadLength);\n\n slice.markAsFinished();\n\n // Remove the slice from the job manager.\n slice.jobManager.removeSlice(slice);\n\n if (this.sliceTiming)\n {\n slice['resultDelta'] = Date.now() - slice['resultDelta'];\n console.debug(`recordResult(${slice['queueingDelta']}, ${slice['executionDelta']}, ${slice['resultDelta']}): Completed slice ${slice.identifier}.`, Date.now() - this.lastTime);\n }\n if (false)\n {}\n\n return resp;\n })\n .catch ((error) => {\n handleRSError (error, slice, payloadData);\n });\n }\n catch (error)\n {\n handleRSError (error, slice, payloadData);\n }\n }\n\n /**\n * Send a work function's result to a server that speaks our DCP Remote Data Server protocol.\n * E.g. https://gitlab.com/Distributed-Compute-Protocol/dcp-rds\n *\n * @param {Slice} slice - Slice object whose result we are sending.\n * @returns {Promise<string>}\n * @throws When HTTP status not in the 2xx range.\n */\n sendResultToRemote (slice)\n {\n return supShared.sendResultToRemote(this, slice);\n }\n\n // _Idx\n //\n // handleWorkReject\n //\n\n /**\n * Handles reassigning or returning a slice that rejected.\n *\n * If error.message === 'false' and slice.hasBeenRejected is false, reschedule the slice.\n * Set the slice.hasBeenRejected to be true.\n *\n * If error.message !== 'false' or slice.hasBeenRejected is true (i.e. has been rejected once already)\n * zthen return all slices from the job to the scheduler and terminate all sandboxes with that jobAddress.\n *\n * @param {Slice} slice\n * @param {Error} error\n */\n handleWorkReject (slice, error)\n {\n debugging() && console.debug('handleWorkReject', error.message, slice.hasBeenRejected, slice.identifier);\n\n const jobManager = slice.jobManager;\n jobManager.rejectedJobReasons.push(error.message); // memoize reasons\n\n // First time rejecting without a reason; try rescheduling the slice.\n if (error.message === 'false' && !slice.hasBeenRejected)\n {\n // Mark slice as rejected.\n slice.hasBeenRejected = true;\n // Reset slice state to allow re-execution.\n slice.resetState();\n }\n else\n {\n // Slice has been rejected twice, so add to array of rejected jobs.\n const rejectedJob = {\n address: slice.jobAddress,\n reasons: jobManager.rejectedJobReasons,\n };\n this.rejectedJobs.push(rejectedJob);\n // Broadcast failure.\n this.workerEmit( 'result', error);\n this.jobEmit(slice, 'result', error);\n // Purge the job.\n this.purgeJob(jobManager);\n // Tell everyone all about it, when allowed.\n if (jobManager.displayMaxDiagInfo)\n {\n const suffixMsg = 'All slices and sandboxes with the same jobAddress returned to the scheduler or terminated.';\n if (slice.hasBeenRejected)\n this.warning(`work.reject: The slice ${slice.identifier} was rejected twice.`, suffixMsg);\n else\n this.warning(`work.reject: The slice ${slice.identifier} was rejected with reason: ${error.message}.`, suffixMsg);\n }\n }\n }\n\n}\nexports.Supervisor = Supervisor;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/supervisor2/index.js?");
4728
4728
 
4729
4729
  /***/ }),
4730
4730
 
@@ -4735,7 +4735,7 @@ eval("/**\n * @file dcp/src/dcp-client/worker/supervisor2/index.js\n *
4735
4735
  /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
4736
4736
 
4737
4737
  "use strict";
4738
- eval("/**\n * @file dcp-client/worker/supervisor2/job-manager.js\n *\n * A support class for Supervisor2.\n * It is a wrapper for the job object returned from the requestTask,\n * along with tracking slices and sandboxes associated with the job. \n *\n * @author Wes Garland, wes@distributive.network,\n * Paul, paul@distributive.network,\n * @date Dec 2020,\n * June 2022, Jan-April 2023\n * @module job-manager\n * @copyright Copyright (c) 2018-2023, Distributive Corp. All Rights Reserved\n */\n// @ts-check\n\n\n/** @typedef {string} opaqueId */ // 22 character base64 string\n/** @typedef {string} address */ // String(Address)\n/** @typedef {import('./sandbox2').Sandbox} Sandbox */\n/** @typedef {import('./index').Supervisor} Supervisor */\n/** @typedef {import('dcp/utils/jsdoc-types').SliceMessage} SliceMessage */\n/** @typedef {import('dcp/utils/jsdoc-types').Auth} Auth */\n/** @typedef {import('dcp/utils/jsdoc-types').Signature} Signature */\n/** @typedef {import('dcp/utils/jsdoc-types').AuthMessage} AuthMessage */\n/** @typedef {import('dcp/dcp-client/worker/origin-access-manager').OriginAccessManager} OriginAccessManager */\n\nconst inspect = Symbol.for('nodejs.util.inspect.custom');\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst EventEmitter = __webpack_require__(/*! events */ \"./node_modules/events/events.js\");\nconst { Inventory } = __webpack_require__(/*! dcp/utils/inventory */ \"./src/utils/inventory.js\");\nconst { Synchronizer } = __webpack_require__(/*! dcp/common/concurrency */ \"./src/common/concurrency.js\");\nconst { rehydrateRange } = __webpack_require__(/*! dcp/dcp-client/range-object */ \"./src/dcp-client/range-object.js\");\nconst { fetchURIAndLength } = __webpack_require__(/*! dcp/utils/fetch-uri */ \"./src/utils/fetch-uri.js\");\nconst utils = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst { stringify, a$sleepMs, constructAlpha, nextEma } = utils;\nconst { allowOriginsPurposes } = __webpack_require__(/*! dcp/common/worker-constants */ \"./src/common/worker-constants.js\");\n\nconst { Slice } = __webpack_require__(/*! ./slice2 */ \"./src/dcp-client/worker/supervisor2/slice2.js\");\nconst { Promise_any } = __webpack_require__(/*! ./promise_any */ \"./src/dcp-client/worker/supervisor2/promise_any.js\");\nconst { Statistics } = __webpack_require__(/*! ./rolling-statistics */ \"./src/dcp-client/worker/supervisor2/rolling-statistics.js\");\nconst { SandboxError, UncaughtExceptionError } = __webpack_require__(/*! ./sandbox2 */ \"./src/dcp-client/worker/supervisor2/sandbox2.js\");\nconst { updateGPUStatistics } = __webpack_require__(/*! ./gpu_support */ \"./src/dcp-client/worker/supervisor2/gpu_support.js\");\nconst common = __webpack_require__(/*! ./common */ \"./src/dcp-client/worker/supervisor2/common.js\");\nconst { selectiveDebug, selectiveDebug2, debugBuild } = common;\n\nconst INITIAL = 'initial';\nconst READY = 'ready';\nconst STOP = 'stop';\nconst REFUSE = 'refuse';\nconst BROKEN = 'broken';\n\nclass BadOriginError extends SandboxError { constructor(msg) { super('EPERM_ORIGIN', msg); } }\nclass RemoteFetchError extends SandboxError { constructor(msg) { super('EFETCH', msg); } }\n\n/**\n * Public event emitter.\n * https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/worker/worker-events.md\n */\nclass JobHandle extends EventEmitter\n{\n /** @type {JobManager} */\n #jobManager;\n \n /**\n * @constructor\n * @param {JobManager} jobManager\n */\n constructor (jobManager)\n {\n super({ captureRejections: false });\n this.#jobManager = jobManager;\n }\n /** @type {string} */\n get address () { return this.#jobManager.address; }\n /** @type {string} */\n get name () { return this.#jobManager.public ? this.#jobManager.public.name : '<unknown>'; }\n /** @type {string} */\n get description () { return this.#jobManager.public ? this.#jobManager.public.description : '<unknown>'; }\n /** @type {string} */\n get link () { return this.#jobManager.public ? this.#jobManager.public.link : '<unknown>'; }\n}\nexports.JobHandle = JobHandle;\n\n//\n// Index to functionality -- search for '_Idx' to toggle through the index.\n//\n// 1) class JobManager\n// 2) addSlices, fetchDependencies, updateStatistics, update, predictLoad\n// 3) reserveOneSlice, resetSlices, destroy, evaluatorDownCleanup\n// 4) assignSandbox, runSlice, runSliceOnSandbox\n// 5) removeSlice\n// 6) fetchError, fetchJob, fetchSliceData\n// 7) jobEmit, sandboxEmit, error, warning. countSliceStr, dumpSlices.\n//\n\n// _Idx\n//\n// class JobManager\n// A JobManager handles all scheduling of slices for a given job.\n// It's also responsible for fetching slice data, work functions and arguments.\n// And it collects statistics about slice completion times and resource usage.\n// All functionality across jobs is handled by Supervisor.\n//\n\n/**\n * JobManager Constructor. An instance of JobManager knows everything about a given\n * job within the supervisor, including:\n * - work function code\n * - work function arguments\n * - all the SliceManager instances that go with this job\n * - how long a slice of this job is expected to run\n *\n * Instances of JobManager emit the following events:\n * - addSlice(sliceHandle)\n * - deleteSlice(sliceHandle)\n * - statusChange(new state, old state);\n *\n * JobManager States\n *\n * Start state:\n * initial\n *\n * Intermediate states:\n * ready\n *\n * Terminal states:\n * broken - job could not be initialized\n * refuse - for some reason, we have decided that we don't want work for this job\n * stop - job manager has been stopped\n *\n * Valid transitions:\n * initial -> broken\n * initial -> ready\n * initial -> ready -> stop\n * \n * NOTE: If you ever use a property with a leading underscore you are probably making a mistake.\n * But if you must, please ask paul, yarn, bryan or eddie for a CR.\n */\nclass JobManager\n{\n /** @type {Supervisor} */\n #supervisor;\n /** @type {Slice[]} */\n #sliceInventory;\n /** @type {Synchronizer} */\n #state;\n /** @type {*} */ // Figure out the type\n #jobMessage;\n /** @type {string} */\n #address;\n /** @type {Statistics} */\n #statistics;\n /** @type {Promise<*>} */\n #jobPromise;\n\n /**\n * @constructor\n * @param {Supervisor} parent - Owning Supervisor.\n * @param {object} jobMessage - Job Descriptor from getJobsForTask.\n * @param {SliceMessage[]} sliceMessages - Messages from task distributor describing slices.\n * @param {AuthMessage} authorizationMessage - The signature that shipped with the task authorizing this worker.\n */\n constructor(parent, jobMessage, sliceMessages, authorizationMessage)\n {\n this.#supervisor = parent;\n this.#sliceInventory = []; // All slices for this.address.\n this.#state = new Synchronizer(INITIAL, [ INITIAL, READY, STOP, REFUSE, BROKEN ]);\n this.#jobMessage = { ...jobMessage };\n this.#address = String(this.jobMessage.address);\n this.#statistics = new Statistics(0.5); // Effective period of 12 or 13\n this.#jobPromise = null;\n\n /** @type {{CPUTime: number, GPUTime: number, TotalTime: number, CPUDensity: number, GPUDensity: number, InDataSize: number, OutDataSize: number }} */\n this.localMetrics = null;\n /** @type {string[]} */\n this.rejectedJobReasons = [];\n /** @type {boolean} */\n this.isEstimation = true;\n /** @type {number} */\n this.inputDataSize = 0;\n /**\n * jobMan.evaluatorDownCleaup returns this.evaluatorDownHandle=a$sleepMs which is\n * interruptible by calling this.evaluatorDownHandle.intr().\n * @type {Promise<void>}\n */\n this.evaluatorDownHandle = null;\n\n /*\n * Make sure this.metrics contains valid data.\n * With old legacy schedulers, this may not be so.\n */\n this.fixUpMetrics();\n\n /** \n * Start loading dependencies in the background. Once these are loaded, this.state will \n * transition to 'ready' and the job will be ready for work.\n */\n this.#jobPromise = this.fetchDependencies(sliceMessages, authorizationMessage);\n\n /* Check for GPU support. */\n const _env = this.requirements?.environment;\n /** @type {boolean} */\n this.useGPU = _env ? (_env.webgpu || _env.offscreenCanvas) : false;\n \n /**\n * Event emitter containing info that describes the job.\n * @type {JobHandle}\n */\n this.jobHandle = new JobHandle(this);\n\n /**\n * Emit the 'job' event on the worker event emitter.\n */\n this.supervisor.safeEmit(this.supervisor.worker, 'job', this.jobHandle);\n }\n\n /** @type {*} */ // Figure out the type\n get jobMessage () { return this.#jobMessage; }\n /** @type {string} */\n get address () { return this.#address; }\n /** @type {string} */\n get uuid () { return this.jobMessage.uuid; }\n /** @type {boolean} */\n get workerConsole () { return this.jobMessage.workerConsole; }\n /** @type {object} */\n get requirements () { return this.jobMessage.requirements; }\n /** @type {boolean} */\n get almostDone () { return this.jobMessage.almostDone; }\n\n // These 3 properties have type object.\n get mro () { return this.jobMessage.mro; }\n get arguments () { return this.jobMessage.arguments; }\n get workFunction () { return this.jobMessage.workFunction; }\n\n /** @type {{computeGroups: Array<{opaqueId: string, name: string, description: string}>, name: string, description: string, link: string}} */\n get public () { return this.jobMessage.public; }\n /** @type {{sliceCPUTime: number, sliceGPUTime: number, sliceCPUDensity: number, sliceGPUDensity: number, sliceInDataSize: number, sliceOutDataSize: number, lastSliceNumber: number, measuredSlices: number, alpha: number}} */\n get metrics () { return this.jobMessage.metrics; }\n /** @type {Supervisor} */\n get supervisor () { return this.#supervisor; }\n /** @type {Synchronizer} */\n get state () { return this.#state; }\n\n /** @type {boolean} */\n get ready () { return this.state.is(READY); }\n\n // These 4 aren't needed yet.\n ///** @type {boolean} */\n //get initial () { return this.state.is(INITIAL); }\n ///** @type {boolean} */\n //get stopped () { return this.state.is(STOP); }\n ///** @type {boolean} */\n //get refuse () { return this.state.is(REFUSE); }\n ///** @type {boolean} */\n //get broken () { return this.state.is(BROKEN); }\n\n /** @type {Sandbox[]} */\n get sandboxInventory () { return this.supervisor.sandboxInventory.filter((sandbox) => sandbox.jobAddress === this.#address && !sandbox.isTerminated); }\n /** @type {Sandbox[]} */\n get assignedSandboxes () { return this.supervisor.sandboxInventory.filter((sandbox) => sandbox.jobAddress === this.#address && sandbox.isAssigned); }\n /** @type {Sandbox[]} */\n get workingSandboxes () { return this.supervisor.sandboxInventory.filter((sandbox) => sandbox.jobAddress === this.#address && sandbox.isWorking); }\n /** @type {Slice[]} */\n get sliceInventory () { return this.#sliceInventory; }\n /** @type {Slice[]} */\n get readySlices () { return this.sliceInventory.filter((slice) => slice.isReady); }\n /** @type {Slice[]} */\n get queuedSlices () { return this.sliceInventory.filter((slice) => slice.isQueued); } // Ready and soon-to-be-ready\n /** @type {Slice[]} */\n get reservedSlices () { return this.sliceInventory.filter((slice) => slice.isReserved); }\n /** @type {Slice[]} */\n get activeSlices () { return this.sliceInventory.filter((slice) => slice.isActive); } // Reserved, working, workdone and complete\n /** @type {Slice[]} */\n get workingSlices () { return this.sliceInventory.filter((slice) => slice.isWorking || slice.isReserved); } // Working and soon-to-be-working\n /** @type {Slice[]} */\n get unFinishedSlices () { return this.sliceInventory.filter((slice) => !slice.isFinished); }\n\n /** @type {Statistics} */\n get statistics () { return this.#statistics; }\n /** @type {Promise<*>} */\n get jobPromise () { return this.#jobPromise; }\n /** @type {number} */\n get globalTime () { return utils.globalTime(this.metrics); }\n /** @type {number} */\n get emaSliceTime () { return this.localMetrics ? utils.localTime(this.localMetrics) : (utils.globalTime(this.metrics) * this.supervisor.defaultQuanta); }\n /** @type {number} */\n get emaTotalTime () { return this.localMetrics?.TotalTime || (utils.globalTotalTime(this.metrics) * this.supervisor.defaultQuanta); }\n /** @type {number} */\n get adjSliceTime () { return Math.min(this.emaSliceTime, this.supervisor.options.targetTaskDuration); }\n /** @type {string} */\n get identifier () { return `${common.truncateAddress(this.address)}.${this.state}`; }\n /** @type {string} */\n get sliceState () { return `${this.queuedSlices.length}.${this.workingSliceCount}.${this.sliceInventory.length}.${this.workRemaining}`; }\n /** @type {string} */\n get [inspect] () { return `[Object JobManager <${this.public.name}::${common.truncateAddress(this.address)}::${this.state}>]`; }\n /** @type {OriginAccessManager} */\n get originManager() { return this.supervisor.originManager; }\n /**\n * Expected density of a slice.\n * When we get the GPU stuff codified, the definition might be\n * { cpu: this.localMetrics?.CPUDensity ?? 1.0, gpu: this.localMetrics?.GPUDensity ?? 1.0}\n * @type {number}\n */\n get estimateDensity () { return this.localMetrics?.CPUDensity || utils.globalDensity(this.metrics); }\n /** @type {number} */\n get estimateGPUDensity () { return this.localMetrics?.GPUDensity || utils.globalGPUDensity(this.metrics); }\n /** @type {number} */\n get cpuTime () { return this.localMetrics?.CPUTime || this.metrics?.sliceCPUTime || 0; }\n /**\n * Estimate the number of milliseconds of total execution time of the job.\n * @type {number}\n */\n get workRemaining ()\n {\n const _adjSliceTime = this.adjSliceTime;\n const queuedTime = _adjSliceTime * this.queuedSlices.length;\n let workingTime = 0;\n for (const slice of this.workingSlices)\n {\n workingTime += Math.max(0, _adjSliceTime - (Date.now() - slice.startTime));\n selectiveDebug2() && console.debug(`JobManager.workRemaining: _adjSliceTime ${_adjSliceTime}, slice-time-elapsed ${Date.now() - slice.startTime}`);\n }\n selectiveDebug() && console.debug(`JobManager.workRemaining: queuedTime ${queuedTime}, queuedCount ${this.queuedSlices.length}, workingTime ${workingTime}, workingCount ${this.workingSliceCount}`);\n return workingTime + queuedTime;\n }\n /**\n * Are there no working slices?\n * @type {boolean}\n */\n get isNotWorking ()\n {\n for (const slice of this.sliceInventory)\n if (slice.isReserved || slice.isWorking)\n return false;\n return true;\n }\n /**\n * Estimated count of working (or reserved) slices.\n * @type {number}\n */\n get workingSliceCount ()\n {\n let count = 0;\n for (const slice of this.sliceInventory)\n if (slice.isReserved || slice.isWorking)\n ++count;\n return count;\n }\n /**\n * Estimated density of working (or reserved) slices.\n * @type {number}\n */\n get workingSliceDensity () { return this.workingSliceCount * this.estimateDensity; }\n /**\n * Estimated density of working (or reserved) slices.\n * @type {number}\n */\n get workingGPUDensity () { return this.workingSliceCount * this.estimateGPUDensity; }\n /**\n * Always display max info under debug builds, otherwise maximal error\n * messages are displayed to the worker, only if both worker and client agree.\n * When displayMaxDiagInfo is true, display stack trace and other enhanced diag info.\n * @type {boolean}\n **/\n get displayMaxDiagInfo ()\n {\n return common.displayMaxDiagInfo() || this.workerConsole && this.supervisor.options.allowConsoleAccess;\n }\n\n // _Idx\n //\n // addSlices, fetchDependencies, updateStatistics, update, predictLoad\n //\n\n /**\n * Add slices to the job manager's inventory.\n *\n * @param {SliceMessage[]} sliceMessages - Messages from task distributor describing slices.\n * @param {AuthMessage} authorizationMessage - The signature that shipped with the task authorizing this worker.\n * @returns {Promise<*>}\n */\n addSlices (sliceMessages, authorizationMessage)\n {\n /** @type {Array<Promise<void>>} */\n const slicePromises = [];\n sliceMessages.forEach((sliceMessage) => {\n const slice = new Slice(this, sliceMessage, authorizationMessage);\n if (!slice.isEstimation)\n this.isEstimation = false;\n this.sliceInventory.push(slice);\n slicePromises.push(slice.fetchSliceDataPromise);\n });\n return Promise_any(slicePromises);\n }\n\n /**\n * Fetch dependencies.\n * @param {SliceMessage[]} sliceMessages - Messages from task distributor describing slices.\n * @param {AuthMessage} authorizationMessage - The signature that shipped with the task authorizing this worker.\n * @returns {Promise<*>}\n */\n fetchDependencies (sliceMessages, authorizationMessage)\n {\n return this.fetchJob(this.jobMessage)\n .then (() => {\n debugging('supervisor') && console.debug('JobManager is transitioning to READY', this.identifier, Date.now() - this.supervisor.lastTime);\n this.state.set(INITIAL, READY);\n return this.addSlices(sliceMessages, authorizationMessage);\n }, (error) => {\n selectiveDebug() && console.error('fetchJob has failed', error);\n this.state.set(INITIAL, BROKEN);\n throw error;\n });\n }\n\n /** XXXpfr @todo Factor out statistics and EMA into another class. */\n /**\n * @param {number} x\n * @param {boolean} isDensity\n * @returns {boolean}\n */\n static isValid (x, isDensity = false)\n {\n return Number.isFinite(x) && !(x < 0) && !(isDensity && x > 1);\n }\n\n /**\n * Old legacy DCP schedulers can send a corrupt this.metrics .\n * We should get rid of this when we don't support old legacy DCP schedulers.\n */\n fixUpMetrics ()\n {\n if (!this.metrics)\n this.jobMessage.metrics = { sliceCPUTime: 0, sliceCPUDensity: 1, sliceGPUTime: 0, sliceGPUDensity: 0, sliceInDataSize: 0, sliceOutDataSize: 0 };\n else\n {\n if (!(JobManager.isValid(this.metrics.sliceCPUTime) && JobManager.isValid(this.metrics.sliceCPUDensity, true)))\n {\n this.metrics.sliceCPUTime = 1;\n this.metrics.sliceCPUDensity = 0.5;\n }\n if (!(JobManager.isValid(this.metrics.sliceGPUTime) && JobManager.isValid(this.metrics.sliceGPUDensity, true)))\n {\n this.metrics.sliceGPUTime = 1;\n this.metrics.sliceGPUDensity = 0.5;\n }\n if (!JobManager.isValid(this.metrics.sliceInDataSize))\n this.metrics.sliceInDataSize = 0;\n if (!JobManager.isValid(this.metrics.sliceOutDataSize))\n this.metrics.sliceOutDataSize = 0;\n }\n }\n\n /**\n * @param {Slice} slice\n * @param {Sandbox} sandbox\n * @returns {{CPUTime: number, GPUTime: number, CPUDensity: number, GPUDensity: number, InDataSize: number, OutDataSize: number}}\n */\n updateStatistics (slice, sandbox)\n {\n /**\n * @todo XXXpfr Make this only check when debugBuild, but not until the Sup2 churn has stopped.\n * @param {{CPUTime: number, GPUTime: number, CPUDensity: number, GPUDensity: number, InDataSize: number, OutDataSize: number}} metrics\n * @returns {boolean}\n */\n const isValidMetrics = (metrics) => {\n return JobManager.isValid(metrics.CPUTime) && JobManager.isValid(metrics.CPUDensity, true)\n && JobManager.isValid(metrics.GPUTime) && JobManager.isValid(metrics.GPUDensity, true)\n && JobManager.isValid(metrics.InDataSize) && JobManager.isValid(metrics.OutDataSize);\n }\n\n const timeReport = slice.timeReport;\n const dataReport = slice.dataReport;\n selectiveDebug() && console.debug('updateStatistics', slice.identifier, timeReport, dataReport);\n\n // Sanity\n if (!(timeReport.CPU >= 1))\n {\n timeReport.CPU = 1;\n timeReport.total += 1;\n }\n if (!(timeReport.total >= 1))\n timeReport.total = 1;\n\n // Construct metrics.\n const GPUTime = timeReport.webGPU + timeReport.webGL;\n if (this.useGPU)\n updateGPUStatistics(GPUTime);\n const metrics = {\n CPUTime: timeReport.CPU,\n GPUTime,\n CPUDensity: timeReport.CPU / timeReport.total,\n GPUDensity: GPUTime / timeReport.total,\n InDataSize: dataReport.InDataSize,\n OutDataSize: dataReport.OutDataSize,\n };\n //console.debug('JM.updateStatistics metrics:', metrics, timeReport.total);\n\n // Create the measurement object which will be passed to the metrics event.\n const eventMeasurements = {\n elapsed: timeReport.total / 1000,\n CPU: metrics.CPUTime / 1000,\n GPU: metrics.GPUTime / 1000,\n in: metrics.InDataSize,\n out: metrics.OutDataSize,\n };\n\n // Emit metrics event for both jobHandle and sandboxHandle\n this.jobEmit( 'metrics', slice.sliceNumber, eventMeasurements);\n this.sandboxEmit(sandbox, 'metrics', slice.sliceNumber, eventMeasurements);\n selectiveDebug() && console.debug(`updateStatistics: slice ${slice.identifier}, eventMeasurements ${stringify(eventMeasurements)}`);\n\n if (!isValidMetrics(metrics))\n throw new Error(`JobManager.updateStatistics: metrics are in an inconsistent state ${JSON.stringify(metrics)}`);\n\n if (!this.localMetrics)\n this.localMetrics = { CPUTime: 0, GPUTime: 0, TotalTime: 0, CPUDensity: 0, GPUDensity: 0, InDataSize: 0, OutDataSize: 0 };\n\n // measuredSlices+1 because the current slice hasn't been measured yet in RS.updateMetrics.\n const alpha = this.metrics?.lastSliceNumber\n ? constructAlpha(this.metrics.lastSliceNumber, this.metrics.measuredSlices + 1)\n : 0.5;\n selectiveDebug2() && console.debug(`updateStatistics: local alpha ${alpha}, global alpha ${this.metrics?.alpha}`);\n this.localMetrics.CPUTime = nextEma(this.localMetrics.CPUTime, metrics.CPUTime, alpha);\n this.localMetrics.GPUTime = nextEma(this.localMetrics.GPUTime, metrics.GPUTime, alpha);\n this.localMetrics.TotalTime = nextEma(this.localMetrics.TotalTime, timeReport.total, alpha);\n this.localMetrics.CPUDensity = nextEma(this.localMetrics.CPUDensity, metrics.CPUDensity, alpha);\n this.localMetrics.GPUDensity = nextEma(this.localMetrics.GPUDensity, metrics.GPUDensity, alpha);\n this.localMetrics.InDataSize = nextEma(this.localMetrics.InDataSize, metrics.InDataSize, alpha); // Don't need this yet.\n this.localMetrics.OutDataSize = nextEma(this.localMetrics.OutDataSize, metrics.OutDataSize, alpha); // Don't need this yet.\n\n if (common.debugQuanta())\n {\n this.supervisor.dbg.addRawLocal(slice, metrics, alpha);\n this.supervisor.dbg.addLocal(slice, this.emaSliceTime, alpha);\n }\n\n if (!isValidMetrics(this.localMetrics))\n throw new Error(`JobManager.updateStatistics: localMetrics are in an inconsistent state ${JSON.stringify(this.localMetrics)}`);\n\n debugging('supervisor') && console.debug('updateStatistics: ema', this.statistics.ema, 'emaSliceTime', this.emaSliceTime, 'estimateDensity', this.estimateDensity, 'metrics', metrics, 'g-metrics', this.metrics, 'alpha', alpha);\n //if (!Number.isFinite(this.statistics.ma))\n // console.log(this.statistics.x);\n return metrics;\n }\n\n /**\n * Update jobMessage, add some slices to inventory and possibly update the initial seed of the statistics.\n * @param {object} jobMessage - Job Descriptor from getJobsForTask.\n * @param {SliceMessage[]} [sliceMessages] - Messages from task distributor describing slices.\n * @param {AuthMessage} [authorizationMessage] - The signature that shipped with the task authorizing this worker.\n * @returns {Promise<*>}\n */\n update (jobMessage, sliceMessages, authorizationMessage)\n {\n Object.assign(this.jobMessage, jobMessage);\n /*\n * Make sure this.metrics contains valid data.\n * With old legacy schedulers, this may not be so.\n */\n this.fixUpMetrics();\n selectiveDebug2() && console.debug('JM.update: new this.jobMessage', this.jobMessage.metrics);\n\n if (sliceMessages)\n return (this.#jobPromise = this.addSlices(sliceMessages, authorizationMessage));\n }\n\n /**\n * Predict the total reduction in density of working sandboxes timeSpanMs from now.\n * This function is called right before fetchTask, in order to calculate how much space is available.\n * In pratice, the returned property queued will have length <= 1, because when it is more we won't call fetchWork.\n * @param {number} timeSpanMs\n * @returns {{ queued: Slice[], working: number }}\n */\n predictLoad (timeSpanMs)\n {\n let queuedCount = this.queuedSlices.length;\n // Optimize to short-circuit when queued > 1, because we won't call fetchWork in that case.\n if (queuedCount > this.workingSliceCount + 1)\n {\n selectiveDebug() && console.debug('JM.predictLoad: bail early', queuedCount, this.workingSliceCount);\n return { queued: [null, null], working: this.workingSliceDensity };\n }\n let space = 0;\n // 1) Predict the current working slices that will finish in timeSpanMs.\n // 2) Compensate for queued slices that may begin working.\n for (const slice of this.sliceInventory)\n {\n if (!slice.isReserved && !slice.isWorking)\n continue;\n const remainingTime = Math.max(0, this.emaSliceTime - (Date.now() - slice.startTime));\n selectiveDebug2() && console.debug('JM.predictLoad: predicting...', queuedCount, space, this.emaSliceTime, remainingTime, this.emaSliceTime <= timeSpanMs - remainingTime);\n // If slice will end before timeSpanMs elapses, it may open up space.\n if (remainingTime <= timeSpanMs)\n {\n // Check to see whether a new queued slices can start running and finish in the remaining time.\n // Somewhat inaccurate because it's possible multiple slices could finish in the remaining time.\n // e.g. Suppose timeSpanMs = 5 * 1000 and this.emaSliceTime is 1000, then there's room\n // to successively schedule 5 slices, one after another.\n /** XXXpfr @todo Fix to incorporate successive scheduling. */\n if (queuedCount > 0)\n {\n --queuedCount;\n // If the new slice finishes in time, count it as opening up space.\n if (this.emaSliceTime <= timeSpanMs - remainingTime)\n ++space;\n }\n else // No queued slice can run so space has opened up.\n ++space;\n }\n }\n // Optimize to short-circuit when queued > 1, because we won't call fetchWork in that case.\n let queued = [null, null];\n if (queuedCount < 2)\n queued = (queuedCount === 1) ? [this.queuedSlices[0]] : [];\n return { queued, working: (this.workingSliceCount - space) * this.estimateDensity };\n }\n\n // _Idx\n //\n // reserveOneSlice, resetSlices, destroy, evaluatorDownCleanup\n //\n\n /**\n * Find one ready slice, mark it as working and return.\n * If there are no ready slices return undefined.\n * @returns {Slice}\n */\n reserveOneSlice ()\n {\n for (const slice of this.sliceInventory)\n if (slice.isReady)\n return slice.markAsReserved();\n return null;\n }\n\n /**\n * Error recovery of slices.\n * When evaluator is down and it goes up again, there may be\n * working slices from working sandboxes that were terminated.\n * @param {string} [tag]\n */\n resetSlices (tag)\n {\n this.sliceInventory.forEach((slice) => {\n if (this.evaluatorDownHandle)\n this.evaluatorDownHandle.intr(); // Interrupt a$sleep in evaluatorDownCleanup\n this.evaluatorDownHandle = null;\n if (slice.isWorking || slice.isFailed)\n slice.resetState();\n });\n //this.dumpSlices(tag); // SAVE\n }\n\n /**\n * Destructor.\n * @returns {Promise<*>}\n */\n destroy ()\n {\n selectiveDebug() && console.debug(`JobManager.destroy: terminating sandboxes and returning slices to scheduler for job manager ${this.identifier}.`);\n this.sandboxInventory.forEach((sandbox) => { this.supervisor.returnSandbox(sandbox); });\n const reason = 'JobManager destroy';\n return this.supervisor.returnSlices(this.unFinishedSlices, reason)\n .finally (() => {\n this.#sliceInventory = [];\n this.state.removeAllListeners();\n if (this.state.isNot(BROKEN))\n this.state.set([ INITIAL, READY, STOP, REFUSE ], BROKEN);\n this.jobEmit('flush');\n });\n }\n\n /**\n * Destructor.\n * Called when the evaluator goes down (or screensaver goes up), in the sandbox 'terminated' event handler in Supervisor.\n * A slice marked as COMPLETE but not FINISHED means it still needs to submit results.\n * This function allows such slices to still submit results.\n * Because sometimes a screensaver only goes down for a minute or less (e.g. the user needs to look at something quick),\n * we delay an average of 90 (or 1.5 * delay) seconds before returning non-complete slices.\n * param {number} [delay=60] - Average delay is 1.5 * delay seconds.\n */\n evaluatorDownCleanup (delay = 60)\n {\n selectiveDebug() && console.debug(`JobManager.evaluatorDownCleanup: delay returning non-complete slices for on average ${1.5 * delay} seconds, ${this.identifier}.`);\n if (this.sandboxInventory.length > 0)\n this.warning('JobManager.evaluatorDownCleanup: When the evaluator is down there should not be any non-terminated sandboxes left.');\n // Call retVal.intr() to interrupt and cancel the timer inside a$sleepMs.\n return (this.evaluatorDownHandle = a$sleepMs(delay * (1.0 + Math.random()) * 1000)\n .finally(() => {\n const slicesToReturn = this.sliceInventory.filter((slice) => !(slice.isComplete || slice.isFinished));\n const reason = 'JobManager evaluatorDownCleanup';\n return this.supervisor.returnSlices(slicesToReturn, reason);\n }));\n }\n\n // _Idx\n //\n // assignSandbox, runSlice, runSliceOnSandbox\n // Functions chain and return non-awaited promises.\n //\n\n /**\n * Create a Sandbox.\n * Start it. Assign it. Add it to jobManager inventory.\n * @param {Slice} slice\n * @returns {Promise<Sandbox>}\n */\n async assignSandbox (slice)\n {\n try\n {\n // We want to await createSandbox so we don't create a large number of sandboxes all at once.\n const sandbox = await this.supervisor.createSandbox()\n .catch((error) => {\n // Catches exception from createSandbox.\n // Shound be rare unless evaluator or screensaver goes down.\n selectiveDebug2() && !this.supervisor.evaluator.down && console.debug('assignSandbox: createSandbox error', slice.identifier, error);\n slice.unReserve(); // reuse slice\n throw error;\n });\n selectiveDebug2() && console.debug('assignSandbox', slice.identifier);\n sandbox.slice = slice; // Must be before sandbox.assign; cf. fetchModule exception in handleRing2Message.\n return sandbox.assign(this)\n .then((assignedSandbox) => {\n selectiveDebug2() && console.debug('assignSandbox: success', sandbox.identifier);\n this.supervisor.sandboxInventory.push(assignedSandbox);\n return assignedSandbox;\n })\n .catch((error) => {\n // Exception is from sandbox.assign.\n // It could be a string coming from evaluator.\n this.error(`Failed to assign job to sandbox ${sandbox.identifier}`, error);\n slice.unReserve(); // reuse slice\n throw error;\n });\n }\n catch (error)\n {\n this.error(`JobManager.assignSandbox failed to find sandbox to run slice ${slice.identifier}`, error);\n throw error;\n }\n }\n\n /**\n * Create or reuse a sandbox and run a slice on it.\n * @param {Slice} slice - The slice to execute on a sandbox.\n * @returns {Promise<any>}\n */\n runSlice (slice)\n {\n try\n {\n selectiveDebug() && console.debug('runSlice:', slice.identifier, 'assigned/working', this.assignedSandboxes.length, this.workingSandboxes.length);\n\n if (this.supervisor.sliceTiming)\n slice['queueingDelta'] = Date.now();\n\n const _assignedSandboxes = this.assignedSandboxes;\n if (_assignedSandboxes.length > 0)\n {\n const sandbox = _assignedSandboxes[0];\n selectiveDebug2() && console.debug(`runSlice(assigned): sandbox ${sandbox.identifier}`, Date.now() - this.supervisor.lastTime);\n // When evaluator goes down, all sandboxes are terminated.\n // A ready slice has a finite lifetime and if exceeded may transition back to READY, in which\n // case we bail since it might already be in the process of being selected in roundRobinSlices.\n if (slice.isReady || sandbox.isTerminated)\n return Promise.resolve();\n slice.markAsWorking();\n sandbox.markAsWorking();\n sandbox.slice = slice; // Reusing assigned sandbox, so set the slice.\n return this.runSliceOnSandbox(slice, sandbox)\n .catch((error) => {\n // Catches exception from runSliceOnSandbox; s.b. rare.\n this.error(`JobManager.runSlice: Failure trying to run slice ${slice.identifier} on a assigned sandbox.`, error);\n return this.supervisor.returnSlice(slice, 'JobManager.runSlice');\n });\n }\n\n return this.assignSandbox(slice)\n .then((sandbox) => {\n selectiveDebug2() && console.debug(`runSlice: sandbox ${sandbox.identifier}`, Date.now() - this.supervisor.lastTime);\n // When evaluator goes down, all sandboxes are terminated.\n // A ready slice has a finite lifetime and if exceeded may transition back to READY, in which\n // case we bail since it might already be in the process of being selected in roundRobinSlices.\n if (slice.isReady || sandbox.isTerminated)\n return Promise.resolve();\n slice.markAsWorking();\n sandbox.markAsWorking();\n return this.runSliceOnSandbox(slice, sandbox);\n }, (error) => {\n // Catches exception from assignSandbox for error control flow.\n selectiveDebug() && console.error(`JobManager.runSlice: Failure from assignSandbox when trying to run slice ${slice.identifier} on a sandbox.`, error);\n })\n .catch((error) => {\n // Catches exception from runSliceOnSandbox; s.b. rare.\n this.error(`JobManager.runSlice: Failure trying to run slice ${slice.identifier} on a sandbox.`, error);\n return this.supervisor.returnSlice(slice, 'JobManager.runSlice');\n });\n }\n catch (error)\n {\n this.error(`JobManager.runSlice failed to run slice ${slice.identifier}`, error);\n return this.supervisor.returnSlice(slice, 'JobManager.runSlice_static_catch');\n }\n }\n\n /**\n * Execute slice on sandbox, collect results or handle errors, and clean up.\n * @param {Slice} slice - The slice to execute on the sandbox.\n * @param {Sandbox} sandbox - The sandbox on which to execute the slice.\n * @returns {Promise<any>}\n */\n runSliceOnSandbox (slice, sandbox)\n {\n const handleError = (error, tag) => {\n if (sandbox.isTerminated)\n {\n this.error(`The screensaver or evaluator may be down: slice ${slice.identifier}`, error);\n return this.supervisor.returnSlice(slice, `${tag}: sandbox terminated`);\n }\n else\n {\n this.error(`Failure in ${tag} for slice ${slice.identifier}`, error);\n const reason = (error.code === 'EPERM_ORIGIN' || error.code === 'EFETCH_BAD_ORIGIN' || error.code === 'EFETCH') ? error.code : tag;\n this.supervisor.returnSlice(slice, reason);\n }\n };\n debugging('supervisor') && console.debug(`runSliceOnSandbox ${sandbox.identifier} #(r/rsv/w/wsbx/sbx), ${this.supervisor.dbg.workingSliceSandboxStr}, #assigned, ${this.assignedSandboxes.length}, #localSBs, ${this.sandboxInventory.map(s => Number(s.id)).sort((x,y)=>x-y)}`);\n selectiveDebug() && console.debug('runSliceOnSandbox', sandbox.identifier, this.workingSliceCount, Date.now() - this.supervisor.lastTime);\n\n if (sandbox.isTerminated) // When evaluator goes down, all sandboxes are terminated.\n return Promise.resolve();\n\n this.sandboxEmit(sandbox, 'slice', slice.sliceNumber);\n\n if (this.supervisor.sliceTiming)\n {\n slice['queueingDelta'] = Date.now() - slice['queueingDelta'];\n slice['executionDelta'] = Date.now();\n }\n slice.startTime = Date.now();\n\n return sandbox.work()\n .finally(() => {\n if (this.supervisor.sliceTiming)\n {\n slice['executionDelta'] = Date.now() - slice['executionDelta'];\n slice['resultDelta'] = Date.now();\n }\n })\n .then((result) => {\n selectiveDebug2() && console.debug(`runSliceOnSandbox: sandbox${sandbox.id} - success: ${slice.identifier}`, Boolean(result), Date.now() - slice.startTime);\n\n if (sandbox.isTerminated)\n throw new Error(`Slice ${slice.identifier} completed work, but the sandbox ${sandbox.id} is terminated.`);\n\n // Allow reuse of sandbox up to a limit of dcpConfig.supervisor.sandbox.maxSandboxUse .\n sandbox.checkSandboxReUse();\n // Transition slice to COMPLETE\n slice.collectResult(result, true /*success*/);\n\n // Record result.\n return this.supervisor.recordResult(slice, sandbox)\n .catch((error) => {\n // Catches exception from supervisor.recordResult .\n return handleError(error, 'recordResult');\n });\n\n }, (error) => {\n // Catches exception from sandbox.work.\n // Kills sandbox and either retries the slice or transitions slice to FAILED and returns to scheduler.\n const reason = this.supervisor.handleSandboxWorkError(sandbox, slice, error);\n selectiveDebug() && console.error(`runSliceOnSandbox: Failure executing slice on sandbox: ${sandbox.identifier}`, reason, error);\n })\n .catch((error) => {\n // Catches exception from sandbox.work or then-clause.\n return handleError(error, 'runSliceOnSandbox');\n });\n }\n\n // _Idx\n //\n // removeSlice\n //\n\n /**\n * Remove slice from this.sliceInventory.\n * @param {Slice} slice - The slice to remove from this.sliceInventory.\n * @return {boolean} - False when slice is not in this.sliceInventory.\n */\n removeSlice (slice)\n {\n debugging('supervisor') && console.debug(`removeSlice slice ${slice.sliceNumber}`);\n return common.removeElement(this.sliceInventory, slice);\n }\n\n // _Idx\n //\n // fetchError, fetchJob, fetchSliceData\n //\n\n fetchError (error)\n {\n if (error.code === 'EPERM_ORIGIN')\n throw new BadOriginError(error);\n if (error.code === 'EFETCH')\n throw new RemoteFetchError(error);\n throw new UncaughtExceptionError(error);\n }\n\n /**\n * Fetch work function, work function arguments and possibly the range object describing the jobs slices..\n * @param {object} mpe - messagePayloadElement: job object returned by task-jobs.js . \n * @returns {Promise<*>}\n */\n fetchJob (mpe)\n {\n debugging('supervisor') && utils.dumpObject(mpe, 'JobManager.fetchJob: mpe', 512);\n\n const promises = [];\n try\n {\n // Get workFn.\n if (!mpe.workFunction)\n {\n const workFunctionPromise = this.fetchWorkFunctions(mpe.codeLocation)\n .then((workFunction) => {\n this.inputDataSize += workFunction.size;\n mpe.workFunction = workFunction.data;\n if (mpe.requirements.useStrict)\n mpe.useStrict = true;\n delete mpe.codeLocation;\n }, (error) => {\n this.fetchError(error);\n });\n promises.push(workFunctionPromise);\n }\n\n if (!mpe.mro && mpe.MROLocation)\n {\n const mroPromise = this.fetchData(mpe.MROLocation)\n .then((mro) => {\n // It's not ready for computing the inputDataSize yet -- that's done in JobManager.fetchSliceData .\n mpe.mro = mro.data;\n delete mpe.MROLocation;\n }, (error) => {\n this.fetchError(error);\n });\n promises.push(mroPromise)\n }\n\n // Get workFn args.\n if (!mpe.arguments && mpe.argumentsLocation)\n {\n mpe.arguments = new Array(mpe.argumentsLocation.length);\n for (let k = 0; k < mpe.argumentsLocation.length; k++)\n promises.push(this.fetchArguments(mpe.argumentsLocation[k].value)\n .then((arg) => {\n this.inputDataSize += arg.size;\n mpe.arguments[k] = arg.data;\n }, (error) => {\n this.fetchError(error);\n })\n );\n }\n\n return Promise.all(promises)\n .then(() => {\n if (mpe.argumentsLocation)\n delete mpe.argumentsLocation;\n return mpe;\n }, (error) => {\n this.fetchError(error);\n });\n }\n catch (error)\n {\n this.fetchError(error);\n }\n }\n\n /**\n * Look up slice.datumUri or use the range object this.mro, constructed by fetchJob.\n * @param {Slice} slice \n * @returns {Promise<{ data: *, size: number }>}\n */\n fetchSliceData (datumUri, slice)\n {\n try\n {\n if (!datumUri)\n {\n if (!this.mro)\n throw new Error('Must complete call to JobManager.fetchJob before calling JobManager.fetchSliceData.');\n /** XXXpfr @todo Inefficient, we're rehydrating the whole range for a single slice datum. */\n const ro = rehydrateRange(this.mro);\n // Slice numbers start at 1.\n const sliceDatum = ro[slice.sliceNumber - 1];\n debugging('supervisor') && console.debug(`Fetched mro datum: ${stringify(sliceDatum, 512)}`);\n return Promise.resolve({ data: sliceDatum, size: JSON.stringify(sliceDatum).length });\n }\n\n return this.fetchData(datumUri)\n .catch((error) => {\n throw this.fetchError(error);\n });\n }\n catch (error)\n {\n this.fetchError(error);\n }\n }\n\n // _Idx\n //\n // jobEmit, sandboxEmit, error, warning. countSliceStr, dumpSlices\n //\n\n /**\n * Safe event emitter on jobHandle.\n * @param {string} event\n * @param {...any} args\n */\n jobEmit(event, ...args)\n {\n this.supervisor.safeEmit(this.jobHandle, event, ...args);\n }\n\n /**\n * Safe event emitter on sandboxHandle.\n * @param {Sandbox} sandbox\n * @param {string} event\n * @param {...any} args\n */\n sandboxEmit(sandbox, event, ...args)\n {\n this.supervisor.safeEmit(sandbox.sandboxHandle, event, ...args);\n }\n\n /**\n * Error feedback to user.\n * @param {string} message\n * @param {Array<Error>|Error|string} [coreError]\n * @param {string} [additionalInfo]\n */\n error (message, coreError, additionalInfo)\n {\n this.supervisor.error(message, coreError, additionalInfo);\n }\n\n /**\n * Warning feedback to user.\n * @param {string[]} messages\n */\n warning (...messages)\n {\n this.supervisor.warning(...messages);\n }\n\n /**\n * Debugging helper.\n * @param {string} [tag='']\n * @returns {string}\n */\n countSliceStr (tag)\n {\n const { unassigned, ready, reserved, working, workdone, complete, failed, finished } = this.dumpSlices (tag, false, false);\n return `${tag}: u/r/rsv/w/wd/c/f/fsh ${unassigned}/${ready}/${reserved}/${working}/${workdone}/${complete}/${failed}/${finished}`;\n }\n\n /**\n * @param {URL|string} uri\n * @returns {Promise<*>}\n */\n fetchData (uri) { return fetchURIAndLength(uri, this.originManager, allowOriginsPurposes.fetchData); }\n /**\n * @param {URL|string} uri\n * @returns {Promise<*>}\n */\n fetchArguments (uri) { return fetchURIAndLength(uri, this.originManager, allowOriginsPurposes.fetchArguments); }\n /**\n * @param {URL|string} uri\n * @returns {Promise<*>}\n */\n fetchWorkFunctions (uri) { return fetchURIAndLength(uri, this.originManager, allowOriginsPurposes.fetchWorkFunctions); }\n\n /**\n * Debugging helper.\n * @param {string} [tag='']\n * @param {boolean} [details=false]\n * @param {boolean} [display=true]\n * @returns {{ ready, reserved, working, workdone, complete, failed, finished, unassigned }}\n */\n dumpSlices (tag = '', details = false, display = true)\n {\n if (display)\n console.log(`dumpSlices(${tag}): ${this.identifier}, sliceCount ${this.sliceInventory.length}, sandboxCount ${this.sandboxInventory.length}`, Date.now() - this.supervisor.lastTime);\n if (this.sliceInventory.length < 1) return { unassigned:0, ready:0, reserved:0, working:0, workdone:0, complete:0, failed:0, finished:0 };\n if (details)\n {\n // Detailed classification of slices.\n const unassigned = [], ready = [], reserved = [], working = [], workdone = [], complete = [], failed = [], finished = [];\n for (const slice of this.sliceInventory)\n {\n if (slice.isReady)\n ready.push(slice);\n else if (slice.isReserved)\n reserved.push(slice);\n else if (slice.isWorking)\n working.push(slice);\n else if (slice.isWorkDone)\n workdone.push(slice);\n else if (slice.isComplete)\n complete.push(slice);\n else if (slice.isFailed)\n failed.push(slice);\n else if (slice.isFinished)\n finished.push(slice);\n else if (slice.isUnassigned)\n unassigned.push(slice);\n else\n throw new Error(`dumpSlices: Unexpected kind a slice; ${slice.identifier}`);\n }\n if (display)\n {\n const dmpSlices = (name, slices) => {\n console.log(`-----${name}(${slices.length})-----------------------------------------------------------------`);\n for (const slice of slices)\n console.log(slice.identifier);\n }\n dmpSlices('unassigned', unassigned);\n dmpSlices('ready', ready);\n dmpSlices('reserved', reserved);\n dmpSlices('working', working);\n dmpSlices('workdone', workdone);\n dmpSlices('complete', complete);\n dmpSlices('failed', failed);\n dmpSlices('finished', finished);\n console.log('-----------------------------------------------------------------------------------');\n }\n return {\n ready: ready.length,\n reserved: reserved.length,\n working: working.length,\n workdone: workdone.length,\n complete: complete.length,\n failed: failed.length,\n finished: finished.length,\n unassigned: unassigned.length,\n };\n }\n else\n {\n // Quick classification of slices.\n const { ready,\n reserved,\n working,\n workdone,\n complete,\n failed,\n finished,\n unassigned } = common.sliceCounts(this.sliceInventory);\n if (display)\n {\n const dumpCount = (name, count) => {\n console.log(`-----${name}(${count})-----------------------------------------------------------------`);\n }\n dumpCount('unassigned', unassigned);\n dumpCount('ready', ready);\n dumpCount('reserved', reserved);\n dumpCount('working', working);\n dumpCount('workdone', workdone);\n dumpCount('complete', complete);\n dumpCount('failed', failed);\n dumpCount('finished', finished);\n console.log('-----------------------------------------------------------------------------------');\n }\n return { ready, reserved, working, workdone, complete, failed, finished, unassigned };\n }\n }\n\n}\nexports.JobManager = JobManager;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/supervisor2/job-manager.js?");
4738
+ eval("/**\n * @file dcp-client/worker/supervisor2/job-manager.js\n *\n * A support class for Supervisor2.\n * It is a wrapper for the job object returned from the requestTask,\n * along with tracking slices and sandboxes associated with the job. \n *\n * @author Wes Garland, wes@distributive.network,\n * Paul, paul@distributive.network,\n * @date Dec 2020,\n * June 2022, Jan-April 2023\n * @module job-manager\n * @copyright Copyright (c) 2018-2023, Distributive Corp. All Rights Reserved\n */\n// @ts-check\n\n\n/** @typedef {string} opaqueId */ // 22 character base64 string\n/** @typedef {string} address */ // String(Address)\n/** @typedef {import('./sandbox2').Sandbox} Sandbox */\n/** @typedef {import('./index').Supervisor} Supervisor */\n/** @typedef {import('dcp/utils/jsdoc-types').SliceMessage} SliceMessage */\n/** @typedef {import('dcp/utils/jsdoc-types').Auth} Auth */\n/** @typedef {import('dcp/utils/jsdoc-types').Signature} Signature */\n/** @typedef {import('dcp/utils/jsdoc-types').AuthMessage} AuthMessage */\n/** @typedef {import('dcp/dcp-client/worker/origin-access-manager').OriginAccessManager} OriginAccessManager */\n\nconst inspect = Symbol.for('nodejs.util.inspect.custom');\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst EventEmitter = __webpack_require__(/*! events */ \"./node_modules/events/events.js\");\nconst { Inventory } = __webpack_require__(/*! dcp/utils/inventory */ \"./src/utils/inventory.js\");\nconst { Synchronizer } = __webpack_require__(/*! dcp/common/concurrency */ \"./src/common/concurrency.js\");\nconst { rehydrateRange } = __webpack_require__(/*! dcp/dcp-client/range-object */ \"./src/dcp-client/range-object.js\");\nconst { fetchURIAndLength } = __webpack_require__(/*! dcp/utils/fetch-uri */ \"./src/utils/fetch-uri.js\");\nconst utils = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst { stringify, a$sleepMs, constructAlpha, nextEma } = utils;\nconst { allowOriginsPurposes } = __webpack_require__(/*! dcp/common/worker-constants */ \"./src/common/worker-constants.js\");\n\nconst { Slice } = __webpack_require__(/*! ./slice2 */ \"./src/dcp-client/worker/supervisor2/slice2.js\");\nconst { Promise_any } = __webpack_require__(/*! ./promise_any */ \"./src/dcp-client/worker/supervisor2/promise_any.js\");\nconst { Statistics } = __webpack_require__(/*! ./rolling-statistics */ \"./src/dcp-client/worker/supervisor2/rolling-statistics.js\");\nconst { SandboxError, UncaughtExceptionError } = __webpack_require__(/*! ./sandbox2 */ \"./src/dcp-client/worker/supervisor2/sandbox2.js\");\nconst { updateGPUStatistics } = __webpack_require__(/*! ./gpu_support */ \"./src/dcp-client/worker/supervisor2/gpu_support.js\");\nconst common = __webpack_require__(/*! ./common */ \"./src/dcp-client/worker/supervisor2/common.js\");\nconst { selectiveDebug, selectiveDebug2, debugBuild } = common;\n\nconst INITIAL = 'initial';\nconst READY = 'ready';\nconst STOP = 'stop';\nconst REFUSE = 'refuse';\nconst BROKEN = 'broken';\n\nclass BadOriginError extends SandboxError { constructor(msg) { super('EPERM_ORIGIN', msg); } }\nclass RemoteFetchError extends SandboxError { constructor(msg) { super('EFETCH', msg); } }\n\n/**\n * Public event emitter.\n * https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/worker/worker-events.md\n */\nclass JobHandle extends EventEmitter\n{\n /** @type {JobManager} */\n #jobManager;\n \n /**\n * @constructor\n * @param {JobManager} jobManager\n */\n constructor (jobManager)\n {\n super({ captureRejections: false });\n this.#jobManager = jobManager;\n }\n /** @type {string} */\n get address () { return this.#jobManager.address; }\n /** @type {string} */\n get name () { return this.#jobManager.public ? this.#jobManager.public.name : '<unknown>'; }\n /** @type {string} */\n get description () { return this.#jobManager.public ? this.#jobManager.public.description : '<unknown>'; }\n /** @type {string} */\n get link () { return this.#jobManager.public ? this.#jobManager.public.link : '<unknown>'; }\n}\nexports.JobHandle = JobHandle;\n\n//\n// Index to functionality -- search for '_Idx' to toggle through the index.\n//\n// 1) class JobManager\n// 2) addSlices, fetchDependencies, updateStatistics, update, predictLoad\n// 3) reserveOneSlice, resetSlices, destroy, evaluatorDownCleanup\n// 4) assignSandbox, runSlice, runSliceOnSandbox\n// 5) removeSlice\n// 6) fetchError, fetchJob, fetchSliceData\n// 7) jobEmit, sandboxEmit, error, warning. countSliceStr, dumpSlices.\n//\n\n// _Idx\n//\n// class JobManager\n// A JobManager handles all scheduling of slices for a given job.\n// It's also responsible for fetching slice data, work functions and arguments.\n// And it collects statistics about slice completion times and resource usage.\n// All functionality across jobs is handled by Supervisor.\n//\n\n/**\n * JobManager Constructor. An instance of JobManager knows everything about a given\n * job within the supervisor, including:\n * - work function code\n * - work function arguments\n * - all the SliceManager instances that go with this job\n * - how long a slice of this job is expected to run\n *\n * Instances of JobManager emit the following events:\n * - addSlice(sliceHandle)\n * - deleteSlice(sliceHandle)\n * - statusChange(new state, old state);\n *\n * JobManager States\n *\n * Start state:\n * initial\n *\n * Intermediate states:\n * ready\n *\n * Terminal states:\n * broken - job could not be initialized\n * refuse - for some reason, we have decided that we don't want work for this job\n * stop - job manager has been stopped\n *\n * Valid transitions:\n * initial -> broken\n * initial -> ready\n * initial -> ready -> stop\n * \n * NOTE: If you ever use a property with a leading underscore you are probably making a mistake.\n * But if you must, please ask paul, yarn, bryan or eddie for a CR.\n */\nclass JobManager\n{\n /** @type {Supervisor} */\n #supervisor;\n /** @type {Slice[]} */\n #sliceInventory;\n /** @type {Synchronizer} */\n #state;\n /** @type {*} */ // Figure out the type\n #jobMessage;\n /** @type {string} */\n #address;\n /** @type {Statistics} */\n #statistics;\n /** @type {Promise<*>} */\n #jobPromise;\n\n /**\n * @constructor\n * @param {Supervisor} parent - Owning Supervisor.\n * @param {object} jobMessage - Job Descriptor from getJobsForTask.\n * @param {SliceMessage[]} sliceMessages - Messages from task distributor describing slices.\n * @param {AuthMessage} authorizationMessage - The signature that shipped with the task authorizing this worker.\n */\n constructor(parent, jobMessage, sliceMessages, authorizationMessage)\n {\n this.#supervisor = parent;\n this.#sliceInventory = []; // All slices for this.address.\n this.#state = new Synchronizer(INITIAL, [ INITIAL, READY, STOP, REFUSE, BROKEN ]);\n this.#jobMessage = { ...jobMessage };\n this.#address = String(this.jobMessage.address);\n this.#statistics = new Statistics(0.5); // Effective period of 12 or 13\n this.#jobPromise = null;\n\n /** @type {{CPUTime: number, GPUTime: number, TotalTime: number, CPUDensity: number, GPUDensity: number, InDataSize: number, OutDataSize: number }} */\n this.localMetrics = null;\n /** @type {string[]} */\n this.rejectedJobReasons = [];\n /** @type {boolean} */\n this.isEstimation = true;\n /** @type {number} */\n this.inputDataSize = 0;\n /**\n * jobMan.evaluatorDownCleaup returns this.evaluatorDownHandle=a$sleepMs which is\n * interruptible by calling this.evaluatorDownHandle.intr().\n * @type {Promise<void>}\n */\n this.evaluatorDownHandle = null;\n\n /*\n * Make sure this.metrics contains valid data.\n * With old legacy schedulers, this may not be so.\n */\n this.fixUpMetrics();\n\n /** \n * Start loading dependencies in the background. Once these are loaded, this.state will\n * transition to 'ready' and the job will be ready for work. Do not place this job manager\n * in inventory until 'ready'.\n */\n this.#jobPromise = this.fetchDependencies(sliceMessages, authorizationMessage)\n .then(() => {\n this.supervisor.jobManagerInventory.push(this);\n });\n\n /* Check for GPU support. */\n const _env = this.requirements?.environment;\n /** @type {boolean} */\n this.useGPU = _env ? (_env.webgpu || _env.offscreenCanvas) : false;\n \n /**\n * Event emitter containing info that describes the job.\n * @type {JobHandle}\n */\n this.jobHandle = new JobHandle(this);\n\n /**\n * Emit the 'job' event on the worker event emitter.\n */\n this.supervisor.safeEmit(this.supervisor.worker, 'job', this.jobHandle);\n }\n\n /** @type {*} */ // Figure out the type\n get jobMessage () { return this.#jobMessage; }\n /** @type {string} */\n get address () { return this.#address; }\n /** @type {string} */\n get uuid () { return this.jobMessage.uuid; }\n /** @type {boolean} */\n get workerConsole () { return this.jobMessage.workerConsole; }\n /** @type {object} */\n get requirements () { return this.jobMessage.requirements; }\n /** @type {boolean} */\n get almostDone () { return this.jobMessage.almostDone; }\n\n // These 3 properties have type object.\n get mro () { return this.jobMessage.mro; }\n get arguments () { return this.jobMessage.arguments; }\n get workFunction () { return this.jobMessage.workFunction; }\n\n /** @type {{computeGroups: Array<{opaqueId: string, name: string, description: string}>, name: string, description: string, link: string}} */\n get public () { return this.jobMessage.public; }\n /** @type {{sliceCPUTime: number, sliceGPUTime: number, sliceCPUDensity: number, sliceGPUDensity: number, sliceInDataSize: number, sliceOutDataSize: number, lastSliceNumber: number, measuredSlices: number, alpha: number}} */\n get metrics () { return this.jobMessage.metrics; }\n /** @type {Supervisor} */\n get supervisor () { return this.#supervisor; }\n /** @type {Synchronizer} */\n get state () { return this.#state; }\n\n /** @type {boolean} */\n get ready () { return this.state.is(READY); }\n\n // These 4 aren't needed yet.\n ///** @type {boolean} */\n //get initial () { return this.state.is(INITIAL); }\n ///** @type {boolean} */\n //get stopped () { return this.state.is(STOP); }\n ///** @type {boolean} */\n //get refuse () { return this.state.is(REFUSE); }\n ///** @type {boolean} */\n //get broken () { return this.state.is(BROKEN); }\n\n /** @type {Sandbox[]} */\n get sandboxInventory () { return this.supervisor.sandboxInventory.filter((sandbox) => sandbox.jobAddress === this.#address && !sandbox.isTerminated); }\n /** @type {Sandbox[]} */\n get assignedSandboxes () { return this.supervisor.sandboxInventory.filter((sandbox) => sandbox.jobAddress === this.#address && sandbox.isAssigned); }\n /** @type {Sandbox[]} */\n get workingSandboxes () { return this.supervisor.sandboxInventory.filter((sandbox) => sandbox.jobAddress === this.#address && sandbox.isWorking); }\n /** @type {Slice[]} */\n get sliceInventory () { return this.#sliceInventory; }\n /** @type {Slice[]} */\n get readySlices () { return this.sliceInventory.filter((slice) => slice.isReady); }\n /** @type {Slice[]} */\n get queuedSlices () { return this.sliceInventory.filter((slice) => slice.isQueued); } // Ready and soon-to-be-ready\n /** @type {Slice[]} */\n get reservedSlices () { return this.sliceInventory.filter((slice) => slice.isReserved); }\n /** @type {Slice[]} */\n get activeSlices () { return this.sliceInventory.filter((slice) => slice.isActive); } // Reserved, working, workdone and complete\n /** @type {Slice[]} */\n get workingSlices () { return this.sliceInventory.filter((slice) => slice.isWorking || slice.isReserved); } // Working and soon-to-be-working\n /** @type {Slice[]} */\n get unFinishedSlices () { return this.sliceInventory.filter((slice) => !slice.isFinished); }\n\n /** @type {Statistics} */\n get statistics () { return this.#statistics; }\n /** @type {Promise<*>} */\n get jobPromise () { return this.#jobPromise; }\n /** @type {number} */\n get globalTime () { return utils.globalTime(this.metrics); }\n /** @type {number} */\n get emaSliceTime () { return this.localMetrics ? utils.localTime(this.localMetrics) : (utils.globalTime(this.metrics) * this.supervisor.defaultQuanta); }\n /** @type {number} */\n get emaTotalTime () { return this.localMetrics?.TotalTime || (utils.globalTotalTime(this.metrics) * this.supervisor.defaultQuanta); }\n /** @type {number} */\n get adjSliceTime () { return Math.min(this.emaSliceTime, this.supervisor.options.targetTaskDuration); }\n /** @type {string} */\n get identifier () { return `${common.truncateAddress(this.address)}.${this.state}`; }\n /** @type {string} */\n get sliceState () { return `${this.queuedSlices.length}.${this.workingSliceCount}.${this.sliceInventory.length}.${this.workRemaining}`; }\n /** @type {string} */\n get [inspect] () { return `[Object JobManager <${this.public.name}::${common.truncateAddress(this.address)}::${this.state}>]`; }\n /** @type {OriginAccessManager} */\n get originManager() { return this.supervisor.originManager; }\n /**\n * Expected density of a slice.\n * When we get the GPU stuff codified, the definition might be\n * { cpu: this.localMetrics?.CPUDensity ?? 1.0, gpu: this.localMetrics?.GPUDensity ?? 1.0}\n * @type {number}\n */\n get estimateDensity () { return this.localMetrics?.CPUDensity || utils.globalDensity(this.metrics); }\n /** @type {number} */\n get estimateGPUDensity () { return this.localMetrics?.GPUDensity || utils.globalGPUDensity(this.metrics); }\n /** @type {number} */\n get cpuTime () { return this.localMetrics?.CPUTime || this.metrics?.sliceCPUTime || 0; }\n /**\n * Estimate the number of milliseconds of total execution time of the job.\n * @type {number}\n */\n get workRemaining ()\n {\n const _adjSliceTime = this.adjSliceTime;\n const queuedTime = _adjSliceTime * this.queuedSlices.length;\n let workingTime = 0;\n for (const slice of this.workingSlices)\n {\n workingTime += Math.max(0, _adjSliceTime - (Date.now() - slice.startTime));\n selectiveDebug2() && console.debug(`JobManager.workRemaining: _adjSliceTime ${_adjSliceTime}, slice-time-elapsed ${Date.now() - slice.startTime}`);\n }\n selectiveDebug() && console.debug(`JobManager.workRemaining: queuedTime ${queuedTime}, queuedCount ${this.queuedSlices.length}, workingTime ${workingTime}, workingCount ${this.workingSliceCount}`);\n return workingTime + queuedTime;\n }\n /**\n * Are there no working slices?\n * @type {boolean}\n */\n get isNotWorking ()\n {\n for (const slice of this.sliceInventory)\n if (slice.isReserved || slice.isWorking)\n return false;\n return true;\n }\n /**\n * Estimated count of working (or reserved) slices.\n * @type {number}\n */\n get workingSliceCount ()\n {\n let count = 0;\n for (const slice of this.sliceInventory)\n if (slice.isReserved || slice.isWorking)\n ++count;\n return count;\n }\n /**\n * Estimated density of working (or reserved) slices.\n * @type {number}\n */\n get workingSliceDensity () { return this.workingSliceCount * this.estimateDensity; }\n /**\n * Estimated density of working (or reserved) slices.\n * @type {number}\n */\n get workingGPUDensity () { return this.workingSliceCount * this.estimateGPUDensity; }\n /**\n * Always display max info under debug builds, otherwise maximal error\n * messages are displayed to the worker, only if both worker and client agree.\n * When displayMaxDiagInfo is true, display stack trace and other enhanced diag info.\n * @type {boolean}\n **/\n get displayMaxDiagInfo ()\n {\n return common.displayMaxDiagInfo() || this.workerConsole && this.supervisor.options.allowConsoleAccess;\n }\n\n // _Idx\n //\n // addSlices, fetchDependencies, updateStatistics, update, predictLoad\n //\n\n /**\n * Add slices to the job manager's inventory.\n *\n * @param {SliceMessage[]} sliceMessages - Messages from task distributor describing slices.\n * @param {AuthMessage} authorizationMessage - The signature that shipped with the task authorizing this worker.\n * @returns {Promise<*>}\n */\n addSlices (sliceMessages, authorizationMessage)\n {\n /** @type {Array<Promise<void>>} */\n const slicePromises = [];\n sliceMessages.forEach((sliceMessage) => {\n const slice = new Slice(this, sliceMessage, authorizationMessage);\n if (!slice.isEstimation)\n this.isEstimation = false;\n this.sliceInventory.push(slice);\n slicePromises.push(slice.fetchSliceDataPromise);\n });\n return Promise_any(slicePromises);\n }\n\n /**\n * Fetch dependencies.\n * @param {SliceMessage[]} sliceMessages - Messages from task distributor describing slices.\n * @param {AuthMessage} authorizationMessage - The signature that shipped with the task authorizing this worker.\n * @returns {Promise<*>}\n */\n fetchDependencies (sliceMessages, authorizationMessage)\n {\n return this.fetchJob(this.jobMessage)\n .then (() => {\n debugging('supervisor') && console.debug('JobManager is transitioning to READY', this.identifier, Date.now() - this.supervisor.lastTime);\n this.state.set(INITIAL, READY);\n return this.addSlices(sliceMessages, authorizationMessage);\n }, (error) => {\n selectiveDebug() && console.error('fetchJob has failed', error);\n this.state.set(INITIAL, BROKEN);\n throw error;\n });\n }\n\n /** XXXpfr @todo Factor out statistics and EMA into another class. */\n /**\n * @param {number} x\n * @param {boolean} isDensity\n * @returns {boolean}\n */\n static isValid (x, isDensity = false)\n {\n return Number.isFinite(x) && !(x < 0) && !(isDensity && x > 1);\n }\n\n /**\n * Old legacy DCP schedulers can send a corrupt this.metrics .\n * We should get rid of this when we don't support old legacy DCP schedulers.\n */\n fixUpMetrics ()\n {\n if (!this.metrics)\n this.jobMessage.metrics = { sliceCPUTime: 0, sliceCPUDensity: 1, sliceGPUTime: 0, sliceGPUDensity: 0, sliceInDataSize: 0, sliceOutDataSize: 0 };\n else\n {\n if (!(JobManager.isValid(this.metrics.sliceCPUTime) && JobManager.isValid(this.metrics.sliceCPUDensity, true)))\n {\n this.metrics.sliceCPUTime = 1;\n this.metrics.sliceCPUDensity = 0.5;\n }\n if (!(JobManager.isValid(this.metrics.sliceGPUTime) && JobManager.isValid(this.metrics.sliceGPUDensity, true)))\n {\n this.metrics.sliceGPUTime = 1;\n this.metrics.sliceGPUDensity = 0.5;\n }\n if (!JobManager.isValid(this.metrics.sliceInDataSize))\n this.metrics.sliceInDataSize = 0;\n if (!JobManager.isValid(this.metrics.sliceOutDataSize))\n this.metrics.sliceOutDataSize = 0;\n }\n }\n\n /**\n * @param {Slice} slice\n * @param {Sandbox} sandbox\n * @returns {{CPUTime: number, GPUTime: number, CPUDensity: number, GPUDensity: number, InDataSize: number, OutDataSize: number}}\n */\n updateStatistics (slice, sandbox)\n {\n /**\n * @todo XXXpfr Make this only check when debugBuild, but not until the Sup2 churn has stopped.\n * @param {{CPUTime: number, GPUTime: number, CPUDensity: number, GPUDensity: number, InDataSize: number, OutDataSize: number}} metrics\n * @returns {boolean}\n */\n const isValidMetrics = (metrics) => {\n return JobManager.isValid(metrics.CPUTime) && JobManager.isValid(metrics.CPUDensity, true)\n && JobManager.isValid(metrics.GPUTime) && JobManager.isValid(metrics.GPUDensity, true)\n && JobManager.isValid(metrics.InDataSize) && JobManager.isValid(metrics.OutDataSize);\n }\n\n const timeReport = slice.timeReport;\n const dataReport = slice.dataReport;\n selectiveDebug() && console.debug('updateStatistics', slice.identifier, timeReport, dataReport);\n\n // Sanity\n if (!(timeReport.CPU >= 1))\n {\n timeReport.CPU = 1;\n timeReport.total += 1;\n }\n if (!(timeReport.total >= 1))\n timeReport.total = 1;\n\n // Construct metrics.\n const GPUTime = timeReport.webGPU + timeReport.webGL;\n if (this.useGPU)\n updateGPUStatistics(GPUTime);\n const metrics = {\n CPUTime: timeReport.CPU,\n GPUTime,\n CPUDensity: timeReport.CPU / timeReport.total,\n GPUDensity: GPUTime / timeReport.total,\n InDataSize: dataReport.InDataSize,\n OutDataSize: dataReport.OutDataSize,\n };\n //console.debug('JM.updateStatistics metrics:', metrics, timeReport.total);\n\n // Create the measurement object which will be passed to the metrics event.\n const eventMeasurements = {\n elapsed: timeReport.total / 1000,\n CPU: metrics.CPUTime / 1000,\n GPU: metrics.GPUTime / 1000,\n in: metrics.InDataSize,\n out: metrics.OutDataSize,\n };\n\n // Emit metrics event for both jobHandle and sandboxHandle\n this.jobEmit( 'metrics', slice.sliceNumber, eventMeasurements);\n this.sandboxEmit(sandbox, 'metrics', slice.sliceNumber, eventMeasurements);\n selectiveDebug() && console.debug(`updateStatistics: slice ${slice.identifier}, eventMeasurements ${stringify(eventMeasurements)}`);\n\n if (!isValidMetrics(metrics))\n throw new Error(`JobManager.updateStatistics: metrics are in an inconsistent state ${JSON.stringify(metrics)}`);\n\n if (!this.localMetrics)\n this.localMetrics = { CPUTime: 0, GPUTime: 0, TotalTime: 0, CPUDensity: 0, GPUDensity: 0, InDataSize: 0, OutDataSize: 0 };\n\n // measuredSlices+1 because the current slice hasn't been measured yet in RS.updateMetrics.\n const alpha = this.metrics?.lastSliceNumber\n ? constructAlpha(this.metrics.lastSliceNumber, this.metrics.measuredSlices + 1)\n : 0.5;\n selectiveDebug2() && console.debug(`updateStatistics: local alpha ${alpha}, global alpha ${this.metrics?.alpha}`);\n this.localMetrics.CPUTime = nextEma(this.localMetrics.CPUTime, metrics.CPUTime, alpha);\n this.localMetrics.GPUTime = nextEma(this.localMetrics.GPUTime, metrics.GPUTime, alpha);\n this.localMetrics.TotalTime = nextEma(this.localMetrics.TotalTime, timeReport.total, alpha);\n this.localMetrics.CPUDensity = nextEma(this.localMetrics.CPUDensity, metrics.CPUDensity, alpha);\n this.localMetrics.GPUDensity = nextEma(this.localMetrics.GPUDensity, metrics.GPUDensity, alpha);\n this.localMetrics.InDataSize = nextEma(this.localMetrics.InDataSize, metrics.InDataSize, alpha); // Don't need this yet.\n this.localMetrics.OutDataSize = nextEma(this.localMetrics.OutDataSize, metrics.OutDataSize, alpha); // Don't need this yet.\n\n if (common.debugQuanta())\n {\n this.supervisor.dbg.addRawLocal(slice, metrics, alpha);\n this.supervisor.dbg.addLocal(slice, this.emaSliceTime, alpha);\n }\n\n if (!isValidMetrics(this.localMetrics))\n throw new Error(`JobManager.updateStatistics: localMetrics are in an inconsistent state ${JSON.stringify(this.localMetrics)}`);\n\n debugging('supervisor') && console.debug('updateStatistics: ema', this.statistics.ema, 'emaSliceTime', this.emaSliceTime, 'estimateDensity', this.estimateDensity, 'metrics', metrics, 'g-metrics', this.metrics, 'alpha', alpha);\n //if (!Number.isFinite(this.statistics.ma))\n // console.log(this.statistics.x);\n return metrics;\n }\n\n /**\n * Update jobMessage, add some slices to inventory and possibly update the initial seed of the statistics.\n * @param {object} jobMessage - Job Descriptor from getJobsForTask.\n * @param {SliceMessage[]} [sliceMessages] - Messages from task distributor describing slices.\n * @param {AuthMessage} [authorizationMessage] - The signature that shipped with the task authorizing this worker.\n * @returns {Promise<*>}\n */\n update (jobMessage, sliceMessages, authorizationMessage)\n {\n Object.assign(this.jobMessage, jobMessage);\n /*\n * Make sure this.metrics contains valid data.\n * With old legacy schedulers, this may not be so.\n */\n this.fixUpMetrics();\n selectiveDebug2() && console.debug('JM.update: new this.jobMessage', this.jobMessage.metrics);\n\n if (sliceMessages)\n return (this.#jobPromise = this.addSlices(sliceMessages, authorizationMessage));\n }\n\n /**\n * Predict the total reduction in density of working sandboxes timeSpanMs from now.\n * This function is called right before fetchTask, in order to calculate how much space is available.\n * In pratice, the returned property queued will have length <= 1, because when it is more we won't call fetchWork.\n * @param {number} timeSpanMs\n * @returns {{ queued: Slice[], working: number }}\n */\n predictLoad (timeSpanMs)\n {\n let queuedCount = this.queuedSlices.length;\n // Optimize to short-circuit when queued > 1, because we won't call fetchWork in that case.\n if (queuedCount > this.workingSliceCount + 1)\n {\n selectiveDebug() && console.debug('JM.predictLoad: bail early', queuedCount, this.workingSliceCount);\n return { queued: [null, null], working: this.workingSliceDensity };\n }\n let space = 0;\n // 1) Predict the current working slices that will finish in timeSpanMs.\n // 2) Compensate for queued slices that may begin working.\n for (const slice of this.sliceInventory)\n {\n if (!slice.isReserved && !slice.isWorking)\n continue;\n const remainingTime = Math.max(0, this.emaSliceTime - (Date.now() - slice.startTime));\n selectiveDebug2() && console.debug('JM.predictLoad: predicting...', queuedCount, space, this.emaSliceTime, remainingTime, this.emaSliceTime <= timeSpanMs - remainingTime);\n // If slice will end before timeSpanMs elapses, it may open up space.\n if (remainingTime <= timeSpanMs)\n {\n // Check to see whether a new queued slices can start running and finish in the remaining time.\n // Somewhat inaccurate because it's possible multiple slices could finish in the remaining time.\n // e.g. Suppose timeSpanMs = 5 * 1000 and this.emaSliceTime is 1000, then there's room\n // to successively schedule 5 slices, one after another.\n /** XXXpfr @todo Fix to incorporate successive scheduling. */\n if (queuedCount > 0)\n {\n --queuedCount;\n // If the new slice finishes in time, count it as opening up space.\n if (this.emaSliceTime <= timeSpanMs - remainingTime)\n ++space;\n }\n else // No queued slice can run so space has opened up.\n ++space;\n }\n }\n // Optimize to short-circuit when queued > 1, because we won't call fetchWork in that case.\n let queued = [null, null];\n if (queuedCount < 2)\n queued = (queuedCount === 1) ? [this.queuedSlices[0]] : [];\n return { queued, working: (this.workingSliceCount - space) * this.estimateDensity };\n }\n\n // _Idx\n //\n // reserveOneSlice, resetSlices, destroy, evaluatorDownCleanup\n //\n\n /**\n * Find one ready slice, mark it as working and return.\n * If there are no ready slices return undefined.\n * @returns {Slice}\n */\n reserveOneSlice ()\n {\n for (const slice of this.sliceInventory)\n if (slice.isReady)\n return slice.markAsReserved();\n return null;\n }\n\n /**\n * Error recovery of slices.\n * When evaluator is down and it goes up again, there may be\n * working slices from working sandboxes that were terminated.\n * @param {string} [tag]\n */\n resetSlices (tag)\n {\n this.sliceInventory.forEach((slice) => {\n if (this.evaluatorDownHandle)\n this.evaluatorDownHandle.intr(); // Interrupt a$sleep in evaluatorDownCleanup\n this.evaluatorDownHandle = null;\n if (slice.isWorking || slice.isFailed)\n slice.resetState();\n });\n //this.dumpSlices(tag); // SAVE\n }\n\n /**\n * Destructor.\n * @returns {Promise<*>}\n */\n destroy ()\n {\n selectiveDebug() && console.debug(`JobManager.destroy: terminating sandboxes and returning slices to scheduler for job manager ${this.identifier}.`);\n this.sandboxInventory.forEach((sandbox) => { this.supervisor.returnSandbox(sandbox); });\n const reason = 'JobManager destroy';\n return this.supervisor.returnSlices(this.unFinishedSlices, reason)\n .finally (() => {\n this.#sliceInventory = [];\n this.state.removeAllListeners();\n if (this.state.isNot(BROKEN))\n this.state.set([ INITIAL, READY, STOP, REFUSE ], BROKEN);\n this.jobEmit('flush');\n });\n }\n\n /**\n * Destructor.\n * Called when the evaluator goes down (or screensaver goes up), in the sandbox 'terminated' event handler in Supervisor.\n * A slice marked as COMPLETE but not FINISHED means it still needs to submit results.\n * This function allows such slices to still submit results.\n * Because sometimes a screensaver only goes down for a minute or less (e.g. the user needs to look at something quick),\n * we delay an average of 90 (or 1.5 * delay) seconds before returning non-complete slices.\n * param {number} [delay=60] - Average delay is 1.5 * delay seconds.\n */\n evaluatorDownCleanup (delay = 60)\n {\n selectiveDebug() && console.debug(`JobManager.evaluatorDownCleanup: delay returning non-complete slices for on average ${1.5 * delay} seconds, ${this.identifier}.`);\n if (this.sandboxInventory.length > 0)\n this.warning('JobManager.evaluatorDownCleanup: When the evaluator is down there should not be any non-terminated sandboxes left.');\n // Call retVal.intr() to interrupt and cancel the timer inside a$sleepMs.\n return (this.evaluatorDownHandle = a$sleepMs(delay * (1.0 + Math.random()) * 1000)\n .finally(() => {\n const slicesToReturn = this.sliceInventory.filter((slice) => !(slice.isComplete || slice.isFinished));\n const reason = 'JobManager evaluatorDownCleanup';\n return this.supervisor.returnSlices(slicesToReturn, reason);\n }));\n }\n\n // _Idx\n //\n // assignSandbox, runSlice, runSliceOnSandbox\n // Functions chain and return non-awaited promises.\n //\n\n /**\n * Create a Sandbox.\n * Start it. Assign it. Add it to jobManager inventory.\n * @param {Slice} slice\n * @returns {Promise<Sandbox>}\n */\n async assignSandbox (slice)\n {\n try\n {\n // We want to await createSandbox so we don't create a large number of sandboxes all at once.\n const sandbox = await this.supervisor.createSandbox()\n .catch((error) => {\n // Catches exception from createSandbox.\n // Shound be rare unless evaluator or screensaver goes down.\n selectiveDebug2() && !this.supervisor.evaluator.down && console.debug('assignSandbox: createSandbox error', slice.identifier, error);\n slice.unReserve(); // reuse slice\n throw error;\n });\n selectiveDebug2() && console.debug('assignSandbox', slice.identifier);\n sandbox.slice = slice; // Must be before sandbox.assign; cf. fetchModule exception in handleRing2Message.\n return sandbox.assign(this)\n .then((assignedSandbox) => {\n selectiveDebug2() && console.debug('assignSandbox: success', sandbox.identifier);\n this.supervisor.sandboxInventory.push(assignedSandbox);\n return assignedSandbox;\n })\n .catch((error) => {\n // Exception is from sandbox.assign.\n // It could be a string coming from evaluator.\n this.error(`Failed to assign job to sandbox ${sandbox.identifier}`, error);\n slice.unReserve(); // reuse slice\n throw error;\n });\n }\n catch (error)\n {\n this.error(`JobManager.assignSandbox failed to find sandbox to run slice ${slice.identifier}`, error);\n throw error;\n }\n }\n\n /**\n * Create or reuse a sandbox and run a slice on it.\n * @param {Slice} slice - The slice to execute on a sandbox.\n * @returns {Promise<any>}\n */\n runSlice (slice)\n {\n try\n {\n selectiveDebug() && console.debug('runSlice:', slice.identifier, 'assigned/working', this.assignedSandboxes.length, this.workingSandboxes.length);\n\n if (this.supervisor.sliceTiming)\n slice['queueingDelta'] = Date.now();\n\n const _assignedSandboxes = this.assignedSandboxes;\n if (_assignedSandboxes.length > 0)\n {\n const sandbox = _assignedSandboxes[0];\n selectiveDebug2() && console.debug(`runSlice(assigned): sandbox ${sandbox.identifier}`, Date.now() - this.supervisor.lastTime);\n // When evaluator goes down, all sandboxes are terminated.\n // A ready slice has a finite lifetime and if exceeded may transition back to READY, in which\n // case we bail since it might already be in the process of being selected in roundRobinSlices.\n if (slice.isReady || sandbox.isTerminated)\n return Promise.resolve();\n slice.markAsWorking();\n sandbox.markAsWorking();\n sandbox.slice = slice; // Reusing assigned sandbox, so set the slice.\n return this.runSliceOnSandbox(slice, sandbox)\n .catch((error) => {\n // Catches exception from runSliceOnSandbox; s.b. rare.\n this.error(`JobManager.runSlice: Failure trying to run slice ${slice.identifier} on a assigned sandbox.`, error);\n return this.supervisor.returnSlice(slice, 'JobManager.runSlice');\n });\n }\n\n return this.assignSandbox(slice)\n .then((sandbox) => {\n selectiveDebug2() && console.debug(`runSlice: sandbox ${sandbox.identifier}`, Date.now() - this.supervisor.lastTime);\n // When evaluator goes down, all sandboxes are terminated.\n // A ready slice has a finite lifetime and if exceeded may transition back to READY, in which\n // case we bail since it might already be in the process of being selected in roundRobinSlices.\n if (slice.isReady || sandbox.isTerminated)\n return Promise.resolve();\n slice.markAsWorking();\n sandbox.markAsWorking();\n return this.runSliceOnSandbox(slice, sandbox);\n }, (error) => {\n // Catches exception from assignSandbox for error control flow.\n selectiveDebug() && console.error(`JobManager.runSlice: Failure from assignSandbox when trying to run slice ${slice.identifier} on a sandbox.`, error);\n })\n .catch((error) => {\n // Catches exception from runSliceOnSandbox; s.b. rare.\n this.error(`JobManager.runSlice: Failure trying to run slice ${slice.identifier} on a sandbox.`, error);\n return this.supervisor.returnSlice(slice, 'JobManager.runSlice');\n });\n }\n catch (error)\n {\n this.error(`JobManager.runSlice failed to run slice ${slice.identifier}`, error);\n return this.supervisor.returnSlice(slice, 'JobManager.runSlice_static_catch');\n }\n }\n\n /**\n * Execute slice on sandbox, collect results or handle errors, and clean up.\n * @param {Slice} slice - The slice to execute on the sandbox.\n * @param {Sandbox} sandbox - The sandbox on which to execute the slice.\n * @returns {Promise<any>}\n */\n runSliceOnSandbox (slice, sandbox)\n {\n const handleError = (error, tag) => {\n if (sandbox.isTerminated)\n {\n this.error(`The screensaver or evaluator may be down: slice ${slice.identifier}`, error);\n return this.supervisor.returnSlice(slice, `${tag}: sandbox terminated`);\n }\n else\n {\n this.error(`Failure in ${tag} for slice ${slice.identifier}`, error);\n const reason = (error.code === 'EPERM_ORIGIN' || error.code === 'EFETCH_BAD_ORIGIN' || error.code === 'EFETCH') ? error.code : tag;\n this.supervisor.returnSlice(slice, reason);\n }\n };\n debugging('supervisor') && console.debug(`runSliceOnSandbox ${sandbox.identifier} #(r/rsv/w/wsbx/sbx), ${this.supervisor.dbg.workingSliceSandboxStr}, #assigned, ${this.assignedSandboxes.length}, #localSBs, ${this.sandboxInventory.map(s => Number(s.id)).sort((x,y)=>x-y)}`);\n selectiveDebug() && console.debug('runSliceOnSandbox', sandbox.identifier, this.workingSliceCount, Date.now() - this.supervisor.lastTime);\n\n if (sandbox.isTerminated) // When evaluator goes down, all sandboxes are terminated.\n return Promise.resolve();\n\n this.sandboxEmit(sandbox, 'slice', slice.sliceNumber);\n\n if (this.supervisor.sliceTiming)\n {\n slice['queueingDelta'] = Date.now() - slice['queueingDelta'];\n slice['executionDelta'] = Date.now();\n }\n slice.startTime = Date.now();\n\n return sandbox.work()\n .finally(() => {\n if (this.supervisor.sliceTiming)\n {\n slice['executionDelta'] = Date.now() - slice['executionDelta'];\n slice['resultDelta'] = Date.now();\n }\n })\n .then((result) => {\n selectiveDebug2() && console.debug(`runSliceOnSandbox: sandbox${sandbox.id} - success: ${slice.identifier}`, Boolean(result), Date.now() - slice.startTime);\n\n if (sandbox.isTerminated)\n throw new Error(`Slice ${slice.identifier} completed work, but the sandbox ${sandbox.id} is terminated.`);\n\n // Allow reuse of sandbox up to a limit of dcpConfig.supervisor.sandbox.maxSandboxUse .\n sandbox.checkSandboxReUse();\n // Transition slice to COMPLETE\n slice.collectResult(result, true /*success*/);\n\n // Record result.\n return this.supervisor.recordResult(slice, sandbox)\n .catch((error) => {\n // Catches exception from supervisor.recordResult .\n return handleError(error, 'recordResult');\n });\n\n }, (error) => {\n // Catches exception from sandbox.work.\n // Kills sandbox and either retries the slice or transitions slice to FAILED and returns to scheduler.\n const reason = this.supervisor.handleSandboxWorkError(sandbox, slice, error);\n selectiveDebug() && console.error(`runSliceOnSandbox: Failure executing slice on sandbox: ${sandbox.identifier}`, reason, error);\n })\n .catch((error) => {\n // Catches exception from sandbox.work or then-clause.\n return handleError(error, 'runSliceOnSandbox');\n });\n }\n\n // _Idx\n //\n // removeSlice\n //\n\n /**\n * Remove slice from this.sliceInventory.\n * @param {Slice} slice - The slice to remove from this.sliceInventory.\n * @return {boolean} - False when slice is not in this.sliceInventory.\n */\n removeSlice (slice)\n {\n debugging('supervisor') && console.debug(`removeSlice slice ${slice.sliceNumber}`);\n return common.removeElement(this.sliceInventory, slice);\n }\n\n // _Idx\n //\n // fetchError, fetchJob, fetchSliceData\n //\n\n fetchError (error)\n {\n if (error.code === 'EPERM_ORIGIN')\n throw new BadOriginError(error);\n if (error.code === 'EFETCH')\n throw new RemoteFetchError(error);\n throw new UncaughtExceptionError(error);\n }\n\n /**\n * Fetch work function, work function arguments and possibly the range object describing the jobs slices..\n * @param {object} mpe - messagePayloadElement: job object returned by task-jobs.js . \n * @returns {Promise<*>}\n */\n fetchJob (mpe)\n {\n debugging('supervisor') && utils.dumpObject(mpe, 'JobManager.fetchJob: mpe', 512);\n\n const promises = [];\n try\n {\n // Get workFn.\n if (!mpe.workFunction)\n {\n const workFunctionPromise = this.fetchWorkFunctions(mpe.codeLocation)\n .then((workFunction) => {\n this.inputDataSize += workFunction.size;\n mpe.workFunction = workFunction.data;\n if (mpe.requirements.useStrict)\n mpe.useStrict = true;\n delete mpe.codeLocation;\n }, (error) => {\n this.fetchError(error);\n });\n promises.push(workFunctionPromise);\n }\n\n if (!mpe.mro && mpe.MROLocation)\n {\n const mroPromise = this.fetchData(mpe.MROLocation)\n .then((mro) => {\n // It's not ready for computing the inputDataSize yet -- that's done in JobManager.fetchSliceData .\n mpe.mro = mro.data;\n delete mpe.MROLocation;\n }, (error) => {\n this.fetchError(error);\n });\n promises.push(mroPromise)\n }\n\n // Get workFn args.\n if (!mpe.arguments && mpe.argumentsLocation)\n {\n mpe.arguments = new Array(mpe.argumentsLocation.length);\n for (let k = 0; k < mpe.argumentsLocation.length; k++)\n promises.push(this.fetchArguments(mpe.argumentsLocation[k].value)\n .then((arg) => {\n this.inputDataSize += arg.size;\n mpe.arguments[k] = arg.data;\n }, (error) => {\n this.fetchError(error);\n })\n );\n }\n\n return Promise.all(promises)\n .then(() => {\n if (mpe.argumentsLocation)\n delete mpe.argumentsLocation;\n return mpe;\n }, (error) => {\n this.fetchError(error);\n });\n }\n catch (error)\n {\n this.fetchError(error);\n }\n }\n\n /**\n * Look up slice.datumUri or use the range object this.mro, constructed by fetchJob.\n * @param {Slice} slice \n * @returns {Promise<{ data: *, size: number }>}\n */\n fetchSliceData (datumUri, slice)\n {\n try\n {\n if (!datumUri)\n {\n if (!this.mro)\n throw new Error('Must complete call to JobManager.fetchJob before calling JobManager.fetchSliceData.');\n /** XXXpfr @todo Inefficient, we're rehydrating the whole range for a single slice datum. */\n const ro = rehydrateRange(this.mro);\n // Slice numbers start at 1.\n const sliceDatum = ro[slice.sliceNumber - 1];\n debugging('supervisor') && console.debug(`Fetched mro datum: ${stringify(sliceDatum, 512)}`);\n return Promise.resolve({ data: sliceDatum, size: JSON.stringify(sliceDatum).length });\n }\n\n return this.fetchData(datumUri)\n .catch((error) => {\n throw this.fetchError(error);\n });\n }\n catch (error)\n {\n this.fetchError(error);\n }\n }\n\n // _Idx\n //\n // jobEmit, sandboxEmit, error, warning. countSliceStr, dumpSlices\n //\n\n /**\n * Safe event emitter on jobHandle.\n * @param {string} event\n * @param {...any} args\n */\n jobEmit(event, ...args)\n {\n this.supervisor.safeEmit(this.jobHandle, event, ...args);\n }\n\n /**\n * Safe event emitter on sandboxHandle.\n * @param {Sandbox} sandbox\n * @param {string} event\n * @param {...any} args\n */\n sandboxEmit(sandbox, event, ...args)\n {\n this.supervisor.safeEmit(sandbox.sandboxHandle, event, ...args);\n }\n\n /**\n * Error feedback to user.\n * @param {string} message\n * @param {Array<Error>|Error|string} [coreError]\n * @param {string} [additionalInfo]\n */\n error (message, coreError, additionalInfo)\n {\n this.supervisor.error(message, coreError, additionalInfo);\n }\n\n /**\n * Warning feedback to user.\n * @param {string[]} messages\n */\n warning (...messages)\n {\n this.supervisor.warning(...messages);\n }\n\n /**\n * Debugging helper.\n * @param {string} [tag='']\n * @returns {string}\n */\n countSliceStr (tag)\n {\n const { unassigned, ready, reserved, working, workdone, complete, failed, finished } = this.dumpSlices (tag, false, false);\n return `${tag}: u/r/rsv/w/wd/c/f/fsh ${unassigned}/${ready}/${reserved}/${working}/${workdone}/${complete}/${failed}/${finished}`;\n }\n\n /**\n * @param {URL|string} uri\n * @returns {Promise<*>}\n */\n fetchData (uri) { return fetchURIAndLength(uri, this.originManager, allowOriginsPurposes.fetchData); }\n /**\n * @param {URL|string} uri\n * @returns {Promise<*>}\n */\n fetchArguments (uri) { return fetchURIAndLength(uri, this.originManager, allowOriginsPurposes.fetchArguments); }\n /**\n * @param {URL|string} uri\n * @returns {Promise<*>}\n */\n fetchWorkFunctions (uri) { return fetchURIAndLength(uri, this.originManager, allowOriginsPurposes.fetchWorkFunctions); }\n\n /**\n * Debugging helper.\n * @param {string} [tag='']\n * @param {boolean} [details=false]\n * @param {boolean} [display=true]\n * @returns {{ ready, reserved, working, workdone, complete, failed, finished, unassigned }}\n */\n dumpSlices (tag = '', details = false, display = true)\n {\n if (display)\n console.log(`dumpSlices(${tag}): ${this.identifier}, sliceCount ${this.sliceInventory.length}, sandboxCount ${this.sandboxInventory.length}`, Date.now() - this.supervisor.lastTime);\n if (this.sliceInventory.length < 1) return { unassigned:0, ready:0, reserved:0, working:0, workdone:0, complete:0, failed:0, finished:0 };\n if (details)\n {\n // Detailed classification of slices.\n const unassigned = [], ready = [], reserved = [], working = [], workdone = [], complete = [], failed = [], finished = [];\n for (const slice of this.sliceInventory)\n {\n if (slice.isReady)\n ready.push(slice);\n else if (slice.isReserved)\n reserved.push(slice);\n else if (slice.isWorking)\n working.push(slice);\n else if (slice.isWorkDone)\n workdone.push(slice);\n else if (slice.isComplete)\n complete.push(slice);\n else if (slice.isFailed)\n failed.push(slice);\n else if (slice.isFinished)\n finished.push(slice);\n else if (slice.isUnassigned)\n unassigned.push(slice);\n else\n throw new Error(`dumpSlices: Unexpected kind a slice; ${slice.identifier}`);\n }\n if (display)\n {\n const dmpSlices = (name, slices) => {\n console.log(`-----${name}(${slices.length})-----------------------------------------------------------------`);\n for (const slice of slices)\n console.log(slice.identifier);\n }\n dmpSlices('unassigned', unassigned);\n dmpSlices('ready', ready);\n dmpSlices('reserved', reserved);\n dmpSlices('working', working);\n dmpSlices('workdone', workdone);\n dmpSlices('complete', complete);\n dmpSlices('failed', failed);\n dmpSlices('finished', finished);\n console.log('-----------------------------------------------------------------------------------');\n }\n return {\n ready: ready.length,\n reserved: reserved.length,\n working: working.length,\n workdone: workdone.length,\n complete: complete.length,\n failed: failed.length,\n finished: finished.length,\n unassigned: unassigned.length,\n };\n }\n else\n {\n // Quick classification of slices.\n const { ready,\n reserved,\n working,\n workdone,\n complete,\n failed,\n finished,\n unassigned } = common.sliceCounts(this.sliceInventory);\n if (display)\n {\n const dumpCount = (name, count) => {\n console.log(`-----${name}(${count})-----------------------------------------------------------------`);\n }\n dumpCount('unassigned', unassigned);\n dumpCount('ready', ready);\n dumpCount('reserved', reserved);\n dumpCount('working', working);\n dumpCount('workdone', workdone);\n dumpCount('complete', complete);\n dumpCount('failed', failed);\n dumpCount('finished', finished);\n console.log('-----------------------------------------------------------------------------------');\n }\n return { ready, reserved, working, workdone, complete, failed, finished, unassigned };\n }\n }\n\n}\nexports.JobManager = JobManager;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/supervisor2/job-manager.js?");
4739
4739
 
4740
4740
  /***/ }),
4741
4741
 
@@ -4798,7 +4798,7 @@ eval("/**\n * @file dcp/src/dcp-client/worker/supervisor2/rolling-statistics.js\
4798
4798
  /***/ ((__unused_webpack_module, exports, __webpack_require__) => {
4799
4799
 
4800
4800
  "use strict";
4801
- eval("/**\n * @file dcp/src/dcp-client/worker/supervisor2/sandbox2.js\n *\n * A sandbox that when constructed and assigned can do work for\n * a distributed slice. A sandbox runs for a single slice at a time.\n *\n * Usage (simplified...):\n * const sandbox = new Sandbox(this, { ...this.options.sandboxOptions });\n * await sandbox.start();\n * sandbox.slice = slice;\n * await sandbox.assign(jobManager);\n * return sandbox.work()\n * .then((result) => {\n * slice.collectResult(result, true);\n * sandbox.checkSandboxReUse();\n * this.supervisor.recordResult(slice)\n * })\n * .catch((error) => {\n * slice.collectResult(error, false);\n * const reason = this.supervisor.handleSandboxWorkError(sandbox, slice, error);\n * this.supervisor.returnSlice(slice, reason);\n * this.returnSandbox(sandbox);\n * });\n *\n * Debug flags:\n * Sandbox.debugWork = true // - turns off 30 second timeout to let user debug sandbox innards more easily\n * Sandbox.debugState = true // - logs all state transitions for this sandbox\n * Sandbox.debugEvents = true // - logs all events received from the sandbox\n *\n * Initial states:\n * UNREADY\n *\n * Terminal states:\n * TERMINATED\n *\n * Valid transitions:\n * ( sandbox.start )\n * UNREADY -> READYING -> READY_FOR_ASSIGN\n * READYING -> TERMINATED\n * ( sandbox.assign )\n * READY_FOR_ASSIGN -> ASSIGNING -> ASSIGNED\n * ASSIGNING -> TERMINATED\n * ( sandbox.markAsWorking )\n * ASSIGEND -> WORKING\n * ( sandbox.work )\n * WORKING -> ASSIGNED\n * -> TERMINATED\n * ( sandbox.terminate )\n * any -> TERMINATED\n *\n * @author Matthew Palma, mpalma@kingsds.network\n * Ryan Rossiter, ryan@kingsds.network\n * Wes Garland, wes@distributive.network\n * Paul, paul@distributive.network\n * @date May 2019\n * May 2019\n * Decemeber 2020\n * June, Dec 2022, Jan-May 2023\n * @module sandbox\n * @copyright Copyright (c) 2018-2023, Distributive Corp. All Rights Reserved\n */\n// @ts-check\n\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst dcp_timers = __webpack_require__(/*! dcp/common/dcp-timers */ \"./src/common/dcp-timers.js\");\nconst { assert, assertEq3 } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\nconst { Synchronizer } = __webpack_require__(/*! dcp/common/concurrency */ \"./src/common/concurrency.js\");\nconst nanoid = (__webpack_require__(/*! nanoid */ \"./node_modules/nanoid/index.browser.js\").nanoid);\nconst EventEmitter = __webpack_require__(/*! events */ \"./node_modules/events/events.js\");\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\nconst { Config } = __webpack_require__(/*! ./config */ \"./src/dcp-client/worker/supervisor2/config.js\");\nconst common = __webpack_require__(/*! ./common */ \"./src/dcp-client/worker/supervisor2/common.js\");\nconst { selectiveDebug, truncateAddress, timeDilation, selectiveDebug2 } = common;\nconst { stringify } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\n\n/**\n * Wraps console.debug to emulate debug module prefixing messages on npm.\n * @param {...any} args\n */\nconst debug = (...args) => {\n if (debugging())\n console.debug('Sandbox:', ...args);\n};\n\n// Sandbox states\nconst UNREADY = 'UNREADY' // No Sandbox (web worker, saworker, etc) has been constructed yet\nconst READYING = 'READYING' // Sandbox is being constructed and environment (bravojs, env) is being set up\nconst READY_FOR_ASSIGN = 'READY_FOR_ASSIGN' // Sandbox is ready to be assigned\nconst ASSIGNED = 'ASSIGNED' // Sandbox is assigned but not working\nconst ASSIGNING = 'ASSIGNING' // Sandbox is in the process of being ASSIGNED\nconst WORKING = 'WORKING' // Sandbox is working\nconst TERMINATED = 'TERMINATED' // Sandbox is terminated.\nconst EVAL_RESULT_PREFIX = 'evalResult::';\n\nclass SandboxError extends Error\n{\n /**\n * @param {string} errorCode\n * @param {string|Error} msg\n */\n constructor(errorCode, msg)\n {\n super((msg.constructor?.name === 'String') ? msg : msg['message']);\n /** @type {string} */\n this.errorCode = errorCode;\n if (msg.constructor?.name !== 'String')\n for (const prop of [ 'name', 'code', 'stack', 'lineNumber', 'columnNumber' ])\n if (msg[prop]) this[prop] = msg[prop];\n }\n}\nclass NoProgressError extends SandboxError { constructor(msg) { super('ENOPROGRESS', msg); } }\nclass SliceTooSlowError extends SandboxError { constructor(msg) { super('ESLICETOOSLOW', msg); } }\nclass UncaughtExceptionError extends SandboxError { constructor(msg) { super('EUNCAUGHT', msg); } }\n\n/** @typedef {import('./slice2').Slice} Slice */\n/** @typedef {import('./index').Supervisor} Supervisor */\n/** @typedef {import('./job-manager').JobManager} JobManager */\n/** @typedef {import('./module-cache').ModuleCache} ModuleCache */\n/** @typedef {import('dcp/utils/jsdoc-types').SandboxOptions} SandboxOptions */\n\n/**\n * Public event emitter.\n * https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/worker/worker-events.md\n */\nclass SandboxHandle extends EventEmitter\n{\n /** @type {{ id: number, public: { name: string, description: string, link: string }, jobManager: JobManager, slice: Slice }} */\n #info;\n\n /**\n * @constructor\n * @param {Sandbox} sandbox\n */\n constructor (sandbox)\n {\n super({ captureRejections: false });\n this.#info = sandbox.info;\n }\n /** @type {number} */\n get id () { return this.#info.id; }\n /** @type {{ name: string, description: string, link: string }} */\n get public () { return this.#info.public ?? { name: '<unassigned>', description: '', link: '' }; }\n /** @type {string} */\n get jobAddress () { return this.#info.jobManager?.address; }\n /** @type {number} */\n get sliceNumber () { return this.#info.slice?.sliceNumber ?? -1; }\n}\nexports.SandboxHandle = SandboxHandle;\n\n//\n// Index to functionality -- search for '_Idx' to toggle through the index.\n//\n// 1) class Sandbox\n// 2) checkSandboxReUse, postMessageToEvaluator, changeState,\n// and punctuatedTimer is expiremental for replacing hard-coded timeouts.\n// 3) start, describe, assign, applyRequirements, assignEvaluator\n// 4) eval, resetState, work, resetProgressTimeout, resetSliceTimeout\n// 5) handleRing0Message, handleRing1Message, handleRing2Message, handleRing3Message\n// 6) onmessage, onerror, terminate\n// 7) updateTime, resetSliceReport, sandboxEmit, error, warning\n//\n\n// _Idx\n//\n// class Sandbox\n//\n\nclass Sandbox extends EventEmitter\n{\n /**\n * A Sandbox (i.e. a worker sandbox) which executes distributed slices.\n *\n * @constructor\n * @param {Supervisor} supervisor\n * @param {SandboxOptions} options\n */\n constructor (supervisor, options)\n {\n super({ captureRejections: false });\n /** @type {Supervisor} */\n this.supervisor = supervisor;\n /** @type {ModuleCache} */\n this.moduleCache = supervisor.moduleCache;\n /** @type {SandboxOptions} */\n this.options = {\n ignoreNoProgress: false,\n ...options,\n SandboxConstructor: options.SandboxConstructor || (__webpack_require__(/*! ../evaluators */ \"./src/dcp-client/worker/evaluators/index.js\").BrowserEvaluator),\n }\n /** @type {Synchronizer} */\n this.state = new Synchronizer(UNREADY, [ UNREADY, READYING, READY_FOR_ASSIGN, ASSIGNING, ASSIGNED, WORKING, TERMINATED ]);\n\n /** @type {{ id: number, public: { name: string, description: string, link: string }, jobManager: JobManager, slice: Slice }} */\n this.info = {\n id: Sandbox.getNewId(),\n public: null,\n jobManager: null,\n slice: null,\n };\n\n /**\n * Event emitter containing info that describes the sandbox.\n * @type {SandboxHandle}\n */\n this.sandboxHandle = new SandboxHandle(this);\n\n /** Properties of type object. */\n this.evaluatorHandle = null;\n this.capabilities = null;\n this.progressTimeout = null;\n this.sliceTimeout = null;\n this.rejectionData = null;\n\n /** @type {number?} */\n this.progress = 100;\n /** @type {{ last: { deltaMs: number, value: any, throttledReports: number }, lastDeterministic: { deltaMs: number, progress: number, value: any, throttledReports: number } }} */\n this.progressReports = null; // cf. job-noProgress.js\n /** @type {object} */\n this.progressTimeout = null;\n /** @type {object} */\n this.sliceTimeout = null;\n\n /** @type {{ total: number, CPU: number, webGL: number, webGPU: number }} */\n this.sliceTimeReport = null;\n /** @type {number} */\n this.moduleInDataSize = 0; // Sandbox level input size; set during assign, never reset.\n /** @type {number} */\n this.sliceOutDataSize = 0; // Slice level output size; reset for every slice executed.\n\n /** @type {number?} */\n this.sliceStartTime = null;\n /** @type {number} */\n this.useCounter = 1; // Anticipating the initial use.\n /** @type {Config} */\n this.hive = new Config();\n\n ///** @type {((data: any) => Promise<void>)[]} */\n this.ringMessageHandlers = [\n this.handleRing0Message,\n this.handleRing1Message,\n this.handleRing2Message,\n this.handleRing3Message,\n ];\n\n this.resetSliceReport();\n }\n\n /** @type {number} */\n get id () { return this.info.id; }\n /** @type {{ name: string, description: string, link: string }} */\n get public () { return this.info.public; }\n /** @type {{ name: string, description: string, link: string }} */\n set public (data) { this.info.public = data; }\n /** @type {JobManager} */\n get jobManager () { return this.info.jobManager; }\n /** @type {string} */\n get jobAddress () { return this.jobManager?.address; }\n /** @type {Slice} */\n get slice () { return this.info.slice; }\n /** @type {Slice} */\n set slice (slice) { this.info.slice = slice; }\n /** @type {number} */\n get sliceNumber () { return this.slice ? this.slice.sliceNumber : -1; }\n /** @type {number} */\n get generalTimeout () { return 2 * this.hive.generalTimeout; }\n /** @type {number} */\n get punctuatedTimeout () { return this.hive.generalTimeout; }\n\n /**\n * Debug string that characterizes sandbox.\n * @type {string}\n */\n get identifier()\n {\n if (!this.jobAddress)\n return `${this.id}.${this.state}`;\n const address = truncateAddress(this.jobAddress);\n if (this.slice)\n return `${this.id}.${address}.${this.state}~${this.slice.sliceNumber}`;\n return `${this.id}.${address}.${this.state}`;\n }\n\n /** @returns {number} */\n static getNewId() { return Sandbox.idCounter++; }\n\n /** @type {boolean} */\n get isReadyForAssign () { return this.state.is(READY_FOR_ASSIGN); }\n /** @type {boolean} */\n get isAssigned () { return this.state.is(ASSIGNED); }\n /** @type {boolean} */\n get isWorking () { return this.state.is(WORKING); }\n /** @type {boolean} */\n get isTerminated () { return this.state.is(TERMINATED); }\n\n // _Idx\n //\n // checkSandboxReUse, postMessageToEvaluator, changeState,\n // punctuatedTimer is expiremental for replacing hard-coded timeouts.\n //\n\n /**\n * Mark WORKING sandbox as ASSIGNED in preparation for possible reuse.\n * Allow use of sandbox on a given job up to a limit of dcpConfig.supervisor.sandbox.maxSandboxUse .\n */\n checkSandboxReUse ()\n {\n selectiveDebug2() && console.debug(`Sandbox2.checkSandboxReUse: useCounter ${this.useCounter}, ${this.identifier}`);\n if (this.useCounter++ < this.hive.maxSandboxUse)\n {\n this.state.set(WORKING, ASSIGNED);\n this.sandboxEmit('ready');\n }\n else\n {\n this.terminate(false);\n common.removeElement(this.supervisor.sandboxInventory, this);\n }\n }\n\n /** Transitions: ASSIGNED --> WORKING. */\n markAsWorking ()\n {\n if (!this.isAssigned)\n throw new Error(`Sandbox ${this.identifier} is not ready to work`);\n this.state.set(ASSIGNED, WORKING);\n }\n \n /**\n * Safely post message to evaluator.\n * @param {object} message\n */\n postMessageToEvaluator (message)\n {\n if (this.isTerminated) // When evaluator goes down, all sandboxes are terminated.\n throw new Error(`postMessageToEvaluator: Sandbox ${this.identifier} has been terminated.`);\n return this.evaluatorHandle.postMessage(message);\n }\n \n /**\n * Safely change state.\n * @param {string} currentState\n * @param {string} nextState\n */\n changeState (currentState, nextState)\n {\n if (this.isTerminated) // When evaluator goes down, all sandboxes are terminated.\n throw new Error(`changeState: Sandbox ${this.identifier} has been terminated.`);\n this.state.set(currentState, nextState);\n }\n\n /** Upon fatal error return slice to scheduler. */\n returnSlice ()\n {\n selectiveDebug() && console.debug('Sandbox.returnSlice', this.identifier);\n return this.supervisor.returnSlice(this.slice, 'Sandbox.returnSlice');\n }\n\n /**\n * @callback cbFn\n * @returns {void}\n */\n\n /**\n * UNUSED.\n * Future work.\n * Replaces the timers in:\n * describe,\n * applyRequirements,\n * resetState,\n * The idea is to have a long timeout with a warning every\n * 6 seconds saying why it is waiting.\n * @param {cbFn} body\n * @param {string} waitMessage\n * @param {string} timerExpiredMessage\n * @returns {Promise<{ closeIntervalTimer: cbFn }>}\n */\n punctuatedTimer(body, waitMessage, timerExpiredMessage)\n {\n const that = this;\n return new Promise((resolve, reject) => {\n let intervalCounter = 0;\n let intervalHandle = null;\n function closeIntervalTimer()\n {\n if (intervalHandle !== null)\n dcp_timers.clearTimeout(intervalHandle);\n intervalHandle = null;\n }\n intervalHandle = dcp_timers.setInterval(() => {\n if (++intervalCounter > 12)\n {\n closeIntervalTimer();\n that.error(timerExpiredMessage);\n }\n that.warning(waitMessage);\n body();\n }, this.punctuatedTimeout)\n // Allow workers and localExec to exit.\n intervalHandle.unref();\n resolve({ closeIntervalTimer });\n });\n }\n\n // _Idx\n //\n // start, describe, assign, applyRequirements, assignEvaluator\n //\n\n /**\n * Readies the sandbox. This will result in the sandbox being ready and not assigned.\n * It will need to be assigned with a job before it is able to do work.\n * Sandbox.start will terminate the sandbox upon failure.\n * @todo maybe preload specific modules or let the cache pass in what modules to load?\n *\n * @returns {Promise<void>}\n * @throws on failure to ready\n */\n async start ()\n {\n debug('Sandbox.start begin');\n await this.supervisor.delayManager.nextDelay('sandboxStart');\n this.changeState(UNREADY, READYING);\n\n try\n {\n // RING 0\n this.evaluatorHandle = new this.options.SandboxConstructor({\n name: `DCP Sandbox #${this.id}`,\n });\n // Annoying! onerror terminates sandbox which can happen independent of whether the slice\n // is ok or not. Since we don't know, we have to return the slice when onerror is called\n // during sandbox.work .\n /** @todo XXXpfr Beware of onerror firing often. */\n this.evaluatorHandle.onerror = this.onerror.bind(this);\n\n const messageHandler = this.onmessage.bind(this);\n this.evaluatorHandle.onmessage = function onmessage(event)\n {\n const data = (event.data.serialized)\n ? kvin.parse(event.data.message)\n : kvin.unmarshal(event.data);\n messageHandler({ data });\n }\n\n const evaluatorPostMessage = this.evaluatorHandle.postMessage.bind(this.evaluatorHandle);\n this.evaluatorHandle.postMessage = function postMessage(message)\n {\n evaluatorPostMessage(kvin.marshal(message));\n }\n\n const that = this;\n this.evaluatorHandle.addEventListener('end', function sandbox$start$addEventListener() {\n selectiveDebug() && console.debug(\"END:Sandbox evaluatorHandle end-handler\", that.identifier, new Date());\n that.supervisor.evaluator.shuttingDown = true;\n that.terminate(true);\n });\n\n // Don't let an open sockets prevent clean worker exit.\n if (this.evaluatorHandle.unref)\n this.evaluatorHandle.unref();\n\n // Now in RING 1\n\n // Now in RING 2\n await this.describe();\n this.changeState(READYING, READY_FOR_ASSIGN);\n\n // Emit the 'sandbox' event on the worker event emitter.\n this.supervisor.safeEmit(this.supervisor.worker, 'sandbox', this.sandboxHandle);\n }\n catch (error)\n {\n if (this.isTerminated)\n debug(`Failed to start sandbox because it is already terminated: ${this.identifier}.\\n\\tMay be due to screensaver worker being down or evaluator was stopped.`);\n else\n {\n debug(`Failed to start sandbox ${this.identifier}.`, error.message); // FIX s.b. error\n this.terminate(false);\n }\n throw error;\n }\n }\n\n /**\n * Sends a post message to describe its capabilities.\n * Side effect: Sets the capabilities property of the current sandbox.\n *\n * @returns {Promise<any>} Resolves with the sandbox's capabilities.\n * Rejects with an error saying a response was not received.\n * @memberof Sandbox\n */\n describe ()\n {\n debugging('sandbox') && debug('Beginning to describe evaluator', this.identifier);\n const that = this;\n\n return new Promise(function sandbox$describePromise(resolve, reject) {\n let describeTimeout;\n\n if (that.isTerminated) // When evaluator goes down, all sandboxes are terminated.\n reject(new Error(`Sandbox ${that.identifier} has been terminated.`));\n\n if (that.evaluatorHandle === null)\n reject(new Error(`Evaluator has not been initialized: ${that.identifier}`));\n\n function sandbox$describe$success(data)\n {\n if (describeTimeout !== false)\n {\n dcp_timers.clearTimeout(describeTimeout);\n describeTimeout = false;\n\n const { capabilities } = data;\n if (typeof capabilities === 'undefined')\n reject(new Error(`Did not receive capabilities from describe response: ${that.identifier}`));\n that.capabilities = capabilities;\n\n debugging('sandbox') && debug('Evaluator has been described');\n resolve(capabilities);\n }\n }\n // Emitted by handleRing2Message.\n that.once('describe', sandbox$describe$success);\n\n describeTimeout = dcp_timers.setTimeout(function sandbox$describe$fail() {\n if (describeTimeout !== false)\n {\n describeTimeout = false;\n that.removeListener('describe', sandbox$describe$success);\n reject(new Error( `Describe message timed-out. No describe response was received from the describe command: ${that.identifier}`));\n }\n }, that.generalTimeout);\n // Allow workers and localExec to exit.\n describeTimeout.unref();\n\n const message = {\n request: 'describe',\n };\n that.postMessageToEvaluator(message);\n });\n }\n\n /**\n * This will assign the sandbox with a job, loading its sandbox code into the sandbox.\n * Sandbox.assign will not terminate the sandbox upon failure.\n * The sandbox will be terminated in JobManager.assignSandbox .\n * @param {JobManager} jobManager - The job manager that will be the owner of this sandbox.\n * @returns {Promise<Sandbox>}\n * @throws on initialization failure\n */\n async assign (jobManager)\n {\n if (!this.slice) // Design assumption.\n throw new Error(`Must have valid sandbox.slice before sandbox.assign is called: ${this.identifier}`);\n\n await this.supervisor.delayManager.nextDelay('sandboxAssign');\n debug('Sandbox.assign', this.identifier, Date.now() - this.supervisor.lastTime);\n\n try\n {\n this.changeState(READY_FOR_ASSIGN, ASSIGNING);\n this.info.jobManager = jobManager;\n this.job = this.jobManager.jobMessage;\n\n /* At this point, the worker has decided that this sandbox will be associated with a specific job. \n Therefore, we emit the SandboxHandle<job> event*/\n this.sandboxEmit('job', jobManager.jobHandle);\n\n assertEq3(this.job.address, this.jobAddress);\n assert(typeof this.job === 'object');\n assert(typeof this.job.requirements === 'object');\n assert(Array.isArray(this.job.dependencies));\n assert(Array.isArray(this.job.requirePath));\n\n // Extract public data from job, with defaults\n this.public = Object.assign({\n name: `Anonymous Job ${truncateAddress(this.jobAddress)}`,\n description: 'Discreetly helping make the world smarter.',\n link: 'https://distributed.computer/about',\n }, this.job.public);\n\n // Future: We may want other filename tags for appliances // RR Nov 2019\n\n // Important: The order of applying requirements before loading the sandbox code\n // is important for modules and sandbox code to set globals over the whitelist.\n await this.applyRequirements(this.job.requirements);\n //const _t0 = Date.now();\n await this.assignEvaluator();\n //console.log('Finished Sandbox.assignEvaluator', Date.now() - _t0);\n this.changeState(ASSIGNING, ASSIGNED);\n this.sandboxEmit('ready');\n }\n catch (error)\n {\n if (this.isTerminated)\n debug(`Failed to assign sandbox ${this.identifier} to evaluator because it is already terminated.\\n\\tMay be due to screensaver worker being down or evaluator was stopped.`);\n else\n {\n debug(`Failed to assign sandbox ${this.identifier} to evaluator.`);\n this.terminate(false);\n }\n throw error;\n }\n\n return this;\n }\n\n /**\n * Passes the job's requirements object into the sandbox so that the global access lists can be updated accordingly.\n * E.g. disallow access to OffscreenCanvas without environment.offscreenCanvas=true present.\n * Must be called after @start.\n *\n * @returns {Promise<void>} - resolves on success, rejects otherwise\n */\n applyRequirements (requirements)\n {\n assert(typeof requirements === 'object');\n const that = this;\n\n return new Promise(function sandbox$applyRequirementsPromise(resolve, reject) {\n let requirementTimeout;\n\n function sandbox$applyRequirements$success()\n {\n if (requirementTimeout !== false)\n {\n dcp_timers.clearTimeout(requirementTimeout);\n requirementTimeout = false;\n resolve();\n }\n }\n // Emitted by handleRing1Message.\n that.once('applyRequirementsDone', sandbox$applyRequirements$success);\n\n requirementTimeout = dcp_timers.setTimeout(function sandbox$finishApplySandboxRequirements$fail() {\n if (requirementTimeout !== false)\n {\n requirementTimeout = false;\n that.removeListener('applyRequirementsDone', sandbox$applyRequirements$success);\n reject(new Error(`applyRequirements never received 'applyRequirementsDone' response from sandbox: ${that.identifier}`));\n }\n }, that.generalTimeout);\n // Allow workers and localExec to exit.\n requirementTimeout.unref();\n\n const message = {\n requirements,\n request: 'applyRequirements',\n };\n that.postMessageToEvaluator(message);\n });\n }\n\n /**\n * Assign job to the evaluator.\n * @returns {Promise<any>} - resolves on success, rejects otherwise\n */\n assignEvaluator ()\n {\n debugging('sandbox') && console.debug('Begin assigning job to evaluator', this.identifier);\n const that = this;\n\n return new Promise(function sandbox$$assignEvaluatorPromise(resolve, reject) {\n function sandbox$assignEvaluator$success(event)\n {\n that.removeListener('reject', sandbox$assignEvaluator$fail);\n debugging('sandbox') && debug('Job assigned to evaluator');\n resolve(event);\n }\n\n function sandbox$assignEvaluator$fail(error)\n {\n that.removeListener('assigned', sandbox$assignEvaluator$success);\n that.error(`assignEvaluator failed(${that.identifier}): evaluator may be out of memory or the screensaver may be down.`, error);\n selectiveDebug() && console.debug('assignEvaluator failed', that.identifier, error);\n if (that.slice) // Normally the slice hasn't been set yet.\n that.returnSlice();\n reject(error);\n }\n\n // Emitted by handleRing2Message.\n that.once('assigned', sandbox$assignEvaluator$success);\n that.once('reject', sandbox$assignEvaluator$fail);\n\n // Had to add useStrict -- not sure if anything else was missed.\n const jobMessage = {\n address: that.job.address,\n arguments: that.job.arguments,\n dependencies: that.job.dependencies,\n modulePath: that.job.modulePath,\n public: that.job.public,\n requireModules: that.job.requireModules,\n requirePath: that.job.requirePath,\n workFunction: that.job.workFunction,\n useStrict: that.job.useStrict,\n };\n\n const message = {\n request: 'assign',\n job: jobMessage,\n sandboxConfig: that.hive.sandboxConfig,\n };\n that.postMessageToEvaluator(message);\n });\n }\n\n // _Idx\n //\n // eval, resetState, work, resetProgressTimeout, resetSliceTimeout\n //\n\n /**\n * Evaluates a string inside the sandbox.\n * @todo XXXpfr -- I don't understand how this gets called?\n * There's an old comment saying: \"no longer working though?\"\n *\n * @param {string} code - the code to evaluate in the sandbox\n * @param {string} filename - the name of the 'file' to help with debugging,\n * @returns {Promise<any>} - resolves with eval result on success, rejects otherwise\n */\n eval (code, filename)\n {\n const that = this;\n const msgId = nanoid();\n\n return new Promise(function sandbox$$eval$Promise(resolve, reject) {\n const eventId = EVAL_RESULT_PREFIX + msgId;\n\n function sandbox$eval$success(event)\n {\n that.removeListener('reject', sandbox$eval$fail);\n resolve(event);\n };\n\n function sandbox$eval$fail(error)\n {\n that.removeListener(eventId, sandbox$eval$success);\n reject(error);\n };\n\n that.once(eventId, sandbox$eval$success);\n that.once('reject', sandbox$eval$fail);\n\n const message = {\n request: 'eval',\n data: code,\n filename,\n msgId,\n };\n that.postMessageToEvaluator(message);\n });\n }\n\n /**\n * Resets the state of the bootstrap, without resetting the sandbox function if assigned.\n * Mostly used to reset the progress status before reusing a sandbox on another slice.\n * Must be called after @start.\n *\n * @returns {Promise<void>} - resolves with result on success, rejects otherwise\n */\n resetState ()\n {\n const that = this;\n assert(this.isWorking); // Design assumption.\n\n return new Promise(function sandbox$resetStatePromise(resolve, reject) {\n let resetStateTimeout;\n\n function sandbox$resetState$success ()\n {\n if (resetStateTimeout !== false)\n {\n dcp_timers.clearTimeout(resetStateTimeout);\n resetStateTimeout = false;\n resolve();\n }\n }\n that.once('resetStateDone', sandbox$resetState$success);\n\n resetStateTimeout = dcp_timers.setTimeout(function sandbox$resetState$fail() {\n if (resetStateTimeout !== false)\n {\n resetStateTimeout = false;\n that.removeListener('resetStateDone', sandbox$resetState$success);\n reject(new Error(`resetState never received resetStateDone event from sandbox: ${that.identifier}`));\n }\n }, that.generalTimeout);\n // Allow workers and localExec to exit.\n resetStateTimeout.unref();\n\n const message = {\n request: 'resetState',\n };\n that.postMessageToEvaluator(message);\n });\n }\n\n /**\n * Executes a slice received from the supervisor.\n * Must be called after this.start, this.assign and this.markAsWorking .\n * Sandbox.work will not terminate the sandbox upon failure.\n * The sandbox will be terminated in Supervisor.handleSandboxWorkError .\n * @returns {Promise<any>} - resolves with result on success, rejects otherwise\n */\n async work ()\n {\n const that = this;\n\n if (!this.slice) // Design assumption\n throw new Error(`Must have valid sandbox.slice before sandbox.assign is called: ${this.identifier}`);\n\n await this.supervisor.delayManager.nextDelay('sandboxWork');\n debug('Sandbox.work begin', this.identifier, Date.now() - this.supervisor.lastTime);\n\n if (this.isTerminated) // When evaluator goes down, all sandboxes are terminated.\n throw new Error(`Sandbox ${this.identifier} has been terminated.`);\n if (!this.isWorking)\n throw new Error(`Sandbox ${this.identifier} in Sandbox.work must be marked as working.`)\n\n // cf. DCP-1719,1720\n this.resetSliceReport();\n\n // Check that sandbox and slice have the same job.\n if (this.jobAddress !== this.slice.jobAddress)\n throw new Error(`Sandbox.work: sandbox ${this.identifier} and slice ${this.slice.identifier} are from different jobsz`);\n\n /** @todo Should sliceHnd just be replaced with { sandbox: this } since this.public is part of this? */\n let sliceHnd = { job: this.public, sandbox: this };\n await this.resetState();\n if (!this.slice)\n {\n this.error(`Slice for job ${this.jobAddress} vanished during work initialization - aborting`);\n return;\n }\n\n const { datum: inputDatum, error: dataError } = this.slice;\n if (dataError)\n {\n that.postWorkEmit('error', {\n message: dataError.message,\n stack: dataError.stack,\n name: this.public.name\n });\n }\n\n this.resetProgressTimeout();\n this.resetSliceTimeout();\n\n return new Promise(function sandbox$$workPromise(resolve, reject) {\n function sandbox$$work$success (event)\n {\n that.removeListener('reject', sandbox$$work$fail);\n resolve(event);\n }\n\n function sandbox$$work$fail (error)\n {\n that.removeListener('resolve', sandbox$$work$success);\n reject(error);\n }\n\n that.once('resolve', sandbox$$work$success);\n that.once('reject', sandbox$$work$fail);\n\n that.sliceStartTime = Date.now();\n that.slice.startTime = that.sliceStartTime;\n that.progress = null;\n that.progressReports = {\n last: undefined,\n lastDeterministic: undefined,\n };\n\n that.resetProgressTimeout();\n that.resetSliceTimeout();\n that.emit('start', sliceHnd);\n\n if (dataError)\n {\n that.removeListener('resolve', sandbox$$work$success);\n that.removeListener('reject', sandbox$$work$fail);\n dcp_timers.setTimeout(() => reject(dataError), 0)\n }\n else\n {\n // Do the work.\n const message = { request: 'main', data: inputDatum, };\n that.postMessageToEvaluator(message);\n }\n })\n .then(async function sandbox$$work$then(event) {\n // Tell supervisor sandbox slot is available.\n that.slice.markAsWorkDone();\n\n selectiveDebug2() && console.debug('Sandbox.sliceFinish', that.identifier, event?.timeReport);\n that.sandboxEmit('sliceEnd', that.slice?.sliceNumber)\n that.emit('complete', that.jobAddress);\n\n // Reset slice property.\n that.slice = null;\n\n // JobManager.runSliceOnSandbox will transition WORKDONE -> ASSIGNED\n return event;\n })\n .catch(async function sandbox$$work$catch(error) {\n selectiveDebug() && console.debug('Sandbox.work catch', that.identifier, error);\n // Tell supervisor sandbox slot is available.\n if (that.slice)\n that.slice.markAsWorkDone();\n // Current sandbox will not be reused.\n // Do not overwrite that.slice because it is needed in subsequent error reporting.\n\n if (error instanceof NoProgressError)\n {\n const payload = {\n name: that.public.name,\n message: error.message,\n timestamp: Date.now() - that.sliceStartTime,\n };\n that.postWorkEmit('error', payload);\n that.postWorkEmit('noProgress', { ...payload, progressReports: that.progressReports });\n }\n if (error.name === 'EWORKREJECT')\n that.handleRejectedWork(that.sliceTimeReport);\n\n // Otherwise sandbox will be terminated in Supervisor.handleSandboxWorkError\n debugging('sandbox') && debug(`Sandbox ${that.identifier} failed to execute slice`, error);\n\n throw error;\n });\n }\n\n resetProgressTimeout()\n {\n const that = this;\n\n if (this.progressTimeout)\n dcp_timers.clearTimeout(this.progressTimeout);\n\n this.progressTimeout = dcp_timers.setTimeout(function sandbox$ProgressTimeout() {\n if (that.options.ignoreNoProgress)\n return that.warning('ENOPROGRESS silenced by localExec: In a remote worker, this slice would be stopped for not calling progress frequently enough.');\n\n that.emit('reject', new NoProgressError(`No progress event was received in the last ${that.hive.progressTimeout / 1000} seconds.`));\n }, this.hive.progressTimeout * timeDilation);\n // Allow workers and localExec to exit.\n this.progressTimeout.unref();\n }\n\n resetSliceTimeout()\n {\n const that = this;\n\n if (this.sliceTimeout)\n dcp_timers.clearTimeout(this.sliceTimeout);\n\n this.sliceTimeout = dcp_timers.setTimeout(function sandbox$SliceTimeout() {\n if (Sandbox.debugWork)\n return that.warning('Sandbox.debugWork: Ignoring slice timeout');\n\n that.emit('reject', new SliceTooSlowError(`Slice took longer than ${that.hive.sliceTimeout / 1000} seconds.`));\n }, this.hive.sliceTimeout * timeDilation);\n // Allow workers and localExec to exit.\n this.sliceTimeout.unref();\n }\n\n /**\n * Send payload to the workEmit endpoint in the event router.\n * @param {string} eventName\n * @param {*} payload\n * @returns {Promise<*>}\n */\n postWorkEmit (eventName, payload)\n {\n // Need to check if the sandbox hasn't been assigned a slice yet.\n if (!this.slice)\n this.error('Sandbox not assigned a slice before sending workEmit message to scheduler', payload, `'workEmit' event originates from '${eventName}' event`);\n else\n {\n const slice = this.slice;\n // Authorization should always be valid.\n if (!slice.authorizationMessage)\n this.warning(`workEmit: missing authorization message for slice ${slice.identifier}`);\n else\n {\n const workEmitPayload = {\n eventName,\n payload,\n job: slice.jobAddress,\n slice: slice.sliceNumber,\n worker: this.supervisor.workerId,\n authorizationMessage : slice.authorizationMessage,\n };\n return this.supervisor.dcp4.safeWorkEmit(workEmitPayload, `Failed to send workEmit (${eventName}) payload for slice ${slice.identifier}`)\n .then((success) => {\n if (!success)\n this.warning(`Message sent to workEmit is unauthorized; not accepted '${eventName}'`);\n });\n }\n }\n }\n\n /**\n * Save rejected slice timeReport data in this.slice.rejectedTimeReport, then when needed in\n * Supervisor.recordResult, merge this.slice.rejectedTimeReport into this.slice.timeReport.\n * @param {{ total: number, CPU: number, webGL: number, webGPU: number }} timeReport\n */\n handleRejectedWork (timeReport)\n {\n selectiveDebug() && console.debug('handleRejectedWork', this.identifier);\n // If the slice already has rejectedTimeReport, add this timeReport to it.\n // If not, assign this timeReport to slices rejectedTimeReport property\n if (this.slice)\n {\n if (!this.slice.rejectedTimeReport)\n this.slice.rejectedTimeReport = timeReport;\n else\n {\n ['total', 'CPU', 'webGL', 'webGPU'].forEach((key) => {\n if (timeReport[key])\n this.slice.rejectedTimeReport[key] += timeReport[key];\n });\n }\n }\n }\n\n /**\n * Attach CGIO to result returned by a slice workFn.\n * @param {*} completeData - results\n */\n attachCGIOToResult (completeData)\n {\n if (!completeData)\n throw new Error('Slice result is not ready'); // Should never fire.\n if (completeData['timeReport'])\n throw new Error('Slice result already has timeReport'); // Should never fire.\n if (completeData['dataReport'])\n throw new Error('Slice result already has dataReport'); // Should never fire.\n if (this.listenerCount('resolve') > 0)\n {\n completeData['timeReport'] = this.sliceTimeReport;\n completeData['dataReport'] = {\n InDataSize: this.moduleInDataSize + this.jobManager.inputDataSize + this.slice.inputDataSize,\n OutDataSize: this.sliceOutDataSize,\n };\n this.emit('resolve', completeData);\n selectiveDebug() && console.debug('attachCGIOToResult', this.moduleInDataSize, this.jobManager.inputDataSize, this.slice.inputDataSize, completeData['dataReport'].InDataSize);\n }\n else\n {\n // If there is no internal listener for 'resolve', the slice was rejected\n // and we need to update this.slice.rejectedTimeReport appropriately.\n this.handleRejectedWork(this.sliceTimeReport);\n }\n // Clear time and data reports so we can catch mistaken writes.\n this.sliceTimeReport = null;\n this.sliceOutDataSize = 0;\n }\n\n // _Idx\n //\n // handleRing0Message, handleRing1Message, handleRing2Message, handleRing3Message\n //\n\n async handleRing0Message(data) // eslint-disable-line require-await\n {\n debugging('ring0') && debug('Ring0', this.identifier, data.request);\n\n switch (data.request)\n {\n case 'scriptLoaded':\n if(data.result !== \"success\")\n this.onerror(data);\n break;\n case 'error':\n debug('Sandbox error in ring0', data.error);\n this.rejectWithCleanup('during initialization', data.error);\n break;\n default:\n this.error('Received unhandled request from sandbox: ' + data.request, null, `data: ${ JSON.stringify(data)}`);\n break;\n }\n }\n\n async handleRing1Message(data) // eslint-disable-line require-await\n {\n debugging('ring1') && debug('Ring1', this.identifier, data.request);\n\n switch (data.request)\n {\n case 'applyRequirementsDone':\n // emit internally\n this.emit(data.request, data)\n break;\n default:\n this.error('Received unhandled request from sandbox ring 1: ' + data.request, null, `data: ${ JSON.stringify(data)}`);\n break; \n }\n }\n\n async handleRing2Message(data)\n {\n debugging('ring2') && debug('Ring2', this.identifier, data.request);\n\n switch (data.request)\n {\n case 'dependency': {\n try\n {\n const moduleData = await this.moduleCache.fetchModule(data.data, this.jobAddress);\n // Success! Restore this['packageManager'] delay to retryMinSleepMs (currently 32ms.)\n // Is there a better way to reset than explicit calls?\n this.supervisor.delayManager.resetEBO('packageManager');\n // Send module data to be evaluator.\n const message = {\n request: 'moduleGroup',\n data: moduleData,\n id: data.id,\n };\n // Module data is dynamic since it may only be required in a conditional branch.\n // Moreover, on a long job, the published module itself may be updated on the scheduler.\n const moduleLength = kvin.stringify(moduleData).length; /** @TODO - fix per DCP-3750 */\n this.moduleInDataSize += moduleLength;\n selectiveDebug() && console.debug('Sandbox.Ring2.fetchModule size', this.moduleInDataSize, moduleLength);\n this.postMessageToEvaluator(message);\n }\n catch (error)\n {\n /*\n * In the event of an error here, we want to let the client know there was a problem in\n * loading their module. In principle we shouldn't need a valid sandbox.slice at sandbox.assign.\n * However, in the implementation of Sup2 there is precisely 1 callsite of sandbox.assign and\n * we do have an associated slice at this point. So we make the assumption that sandbox.slice\n * is valid here.\n */\n if (!this.slice) // Design assumption\n throw new Error(`Must have valid slice in sandbox before sandbox.assign is called: ${this.identifier}`);\n\n const payload = {\n name: error.name,\n message: error.message,\n timestamp: error.timestamp ? error.timestamp : new Date(),\n };\n\n this.postWorkEmit('error', payload);\n this.emit('reject', error);\n\n debugging() && console.debug(`Sandbox.Ring2: fetchModule failed ${this.identifier}`, payload, error, Date.now() - this.supervisor.lastTime);\n\n // Close packageManager to start the connection reconnect logic.\n // Should we do a retry loop with fetchModule too?\n this.supervisor.dcp4.resetConnection('packageManager');\n }\n break;\n }\n case 'error':\n /*\n * Ring 2 error messages will only fire for problems inside of the worker that are separate from\n * the work function. In most cases there are other handlers for situations where 'error' may be emitted\n * such as timeouts if the expected message isn't recieved.\n */\n debug('Sandbox error in ring2', data.error);\n this.rejectWithCleanup('during assignment and dependency resolution', data.error);\n break;\n case 'describe':\n case 'evalResult':\n case 'resetStateDone':\n case 'assigned':\n this.emit(data.request, data); // emit internally\n break;\n case 'reject':\n this.emit('reject', data.error); // emit internally\n break;\n default:\n this.error(`Received unhandled request from sandbox ring 2. Data: ${JSON.stringify(data, null, 2)}`);\n break;\n }\n }\n\n async handleRing3Message(data) // eslint-disable-line require-await\n {\n debugging('ring3') && debug('Ring3', this.identifier, data.request);\n\n switch (data.request)\n {\n case 'complete':\n dcp_timers.clearTimeout(this.progressTimeout);\n dcp_timers.clearTimeout(this.sliceTimeout);\n this.progressTimeout = this.sliceTimeout = null;\n\n if (this.progress === null)\n {\n if (this.options.ignoreNoProgress)\n this.warning(\"ENOPROGRESS silenced by localExec: Progress was not called during this slice's execution, in a remote sandbox this would cause the slice to fail.\");\n else\n {\n // If a progress update was never received (progress === null) then reject\n this.emit('reject', new NoProgressError('Sandbox never emitted a progress event.'));\n this.handleRejectedWork(this.sliceTimeReport);\n break;\n }\n }\n \n this.progress = 100;\n this.sliceOutDataSize += kvin.stringify(data.result).length; /** @TODO - fix per DCP-3750 */\n this.attachCGIOToResult(data);\n break;\n case 'progress':\n {\n const { progress, indeterminate, throttledReports, value } = data;\n this.progress = progress;\n // cf. job-noProgress.js\n const progressReport = {\n deltaMs: Date.now() - this.sliceStartTime,\n progress,\n value,\n throttledReports,\n }\n this.progressReports.last = progressReport;\n if (!indeterminate)\n this.progressReports.lastDeterministic = progressReport;\n\n this.resetProgressTimeout();\n this.sandboxEmit('progress', indeterminate || progress < 0 || progress > 100 ? undefined : progress);\n break;\n }\n case 'noProgress':\n this.emit('reject', new NoProgressError(data.message));\n break;\n case 'console':\n data.payload.message = kvin.marshal(data.payload.message);\n this.sliceOutDataSize += JSON.stringify(data.payload.message).length; /** @TODO - fix per DCP-3750 */\n this.postWorkEmit('console', data.payload);\n break;\n case 'emitEvent': /* ad-hoc event from the sandbox (work.emit) */\n this.postWorkEmit('custom', data.payload);\n break;\n case 'measurement':\n this.updateTime(data);\n break;\n case 'sandboxError': /* the sandbox itself has an error condition */\n debug(`Ring3 received a 'sandboxError' event for sandbox ${this.identifier}`, data.error);\n this.emit('sandboxError', data.error);\n this.rejectWithCleanup('internal sandbox error while executing work function', data.error);\n break;\n case 'workError': /* the work function threw/rejected */\n debug(`Ring3 received a 'workError' event for sandbox ${this.identifier}`, data.error);\n this.postWorkEmit('error', data.error);\n const wrappedError = new UncaughtExceptionError(data.error);\n this.rejectWithCleanup('error while executing work function', wrappedError);\n break;\n default:\n this.error('Received unhandled request from sandbox ring 3: ' + data.request, null, `data: ${ JSON.stringify(data)}`);\n break; \n }\n }\n\n /**\n * Try to send the error back to the reject handler in Sandbox.work.\n * But if the reject handler is not available (s.b. rare) then cleanup, emit error and throw.\n * @param {string} message\n * @param {Error|string} error\n */\n rejectWithCleanup (message, error)\n {\n if (this.listenerCount('reject') > 0)\n this.emit('reject', error);\n else\n {\n this.terminate(false);\n this.error(`Sandbox ${this.identifier} ${message}`, error);\n throw error;\n }\n }\n\n // _Idx\n //\n // onmessage, onerror, terminate\n //\n\n /**\n * Handles progress and completion events from sandbox.\n * Unless explicitly returned out of this function will re-emit the event\n * where the name of the event is event.data.request.\n *\n * @param {object} event - event received from the evaaluator sandbox\n * @returns {Promise<void>}\n */\n onmessage (event)\n {\n debugging('event') && debug('onmessage-event', event.data.ringSource);\n if (Sandbox.debugEvents)\n console.debug('sandbox - eventDebug:', { id: this.id, state: this.state.valueOf(), event: JSON.stringify(event) });\n\n const { data } = event;\n const ringLevel = data.ringSource\n\n // Give the data to a handler depending on ring level\n if (ringLevel === -1)\n {\n /** @todo XXXpfr Beware of this happening often. */\n this.error(`Message sent directly from raw postMessage for event ${event}. Terminating worker...`);\n this.terminate(true);\n }\n else\n {\n const handler = this.ringMessageHandlers[ringLevel];\n if (handler)\n return handler.call(this, data.value);\n this.warning(`No handler defined for message from ring ${ringLevel} for event ${event}.`);\n }\n }\n\n /**\n * Error handler for the internal sandbox.\n * Emits error event that gets handled up in the Worker class.\n */\n onerror (event)\n {\n /** @todo XXXpfr Beware of onerror firing often. */\n this.error(`Sandbox.onerror emitted an error: ${event}`);\n this.terminate(true, true);\n }\n\n /**\n * Clears the timeout and terminates the sandbox and sometimes emits a reject event.\n *\n * @param {boolean} [reject=true] - if true emit reject event\n * @param {boolean} [immediate=false] - passed to terminate, used by standaloneWorker to immediately close the connection\n */\n terminate (reject = true, immediate = false)\n {\n if (this.slice)\n this.returnSlice();\n if (this.isTerminated)\n return;\n\n selectiveDebug() && console.debug(`Terminate sandbox ${this.identifier}`);\n\n this.state = new Synchronizer(TERMINATED, [ UNREADY, READYING, READY_FOR_ASSIGN, ASSIGNING, ASSIGNED, WORKING, TERMINATED ]);\n\n dcp_timers.clearTimeout(this.progressTimeout);\n dcp_timers.clearTimeout(this.sliceTimeout);\n this.progressTimeout = this.sliceTimeout = null;\n\n if (this.evaluatorHandle && typeof this.evaluatorHandle.terminate === 'function')\n {\n try\n {\n this.evaluatorHandle.terminate(immediate);\n }\n catch (e)\n {\n this.error(`Error terminating sandbox ${this.id}:`, e);\n }\n finally\n {\n this.evaluatorHandle = null;\n }\n }\n\n if (reject)\n this.emit('reject', new Error(`Sandbox ${this.identifier} was terminated.`));\n\n this.sandboxEmit('end');\n }\n\n // _Idx\n //\n // updateTime, resetSliceReport, sandboxEmit, error, warning\n //\n\n /**\n * ringNPostMessage can send a `measurement` request and update these\n * totals.\n */\n updateTime (measurementEvent)\n {\n ['total', 'CPU', 'webGL', 'webGPU'].forEach((key) => {\n if (measurementEvent[key])\n this.sliceTimeReport[key] += measurementEvent[key];\n });\n }\n\n /**\n * Start over sandbox work timers.\n */\n resetSliceReport ()\n {\n this.sliceTimeReport = {\n total: 0,\n CPU: 0,\n webGL: 0,\n webGPU: 0,\n };\n this.sliceOutDataSize = 0;\n }\n\n /**\n * Safe event emitter on sandboxHandle.\n * @param {string} event\n * @param {...any} args\n */\n sandboxEmit(event, ...args)\n {\n this.supervisor.safeEmit(this.sandboxHandle, event, ...args);\n }\n\n /**\n * Error feedback to user.\n * @param {string} message\n * @param {Array<Error>|Error|string} [coreError]\n * @param {string} [additionalInfo]\n * @param {boolean} [supressStack=false]\n */\n error (message, coreError, additionalInfo, supressStack = false)\n {\n this.supervisor.error(message, coreError, additionalInfo, supressStack);\n }\n\n /**\n * Warning feedback to user.\n * @param {string[]} messages\n */\n warning (...messages)\n {\n this.supervisor.warning(...messages);\n }\n}\n\nSandbox.idCounter = 1;\nSandbox.debugWork = false;\nSandbox.debugState = false;\nSandbox.debugEvents = false;\n\nexports.Sandbox = Sandbox;\nexports.SandboxError = SandboxError;\nexports.NoProgressError = NoProgressError;\nexports.SliceTooSlowError = SliceTooSlowError;\nexports.UncaughtExceptionError = UncaughtExceptionError;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/supervisor2/sandbox2.js?");
4801
+ eval("/**\n * @file dcp/src/dcp-client/worker/supervisor2/sandbox2.js\n *\n * A sandbox that when constructed and assigned can do work for\n * a distributed slice. A sandbox runs for a single slice at a time.\n *\n * Usage (simplified...):\n * const sandbox = new Sandbox(this, { ...this.options.sandboxOptions });\n * await sandbox.start();\n * sandbox.slice = slice;\n * await sandbox.assign(jobManager);\n * return sandbox.work()\n * .then((result) => {\n * slice.collectResult(result, true);\n * sandbox.checkSandboxReUse();\n * this.supervisor.recordResult(slice)\n * })\n * .catch((error) => {\n * slice.collectResult(error, false);\n * const reason = this.supervisor.handleSandboxWorkError(sandbox, slice, error);\n * this.supervisor.returnSlice(slice, reason);\n * this.returnSandbox(sandbox);\n * });\n *\n * Debug flags:\n * Sandbox.debugWork = true // - turns off 30 second timeout to let user debug sandbox innards more easily\n * Sandbox.debugState = true // - logs all state transitions for this sandbox\n * Sandbox.debugEvents = true // - logs all events received from the sandbox\n *\n * Initial states:\n * UNREADY\n *\n * Terminal states:\n * TERMINATED\n *\n * Valid transitions:\n * ( sandbox.start )\n * UNREADY -> READYING -> READY_FOR_ASSIGN\n * READYING -> TERMINATED\n * ( sandbox.assign )\n * READY_FOR_ASSIGN -> ASSIGNING -> ASSIGNED\n * ASSIGNING -> TERMINATED\n * ( sandbox.markAsWorking )\n * ASSIGEND -> WORKING\n * ( sandbox.work )\n * WORKING -> ASSIGNED\n * -> TERMINATED\n * ( sandbox.terminate )\n * any -> TERMINATED\n *\n * @author Matthew Palma, mpalma@kingsds.network\n * Ryan Rossiter, ryan@kingsds.network\n * Wes Garland, wes@distributive.network\n * Paul, paul@distributive.network\n * @date May 2019\n * May 2019\n * Decemeber 2020\n * June, Dec 2022, Jan-May 2023\n * @module sandbox\n * @copyright Copyright (c) 2018-2023, Distributive Corp. All Rights Reserved\n */\n// @ts-check\n\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst dcp_timers = __webpack_require__(/*! dcp/common/dcp-timers */ \"./src/common/dcp-timers.js\");\nconst { assert, assertEq3 } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\nconst { Synchronizer } = __webpack_require__(/*! dcp/common/concurrency */ \"./src/common/concurrency.js\");\nconst nanoid = (__webpack_require__(/*! nanoid */ \"./node_modules/nanoid/index.browser.js\").nanoid);\nconst EventEmitter = __webpack_require__(/*! events */ \"./node_modules/events/events.js\");\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\nconst { Config } = __webpack_require__(/*! ./config */ \"./src/dcp-client/worker/supervisor2/config.js\");\nconst common = __webpack_require__(/*! ./common */ \"./src/dcp-client/worker/supervisor2/common.js\");\nconst { selectiveDebug, truncateAddress, timeDilation, selectiveDebug2 } = common;\nconst { stringify } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\n\n/**\n * Wraps console.debug to emulate debug module prefixing messages on npm.\n * @param {...any} args\n */\nconst debug = (...args) => {\n if (debugging())\n console.debug('Sandbox:', ...args);\n};\n\n// Sandbox states\nconst UNREADY = 'UNREADY' // No Sandbox (web worker, saworker, etc) has been constructed yet\nconst READYING = 'READYING' // Sandbox is being constructed and environment (bravojs, env) is being set up\nconst READY_FOR_ASSIGN = 'READY_FOR_ASSIGN' // Sandbox is ready to be assigned\nconst ASSIGNED = 'ASSIGNED' // Sandbox is assigned but not working\nconst ASSIGNING = 'ASSIGNING' // Sandbox is in the process of being ASSIGNED\nconst WORKING = 'WORKING' // Sandbox is working\nconst TERMINATED = 'TERMINATED' // Sandbox is terminated.\nconst EVAL_RESULT_PREFIX = 'evalResult::';\n\nclass SandboxError extends Error\n{\n /**\n * @param {string} errorCode\n * @param {string|Error} msg\n */\n constructor(errorCode, msg)\n {\n super((msg.constructor?.name === 'String') ? msg : msg['message']);\n /** @type {string} */\n this.errorCode = errorCode;\n if (msg.constructor?.name !== 'String')\n for (const prop of [ 'name', 'code', 'stack', 'lineNumber', 'columnNumber' ])\n if (msg[prop]) this[prop] = msg[prop];\n }\n}\nclass NoProgressError extends SandboxError { constructor(msg) { super('ENOPROGRESS', msg); } }\nclass SliceTooSlowError extends SandboxError { constructor(msg) { super('ESLICETOOSLOW', msg); } }\nclass UncaughtExceptionError extends SandboxError { constructor(msg) { super('EUNCAUGHT', msg); } }\n\n/** @typedef {import('./slice2').Slice} Slice */\n/** @typedef {import('./index').Supervisor} Supervisor */\n/** @typedef {import('./job-manager').JobManager} JobManager */\n/** @typedef {import('./module-cache').ModuleCache} ModuleCache */\n/** @typedef {import('dcp/utils/jsdoc-types').SandboxOptions} SandboxOptions */\n\n/**\n * Public event emitter.\n * https://gitlab.com/Distributed-Compute-Protocol/dcp-docs-wes/-/blob/wip/worker/worker-events.md\n */\nclass SandboxHandle extends EventEmitter\n{\n /** @type {{ id: number, public: { name: string, description: string, link: string }, jobManager: JobManager, slice: Slice }} */\n #info;\n\n /**\n * @constructor\n * @param {Sandbox} sandbox\n */\n constructor (sandbox)\n {\n super({ captureRejections: false });\n this.#info = sandbox.info;\n }\n /** @type {number} */\n get id () { return this.#info.id; }\n /** @type {{ name: string, description: string, link: string }} */\n get public () { return this.#info.public ?? { name: '<unassigned>', description: '', link: '' }; }\n /** @type {string} */\n get jobAddress () { return this.#info.jobManager?.address; }\n /** @type {number} */\n get sliceNumber () { return this.#info.slice?.sliceNumber ?? -1; }\n}\nexports.SandboxHandle = SandboxHandle;\n\n//\n// Index to functionality -- search for '_Idx' to toggle through the index.\n//\n// 1) class Sandbox\n// 2) checkSandboxReUse, postMessageToEvaluator, changeState,\n// and punctuatedTimer is expiremental for replacing hard-coded timeouts.\n// 3) start, describe, assign, applyRequirements, assignEvaluator\n// 4) eval, resetState, work, resetProgressTimeout, resetSliceTimeout\n// 5) handleRing0Message, handleRing1Message, handleRing2Message, handleRing3Message\n// 6) onmessage, onerror, terminate\n// 7) updateTime, resetSliceReport, sandboxEmit, error, warning\n//\n\n// _Idx\n//\n// class Sandbox\n//\n\nclass Sandbox extends EventEmitter\n{\n /**\n * A Sandbox (i.e. a worker sandbox) which executes distributed slices.\n *\n * @constructor\n * @param {Supervisor} supervisor\n * @param {SandboxOptions} options\n */\n constructor (supervisor, options)\n {\n super({ captureRejections: false });\n /** @type {Supervisor} */\n this.supervisor = supervisor;\n /** @type {ModuleCache} */\n this.moduleCache = supervisor.moduleCache;\n /** @type {SandboxOptions} */\n this.options = {\n ignoreNoProgress: false,\n ...options,\n SandboxConstructor: options.SandboxConstructor || (__webpack_require__(/*! ../evaluators */ \"./src/dcp-client/worker/evaluators/index.js\").BrowserEvaluator),\n }\n /** @type {Synchronizer} */\n this.state = new Synchronizer(UNREADY, [ UNREADY, READYING, READY_FOR_ASSIGN, ASSIGNING, ASSIGNED, WORKING, TERMINATED ]);\n\n /** @type {{ id: number, public: { name: string, description: string, link: string }, jobManager: JobManager, slice: Slice }} */\n this.info = {\n id: Sandbox.getNewId(),\n public: null,\n jobManager: null,\n slice: null,\n };\n\n /**\n * Event emitter containing info that describes the sandbox.\n * @type {SandboxHandle}\n */\n this.sandboxHandle = new SandboxHandle(this);\n\n /** Properties of type object. */\n this.evaluatorHandle = null;\n this.capabilities = null;\n this.progressTimeout = null;\n this.sliceTimeout = null;\n this.rejectionData = null;\n\n /** @type {number?} */\n this.progress = 100;\n /** @type {{ last: { deltaMs: number, value: any, throttledReports: number }, lastDeterministic: { deltaMs: number, progress: number, value: any, throttledReports: number } }} */\n this.progressReports = null; // cf. job-noProgress.js\n /** @type {object} */\n this.progressTimeout = null;\n /** @type {object} */\n this.sliceTimeout = null;\n\n /** @type {{ total: number, CPU: number, webGL: number, webGPU: number }} */\n this.sliceTimeReport = null;\n /** @type {number} */\n this.moduleInDataSize = 0; // Sandbox level input size; set during assign, never reset.\n /** @type {number} */\n this.sliceOutDataSize = 0; // Slice level output size; reset for every slice executed.\n\n /** @type {number?} */\n this.sliceStartTime = null;\n /** @type {number} */\n this.useCounter = 1; // Anticipating the initial use.\n /** @type {Config} */\n this.hive = new Config();\n\n ///** @type {((data: any) => Promise<void>)[]} */\n this.ringMessageHandlers = [\n this.handleRing0Message,\n this.handleRing1Message,\n this.handleRing2Message,\n this.handleRing3Message,\n ];\n\n this.resetSliceReport();\n }\n\n /** @type {number} */\n get id () { return this.info.id; }\n /** @type {{ name: string, description: string, link: string }} */\n get public () { return this.info.public; }\n /** @type {{ name: string, description: string, link: string }} */\n set public (data) { this.info.public = data; }\n /** @type {JobManager} */\n get jobManager () { return this.info.jobManager; }\n /** @type {string} */\n get jobAddress () { return this.jobManager?.address; }\n /** @type {Slice} */\n get slice () { return this.info.slice; }\n /** @type {Slice} */\n set slice (slice) { this.info.slice = slice; }\n /** @type {number} */\n get sliceNumber () { return this.slice ? this.slice.sliceNumber : -1; }\n /** @type {number} */\n get generalTimeout () { return 2 * this.hive.generalTimeout; }\n /** @type {number} */\n get punctuatedTimeout () { return this.hive.generalTimeout; }\n\n /**\n * Debug string that characterizes sandbox.\n * @type {string}\n */\n get identifier()\n {\n if (!this.jobAddress)\n return `${this.id}.${this.state}`;\n const address = truncateAddress(this.jobAddress);\n if (this.slice)\n return `${this.id}.${address}.${this.state}~${this.slice.sliceNumber}`;\n return `${this.id}.${address}.${this.state}`;\n }\n\n /** @returns {number} */\n static getNewId() { return Sandbox.idCounter++; }\n\n /** @type {boolean} */\n get isReadyForAssign () { return this.state.is(READY_FOR_ASSIGN); }\n /** @type {boolean} */\n get isAssigned () { return this.state.is(ASSIGNED); }\n /** @type {boolean} */\n get isWorking () { return this.state.is(WORKING); }\n /** @type {boolean} */\n get isTerminated () { return this.state.is(TERMINATED); }\n\n // _Idx\n //\n // checkSandboxReUse, postMessageToEvaluator, changeState,\n // punctuatedTimer is expiremental for replacing hard-coded timeouts.\n //\n\n /**\n * Mark WORKING sandbox as ASSIGNED in preparation for possible reuse.\n * Allow use of sandbox on a given job up to a limit of dcpConfig.supervisor.sandbox.maxSandboxUse .\n */\n checkSandboxReUse ()\n {\n selectiveDebug2() && console.debug(`Sandbox2.checkSandboxReUse: useCounter ${this.useCounter}, ${this.identifier}`);\n if (this.useCounter++ < this.hive.maxSandboxUse)\n {\n this.state.set(WORKING, ASSIGNED);\n this.sandboxEmit('ready');\n }\n else\n {\n this.terminate(false);\n common.removeElement(this.supervisor.sandboxInventory, this);\n }\n }\n\n /** Transitions: ASSIGNED --> WORKING. */\n markAsWorking ()\n {\n if (!this.isAssigned)\n throw new Error(`Sandbox ${this.identifier} is not ready to work`);\n this.state.set(ASSIGNED, WORKING);\n }\n \n /**\n * Safely post message to evaluator.\n * @param {object} message\n */\n postMessageToEvaluator (message)\n {\n if (this.isTerminated) // When evaluator goes down, all sandboxes are terminated.\n throw new Error(`postMessageToEvaluator: Sandbox ${this.identifier} has been terminated.`);\n return this.evaluatorHandle.postMessage(message);\n }\n \n /**\n * Safely change state.\n * @param {string} currentState\n * @param {string} nextState\n */\n changeState (currentState, nextState)\n {\n if (this.isTerminated) // When evaluator goes down, all sandboxes are terminated.\n throw new Error(`changeState: Sandbox ${this.identifier} has been terminated.`);\n this.state.set(currentState, nextState);\n }\n\n /** Upon fatal error return slice to scheduler. */\n returnSlice ()\n {\n selectiveDebug() && console.debug('Sandbox.returnSlice', this.identifier);\n return this.supervisor.returnSlice(this.slice, 'Sandbox.returnSlice');\n }\n\n /**\n * @callback cbFn\n * @returns {void}\n */\n\n /**\n * UNUSED.\n * Future work.\n * Replaces the timers in:\n * describe,\n * applyRequirements,\n * resetState,\n * The idea is to have a long timeout with a warning every\n * 6 seconds saying why it is waiting.\n * @param {cbFn} body\n * @param {string} waitMessage\n * @param {string} timerExpiredMessage\n * @returns {Promise<{ closeIntervalTimer: cbFn }>}\n */\n punctuatedTimer(body, waitMessage, timerExpiredMessage)\n {\n const that = this;\n return new Promise((resolve, reject) => {\n let intervalCounter = 0;\n let intervalHandle = null;\n function closeIntervalTimer()\n {\n if (intervalHandle !== null)\n dcp_timers.clearTimeout(intervalHandle);\n intervalHandle = null;\n }\n intervalHandle = dcp_timers.setInterval(() => {\n if (++intervalCounter > 12)\n {\n closeIntervalTimer();\n that.error(timerExpiredMessage);\n }\n that.warning(waitMessage);\n body();\n }, this.punctuatedTimeout)\n // Allow workers and localExec to exit.\n intervalHandle.unref();\n resolve({ closeIntervalTimer });\n });\n }\n\n // _Idx\n //\n // start, describe, assign, applyRequirements, assignEvaluator\n //\n\n /**\n * Readies the sandbox. This will result in the sandbox being ready and not assigned.\n * It will need to be assigned with a job before it is able to do work.\n * Sandbox.start will terminate the sandbox upon failure.\n * @todo maybe preload specific modules or let the cache pass in what modules to load?\n *\n * @returns {Promise<void>}\n * @throws on failure to ready\n */\n async start ()\n {\n debug('Sandbox.start begin');\n await this.supervisor.delayManager.nextDelay('sandboxStart');\n this.changeState(UNREADY, READYING);\n\n try\n {\n // RING 0\n this.evaluatorHandle = new this.options.SandboxConstructor({\n name: `DCP Sandbox #${this.id}`,\n });\n // Annoying! onerror terminates sandbox which can happen independent of whether the slice\n // is ok or not. Since we don't know, we have to return the slice when onerror is called\n // during sandbox.work .\n /** @todo XXXpfr Beware of onerror firing often. */\n this.evaluatorHandle.onerror = this.onerror.bind(this);\n\n const messageHandler = this.onmessage.bind(this);\n this.evaluatorHandle.onmessage = function onmessage(event)\n {\n const data = (event.data.serialized)\n ? kvin.parse(event.data.message)\n : kvin.unmarshal(event.data);\n messageHandler({ data });\n }\n\n const evaluatorPostMessage = this.evaluatorHandle.postMessage.bind(this.evaluatorHandle);\n this.evaluatorHandle.postMessage = function postMessage(message)\n {\n evaluatorPostMessage(kvin.marshal(message));\n }\n\n const that = this;\n this.evaluatorHandle.addEventListener('end', function sandbox$start$addEventListener() {\n selectiveDebug() && console.debug(\"END:Sandbox evaluatorHandle end-handler\", that.identifier, new Date());\n that.supervisor.evaluator.shuttingDown = true;\n that.terminate(true);\n });\n\n // Don't let an open sockets prevent clean worker exit.\n if (this.evaluatorHandle.unref)\n this.evaluatorHandle.unref();\n\n // Now in RING 1\n\n // Now in RING 2\n await this.describe();\n this.changeState(READYING, READY_FOR_ASSIGN);\n\n // Emit the 'sandbox' event on the worker event emitter.\n this.supervisor.safeEmit(this.supervisor.worker, 'sandbox', this.sandboxHandle);\n }\n catch (error)\n {\n if (this.isTerminated)\n debug(`Failed to start sandbox because it is already terminated: ${this.identifier}.\\n\\tMay be due to screensaver worker being down or evaluator was stopped.`);\n else\n {\n debug(`Failed to start sandbox ${this.identifier}.`, error.message); // FIX s.b. error\n this.terminate(false);\n }\n throw error;\n }\n }\n\n /**\n * Sends a post message to describe its capabilities.\n * Side effect: Sets the capabilities property of the current sandbox.\n *\n * @returns {Promise<any>} Resolves with the sandbox's capabilities.\n * Rejects with an error saying a response was not received.\n * @memberof Sandbox\n */\n describe ()\n {\n debugging('sandbox') && debug('Beginning to describe evaluator', this.identifier);\n const that = this;\n\n return new Promise(function sandbox$describePromise(resolve, reject) {\n let describeTimeout;\n\n if (that.isTerminated) // When evaluator goes down, all sandboxes are terminated.\n reject(new Error(`Sandbox ${that.identifier} has been terminated.`));\n\n if (that.evaluatorHandle === null)\n reject(new Error(`Evaluator has not been initialized: ${that.identifier}`));\n\n function sandbox$describe$success(data)\n {\n if (describeTimeout !== false)\n {\n dcp_timers.clearTimeout(describeTimeout);\n describeTimeout = false;\n\n const { capabilities } = data;\n if (typeof capabilities === 'undefined')\n reject(new Error(`Did not receive capabilities from describe response: ${that.identifier}`));\n that.capabilities = capabilities;\n\n debugging('sandbox') && debug('Evaluator has been described');\n resolve(capabilities);\n }\n }\n // Emitted by handleRing2Message.\n that.once('describe', sandbox$describe$success);\n\n describeTimeout = dcp_timers.setTimeout(function sandbox$describe$fail() {\n if (describeTimeout !== false)\n {\n describeTimeout = false;\n that.removeListener('describe', sandbox$describe$success);\n reject(new Error( `Describe message timed-out. No describe response was received from the describe command: ${that.identifier}`));\n }\n }, that.generalTimeout);\n // Allow workers and localExec to exit.\n describeTimeout.unref();\n\n const message = {\n request: 'describe',\n };\n that.postMessageToEvaluator(message);\n });\n }\n\n /**\n * This will assign the sandbox with a job, loading its sandbox code into the sandbox.\n * Sandbox.assign will not terminate the sandbox upon failure.\n * The sandbox will be terminated in JobManager.assignSandbox .\n * @param {JobManager} jobManager - The job manager that will be the owner of this sandbox.\n * @returns {Promise<Sandbox>}\n * @throws on initialization failure\n */\n async assign (jobManager)\n {\n if (!this.slice) // Design assumption.\n throw new Error(`Must have valid sandbox.slice before sandbox.assign is called: ${this.identifier}`);\n\n await this.supervisor.delayManager.nextDelay('sandboxAssign');\n debug('Sandbox.assign', this.identifier, Date.now() - this.supervisor.lastTime);\n\n try\n {\n this.changeState(READY_FOR_ASSIGN, ASSIGNING);\n this.info.jobManager = jobManager;\n this.job = this.jobManager.jobMessage;\n\n /* At this point, the worker has decided that this sandbox will be associated with a specific job. \n Therefore, we emit the SandboxHandle<job> event*/\n this.sandboxEmit('job', jobManager.jobHandle);\n\n assertEq3(this.job.address, this.jobAddress);\n assert(typeof this.job === 'object');\n assert(typeof this.job.requirements === 'object');\n assert(Array.isArray(this.job.dependencies));\n assert(Array.isArray(this.job.requirePath));\n\n // Extract public data from job, with defaults\n this.public = Object.assign({\n name: `Anonymous Job ${truncateAddress(this.jobAddress)}`,\n description: 'Discreetly helping make the world smarter.',\n link: 'https://distributed.computer/about',\n }, this.job.public);\n\n // Future: We may want other filename tags for appliances // RR Nov 2019\n\n // Important: The order of applying requirements before loading the sandbox code\n // is important for modules and sandbox code to set globals over the whitelist.\n await this.applyRequirements(this.job.requirements);\n //const _t0 = Date.now();\n await this.assignEvaluator();\n //console.log('Finished Sandbox.assignEvaluator', Date.now() - _t0);\n this.changeState(ASSIGNING, ASSIGNED);\n this.sandboxEmit('ready');\n }\n catch (error)\n {\n if (this.isTerminated)\n debug(`Failed to assign sandbox ${this.identifier} to evaluator because it is already terminated.\\n\\tMay be due to screensaver worker being down or evaluator was stopped.`);\n else\n {\n debug(`Failed to assign sandbox ${this.identifier} to evaluator.`);\n this.terminate(false);\n }\n throw error;\n }\n\n return this;\n }\n\n /**\n * Passes the job's requirements object into the sandbox so that the global access lists can be updated accordingly.\n * E.g. disallow access to OffscreenCanvas without environment.offscreenCanvas=true present.\n * Must be called after @start.\n *\n * @returns {Promise<void>} - resolves on success, rejects otherwise\n */\n applyRequirements (requirements)\n {\n assert(typeof requirements === 'object');\n const that = this;\n\n return new Promise(function sandbox$applyRequirementsPromise(resolve, reject) {\n let requirementTimeout;\n\n function sandbox$applyRequirements$success()\n {\n if (requirementTimeout !== false)\n {\n dcp_timers.clearTimeout(requirementTimeout);\n requirementTimeout = false;\n resolve();\n }\n }\n // Emitted by handleRing1Message.\n that.once('applyRequirementsDone', sandbox$applyRequirements$success);\n\n requirementTimeout = dcp_timers.setTimeout(function sandbox$finishApplySandboxRequirements$fail() {\n if (requirementTimeout !== false)\n {\n requirementTimeout = false;\n that.removeListener('applyRequirementsDone', sandbox$applyRequirements$success);\n reject(new Error(`applyRequirements never received 'applyRequirementsDone' response from sandbox: ${that.identifier}`));\n }\n }, that.generalTimeout);\n // Allow workers and localExec to exit.\n requirementTimeout.unref();\n\n const message = {\n requirements,\n request: 'applyRequirements',\n };\n that.postMessageToEvaluator(message);\n });\n }\n\n /**\n * Assign job to the evaluator.\n * @returns {Promise<any>} - resolves on success, rejects otherwise\n */\n assignEvaluator ()\n {\n debugging('sandbox') && console.debug('Begin assigning job to evaluator', this.identifier);\n const that = this;\n\n return new Promise(function sandbox$$assignEvaluatorPromise(resolve, reject) {\n function sandbox$assignEvaluator$success(event)\n {\n that.removeListener('reject', sandbox$assignEvaluator$fail);\n debugging('sandbox') && debug('Job assigned to evaluator');\n resolve(event);\n }\n\n function sandbox$assignEvaluator$fail(error)\n {\n that.removeListener('assigned', sandbox$assignEvaluator$success);\n that.error(`assignEvaluator failed(${that.identifier}): evaluator may be out of memory or the screensaver may be down.`, error);\n selectiveDebug() && console.debug('assignEvaluator failed', that.identifier, error);\n if (that.slice) // Normally the slice hasn't been set yet.\n that.returnSlice();\n reject(error);\n }\n\n // Emitted by handleRing2Message.\n that.once('assigned', sandbox$assignEvaluator$success);\n that.once('reject', sandbox$assignEvaluator$fail);\n\n // Had to add useStrict -- not sure if anything else was missed.\n const jobMessage = {\n address: that.job.address,\n arguments: that.job.arguments,\n dependencies: that.job.dependencies,\n modulePath: that.job.modulePath,\n public: that.job.public,\n requireModules: that.job.requireModules,\n requirePath: that.job.requirePath,\n workFunction: that.job.workFunction,\n useStrict: that.job.useStrict,\n };\n\n const message = {\n request: 'assign',\n job: jobMessage,\n sandboxConfig: that.hive.sandboxConfig,\n };\n that.postMessageToEvaluator(message);\n });\n }\n\n // _Idx\n //\n // eval, resetState, work, resetProgressTimeout, resetSliceTimeout\n //\n\n /**\n * Evaluates a string inside the sandbox.\n * @todo XXXpfr -- I don't understand how this gets called?\n * There's an old comment saying: \"no longer working though?\"\n *\n * @param {string} code - the code to evaluate in the sandbox\n * @param {string} filename - the name of the 'file' to help with debugging,\n * @returns {Promise<any>} - resolves with eval result on success, rejects otherwise\n */\n eval (code, filename)\n {\n const that = this;\n const msgId = nanoid();\n\n return new Promise(function sandbox$$eval$Promise(resolve, reject) {\n const eventId = EVAL_RESULT_PREFIX + msgId;\n\n function sandbox$eval$success(event)\n {\n that.removeListener('reject', sandbox$eval$fail);\n resolve(event);\n };\n\n function sandbox$eval$fail(error)\n {\n that.removeListener(eventId, sandbox$eval$success);\n reject(error);\n };\n\n that.once(eventId, sandbox$eval$success);\n that.once('reject', sandbox$eval$fail);\n\n const message = {\n request: 'eval',\n data: code,\n filename,\n msgId,\n };\n that.postMessageToEvaluator(message);\n });\n }\n\n /**\n * Resets the state of the bootstrap, without resetting the sandbox function if assigned.\n * Mostly used to reset the progress status before reusing a sandbox on another slice.\n * Must be called after @start.\n *\n * @returns {Promise<void>} - resolves with result on success, rejects otherwise\n */\n resetState ()\n {\n const that = this;\n assert(this.isWorking); // Design assumption.\n\n return new Promise(function sandbox$resetStatePromise(resolve, reject) {\n let resetStateTimeout;\n\n function sandbox$resetState$success ()\n {\n if (resetStateTimeout !== false)\n {\n dcp_timers.clearTimeout(resetStateTimeout);\n resetStateTimeout = false;\n resolve();\n }\n }\n that.once('resetStateDone', sandbox$resetState$success);\n\n resetStateTimeout = dcp_timers.setTimeout(function sandbox$resetState$fail() {\n if (resetStateTimeout !== false)\n {\n resetStateTimeout = false;\n that.removeListener('resetStateDone', sandbox$resetState$success);\n reject(new Error(`resetState never received resetStateDone event from sandbox: ${that.identifier}`));\n }\n }, that.generalTimeout);\n // Allow workers and localExec to exit.\n resetStateTimeout.unref();\n\n const message = {\n request: 'resetState',\n };\n that.postMessageToEvaluator(message);\n });\n }\n\n /**\n * Executes a slice received from the supervisor.\n * Must be called after this.start, this.assign and this.markAsWorking .\n * Sandbox.work will not terminate the sandbox upon failure.\n * The sandbox will be terminated in Supervisor.handleSandboxWorkError .\n * @returns {Promise<any>} - resolves with result on success, rejects otherwise\n */\n async work ()\n {\n const that = this;\n\n if (!this.slice) // Design assumption\n throw new Error(`Must have valid sandbox.slice before sandbox.assign is called: ${this.identifier}`);\n\n await this.supervisor.delayManager.nextDelay('sandboxWork');\n debug('Sandbox.work begin', this.identifier, Date.now() - this.supervisor.lastTime);\n\n if (this.isTerminated) // When evaluator goes down, all sandboxes are terminated.\n throw new Error(`Sandbox ${this.identifier} has been terminated.`);\n if (!this.isWorking)\n throw new Error(`Sandbox ${this.identifier} in Sandbox.work must be marked as working.`)\n\n // cf. DCP-1719,1720\n this.resetSliceReport();\n\n // Check that sandbox and slice have the same job.\n if (this.jobAddress !== this.slice.jobAddress)\n throw new Error(`Sandbox.work: sandbox ${this.identifier} and slice ${this.slice.identifier} are from different jobsz`);\n\n /** @todo Should sliceHnd just be replaced with { sandbox: this } since this.public is part of this? */\n let sliceHnd = { job: this.public, sandbox: this };\n await this.resetState();\n if (!this.slice)\n {\n this.error(`Slice for job ${this.jobAddress} vanished during work initialization - aborting`);\n return;\n }\n\n const { datum: inputDatum, error: dataError } = this.slice;\n if (dataError)\n {\n that.postWorkEmit('error', {\n message: dataError.message,\n stack: dataError.stack,\n name: this.public.name\n });\n }\n\n this.resetProgressTimeout();\n this.resetSliceTimeout();\n\n return new Promise(function sandbox$$workPromise(resolve, reject) {\n function sandbox$$work$success (event)\n {\n that.removeListener('reject', sandbox$$work$fail);\n resolve(event);\n }\n\n function sandbox$$work$fail (error)\n {\n that.removeListener('resolve', sandbox$$work$success);\n reject(error);\n }\n\n that.once('resolve', sandbox$$work$success);\n that.once('reject', sandbox$$work$fail);\n\n that.sliceStartTime = Date.now();\n that.slice.startTime = that.sliceStartTime;\n that.progress = null;\n that.progressReports = {\n last: undefined,\n lastDeterministic: undefined,\n };\n\n that.resetProgressTimeout();\n that.resetSliceTimeout();\n that.emit('start', sliceHnd);\n\n if (dataError)\n {\n that.removeListener('resolve', sandbox$$work$success);\n that.removeListener('reject', sandbox$$work$fail);\n dcp_timers.setTimeout(() => reject(dataError), 0)\n }\n else\n {\n // Do the work.\n const message = { request: 'main', data: inputDatum, };\n that.postMessageToEvaluator(message);\n }\n })\n .then(async function sandbox$$work$then(event) {\n // Tell supervisor sandbox slot is available.\n that.slice.markAsWorkDone();\n\n selectiveDebug2() && console.debug('Sandbox.sliceFinish', that.identifier, event?.timeReport);\n that.sandboxEmit('sliceEnd', that.slice?.sliceNumber)\n that.emit('complete', that.jobAddress);\n\n // Reset slice property.\n that.slice = null;\n\n // JobManager.runSliceOnSandbox will transition WORKDONE -> ASSIGNED\n return event;\n })\n .catch(async function sandbox$$work$catch(error) {\n selectiveDebug() && console.debug('Sandbox.work catch', that.identifier, error);\n // Tell supervisor sandbox slot is available.\n if (that.slice)\n that.slice.markAsWorkDone();\n // Current sandbox will not be reused.\n // Do not overwrite that.slice because it is needed in subsequent error reporting.\n\n if (error instanceof NoProgressError)\n {\n const payload = {\n name: that.public.name,\n message: error.message,\n timestamp: Date.now() - that.sliceStartTime,\n };\n that.postWorkEmit('error', payload);\n that.postWorkEmit('noProgress', { ...payload, progressReports: that.progressReports });\n }\n if (error.name === 'EWORKREJECT')\n that.handleRejectedWork(that.sliceTimeReport);\n\n // Otherwise sandbox will be terminated in Supervisor.handleSandboxWorkError\n debugging('sandbox') && debug(`Sandbox ${that.identifier} failed to execute slice`, error);\n\n throw error;\n });\n }\n\n resetProgressTimeout()\n {\n const that = this;\n\n if (this.progressTimeout)\n dcp_timers.clearTimeout(this.progressTimeout);\n\n this.progressTimeout = dcp_timers.setTimeout(function sandbox$ProgressTimeout() {\n if (that.options.ignoreNoProgress)\n return that.warning('ENOPROGRESS silenced by localExec: In a remote worker, this slice would be stopped for not calling progress frequently enough.');\n\n that.emit('reject', new NoProgressError(`No progress event was received in the last ${that.hive.progressTimeout / 1000} seconds.`));\n }, this.hive.progressTimeout * timeDilation);\n // Allow workers and localExec to exit.\n this.progressTimeout.unref();\n }\n\n resetSliceTimeout()\n {\n const that = this;\n\n if (this.sliceTimeout)\n dcp_timers.clearTimeout(this.sliceTimeout);\n\n this.sliceTimeout = dcp_timers.setTimeout(function sandbox$SliceTimeout() {\n if (Sandbox.debugWork)\n return that.warning('Sandbox.debugWork: Ignoring slice timeout');\n\n that.emit('reject', new SliceTooSlowError(`Slice took longer than ${that.hive.sliceTimeout / 1000} seconds.`));\n }, this.hive.sliceTimeout * timeDilation);\n // Allow workers and localExec to exit.\n this.sliceTimeout.unref();\n }\n\n /**\n * Send payload to the workEmit endpoint in the event router.\n * @param {string} eventName\n * @param {*} payload\n * @returns {Promise<*>}\n */\n postWorkEmit (eventName, payload)\n {\n // Need to check if the sandbox hasn't been assigned a slice yet.\n if (!this.slice)\n this.error('Sandbox not assigned a slice before sending workEmit message to scheduler', payload, `'workEmit' event originates from '${eventName}' event`);\n else\n {\n const slice = this.slice;\n // Authorization should always be valid.\n if (!slice.authorizationMessage)\n this.warning(`workEmit: missing authorization message for slice ${slice.identifier}`);\n else\n {\n const workEmitPayload = {\n eventName,\n payload,\n job: slice.jobAddress,\n slice: slice.sliceNumber,\n worker: this.supervisor.workerId,\n authorizationMessage : slice.authorizationMessage,\n };\n return this.supervisor.dcp4.safeWorkEmit(workEmitPayload, `Failed to send workEmit (${eventName}) payload for slice ${slice.identifier}`)\n .then((success) => {\n if (!success)\n this.warning(`Message sent to workEmit is unauthorized; not accepted '${eventName}'`);\n });\n }\n }\n }\n\n /**\n * Save rejected slice timeReport data in this.slice.rejectedTimeReport, then when needed in\n * Supervisor.recordResult, merge this.slice.rejectedTimeReport into this.slice.timeReport.\n * @param {{ total: number, CPU: number, webGL: number, webGPU: number }} timeReport\n */\n handleRejectedWork (timeReport)\n {\n selectiveDebug() && console.debug('handleRejectedWork', this.identifier);\n // If the slice already has rejectedTimeReport, add this timeReport to it.\n // If not, assign this timeReport to slices rejectedTimeReport property\n if (this.slice)\n {\n if (!this.slice.rejectedTimeReport)\n this.slice.rejectedTimeReport = timeReport;\n else\n {\n ['total', 'CPU', 'webGL', 'webGPU'].forEach((key) => {\n if (timeReport[key])\n this.slice.rejectedTimeReport[key] += timeReport[key];\n });\n }\n }\n }\n\n /**\n * Attach CGIO to result returned by a slice workFn.\n * @param {*} completeData - results\n */\n attachCGIOToResult (completeData)\n {\n if (!completeData)\n throw new Error('Slice result is not ready'); // Should never fire.\n if (completeData['timeReport'])\n throw new Error('Slice result already has timeReport'); // Should never fire.\n if (completeData['dataReport'])\n throw new Error('Slice result already has dataReport'); // Should never fire.\n if (this.listenerCount('resolve') > 0)\n {\n completeData['timeReport'] = this.sliceTimeReport;\n completeData['dataReport'] = {\n InDataSize: this.moduleInDataSize + this.jobManager.inputDataSize + this.slice.inputDataSize,\n OutDataSize: this.sliceOutDataSize,\n };\n this.emit('resolve', completeData);\n selectiveDebug() && console.debug('attachCGIOToResult', this.moduleInDataSize, this.jobManager.inputDataSize, this.slice.inputDataSize, completeData['dataReport'].InDataSize);\n }\n else\n {\n // If there is no internal listener for 'resolve', the slice was rejected\n // and we need to update this.slice.rejectedTimeReport appropriately.\n this.handleRejectedWork(this.sliceTimeReport);\n }\n // Clear time and data reports so we can catch mistaken writes.\n this.sliceTimeReport = null;\n this.sliceOutDataSize = 0;\n }\n\n // _Idx\n //\n // handleRing0Message, handleRing1Message, handleRing2Message, handleRing3Message\n //\n\n async handleRing0Message(data) // eslint-disable-line require-await\n {\n debugging('ring0') && debug('Ring0', this.identifier, data.request);\n\n switch (data.request)\n {\n // The 'sandboxLoaded' event is used by dcp-native; do not remove.\n case 'sandboxLoaded': // SAVE\n break;\n case 'scriptLoaded':\n if(data.result !== \"success\")\n this.onerror(data);\n break;\n case 'error':\n debug('Sandbox error in ring0', data.error);\n this.rejectWithCleanup('during initialization', data.error);\n break;\n default:\n this.error('Received unhandled request from sandbox: ' + data.request, null, `data: ${ JSON.stringify(data)}`);\n break;\n }\n }\n\n async handleRing1Message(data) // eslint-disable-line require-await\n {\n debugging('ring1') && debug('Ring1', this.identifier, data.request);\n\n switch (data.request)\n {\n case 'applyRequirementsDone':\n // emit internally\n this.emit(data.request, data)\n break;\n default:\n this.error('Received unhandled request from sandbox ring 1: ' + data.request, null, `data: ${ JSON.stringify(data)}`);\n break; \n }\n }\n\n async handleRing2Message(data)\n {\n debugging('ring2') && debug('Ring2', this.identifier, data.request);\n\n switch (data.request)\n {\n case 'dependency': {\n try\n {\n const moduleData = await this.moduleCache.fetchModule(data.data, this.jobAddress);\n // Success! Restore this['packageManager'] delay to retryMinSleepMs (currently 32ms.)\n // Is there a better way to reset than explicit calls?\n this.supervisor.delayManager.resetEBO('packageManager');\n // Send module data to be evaluator.\n const message = {\n request: 'moduleGroup',\n data: moduleData,\n id: data.id,\n };\n // Module data is dynamic since it may only be required in a conditional branch.\n // Moreover, on a long job, the published module itself may be updated on the scheduler.\n const moduleLength = kvin.stringify(moduleData).length; /** @TODO - fix per DCP-3750 */\n this.moduleInDataSize += moduleLength;\n selectiveDebug() && console.debug('Sandbox.Ring2.fetchModule size', this.moduleInDataSize, moduleLength);\n this.postMessageToEvaluator(message);\n }\n catch (error)\n {\n /*\n * In the event of an error here, we want to let the client know there was a problem in\n * loading their module. In principle we shouldn't need a valid sandbox.slice at sandbox.assign.\n * However, in the implementation of Sup2 there is precisely 1 callsite of sandbox.assign and\n * we do have an associated slice at this point. So we make the assumption that sandbox.slice\n * is valid here.\n */\n if (!this.slice) // Design assumption\n throw new Error(`Must have valid slice in sandbox before sandbox.assign is called: ${this.identifier}`);\n\n const payload = {\n name: error.name,\n message: error.message,\n timestamp: error.timestamp ? error.timestamp : new Date(),\n };\n\n this.postWorkEmit('error', payload);\n this.emit('reject', error);\n\n debugging() && console.debug(`Sandbox.Ring2: fetchModule failed ${this.identifier}`, payload, error, Date.now() - this.supervisor.lastTime);\n\n // Close packageManager to start the connection reconnect logic.\n // Should we do a retry loop with fetchModule too?\n this.supervisor.dcp4.resetConnection('packageManager');\n }\n break;\n }\n case 'error':\n /*\n * Ring 2 error messages will only fire for problems inside of the worker that are separate from\n * the work function. In most cases there are other handlers for situations where 'error' may be emitted\n * such as timeouts if the expected message isn't recieved.\n */\n debug('Sandbox error in ring2', data.error);\n this.rejectWithCleanup('during assignment and dependency resolution', data.error);\n break;\n case 'describe':\n case 'evalResult':\n case 'resetStateDone':\n case 'assigned':\n this.emit(data.request, data); // emit internally\n break;\n case 'reject':\n this.emit('reject', data.error); // emit internally\n break;\n default:\n this.error(`Received unhandled request from sandbox ring 2. Data: ${JSON.stringify(data, null, 2)}`);\n break;\n }\n }\n\n async handleRing3Message(data) // eslint-disable-line require-await\n {\n debugging('ring3') && debug('Ring3', this.identifier, data.request);\n\n switch (data.request)\n {\n case 'complete':\n dcp_timers.clearTimeout(this.progressTimeout);\n dcp_timers.clearTimeout(this.sliceTimeout);\n this.progressTimeout = this.sliceTimeout = null;\n\n if (this.progress === null)\n {\n if (this.options.ignoreNoProgress)\n this.warning(\"ENOPROGRESS silenced by localExec: Progress was not called during this slice's execution, in a remote sandbox this would cause the slice to fail.\");\n else\n {\n // If a progress update was never received (progress === null) then reject\n this.emit('reject', new NoProgressError('Sandbox never emitted a progress event.'));\n this.handleRejectedWork(this.sliceTimeReport);\n break;\n }\n }\n \n this.progress = 100;\n this.sliceOutDataSize += kvin.stringify(data.result).length; /** @TODO - fix per DCP-3750 */\n this.attachCGIOToResult(data);\n break;\n case 'progress':\n {\n const { progress, indeterminate, throttledReports, value } = data;\n this.progress = progress;\n // cf. job-noProgress.js\n const progressReport = {\n deltaMs: Date.now() - this.sliceStartTime,\n progress,\n value,\n throttledReports,\n }\n this.progressReports.last = progressReport;\n if (!indeterminate)\n this.progressReports.lastDeterministic = progressReport;\n\n this.resetProgressTimeout();\n this.sandboxEmit('progress', indeterminate || progress < 0 || progress > 100 ? undefined : progress);\n break;\n }\n case 'noProgress':\n this.emit('reject', new NoProgressError(data.message));\n break;\n case 'console':\n data.payload.message = kvin.marshal(data.payload.message);\n this.sliceOutDataSize += JSON.stringify(data.payload.message).length; /** @TODO - fix per DCP-3750 */\n this.postWorkEmit('console', data.payload);\n break;\n case 'emitEvent': /* ad-hoc event from the sandbox (work.emit) */\n this.postWorkEmit('custom', data.payload);\n break;\n case 'measurement':\n this.updateTime(data);\n break;\n case 'sandboxError': /* the sandbox itself has an error condition */\n debug(`Ring3 received a 'sandboxError' event for sandbox ${this.identifier}`, data.error);\n this.emit('sandboxError', data.error);\n this.rejectWithCleanup('internal sandbox error while executing work function', data.error);\n break;\n case 'workError': /* the work function threw/rejected */\n debug(`Ring3 received a 'workError' event for sandbox ${this.identifier}`, data.error);\n this.postWorkEmit('error', data.error);\n const wrappedError = new UncaughtExceptionError(data.error);\n this.rejectWithCleanup('error while executing work function', wrappedError);\n break;\n default:\n this.error('Received unhandled request from sandbox ring 3: ' + data.request, null, `data: ${ JSON.stringify(data)}`);\n break; \n }\n }\n\n /**\n * Try to send the error back to the reject handler in Sandbox.work.\n * But if the reject handler is not available (s.b. rare) then cleanup, emit error and throw.\n * @param {string} message\n * @param {Error|string} error\n */\n rejectWithCleanup (message, error)\n {\n if (this.listenerCount('reject') > 0)\n this.emit('reject', error);\n else\n {\n this.terminate(false);\n this.error(`Sandbox ${this.identifier} ${message}`, error);\n throw error;\n }\n }\n\n // _Idx\n //\n // onmessage, onerror, terminate\n //\n\n /**\n * Handles progress and completion events from sandbox.\n * Unless explicitly returned out of this function will re-emit the event\n * where the name of the event is event.data.request.\n *\n * @param {object} event - event received from the evaaluator sandbox\n * @returns {Promise<void>}\n */\n onmessage (event)\n {\n debugging('event') && debug('onmessage-event', event.data.ringSource);\n if (Sandbox.debugEvents)\n console.debug('sandbox - eventDebug:', { id: this.id, state: this.state.valueOf(), event: JSON.stringify(event) });\n\n const { data } = event;\n const ringLevel = data.ringSource\n\n // Give the data to a handler depending on ring level\n if (ringLevel === -1)\n {\n /** @todo XXXpfr Beware of this happening often. */\n this.error(`Message sent directly from raw postMessage for event ${event}. Terminating worker...`);\n this.terminate(true);\n }\n else\n {\n const handler = this.ringMessageHandlers[ringLevel];\n if (handler)\n return handler.call(this, data.value);\n this.warning(`No handler defined for message from ring ${ringLevel} for event ${event}.`);\n }\n }\n\n /**\n * Error handler for the internal sandbox.\n * Emits error event that gets handled up in the Worker class.\n */\n onerror (event)\n {\n /** @todo XXXpfr Beware of onerror firing often. */\n this.error(`Sandbox.onerror emitted an error: ${event}`);\n this.terminate(true, true);\n }\n\n /**\n * Clears the timeout and terminates the sandbox and sometimes emits a reject event.\n *\n * @param {boolean} [reject=true] - if true emit reject event\n * @param {boolean} [immediate=false] - passed to terminate, used by standaloneWorker to immediately close the connection\n */\n terminate (reject = true, immediate = false)\n {\n if (this.slice)\n this.returnSlice();\n if (this.isTerminated)\n return;\n\n selectiveDebug() && console.debug(`Terminate sandbox ${this.identifier}`);\n\n this.state = new Synchronizer(TERMINATED, [ UNREADY, READYING, READY_FOR_ASSIGN, ASSIGNING, ASSIGNED, WORKING, TERMINATED ]);\n\n dcp_timers.clearTimeout(this.progressTimeout);\n dcp_timers.clearTimeout(this.sliceTimeout);\n this.progressTimeout = this.sliceTimeout = null;\n\n if (this.evaluatorHandle && typeof this.evaluatorHandle.terminate === 'function')\n {\n try\n {\n this.evaluatorHandle.terminate(immediate);\n }\n catch (e)\n {\n this.error(`Error terminating sandbox ${this.id}:`, e);\n }\n finally\n {\n this.evaluatorHandle = null;\n }\n }\n\n if (reject)\n this.emit('reject', new Error(`Sandbox ${this.identifier} was terminated.`));\n\n this.sandboxEmit('end');\n }\n\n // _Idx\n //\n // updateTime, resetSliceReport, sandboxEmit, error, warning\n //\n\n /**\n * ringNPostMessage can send a `measurement` request and update these\n * totals.\n */\n updateTime (measurementEvent)\n {\n ['total', 'CPU', 'webGL', 'webGPU'].forEach((key) => {\n if (measurementEvent[key])\n this.sliceTimeReport[key] += measurementEvent[key];\n });\n }\n\n /**\n * Start over sandbox work timers.\n */\n resetSliceReport ()\n {\n this.sliceTimeReport = {\n total: 0,\n CPU: 0,\n webGL: 0,\n webGPU: 0,\n };\n this.sliceOutDataSize = 0;\n }\n\n /**\n * Safe event emitter on sandboxHandle.\n * @param {string} event\n * @param {...any} args\n */\n sandboxEmit(event, ...args)\n {\n this.supervisor.safeEmit(this.sandboxHandle, event, ...args);\n }\n\n /**\n * Error feedback to user.\n * @param {string} message\n * @param {Array<Error>|Error|string} [coreError]\n * @param {string} [additionalInfo]\n * @param {boolean} [supressStack=false]\n */\n error (message, coreError, additionalInfo, supressStack = false)\n {\n this.supervisor.error(message, coreError, additionalInfo, supressStack);\n }\n\n /**\n * Warning feedback to user.\n * @param {string[]} messages\n */\n warning (...messages)\n {\n this.supervisor.warning(...messages);\n }\n}\n\nSandbox.idCounter = 1;\nSandbox.debugWork = false;\nSandbox.debugState = false;\nSandbox.debugEvents = false;\n\nexports.Sandbox = Sandbox;\nexports.SandboxError = SandboxError;\nexports.NoProgressError = NoProgressError;\nexports.SliceTooSlowError = SliceTooSlowError;\nexports.UncaughtExceptionError = UncaughtExceptionError;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/supervisor2/sandbox2.js?");
4802
4802
 
4803
4803
  /***/ }),
4804
4804