dcp-client 4.2.9 → 4.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +1 -1
- package/README.md +9 -7
- package/dist/dcp-client-bundle.js +14 -14
- package/generated/sandbox-definitions.json +1 -1
- package/index.js +30 -12
- package/libexec/evaluator-node.js +1 -1
- package/libexec/sandbox/bootstrap.js +1 -1
- package/libexec/sandbox/bravojs-env.js +1 -1
- package/libexec/sandbox/bravojs-init.js +2 -2
- package/libexec/sandbox/calculate-capabilities.js +1 -1
- package/libexec/sandbox/event-loop-virtualization.js +12 -12
- package/libexec/sandbox/gpu-timers.js +2 -1
- package/libexec/sandbox/native-event-loop.js +17 -15
- package/package.json +4 -4
|
@@ -3831,7 +3831,7 @@ eval("// Copyright Joyent, Inc. and other Node contributors.\n//\n// Permission
|
|
|
3831
3831
|
/***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => {
|
|
3832
3832
|
|
|
3833
3833
|
"use strict";
|
|
3834
|
-
eval("__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ \"default\": () => (/* binding */ Modal)\n/* harmony export */ });\n/**\n * A Small Modal Class\n * @module Modal\n */\n/* globals Event dcpConfig */\nclass Modal {\n constructor (title, message, callback = false, exitHandler = false, {\n continueLabel = 'Continue',\n cancelLabel = 'Cancel',\n cancelVisible = true\n } = {}) {\n const modal = document.createElement('div')\n modal.className = 'dcp-modal-container-old day'\n modal.innerHTML = `\n <dialog class=\"dcp-modal-content\">\n <div class=\"dcp-modal-header\">\n <h2>${title}<button type=\"button\" class=\"close\">×</button></h2>\n ${message ? '<p>' + message + '</p>' : ''}\n </div>\n <div class=\"dcp-modal-loading hidden\">\n <div class='loading'></div>\n </div>\n <form onsubmit='return false' method=\"dialog\">\n <div class=\"dcp-modal-body\"></div>\n <div class=\"dcp-modal-footer ${cancelVisible ? '' : 'centered'}\">\n <button type=\"submit\" class=\"continue green-modal-button\">${continueLabel}</button>\n <button type=\"button\" class=\"cancel green-modal-button\">${cancelLabel}</button>\n </div>\n </form>\n </dialog>`\n\n // To give a reference to do developer who wants to override the form submit.\n // May occur if they want to validate the information in the backend\n // without closing the modal prematurely.\n this.form = modal.querySelector('.dcp-modal-content form')\n this.continueButton = modal.querySelector('.dcp-modal-footer button.continue')\n this.cancelButton = modal.querySelector('.dcp-modal-footer button.cancel')\n this.closeButton = modal.querySelector('.dcp-modal-header .close')\n if (!cancelVisible) {\n this.cancelButton.style.display = 'none'\n }\n\n // To remove the event listener, the reference to the original function\n // added is required.\n this.formSubmitHandler = this.continue.bind(this)\n\n modal.addEventListener('keydown', function (event) {\n event.stopPropagation()\n // 27 is the keycode for the escape key.\n if (event.keyCode === 27) this.close()\n }.bind(this))\n\n this.container = modal\n this.callback = callback\n this.exitHandler = exitHandler\n document.body.appendChild(modal)\n }\n\n changeFormSubmitHandler (newFormSubmitHandler) {\n this.formSubmitHandler = newFormSubmitHandler\n }\n\n /**\n * Validates the form values in the modal and calls the modal's callback\n */\n async continue (event) {\n // To further prevent form submission from trying to redirect from the\n // current page.\n if (event instanceof Event) {\n event.preventDefault()\n }\n let fieldsAreValid = true\n let formElements = this.container.querySelectorAll('.dcp-modal-body select, .dcp-modal-body input, .dcp-modal-body textarea')\n\n const formValues = []\n if (typeof formElements.length === 'undefined') formElements = [formElements]\n // Separate into two loops to enable input validation requiring formValues\n // that come after it. e.g. Two password fields matching.\n for (let i = 0; i < formElements.length; i++) {\n switch (formElements[i].type) {\n case 'file':\n formValues.push(formElements[i])\n break\n case 'checkbox':\n formValues.push(formElements[i].checked)\n break\n default:\n formValues.push(formElements[i].value)\n break\n }\n }\n for (let i = 0; i < formElements.length; i++) {\n if (formElements[i].validation) {\n // Optional fields are allowed to be empty but still can't be wrong if not empty.\n if (!(formElements[i].value === '' && !formElements[i].required)) {\n if (typeof formElements[i].validation === 'function') {\n if (!formElements[i].validation(formValues)) {\n fieldsAreValid = false\n formElements[i].classList.add('is-invalid')\n }\n } else if (!formElements[i].validation.test(formElements[i].value)) {\n fieldsAreValid = false\n formElements[i].classList.add('is-invalid')\n }\n }\n }\n }\n\n if (!fieldsAreValid) return\n\n this.loading()\n if (typeof this.callback === 'function') {\n try {\n return this.callback(formValues)\n } catch (error) {\n console.error('Unexpected error in modal.continue:', error);\n return this.close(false)\n }\n }\n this.close(true)\n }\n\n loading () {\n this.container.querySelector('.dcp-modal-loading').classList.remove('hidden')\n this.container.querySelector('.dcp-modal-body').classList.add('hidden')\n this.container.querySelector('.dcp-modal-footer').classList.add('hidden')\n }\n\n open () {\n this.form.addEventListener('submit', async (event) => {\n const success = await this.formSubmitHandler(event)\n if (success === false) {\n return\n }\n this.close(true)\n })\n // When the user clicks on <span> (x), close the modal\n this.closeButton.addEventListener('click', this.close.bind(this))\n this.cancelButton.addEventListener('click', this.close.bind(this))\n\n // Prevent lingering outlines after clicking some form elements.\n this.container.querySelectorAll('.dcp-modal-body button, .dcp-modal-body input[type=\"checkbox\"]').forEach(element => {\n element.addEventListener('click', () => {\n element.blur()\n })\n })\n\n // Show the modal.\n this.container.style.display = 'block'\n\n const formElements = this.container.querySelectorAll('.dcp-modal-body select, .dcp-modal-body input')\n if (formElements.length) {\n formElements[0].focus()\n if (formElements[0].type === 'text') {\n formElements[0].select()\n }\n for (const el of formElements) {\n if (el.realType) {\n el.type = el.realType\n }\n }\n } else {\n // With no form elements to allow for form submission on enter, focus the\n // continue button.\n this.container.querySelector('.dcp-modal-footer button.continue').focus()\n }\n } // TODO: This should return a promise with the action resolving it\n\n /**\n * Shows the modal and returns a promise of the result of the modal (e.g. was\n * it closed, did its action succeed?)\n */\n showModal () {\n return new Promise((resolve, reject) => {\n this.form.addEventListener('submit', handleContinue.bind(this))\n this.cancelButton.addEventListener('click', handleCancel.bind(this))\n this.closeButton.addEventListener('click', handleCancel.bind(this))\n\n // Prevent lingering outlines after clicking some form elements.\n this.container.querySelectorAll('.dcp-modal-body button, .dcp-modal-body input[type=\"checkbox\"]').forEach(element => {\n element.addEventListener('click', () => {\n element.blur()\n })\n })\n\n // Show the modal.\n this.container.style.display = 'block'\n\n const formElements = this.container.querySelectorAll('.dcp-modal-body select, .dcp-modal-body input')\n if (formElements.length) {\n formElements[0].focus()\n if (formElements[0].type === 'text') {\n formElements[0].select()\n }\n for (const el of formElements) {\n if (el.realType) {\n el.type = el.realType\n }\n }\n } else {\n // With no form elements to allow for form submission on enter, focus the\n // continue button.\n this.continueButton.focus()\n }\n\n async function handleContinue (event) {\n let result\n try {\n result = await this.formSubmitHandler(event)\n } catch (error) {\n reject(error)\n }\n this.close(true)\n resolve(result)\n }\n\n async function handleCancel () {\n let result\n try {\n result = await this.close()\n } catch (error) {\n reject(error)\n }\n resolve(result)\n }\n })\n }\n\n close (success = false) {\n this.container.style.display = 'none'\n if (this.container.parentNode) {\n this.container.parentNode.removeChild(this.container)\n }\n\n // @todo this needs to remove eventlisteners to prevent memory leaks\n\n if ((success !== true) && typeof this.exitHandler === 'function') {\n return this.exitHandler(this)\n }\n }\n\n /**\n * Adds different form elements to the modal depending on the case.\n *\n * @param {*} elements - The properties of the form elements to add.\n * @returns {HTMLElement} The input form elements.\n */\n addFormElement (...elements) {\n const body = this.container.querySelector('.dcp-modal-body')\n const inputElements = []\n let label\n for (let i = 0; i < elements.length; i++) {\n let row = document.createElement('div')\n row.className = 'row'\n\n let col, input\n switch (elements[i].type) {\n case 'button':\n col = document.createElement('div')\n col.className = 'col-md-12'\n\n input = document.createElement('button')\n input.innerHTML = elements[i].label\n input.type = 'button'\n input.classList.add('green-modal-button')\n if (!elements[i].onclick) {\n throw new Error('A button in the modal body should have an on click event handler.')\n }\n input.addEventListener('click', elements[i].onclick)\n\n col.appendChild(input)\n row.appendChild(col)\n break\n case 'textarea':\n col = document.createElement('div')\n col.className = 'col-md-12'\n\n input = document.createElement('textarea')\n input.className = 'text-input-field form-control'\n if (elements[i].placeholder) input.placeholder = elements[i].placeholder\n\n col.appendChild(input)\n row.appendChild(col)\n break\n case 'text':\n case 'email':\n case 'number':\n case 'password': {\n const inputCol = document.createElement('div')\n\n input = document.createElement('input')\n input.type = elements[i].type\n input.validation = elements[i].validation\n input.autocomplete = elements[i].autocomplete || (elements[i].type === 'password' ? 'off' : 'on')\n input.className = 'text-input-field form-control'\n\n // Adding bootstraps custom feedback styles.\n let invalidFeedback = null\n if (elements[i].invalidFeedback) {\n invalidFeedback = document.createElement('div')\n invalidFeedback.className = 'invalid-feedback'\n invalidFeedback.innerText = elements[i].invalidFeedback\n }\n\n if (elements[i].type === 'password') {\n elements[i].realType = 'password'\n }\n\n if (elements[i].label) {\n const labelCol = document.createElement('div')\n label = document.createElement('label')\n label.innerText = elements[i].label\n const inputId = 'dcp-modal-input-' + this.container.querySelectorAll('input[type=\"text\"], input[type=\"email\"], input[type=\"number\"], input[type=\"password\"]').length\n label.setAttribute('for', inputId)\n input.id = inputId\n labelCol.classList.add('col-md-6', 'label-column')\n labelCol.appendChild(label)\n row.appendChild(labelCol)\n inputCol.className = 'col-md-6'\n } else {\n inputCol.className = 'col-md-12'\n }\n\n inputCol.appendChild(input)\n if (invalidFeedback !== null) {\n inputCol.appendChild(invalidFeedback)\n }\n row.appendChild(inputCol)\n break\n }\n case 'select':\n col = document.createElement('div')\n col.className = 'col-md-4'\n\n label = document.createElement('span')\n label.innerText = elements[i].label\n\n col.appendChild(label)\n row.appendChild(col)\n\n col = document.createElement('div')\n col.className = 'col-md-8'\n\n input = document.createElement('select')\n\n col.appendChild(input)\n row.appendChild(col)\n break\n case 'checkbox': {\n row.classList.add('checkbox-row')\n const checkboxLabelCol = document.createElement('div')\n checkboxLabelCol.classList.add('label-column', 'checkbox-label-column')\n\n label = document.createElement('label')\n label.innerText = elements[i].label\n label.for = 'dcp-checkbox-input-' + this.container.querySelectorAll('input[type=\"checkbox\"]').length\n label.setAttribute('for', label.for)\n label.className = 'checkbox-label'\n\n checkboxLabelCol.appendChild(label)\n\n const checkboxCol = document.createElement('div')\n checkboxCol.classList.add('checkbox-column')\n\n input = document.createElement('input')\n input.type = 'checkbox'\n input.id = label.for\n if (elements[i].checked) {\n input.checked = true\n }\n\n checkboxCol.appendChild(input)\n\n if (elements[i].labelToTheRightOfCheckbox) {\n checkboxCol.classList.add('col-md-5')\n row.appendChild(checkboxCol)\n checkboxLabelCol.classList.add('col-md-7')\n row.appendChild(checkboxLabelCol)\n } else {\n checkboxLabelCol.classList.add('col-md-6')\n checkboxCol.classList.add('col-md-6')\n row.appendChild(checkboxLabelCol)\n row.appendChild(checkboxCol)\n }\n break\n }\n case 'file':\n [input, row] = this.addFileInput(elements[i], input, row)\n break\n case 'label':\n row.classList.add('label-row')\n label = document.createElement('label')\n label.innerText = elements[i].label\n row.appendChild(label)\n break\n }\n\n // Copy other possibly specified element properties:\n const inputPropertyNames = ['title', 'inputmode', 'value', 'minLength', 'maxLength', 'size', 'required', 'pattern', 'min', 'max', 'step', 'placeholder', 'accept', 'multiple', 'id', 'onkeypress', 'oninput', 'for', 'readonly', 'autocomplete']\n for (const propertyName of inputPropertyNames) {\n if (Object.prototype.hasOwnProperty.call(elements[i], propertyName)) {\n if (propertyName === 'for' && !label.hasAttribute(propertyName)) {\n label.setAttribute(propertyName, elements[i][propertyName])\n }\n if (propertyName.startsWith('on')) {\n input.addEventListener(propertyName.slice(2), elements[i][propertyName])\n } else {\n input.setAttribute(propertyName, elements[i][propertyName])\n }\n }\n }\n\n inputElements.push(input)\n body.appendChild(row)\n }\n\n if (inputElements.length === 1) return inputElements[0]\n else return inputElements\n }\n\n /**\n * Adds a drag and drop file form element to the modal.\n *\n * @param {*} fileInputProperties - An object specifying some of the\n * properties of the file input element.\n * @param {*} fileInput - Placeholders to help create the file\n * input.\n * @param {HTMLDivElement} row - Placeholders to help create the file\n * input.\n */\n addFileInput (fileInputProperties, fileInput, row) {\n // Adding the upload label.\n const uploadLabel = document.createElement('label')\n uploadLabel.innerText = fileInputProperties.label\n row.appendChild(uploadLabel)\n const body = this.container.querySelector('.dcp-modal-body')\n body.appendChild(row)\n const fileSelectionRow = document.createElement('div')\n fileSelectionRow.id = 'file-selection-row'\n\n // Adding the drag and drop file upload input.\n const dropContainer = document.createElement('div')\n dropContainer.id = 'drop-container'\n\n // Adding an image of a wallet\n const imageContainer = document.createElement('div')\n imageContainer.id = 'image-container'\n const walletImage = document.createElement('span')\n walletImage.classList.add('fas', 'fa-wallet')\n imageContainer.appendChild(walletImage)\n\n // Adding some text prompts\n const dropMessage = document.createElement('span')\n dropMessage.innerText = 'Drop a keystore file here'\n const orMessage = document.createElement('span')\n orMessage.innerText = 'or'\n\n // Adding the manual file input element (hiding the default one)\n const fileInputContainer = document.createElement('div')\n const fileInputLabel = document.createElement('label')\n // Linking the label to the file input so that clicking on the label\n // activates the file input.\n fileInputLabel.setAttribute('for', 'file-input')\n fileInputLabel.innerText = 'Browse'\n fileInput = document.createElement('input')\n fileInput.type = fileInputProperties.type\n fileInput.id = 'file-input'\n // To remove the lingering outline after selecting the file.\n fileInput.addEventListener('click', () => {\n fileInput.blur()\n })\n fileInputContainer.append(fileInput, fileInputLabel)\n\n // Creating the final row element to append to the modal body.\n dropContainer.append(imageContainer, dropMessage, orMessage, fileInputContainer)\n fileSelectionRow.appendChild(dropContainer)\n\n // Adding functionality to the drag and drop file input.\n dropContainer.addEventListener('drop', selectDroppedFile.bind(this))\n dropContainer.addEventListener('drop', unhighlightDropArea)\n // Prevent file from being opened by the browser.\n dropContainer.ondragover = highlightDropArea\n dropContainer.ondragenter = highlightDropArea\n dropContainer.ondragleave = unhighlightDropArea\n\n fileInput.addEventListener('change', handleFileChange)\n\n const fileNamePlaceholder = document.createElement('center')\n fileNamePlaceholder.id = 'file-name-placeholder'\n fileNamePlaceholder.className = 'row'\n fileNamePlaceholder.innerText = ''\n fileSelectionRow.appendChild(fileNamePlaceholder)\n fileNamePlaceholder.classList.add('hidden')\n\n // Check if the continue button is invalid on the keystore upload modal and\n // click it if it should no longer be invalid.\n this.continueButton.addEventListener('invalid', () => {\n const fileFormElements = this.container.querySelectorAll('.dcp-modal-body input[type=\"file\"], .dcp-modal-body input[type=\"text\"]')\n const filledInFileFormElements = Array.from(fileFormElements).filter(fileFormElement => fileFormElement.value !== '')\n if (fileFormElements.length !== 0 && filledInFileFormElements.length !== 0) {\n this.continueButton.setCustomValidity('')\n // Clicking instead of dispatching a submit event to ensure other form validation is used before submitting the form.\n this.continueButton.click()\n }\n })\n\n return [fileInput, fileSelectionRow]\n\n /**\n * Checks that the dropped items contain only a single keystore file.\n * If valid, sets the file input's value to the dropped file.\n * @param {DragEvent} event - Contains the files dropped.\n */\n function selectDroppedFile (event) {\n // Prevent file from being opened.\n event.preventDefault()\n\n // Check if only one file was dropped.\n const wasOneFileDropped = event.dataTransfer.items.length === 1 ||\n event.dataTransfer.files.length === 1\n updateFileSelectionStatus(wasOneFileDropped)\n if (!wasOneFileDropped) {\n fileInput.setCustomValidity('Only one file can be uploaded.')\n fileInput.reportValidity()\n return\n } else {\n fileInput.setCustomValidity('')\n }\n\n // Now to use the DataTransfer interface to access the file(s), setting\n // the value of the file input.\n const file = event.dataTransfer.files[0]\n\n if (checkFileExtension(file)) {\n fileInput.files = event.dataTransfer.files\n fileInput.dispatchEvent(new Event('change'))\n }\n }\n\n function handleFileChange () {\n if (checkFileExtension(this.files[0]) && this.files.length === 1) {\n fileNamePlaceholder.innerText = `Selected File: ${this.files[0].name}`\n updateFileSelectionStatus(true)\n // Invoke a callback if additional functionality is required.\n if (typeof fileInputProperties.callback === 'function') {\n fileInputProperties.callback(this.files[0])\n }\n }\n }\n\n /**\n * Checks if the file extension on the inputted file is correct.\n * @param {File} file - The file to check\n * @returns {boolean} True if the file extension is valid, false otherwise.\n */\n function checkFileExtension (file) {\n // If there's no restriction, return true.\n if (!fileInputProperties.extension) {\n return true\n }\n const fileExtension = file.name.split('.').pop()\n const isValidExtension = fileExtension === fileInputProperties.extension\n updateFileSelectionStatus(isValidExtension)\n if (!isValidExtension) {\n fileInput.setCustomValidity(`Only a .${fileInputProperties.extension} file can be uploaded.`)\n fileInput.reportValidity()\n fileNamePlaceholder.classList.add('hidden')\n } else {\n fileInput.setCustomValidity('')\n }\n return isValidExtension\n }\n\n /**\n * Updates the file input to reflect the validity of the current file\n * selection.\n * @param {boolean} isValidFileSelection - True if a single .keystore file\n * was selected. False otherwise.\n */\n function updateFileSelectionStatus (isValidFileSelection) {\n imageContainer.innerHTML = ''\n const statusImage = document.createElement('span')\n statusImage.classList.add('fas', isValidFileSelection ? 'fa-check' : 'fa-times')\n statusImage.style.color = isValidFileSelection ? 'green' : 'red'\n imageContainer.appendChild(statusImage)\n\n if (!isValidFileSelection) {\n fileInput.value = null\n fileNamePlaceholder.classList.add('hidden')\n } else {\n fileNamePlaceholder.classList.remove('hidden')\n }\n\n // If the modal contains a password field for a keystore file, change its\n // visibility.\n const walletPasswordInputContainer = document.querySelector('.dcp-modal-body input[type=\"password\"]').parentElement.parentElement\n if (walletPasswordInputContainer) {\n if (isValidFileSelection) {\n walletPasswordInputContainer.classList.remove('hidden')\n const walletPasswordInput = document.querySelector('.dcp-modal-body input[type=\"password\"]')\n walletPasswordInput.focus()\n } else {\n walletPasswordInputContainer.classList.add('hidden')\n }\n }\n }\n\n function highlightDropArea (event) {\n event.preventDefault()\n this.classList.add('highlight')\n }\n\n function unhighlightDropArea (event) {\n event.preventDefault()\n this.classList.remove('highlight')\n }\n }\n\n /**\n * Sets up a custom tooltip to pop up when the passwords do not match, but are\n * valid otherwise.\n */\n addFormValidationForPasswordConfirmation () {\n const [newPassword, confirmPassword] = document.querySelectorAll('.dcp-modal-body input[type=\"password\"]')\n if (!newPassword || !confirmPassword) {\n throw Error('New Password field and Confirm Password fields not present.')\n }\n\n newPassword.addEventListener('input', checkMatchingPasswords)\n confirmPassword.addEventListener('input', checkMatchingPasswords)\n\n function checkMatchingPasswords () {\n if (newPassword.value !== confirmPassword.value &&\n newPassword.validity.valid &&\n confirmPassword.validity.valid) {\n newPassword.setCustomValidity('Both passwords must match.')\n } else if (newPassword.value === confirmPassword.value ||\n newPassword.validity.tooShort ||\n newPassword.validity.patternMismatch ||\n newPassword.validity.valueMissing ||\n confirmPassword.validity.tooShort ||\n confirmPassword.validity.patternMismatch ||\n confirmPassword.validity.valueMissing) {\n // If the passwords fields match or have become invalidated some other\n // way again, reset the custom message.\n newPassword.setCustomValidity('')\n }\n }\n }\n\n updateInvalidEmailMessage() {\n const email = document.querySelector('.dcp-modal-body input[id=\"email\"')\n if (!email){\n throw Error(\"Email field not present\")\n }\n email.addEventListener('input', checkValidEmail);\n function checkValidEmail() {\n if (!email.validity.patternMismatch &&\n !email.validity.valueMissing) {\n email.setCustomValidity('')\n } else {\n email.setCustomValidity(\"Enter a valid email address.\")\n }\n\n }\n }\n\n /**\n * Adds message(s) to the modal's body.\n * @param {string} messages - The message(s) to add to the modal's body.\n * @returns Paragraph element(s) containing the message(s) added to the\n * modal's body.\n */\n addMessage (...messages) {\n const elements = []\n const body = this.container.querySelector('.dcp-modal-body')\n for (let i = 0; i < messages.length; i++) {\n const row = document.createElement('div')\n row.className = 'row'\n\n const paragraph = document.createElement('p')\n paragraph.innerHTML = messages[i]\n paragraph.classList.add('message')\n row.appendChild(paragraph)\n body.appendChild(row)\n\n elements.push(paragraph)\n }\n\n if (elements.length === 1) return elements[0]\n else return elements\n }\n\n addHorizontalRule () {\n const body = this.container.querySelector('.dcp-modal-body')\n body.appendChild(document.createElement('hr'))\n }\n\n // Does what it says. Still ill advised to use unless you have to.\n addCustomHTML (htmlStr, browseCallback) {\n const elements = []\n const body = this.container.querySelector('.dcp-modal-body')\n body.innerHTML += htmlStr\n body.querySelector('#browse-button').addEventListener('click', browseCallback.bind(this, this))\n\n if (elements.length === 1) return elements[0]\n else return elements\n }\n\n addButton (...buttons) {\n const elements = []\n const body = this.container.querySelector('.dcp-modal-body')\n for (let i = 0; i < buttons.length; i++) {\n const row = document.createElement('div')\n row.className = 'row'\n\n let col = document.createElement('div')\n col.className = 'col-md-4'\n\n const description = document.createElement('span')\n description.innerText = buttons[i].description\n\n col.appendChild(description)\n row.appendChild(col)\n\n col = document.createElement('div')\n col.className = 'col-md-8'\n\n const button = document.createElement('button')\n button.innerText = buttons[i].label\n button.addEventListener('click', buttons[i].callback.bind(this, this))\n\n elements.push(button)\n\n col.appendChild(button)\n row.appendChild(col)\n\n body.appendChild(row)\n }\n\n if (elements.length === 1) return elements[0]\n else return elements\n }\n}\n\n\n// Inject our special stylesheet from dcp-client only if we're on the portal webpage.\nif (typeof window !== 'undefined' && typeof document !== 'undefined' && dcpConfig.portal.location.hostname === window.location.hostname) {\n // <link rel='stylesheet' href='/css/dashboard.css'>\n const stylesheet = document.createElement('link')\n stylesheet.rel = 'stylesheet'\n // Needed for the duplicate check done later.\n stylesheet.id = 'dcp-modal-styles'\n\n const dcpClientBundle = document.getElementById('_dcp_client_bundle')\n let src\n if (dcpClientBundle) {\n src = dcpClientBundle.src.replace('dcp-client-bundle.js', 'dcp-modal-style.css')\n } else {\n src = dcpConfig.portal.location.href + 'dcp-client/dist/dcp-modal-style.css'\n }\n\n stylesheet.href = src\n // If the style was injected before, don't inject it again.\n // Could occur when loading a file that imports Modal.js and loading\n // comput.min.js in the same HTML file.\n if (document.getElementById(stylesheet.id) === null) {\n document.getElementsByTagName('head')[0].appendChild(stylesheet)\n }\n\n if (typeof {\"version\":\"b5864d51c1e7a9b3024698306574b60003a91d62\",\"branch\":\"release\",\"dcpClient\":{\"version\":\"4.2.9\",\"from\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#prod-20220718\",\"resolved\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#15ef46ae27df39b0d1383bd282d36f8c3227905e\"},\"built\":\"Fri Jul 22 2022 09:34:47 GMT-0400 (Eastern Daylight Saving Time)\",\"config\":{\"generated\":\"Fri 22 Jul 2022 09:34:45 AM EDT by erose on lorge\",\"build\":\"debug\"},\"webpack\":\"5.70.0\",\"node\":\"v14.20.0\"} !== 'undefined' && typeof window.Modal === 'undefined') {\n window.Modal = Modal\n }\n}\n\n\n//# sourceURL=webpack://dcp/./portal/www/js/modal.js?");
|
|
3834
|
+
eval("__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ \"default\": () => (/* binding */ Modal)\n/* harmony export */ });\n/**\n * A Small Modal Class\n * @module Modal\n */\n/* globals Event dcpConfig */\nclass Modal {\n constructor (title, message, callback = false, exitHandler = false, {\n continueLabel = 'Continue',\n cancelLabel = 'Cancel',\n cancelVisible = true\n } = {}) {\n const modal = document.createElement('div')\n modal.className = 'dcp-modal-container-old day'\n modal.innerHTML = `\n <dialog class=\"dcp-modal-content\">\n <div class=\"dcp-modal-header\">\n <h2>${title}<button type=\"button\" class=\"close\">×</button></h2>\n ${message ? '<p>' + message + '</p>' : ''}\n </div>\n <div class=\"dcp-modal-loading hidden\">\n <div class='loading'></div>\n </div>\n <form onsubmit='return false' method=\"dialog\">\n <div class=\"dcp-modal-body\"></div>\n <div class=\"dcp-modal-footer ${cancelVisible ? '' : 'centered'}\">\n <button type=\"submit\" class=\"continue green-modal-button\">${continueLabel}</button>\n <button type=\"button\" class=\"cancel green-modal-button\">${cancelLabel}</button>\n </div>\n </form>\n </dialog>`\n\n // To give a reference to do developer who wants to override the form submit.\n // May occur if they want to validate the information in the backend\n // without closing the modal prematurely.\n this.form = modal.querySelector('.dcp-modal-content form')\n this.continueButton = modal.querySelector('.dcp-modal-footer button.continue')\n this.cancelButton = modal.querySelector('.dcp-modal-footer button.cancel')\n this.closeButton = modal.querySelector('.dcp-modal-header .close')\n if (!cancelVisible) {\n this.cancelButton.style.display = 'none'\n }\n\n // To remove the event listener, the reference to the original function\n // added is required.\n this.formSubmitHandler = this.continue.bind(this)\n\n modal.addEventListener('keydown', function (event) {\n event.stopPropagation()\n // 27 is the keycode for the escape key.\n if (event.keyCode === 27) this.close()\n }.bind(this))\n\n this.container = modal\n this.callback = callback\n this.exitHandler = exitHandler\n document.body.appendChild(modal)\n }\n\n changeFormSubmitHandler (newFormSubmitHandler) {\n this.formSubmitHandler = newFormSubmitHandler\n }\n\n /**\n * Validates the form values in the modal and calls the modal's callback\n */\n async continue (event) {\n // To further prevent form submission from trying to redirect from the\n // current page.\n if (event instanceof Event) {\n event.preventDefault()\n }\n let fieldsAreValid = true\n let formElements = this.container.querySelectorAll('.dcp-modal-body select, .dcp-modal-body input, .dcp-modal-body textarea')\n\n const formValues = []\n if (typeof formElements.length === 'undefined') formElements = [formElements]\n // Separate into two loops to enable input validation requiring formValues\n // that come after it. e.g. Two password fields matching.\n for (let i = 0; i < formElements.length; i++) {\n switch (formElements[i].type) {\n case 'file':\n formValues.push(formElements[i])\n break\n case 'checkbox':\n formValues.push(formElements[i].checked)\n break\n default:\n formValues.push(formElements[i].value)\n break\n }\n }\n for (let i = 0; i < formElements.length; i++) {\n if (formElements[i].validation) {\n // Optional fields are allowed to be empty but still can't be wrong if not empty.\n if (!(formElements[i].value === '' && !formElements[i].required)) {\n if (typeof formElements[i].validation === 'function') {\n if (!formElements[i].validation(formValues)) {\n fieldsAreValid = false\n formElements[i].classList.add('is-invalid')\n }\n } else if (!formElements[i].validation.test(formElements[i].value)) {\n fieldsAreValid = false\n formElements[i].classList.add('is-invalid')\n }\n }\n }\n }\n\n if (!fieldsAreValid) return\n\n this.loading()\n if (typeof this.callback === 'function') {\n try {\n return this.callback(formValues)\n } catch (error) {\n console.error('Unexpected error in modal.continue:', error);\n return this.close(false)\n }\n }\n this.close(true)\n }\n\n loading () {\n this.container.querySelector('.dcp-modal-loading').classList.remove('hidden')\n this.container.querySelector('.dcp-modal-body').classList.add('hidden')\n this.container.querySelector('.dcp-modal-footer').classList.add('hidden')\n }\n\n open () {\n this.form.addEventListener('submit', async (event) => {\n const success = await this.formSubmitHandler(event)\n if (success === false) {\n return\n }\n this.close(true)\n })\n // When the user clicks on <span> (x), close the modal\n this.closeButton.addEventListener('click', this.close.bind(this))\n this.cancelButton.addEventListener('click', this.close.bind(this))\n\n // Prevent lingering outlines after clicking some form elements.\n this.container.querySelectorAll('.dcp-modal-body button, .dcp-modal-body input[type=\"checkbox\"]').forEach(element => {\n element.addEventListener('click', () => {\n element.blur()\n })\n })\n\n // Show the modal.\n this.container.style.display = 'block'\n\n const formElements = this.container.querySelectorAll('.dcp-modal-body select, .dcp-modal-body input')\n if (formElements.length) {\n formElements[0].focus()\n if (formElements[0].type === 'text') {\n formElements[0].select()\n }\n for (const el of formElements) {\n if (el.realType) {\n el.type = el.realType\n }\n }\n } else {\n // With no form elements to allow for form submission on enter, focus the\n // continue button.\n this.container.querySelector('.dcp-modal-footer button.continue').focus()\n }\n } // TODO: This should return a promise with the action resolving it\n\n /**\n * Shows the modal and returns a promise of the result of the modal (e.g. was\n * it closed, did its action succeed?)\n */\n showModal () {\n return new Promise((resolve, reject) => {\n this.form.addEventListener('submit', handleContinue.bind(this))\n this.cancelButton.addEventListener('click', handleCancel.bind(this))\n this.closeButton.addEventListener('click', handleCancel.bind(this))\n\n // Prevent lingering outlines after clicking some form elements.\n this.container.querySelectorAll('.dcp-modal-body button, .dcp-modal-body input[type=\"checkbox\"]').forEach(element => {\n element.addEventListener('click', () => {\n element.blur()\n })\n })\n\n // Show the modal.\n this.container.style.display = 'block'\n\n const formElements = this.container.querySelectorAll('.dcp-modal-body select, .dcp-modal-body input')\n if (formElements.length) {\n formElements[0].focus()\n if (formElements[0].type === 'text') {\n formElements[0].select()\n }\n for (const el of formElements) {\n if (el.realType) {\n el.type = el.realType\n }\n }\n } else {\n // With no form elements to allow for form submission on enter, focus the\n // continue button.\n this.continueButton.focus()\n }\n\n async function handleContinue (event) {\n let result\n try {\n result = await this.formSubmitHandler(event)\n } catch (error) {\n reject(error)\n }\n this.close(true)\n resolve(result)\n }\n\n async function handleCancel () {\n let result\n try {\n result = await this.close()\n } catch (error) {\n reject(error)\n }\n resolve(result)\n }\n })\n }\n\n close (success = false) {\n this.container.style.display = 'none'\n if (this.container.parentNode) {\n this.container.parentNode.removeChild(this.container)\n }\n\n // @todo this needs to remove eventlisteners to prevent memory leaks\n\n if ((success !== true) && typeof this.exitHandler === 'function') {\n return this.exitHandler(this)\n }\n }\n\n /**\n * Adds different form elements to the modal depending on the case.\n *\n * @param {*} elements - The properties of the form elements to add.\n * @returns {HTMLElement} The input form elements.\n */\n addFormElement (...elements) {\n const body = this.container.querySelector('.dcp-modal-body')\n const inputElements = []\n let label\n for (let i = 0; i < elements.length; i++) {\n let row = document.createElement('div')\n row.className = 'row'\n\n let col, input\n switch (elements[i].type) {\n case 'button':\n col = document.createElement('div')\n col.className = 'col-md-12'\n\n input = document.createElement('button')\n input.innerHTML = elements[i].label\n input.type = 'button'\n input.classList.add('green-modal-button')\n if (!elements[i].onclick) {\n throw new Error('A button in the modal body should have an on click event handler.')\n }\n input.addEventListener('click', elements[i].onclick)\n\n col.appendChild(input)\n row.appendChild(col)\n break\n case 'textarea':\n col = document.createElement('div')\n col.className = 'col-md-12'\n\n input = document.createElement('textarea')\n input.className = 'text-input-field form-control'\n if (elements[i].placeholder) input.placeholder = elements[i].placeholder\n\n col.appendChild(input)\n row.appendChild(col)\n break\n case 'text':\n case 'email':\n case 'number':\n case 'password': {\n const inputCol = document.createElement('div')\n\n input = document.createElement('input')\n input.type = elements[i].type\n input.validation = elements[i].validation\n input.autocomplete = elements[i].autocomplete || (elements[i].type === 'password' ? 'off' : 'on')\n input.className = 'text-input-field form-control'\n\n // Adding bootstraps custom feedback styles.\n let invalidFeedback = null\n if (elements[i].invalidFeedback) {\n invalidFeedback = document.createElement('div')\n invalidFeedback.className = 'invalid-feedback'\n invalidFeedback.innerText = elements[i].invalidFeedback\n }\n\n if (elements[i].type === 'password') {\n elements[i].realType = 'password'\n }\n\n if (elements[i].label) {\n const labelCol = document.createElement('div')\n label = document.createElement('label')\n label.innerText = elements[i].label\n const inputId = 'dcp-modal-input-' + this.container.querySelectorAll('input[type=\"text\"], input[type=\"email\"], input[type=\"number\"], input[type=\"password\"]').length\n label.setAttribute('for', inputId)\n input.id = inputId\n labelCol.classList.add('col-md-6', 'label-column')\n labelCol.appendChild(label)\n row.appendChild(labelCol)\n inputCol.className = 'col-md-6'\n } else {\n inputCol.className = 'col-md-12'\n }\n\n inputCol.appendChild(input)\n if (invalidFeedback !== null) {\n inputCol.appendChild(invalidFeedback)\n }\n row.appendChild(inputCol)\n break\n }\n case 'select':\n col = document.createElement('div')\n col.className = 'col-md-4'\n\n label = document.createElement('span')\n label.innerText = elements[i].label\n\n col.appendChild(label)\n row.appendChild(col)\n\n col = document.createElement('div')\n col.className = 'col-md-8'\n\n input = document.createElement('select')\n\n col.appendChild(input)\n row.appendChild(col)\n break\n case 'checkbox': {\n row.classList.add('checkbox-row')\n const checkboxLabelCol = document.createElement('div')\n checkboxLabelCol.classList.add('label-column', 'checkbox-label-column')\n\n label = document.createElement('label')\n label.innerText = elements[i].label\n label.for = 'dcp-checkbox-input-' + this.container.querySelectorAll('input[type=\"checkbox\"]').length\n label.setAttribute('for', label.for)\n label.className = 'checkbox-label'\n\n checkboxLabelCol.appendChild(label)\n\n const checkboxCol = document.createElement('div')\n checkboxCol.classList.add('checkbox-column')\n\n input = document.createElement('input')\n input.type = 'checkbox'\n input.id = label.for\n if (elements[i].checked) {\n input.checked = true\n }\n\n checkboxCol.appendChild(input)\n\n if (elements[i].labelToTheRightOfCheckbox) {\n checkboxCol.classList.add('col-md-5')\n row.appendChild(checkboxCol)\n checkboxLabelCol.classList.add('col-md-7')\n row.appendChild(checkboxLabelCol)\n } else {\n checkboxLabelCol.classList.add('col-md-6')\n checkboxCol.classList.add('col-md-6')\n row.appendChild(checkboxLabelCol)\n row.appendChild(checkboxCol)\n }\n break\n }\n case 'file':\n [input, row] = this.addFileInput(elements[i], input, row)\n break\n case 'label':\n row.classList.add('label-row')\n label = document.createElement('label')\n label.innerText = elements[i].label\n row.appendChild(label)\n break\n }\n\n // Copy other possibly specified element properties:\n const inputPropertyNames = ['title', 'inputmode', 'value', 'minLength', 'maxLength', 'size', 'required', 'pattern', 'min', 'max', 'step', 'placeholder', 'accept', 'multiple', 'id', 'onkeypress', 'oninput', 'for', 'readonly', 'autocomplete']\n for (const propertyName of inputPropertyNames) {\n if (Object.prototype.hasOwnProperty.call(elements[i], propertyName)) {\n if (propertyName === 'for' && !label.hasAttribute(propertyName)) {\n label.setAttribute(propertyName, elements[i][propertyName])\n }\n if (propertyName.startsWith('on')) {\n input.addEventListener(propertyName.slice(2), elements[i][propertyName])\n } else {\n input.setAttribute(propertyName, elements[i][propertyName])\n }\n }\n }\n\n inputElements.push(input)\n body.appendChild(row)\n }\n\n if (inputElements.length === 1) return inputElements[0]\n else return inputElements\n }\n\n /**\n * Adds a drag and drop file form element to the modal.\n *\n * @param {*} fileInputProperties - An object specifying some of the\n * properties of the file input element.\n * @param {*} fileInput - Placeholders to help create the file\n * input.\n * @param {HTMLDivElement} row - Placeholders to help create the file\n * input.\n */\n addFileInput (fileInputProperties, fileInput, row) {\n // Adding the upload label.\n const uploadLabel = document.createElement('label')\n uploadLabel.innerText = fileInputProperties.label\n row.appendChild(uploadLabel)\n const body = this.container.querySelector('.dcp-modal-body')\n body.appendChild(row)\n const fileSelectionRow = document.createElement('div')\n fileSelectionRow.id = 'file-selection-row'\n\n // Adding the drag and drop file upload input.\n const dropContainer = document.createElement('div')\n dropContainer.id = 'drop-container'\n\n // Adding an image of a wallet\n const imageContainer = document.createElement('div')\n imageContainer.id = 'image-container'\n const walletImage = document.createElement('span')\n walletImage.classList.add('fas', 'fa-wallet')\n imageContainer.appendChild(walletImage)\n\n // Adding some text prompts\n const dropMessage = document.createElement('span')\n dropMessage.innerText = 'Drop a keystore file here'\n const orMessage = document.createElement('span')\n orMessage.innerText = 'or'\n\n // Adding the manual file input element (hiding the default one)\n const fileInputContainer = document.createElement('div')\n const fileInputLabel = document.createElement('label')\n // Linking the label to the file input so that clicking on the label\n // activates the file input.\n fileInputLabel.setAttribute('for', 'file-input')\n fileInputLabel.innerText = 'Browse'\n fileInput = document.createElement('input')\n fileInput.type = fileInputProperties.type\n fileInput.id = 'file-input'\n // To remove the lingering outline after selecting the file.\n fileInput.addEventListener('click', () => {\n fileInput.blur()\n })\n fileInputContainer.append(fileInput, fileInputLabel)\n\n // Creating the final row element to append to the modal body.\n dropContainer.append(imageContainer, dropMessage, orMessage, fileInputContainer)\n fileSelectionRow.appendChild(dropContainer)\n\n // Adding functionality to the drag and drop file input.\n dropContainer.addEventListener('drop', selectDroppedFile.bind(this))\n dropContainer.addEventListener('drop', unhighlightDropArea)\n // Prevent file from being opened by the browser.\n dropContainer.ondragover = highlightDropArea\n dropContainer.ondragenter = highlightDropArea\n dropContainer.ondragleave = unhighlightDropArea\n\n fileInput.addEventListener('change', handleFileChange)\n\n const fileNamePlaceholder = document.createElement('center')\n fileNamePlaceholder.id = 'file-name-placeholder'\n fileNamePlaceholder.className = 'row'\n fileNamePlaceholder.innerText = ''\n fileSelectionRow.appendChild(fileNamePlaceholder)\n fileNamePlaceholder.classList.add('hidden')\n\n // Check if the continue button is invalid on the keystore upload modal and\n // click it if it should no longer be invalid.\n this.continueButton.addEventListener('invalid', () => {\n const fileFormElements = this.container.querySelectorAll('.dcp-modal-body input[type=\"file\"], .dcp-modal-body input[type=\"text\"]')\n const filledInFileFormElements = Array.from(fileFormElements).filter(fileFormElement => fileFormElement.value !== '')\n if (fileFormElements.length !== 0 && filledInFileFormElements.length !== 0) {\n this.continueButton.setCustomValidity('')\n // Clicking instead of dispatching a submit event to ensure other form validation is used before submitting the form.\n this.continueButton.click()\n }\n })\n\n return [fileInput, fileSelectionRow]\n\n /**\n * Checks that the dropped items contain only a single keystore file.\n * If valid, sets the file input's value to the dropped file.\n * @param {DragEvent} event - Contains the files dropped.\n */\n function selectDroppedFile (event) {\n // Prevent file from being opened.\n event.preventDefault()\n\n // Check if only one file was dropped.\n const wasOneFileDropped = event.dataTransfer.items.length === 1 ||\n event.dataTransfer.files.length === 1\n updateFileSelectionStatus(wasOneFileDropped)\n if (!wasOneFileDropped) {\n fileInput.setCustomValidity('Only one file can be uploaded.')\n fileInput.reportValidity()\n return\n } else {\n fileInput.setCustomValidity('')\n }\n\n // Now to use the DataTransfer interface to access the file(s), setting\n // the value of the file input.\n const file = event.dataTransfer.files[0]\n\n if (checkFileExtension(file)) {\n fileInput.files = event.dataTransfer.files\n fileInput.dispatchEvent(new Event('change'))\n }\n }\n\n function handleFileChange () {\n if (checkFileExtension(this.files[0]) && this.files.length === 1) {\n fileNamePlaceholder.innerText = `Selected File: ${this.files[0].name}`\n updateFileSelectionStatus(true)\n // Invoke a callback if additional functionality is required.\n if (typeof fileInputProperties.callback === 'function') {\n fileInputProperties.callback(this.files[0])\n }\n }\n }\n\n /**\n * Checks if the file extension on the inputted file is correct.\n * @param {File} file - The file to check\n * @returns {boolean} True if the file extension is valid, false otherwise.\n */\n function checkFileExtension (file) {\n // If there's no restriction, return true.\n if (!fileInputProperties.extension) {\n return true\n }\n const fileExtension = file.name.split('.').pop()\n const isValidExtension = fileExtension === fileInputProperties.extension\n updateFileSelectionStatus(isValidExtension)\n if (!isValidExtension) {\n fileInput.setCustomValidity(`Only a .${fileInputProperties.extension} file can be uploaded.`)\n fileInput.reportValidity()\n fileNamePlaceholder.classList.add('hidden')\n } else {\n fileInput.setCustomValidity('')\n }\n return isValidExtension\n }\n\n /**\n * Updates the file input to reflect the validity of the current file\n * selection.\n * @param {boolean} isValidFileSelection - True if a single .keystore file\n * was selected. False otherwise.\n */\n function updateFileSelectionStatus (isValidFileSelection) {\n imageContainer.innerHTML = ''\n const statusImage = document.createElement('span')\n statusImage.classList.add('fas', isValidFileSelection ? 'fa-check' : 'fa-times')\n statusImage.style.color = isValidFileSelection ? 'green' : 'red'\n imageContainer.appendChild(statusImage)\n\n if (!isValidFileSelection) {\n fileInput.value = null\n fileNamePlaceholder.classList.add('hidden')\n } else {\n fileNamePlaceholder.classList.remove('hidden')\n }\n\n // If the modal contains a password field for a keystore file, change its\n // visibility.\n const walletPasswordInputContainer = document.querySelector('.dcp-modal-body input[type=\"password\"]').parentElement.parentElement\n if (walletPasswordInputContainer) {\n if (isValidFileSelection) {\n walletPasswordInputContainer.classList.remove('hidden')\n const walletPasswordInput = document.querySelector('.dcp-modal-body input[type=\"password\"]')\n walletPasswordInput.focus()\n } else {\n walletPasswordInputContainer.classList.add('hidden')\n }\n }\n }\n\n function highlightDropArea (event) {\n event.preventDefault()\n this.classList.add('highlight')\n }\n\n function unhighlightDropArea (event) {\n event.preventDefault()\n this.classList.remove('highlight')\n }\n }\n\n /**\n * Sets up a custom tooltip to pop up when the passwords do not match, but are\n * valid otherwise.\n */\n addFormValidationForPasswordConfirmation () {\n const [newPassword, confirmPassword] = document.querySelectorAll('.dcp-modal-body input[type=\"password\"]')\n if (!newPassword || !confirmPassword) {\n throw Error('New Password field and Confirm Password fields not present.')\n }\n\n newPassword.addEventListener('input', checkMatchingPasswords)\n confirmPassword.addEventListener('input', checkMatchingPasswords)\n\n function checkMatchingPasswords () {\n if (newPassword.value !== confirmPassword.value &&\n newPassword.validity.valid &&\n confirmPassword.validity.valid) {\n newPassword.setCustomValidity('Both passwords must match.')\n } else if (newPassword.value === confirmPassword.value ||\n newPassword.validity.tooShort ||\n newPassword.validity.patternMismatch ||\n newPassword.validity.valueMissing ||\n confirmPassword.validity.tooShort ||\n confirmPassword.validity.patternMismatch ||\n confirmPassword.validity.valueMissing) {\n // If the passwords fields match or have become invalidated some other\n // way again, reset the custom message.\n newPassword.setCustomValidity('')\n }\n }\n }\n\n updateInvalidEmailMessage() {\n const email = document.querySelector('.dcp-modal-body input[id=\"email\"')\n if (!email){\n throw Error(\"Email field not present\")\n }\n email.addEventListener('input', checkValidEmail);\n function checkValidEmail() {\n if (!email.validity.patternMismatch &&\n !email.validity.valueMissing) {\n email.setCustomValidity('')\n } else {\n email.setCustomValidity(\"Enter a valid email address.\")\n }\n\n }\n }\n\n /**\n * Adds message(s) to the modal's body.\n * @param {string} messages - The message(s) to add to the modal's body.\n * @returns Paragraph element(s) containing the message(s) added to the\n * modal's body.\n */\n addMessage (...messages) {\n const elements = []\n const body = this.container.querySelector('.dcp-modal-body')\n for (let i = 0; i < messages.length; i++) {\n const row = document.createElement('div')\n row.className = 'row'\n\n const paragraph = document.createElement('p')\n paragraph.innerHTML = messages[i]\n paragraph.classList.add('message')\n row.appendChild(paragraph)\n body.appendChild(row)\n\n elements.push(paragraph)\n }\n\n if (elements.length === 1) return elements[0]\n else return elements\n }\n\n addHorizontalRule () {\n const body = this.container.querySelector('.dcp-modal-body')\n body.appendChild(document.createElement('hr'))\n }\n\n // Does what it says. Still ill advised to use unless you have to.\n addCustomHTML (htmlStr, browseCallback) {\n const elements = []\n const body = this.container.querySelector('.dcp-modal-body')\n body.innerHTML += htmlStr\n body.querySelector('#browse-button').addEventListener('click', browseCallback.bind(this, this))\n\n if (elements.length === 1) return elements[0]\n else return elements\n }\n\n addButton (...buttons) {\n const elements = []\n const body = this.container.querySelector('.dcp-modal-body')\n for (let i = 0; i < buttons.length; i++) {\n const row = document.createElement('div')\n row.className = 'row'\n\n let col = document.createElement('div')\n col.className = 'col-md-4'\n\n const description = document.createElement('span')\n description.innerText = buttons[i].description\n\n col.appendChild(description)\n row.appendChild(col)\n\n col = document.createElement('div')\n col.className = 'col-md-8'\n\n const button = document.createElement('button')\n button.innerText = buttons[i].label\n button.addEventListener('click', buttons[i].callback.bind(this, this))\n\n elements.push(button)\n\n col.appendChild(button)\n row.appendChild(col)\n\n body.appendChild(row)\n }\n\n if (elements.length === 1) return elements[0]\n else return elements\n }\n}\n\n\n// Inject our special stylesheet from dcp-client only if we're on the portal webpage.\nif (typeof window !== 'undefined' && typeof document !== 'undefined' && dcpConfig.portal.location.hostname === window.location.hostname) {\n // <link rel='stylesheet' href='/css/dashboard.css'>\n const stylesheet = document.createElement('link')\n stylesheet.rel = 'stylesheet'\n // Needed for the duplicate check done later.\n stylesheet.id = 'dcp-modal-styles'\n\n const dcpClientBundle = document.getElementById('_dcp_client_bundle')\n let src\n if (dcpClientBundle) {\n src = dcpClientBundle.src.replace('dcp-client-bundle.js', 'dcp-modal-style.css')\n } else {\n src = dcpConfig.portal.location.href + 'dcp-client/dist/dcp-modal-style.css'\n }\n\n stylesheet.href = src\n // If the style was injected before, don't inject it again.\n // Could occur when loading a file that imports Modal.js and loading\n // comput.min.js in the same HTML file.\n if (document.getElementById(stylesheet.id) === null) {\n document.getElementsByTagName('head')[0].appendChild(stylesheet)\n }\n\n if (typeof {\"version\":\"b5864d51c1e7a9b3024698306574b60003a91d62\",\"branch\":\"release\",\"dcpClient\":{\"version\":\"4.2.10\",\"from\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#prod-20220803\",\"resolved\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#1555d2f6658646b466c9e1cc2c06f673b25a2e53\"},\"built\":\"Fri Aug 05 2022 09:46:39 GMT-0400 (Eastern Daylight Saving Time)\",\"config\":{\"generated\":\"Fri 05 Aug 2022 09:46:36 AM EDT by erose on lorge\",\"build\":\"debug\"},\"webpack\":\"5.70.0\",\"node\":\"v14.20.0\"} !== 'undefined' && typeof window.Modal === 'undefined') {\n window.Modal = Modal\n }\n}\n\n\n//# sourceURL=webpack://dcp/./portal/www/js/modal.js?");
|
|
3835
3835
|
|
|
3836
3836
|
/***/ }),
|
|
3837
3837
|
|
|
@@ -3873,7 +3873,7 @@ eval("/** \n * Factory function which creates instances of the future function t
|
|
|
3873
3873
|
/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
|
|
3874
3874
|
|
|
3875
3875
|
"use strict";
|
|
3876
|
-
eval("/**\n * @file dcp-assert.js\n * Simple assertion module for DCP. Assertions are only\n * evaluated for debug builds, except for the security-named\n * assertions.\n *\n * @author Wes Garland, wes@kingsds.network\n * @date Dec 2019\n */\n\n\nvar isDebugBuild = undefined;\
|
|
3876
|
+
eval("/**\n * @file dcp-assert.js\n * Simple assertion module for DCP. Assertions are only\n * evaluated for debug builds, except for the security-named\n * assertions.\n *\n * @author Wes Garland, wes@kingsds.network\n * @date Dec 2019\n */\n\n\nvar isDebugBuild = undefined;\n\nexports.assertTriggersDebugger = !!(__webpack_require__(/*! process */ \"./node_modules/process/browser.js\").env.DCP_ASSERT_TRIGGERS_DEBUGGER);\nexports.assertTestTriggersDebugger = !!(__webpack_require__(/*! process */ \"./node_modules/process/browser.js\").env.DCP_ASSERT_TEST_TRIGGERS_DEBUGGER);\n\n/** \n * Sets whether this assertion module is running in a debug mode or not. If not called\n * before the first assertion, then we figure this out by checking the module that DCP\n * was configured in with ./configure.sh. The difference is that non-security assertions\n * are ignored during production builds.\n *\n * @note this code test to run very early in the process due to eager module initialization,\n * especially while loading the webpack bundle, which might not have actually set\n * the correct dcpConfig.build yet, so it's stuck at \"bootstrap\", which we treat mostly\n * like a debug build.\n * \n * @param idb {boolean|undefined} falsey if this is release build; truey if this is a debug build; undefined to detect\n */\nexports.setDebugBuild = function dcpAssert$$setDebugBuild(idb)\n{\n if (typeof idb === 'undefined')\n idb = dcpConfig.build !== 'release';\n \n /* In release mode, rewrite the non-security assertions as fast dummy functions */\n if (!idb)\n {\n let dummy = function dcpAssert$dummy(){return};\n for (let assertion of assertionList) {\n if (typeof exports[assertion] === 'function')\n exports[assertion] = dummy;\n }\n }\n else\n {\n for (let assertion of assertionList)\n exports[assertion] = exports.always[assertion];\n }\n\n if (dcpConfig.build !== 'bootstrap')\n isDebugBuild = Boolean(idb); \n}\n \n/** Generic assertion mechanism. Throws if any argument is not true. */\nlet assert = exports.assert = function dcpAssert$$assert() {\n let e;\n\n if (typeof isDebugBuild === 'undefined')\n exports.setDebugBuild();\n \n if (exports.assertTestTriggersDebugger)\n debugger; // allow-debugger\n \n for (let value of arguments) {\n if (!value) {\n if (exports.assertTriggersDebugger)\n debugger; // allow-debugger\n\n try { /* this throws in ES5 strict mode and maybe future-ES */\n let loc = 2;\n if (Object.keys(exports).map((key) => exports[key]).includes(arguments.callee.caller))\n loc++;\n e = new Error('Assertion failure ' + new Error().stack.toString().split('\\n')[loc].trim());\n } catch(error) {\n e = new Error('Assertion failure');\n }\n e.code = 'EASSERT';\n throw e;\n }\n }\n}\n\n/** Evaluate an expression; assert if the result is not true */\nexports.assertEval = function dcpAssert$$assertEval(expr) {\n assert(eval(expr));\n}\n\n/** Assert to that two values are == equivalent */\nexports.assertEq2 = function dcpAssert$$assertEq2(lValue, rValue) {\n assert(lValue == rValue)\n}\n\n/**\n * Asserts that two values are the same in terms of strict equality (===).\n * Can pass an optional message describing the assertion being made.\n *\n * @param {any} expected The expected value to test for equality\n * @param {any} actual The actual value to compare teh expected value against\n * @param {string} [message=''] An message appended to the assertion error\n */\nexports.assertEq3 = function dcpAssert$$assertEq3(\n expected,\n actual,\n message = '',\n) {\n try {\n assert(expected === actual);\n } catch (e) {\n if (message) {\n e.message += `: ${message}`;\n }\n\n e.message += ` (${expected} !== ${actual})`;\n throw e;\n }\n};\n\n/** Assert to that two values are not == equivalent */\nexports.assertNeq2 = function dcpAssert$$assertNeq2(lValue, rValue) {\n assert(lValue != rValue);\n}\n\n/** Assert to that two values are not the same */\nexports.assertNeq3 = function dcpAssert$$assertNeq3(lValue, rValue) {\n assert(lValue !== rValue);\n}\n\n/**\n * Assertion that ensures a given statement will throw a given exception.\n * @param statement {function} function to invoke which is expected to throw\n * @param statement {string} source code of statement which is evaluated with direct-eval\n * and expected to throw\n * @param code [optional] {string} expected value of the exception's code property\n * @param ctor [optional] {function} function which is expected on the exceptions prototype chain\n * @returns true if expectations were met\n */\nexports.assertThrows = function dcpAssert$$assertThrows(statement, code, ctor) {\n var threw;\n \n if (typeof statement === 'string') {\n statement = function shouldThrow_statement() { eval(arguments[0]) };\n }\n if (arguments.length === 2 && typeof code === 'function') {\n ctor = code;\n code = undefined;\n }\n\n try {\n let result = statement();\n threw = false;\n } catch(e) {\n threw = true;\n if (code)\n assert(e.code === code);\n if (ctor)\n assert(e instanceof ctor);\n }\n\n assert(threw === true);\n}\n\n/**\n * Assertion that ensures a given collection contains a given element.\n *\n * @param {any} haystack The collection to search; must be a Set, Map, Array or Array-like object.\n * @param {any} needle The element to search for\n */\nexports.assertHas = function dcpAssert$$assertHas(haystack, needle) {\n if (Array.isArray(haystack))\n assert(haystack.indexOf(needle) !== -1);\n else if (needle instanceof Set || needle instanceof Map)\n assert(haystack.has(needle));\n else\n assert(Array.from(haystack).indexOf(needle) !== -1);\n}\n\n/**\n * Assertion that ensures a given value is of a given type.\n */\nexports.assertIsA = function dcpAssert$$assertIsA(value, type) {\n assert(typeof value === type);\n}\n\n/* *** All assertions must be defined above here *** */\nconst assertionList = Object.keys(exports);\n\n/** Add the security assertions (not disabled by debug build) */\nfor (let assertion of assertionList) {\n let securityAssertion = 'security' + assertion[0].toUpperCase() + assertion.slice(1);\n exports[securityAssertion] = exports[assertion];\n}\n\n/** Add the 'always' assertions (also not disabled by debug build) */\nexports.always = {};\nfor (let assertion of assertionList) {\n exports.always[assertion] = exports[assertion];\n}\n\n\n//# sourceURL=webpack://dcp/./src/common/dcp-assert.js?");
|
|
3877
3877
|
|
|
3878
3878
|
/***/ }),
|
|
3879
3879
|
|
|
@@ -4147,7 +4147,7 @@ eval("/**\n * @file Client facing module that implements Compute Groups API\n
|
|
|
4147
4147
|
\***********************************/
|
|
4148
4148
|
/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
|
|
4149
4149
|
|
|
4150
|
-
eval("/**\n * @file Module that implements Compute API\n * @module dcp/compute\n * @access public\n */\n\n/* global dcpConfig */\n\n/**\n * @access private - Will change soon!\n * @typedef {object} JobRequirements\n * @property {object} environment\n * @property {boolean} environment.fdlibm - Workers express capability 'fdlibm' in order for client applications to have confidence that results will be bitwise identical across workers. This library is recommended, but not required, for implementaiton of Google chrome, Mozilla Firefox and the DCP standalone worker use fdlibm but Microsoft Edge and Apple Safari do not use it.\n * @property {boolean} environment.offscreenCanvas When present, this capability indicates that the worker environment has the OffscreenCanvas symbol defined.\n * @property {object} details\n * @property {object} details.offscreenCanvas\n * @property {boolean} details.offscreenCanvas.bigTexture4096 This capability indicates that the worker's WebGL MAX_TEXTURE_SIZE is at least 4096.\n * @property {boolean} details.offscreenCanvas.bigTexture8192 This capability indicates that the worker's WebGL MAX_TEXTURE_SIZE is at least 8192.\n * @property {boolean} details.offscreenCanvas.bigTexture16384 This capability indicates that the worker's WebGL MAX_TEXTURE_SIZE is at least 16384.\n * @property {boolean} details.offscreenCanvas.bigTexture32768 This capability indicates that the worker's WebGL MAX_TEXTURE_SIZE is at least 32768.\n * @property {object} engine\n * @property {boolean} engine.es7 This capability enforces that the worker is running an es7-compliant JavaScript engine.\n * @property {boolean} engine.spidermonkey The capability enforces that the worker will run in the SpiderMonkey JS engine.\n * @property {boolean} browser.chrome The capability to exclude Chrome browsers.\n */\n\n/**\n * This JSDoc 'namespace' is a convenient way to document some globals that are available only in\n * the sandbox environment provided to work functions.\n * @access public\n * @namespace sandboxEnv\n */\n\n/**\n * This function emits a progress event. Progress events should be emitted approximately once per second; a task which fails to emit a \n * progress event for a certain period of time will be cancelled by the supervisor. The argument to this function is interpreted to six \n * significant digits, and must increase for every call. *All work functions must emit at least one progress event* - this requirement \n * will be enforced by the estimator. * The period of time mentioned above will be at least 30 wall-clock seconds and at least 30 \n * benchmark-adjusted seconds\n * @access public\n * @method progress\n * @param {string|number|undefined} n a number between 0 and 1 (inclusive) that represents a best-guess \n * at the completed portion of the task as a ratio of completed work to total work. If the argument is a string ending in the `%` symbol, \n * it will be interpreted as a number in the usual mathematical sense. If the argument is `undefined`, the slice progress is noted but \n * the amount of progress is considered to be indeterminate.\n * @memberof module:dcp/compute~sandboxEnv\n * @returns {boolean} true\n */\n\n/**\n * Emit a `console` event to the `JobHandle` in the client. If the sandbox is running in an environment with a native console object, the native method may also be invoked.\n * \n * @member console\n * @property {function} log fire console event with 'log' as `level` (see {@link Job#event:console|Job#event:console.level}).\n * @property {function} debug fire console event with 'debug' as `level` (see {@link Job#event:console|Job#event:console.level}).\n * @property {function} info fire console event with 'info' as `level` (see {@link Job#event:console|Job#event:console.level}).\n * @property {function} warn fire console event with 'warn' as `level` (see {@link Job#event:console|Job#event:console.level}).\n * @property {function} error fire console event with 'error' as `level` (see {@link Job#event:console|Job#event:console.level}).\n * \n * **Note** some ES environments (Chrome, Firefox) implement C-style print formatting in this method. This is currently not supported.\n * @memberof module:dcp/compute~sandboxEnv\n * @emits Job#console\n * @access public\n */\n\n/**\n * @member work An object that contains the following properties\n * @property {function} emit (eventName, ...) - send arbitrary args with `eventName`. This event is received client-side with those args (see {@link Job#work}).\n * @property {object} job\n * @property {object} job.public Property set on job handle, see {@link Job#public}.\n * @memberof module:dcp/compute~sandboxEnv\n * @access public\n */\n\n/**\n * @member {OffscreenCanvas} OffscreenCanvas As defined in the {@link https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas|HTML Standard}, provides a canvas which can be rendered off-screen. If this interface is not available in a given worker, the worker will not report capability \"offscreenCanvas\".\n * @memberof module:dcp/compute~sandboxEnv\n * @access public\n */\n\nconst protocolV4 = __webpack_require__(/*! dcp/protocol-v4 */ \"./src/protocol-v4/index.js\");\nconst { EventEmitter } = __webpack_require__(/*! dcp/common/dcp-events/event-emitter */ \"./src/common/dcp-events/event-emitter.js\");\nconst { RangeObject, rehydrateRange } = __webpack_require__(/*! dcp/dcp-client/range-object */ \"./src/dcp-client/range-object.js\");\nconst { RemoteDataSet } = __webpack_require__(/*! dcp/dcp-client/remote-data-set */ \"./src/dcp-client/remote-data-set.js\");\nconst { RemoteDataPattern } = __webpack_require__(/*! dcp/dcp-client/remote-data-pattern */ \"./src/dcp-client/remote-data-pattern.js\");\nconst { Job, ResultHandle } = __webpack_require__(/*! dcp/dcp-client/job */ \"./src/dcp-client/job/index.js\");\nconst { jobStatuses } = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst wallet = __webpack_require__(/*! dcp/dcp-client/wallet */ \"./src/dcp-client/wallet/index.js\");\nconst { DcpURL } = __webpack_require__(/*! dcp/common/dcp-url */ \"./src/common/dcp-url.js\");\nconst { disableWorker } = __webpack_require__(/*! dcp/dcp-client/worker */ \"./src/dcp-client/worker/index.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp-client');\n\nconst debug = (...args) => {\n if (debugging()) {\n args.unshift('dcp-client:compute');\n console.debug(...args);\n }\n};\n\nconst MARKET_VALUE = Symbol('Market Value');\nconst indirectEval = eval; // eslint-disable-line no-eval\n\n/**\n * Instance methods of this class are exported as static members of {@link module:dcp/compute|dcp/compute}.\n * @example\n * let job = compute.for(1, 3, function (i) {\n * progress('100%');\n * return i*10;\n * })\n * let results = await job.exec();\n */\nclass Compute extends EventEmitter {\n constructor () {\n super('Compute');\n\n /** @todo move these into some central config */\n this.defaultMaxGPUs = 1;\n this.GPUsAssigned = 0;\n this.allowSlicesToFinishBeforeStopping = false;\n\n this.schedulerURL = undefined; // Will default to dcpConfig.scheduler.location if not overidden\n this.bankURL = undefined; // Will default to dcpConfig.bank.services.bankTeller.location if not overidden\n\n Object.defineProperty(this.marketValue, MARKET_VALUE, {\n value: true,\n writable: false,\n configurable: false,\n enumerable: false,\n });\n\n const ceci = this;\n this.RemoteDataSet = RemoteDataSet;\n this.RemoteDataPattern = RemoteDataPattern;\n // Initialize to null so these properties are recognized for the Job class\n this.phemeConnection = null;\n this.phemeUses = 0;\n this.deployConnection = null;\n this.openPhemeConn = function openPhemeConn()\n {\n ceci.phemeConnection = new protocolV4.Connection(dcpConfig.scheduler.services.pheme);\n ceci.phemeConnection.on('close', ceci.openPhemeConn);\n }\n\n this.openDeployConn = function openDeployConn()\n {\n ceci.deployConnection = new protocolV4.Connection(dcpConfig.scheduler.services.jobSubmit);\n ceci.deployConnection.on('close', ceci.openDeployConn);\n }\n }\n\n get defaultMaxWorkers () {\n return portalWorker.supervisor.maxWorkingSandboxes\n }\n\n set defaultMaxWorkers (value) {\n portalWorker.supervisor.maxWorkingSandboxes = value\n }\n\n get paymentAddress () {\n return portalWorker.supervisor.paymentAddress\n }\n\n set paymentAddress (value) {\n portalWorker.supervisor.paymentAddress = value\n }\n\n get mining () {\n return portalWorker.working\n }\n\n /** Return a special object that represents the current market value\n * @param {number} factor - OPTIONAL multiplier. default: 1.0\n */\n marketValue (factor = 1) {\n return {\n [MARKET_VALUE]: factor\n };\n }\n\n /** Describes the payment required to compute such a slice on a worker\n * or in a market working at the rates described in the WorkValue object.\n * This function does not take into account job-related overhead.\n * \n * @param {object} sliceProfile A SliceProfile-shaped object\n * @param {object} workValue A WorkValue-shaped object\n * @returns {number} Payment value in DCC describing rate to compute slice\n */\n calculateSlicePayment(sliceProfile, workValue) {\n return sliceProfile.cpuHours * workValue.cpuHour\n + sliceProfile.gpuHours * workValue.gpuHour\n + sliceProfile.inputMBytes * workValue.inputMByte\n + sliceProfile.outputMBytes * workValue.outputMByte;\n }\n\n \n /**\n * Deprecated `mine` API. Use `work` instead\n */\n async mine (numberOfCores, paymentAddress = this.paymentAddress) {\n return this.work(numberOfCores, paymentAddress);\n }\n\n /**\n * Instruct the Worker to stop working, then emit workerInactive\n */\n async stopWorking() {\n console.warn('Using the Compute API for controlling workers is deprecated. Use the Worker API instead.')\n await portalWorker.stop(!this.allowSlicesToFinishBeforeStopping)\n }\n\n /**\n * Deprecated `stopMining` API. Use `stopWorking` instead\n */\n stopMining () {\n console.log(\"Warning: used deprecated 'stopMining' API\");\n this.stopWorking();\n }\n \n /**\n * Flag worker as disabled\n */\n disableWorker() {\n disableWorker();\n }\n\n /**\n * Deprecated `disableMiner` API. Use `disableWorker` instead\n */\n disableMiner() {\n this.disableWorker();\n }\n\n /**\n * Experimental replacement for mine() /* wg aug 2019\n * @param {number} [numberOfCores = this.defaultMaxWorkers] - Number of parallel processes to run\n * @param {string} [paymentAddress = this.paymentAddress || wallet.get().address] - The address of the account to work for\n */\n async work(numCores, paymentAddress) {\n console.warn('Using the Compute API for controlling workers is deprecated. Use the Worker API instead.');\n\n portalWorker.supervisor.paymentAddress = paymentAddress || this.paymentAddress || (await wallet.get()).address;\n portalWorker.supervisor.maxWorkingSandboxes = numCores || this.defaultMaxWorkers || 1;\n await portalWorker.start();\n\n return portalWorker;\n }\n\n /**\n * Form 1. Create a Job for the distributed computer by specifying the start,\n * end, and step for a RangeObject with a given work function and arguments.\n *\n * @param {number} start First number in a range (see {@link RangeObject})\n * @param {number} end Last number in a range (see {@link RangeObject})\n * @param {number} [step=1] the space between numbers in a range (see\n * {@link RangeObject})\n * @param {function | string | URL | DcpURL} work Work function as string or\n * function literal.\n * @param {object} [arguments=[]] optional Array-like object which contains\n * arguments that are passed to the work function\n * @returns {Job}\n * @access public\n * @example\n * let job = compute.for(1, 3, function (i) {\n * progress('100%')\n * return i*10\n * })\n */\n /**\n * Form 2.a Create a Job for the distributed computer by specifying the input\n * as a RemoteDataSet with a given work function and arguments.\n *\n * @param {RemoteDataSet} remoteDataSet Array, ES6 function* generator, or any\n * other type of iterable object.\n * \n * Form 2.b Create a Job for the distributed computer by specifying the input\n * as a RemoteDataPattern with a given work function and arguments.\n * @param {RemoteDataPattern} RemoteDataPattern Array, ES6 function* generator, or any\n * other type of iterable object.\n * @param {function | string | URL | DcpURL} work Work function as string or\n * function literal.\n * @param {object} [arguments=[]] optional Array-like object which contains\n * arguments that are passed to the work function\n * @returns {Job}\n * @access public\n * @example\n * const remoteData = ['https://example.com/data'];\n * const job = compute.for(new RemoteDataSet(...remoteData), function(i) {\n * progress(1)\n * return i\n * })\n * \n * * @example\n * const pattern = 'https://example.com/data/{slice}.json';\n * const NumberOfSlices = 5;\n * const job = compute.for(new RemoteDataSet(pattern, NumberOfSlices), function(i) {\n * progress(1)\n * return i\n * })\n */\n /**\n * Form 3. Create a Job for the distributed computer by specifying the input\n * as an Iterable object with a given work function and arguments.\n *\n * @param {Iterable} iterableObject Array, ES6 function* generator, or any\n * other type of iterable object.\n * @param {function | string | URL | DcpURL} work Work function as string or\n * function literal.\n * @param {object} [arguments=[]] optional Array-like object which contains\n * arguments that are passed to the work function\n * @returns {Job}\n * @access public\n * @example\n * let job = compute.for([123,456], function(i) {\n * progress(1)\n * return i\n * })\n */\n /**\n * Form 4. Create a Job for the distributed computer by specifying the input\n * as a RangeObject with a given work function and arguments.\n *\n * @param {RangeObject} rangeObject Either a RangeObject or an object literal\n * which can be used to create a RangeObject (see {@link RangeObject}).\n * @param {function | string | URL | DcpURL} work Work function as string or\n * function literal.\n * @param {object} [arguments=[]] optional Array-like object which contains\n * arguments that are passed to the work function\n * @returns {Job}\n * @access public\n * @example\n * let job = compute.for({start: 10, end: 13, step: 2}, (i) => progress(1) && i)\n */\n /**\n * Form 5. Similar to Form 4, except with a multi range object containing an\n * array of range objects in the key ranges. These are used to create\n * multi-dimensional ranges, like nested loops. If they were written as\n * traditional loops, the outermost loop would be the leftmost range object,\n * and the innermost loop would be the rightmost range object.\n *\n * @param {MultiRangeObject} multiRange Either a MultiRangeObject or any valid\n * argument to the constructor.\n * @param {function | string | URL | DcpURL} work Work function as string or\n * function literal.\n * @param {object} [arguments=[]] optional Array-like object which contains\n * arguments that are passed to the work function\n * @returns {Job}\n * @access public\n * @example\n * let job = compute.for([{start: 1, end: 2}, {start: 3, end: 5}], function(i,j) {\n * return [i, j, i*j]\n * })\n */\n for(...args) {\n // args, or work function\n const lastArg = args[args.length - 1];\n // work function, if args provided\n const secondLastArg = args[args.length - 2];\n let range = null;\n let work;\n let genArgs;\n\n // All forms: extract work and optional args from the end and validate it.\n if (isValidWorkFunctionType(secondLastArg)) {\n // for(..., workFunction, arguments)\n work = secondLastArg\n if (lastArg && Array.from(lastArg).length > 0 || typeof lastArg === 'string') {\n // array or string, use as-is (string should be a URI)\n genArgs = lastArg;\n } else if (lastArg === undefined) {\n genArgs = [];\n } else {\n throw new TypeError(\n 'Work function arguments (the last parameter), should be an array.',\n );\n }\n } else if (isValidWorkFunctionType(lastArg)) {\n // for(..., workFunction)\n work = lastArg;\n genArgs = [];\n }\n\n if (!isWorkFunctionValid(work)) {\n throw new TypeError(\n `Work parameter must be a function or a string that evaluates to a function, or a URL that points to the location of a function.\\nReceived type ${typeof work} (${work})`,\n );\n }\n\n // Extract the proper information from every form.\n const [firstArg, secondArg, thirdArg] = args;\n if (['object', 'string'].includes(typeof firstArg)) {\n // Forms 2 & 3 (data is iterable, eg. an Array or RemoteDataSet)\n // Forms 4 & 5 (data is RangeObject & Friends)\n range = rehydrateRange(firstArg);\n } else if (typeof firstArg === 'number' && typeof secondArg === 'number') {\n // Form 1 (data is start, end[, step])\n const start = firstArg;\n const end = secondArg;\n\n /**\n * If there is an argument between the end and the work fn, use it as step\n * otherwise set step to undefined.\n */\n const step = typeof thirdArg === 'number' ? thirdArg : undefined;\n range = new RangeObject(start, end, step);\n } else {\n throw new TypeError(\n 'Parameters do not match one of the accepted call forms.',\n );\n }\n\n if (range.length === 0) {\n throw new Error('Length of input set must be greater than zero.');\n }\n\n debug('Work:', work);\n debug('Input Set:', range);\n debug('Arguments:', genArgs);\n return new Job(work, range, genArgs);\n\n /**\n * @param {any} workFunction\n */\n function isValidWorkFunctionType(workFunction) {\n return (\n typeof workFunction === 'function' ||\n typeof workFunction === 'string' ||\n DcpURL.isURL(workFunction)\n );\n }\n\n /**\n * Checks if the workFunction can be evaluated to a function or if it's\n * protocol is HTTPS,\n * @param {function | string | URL | DcpURL} workFunction\n */\n function isWorkFunctionValid(workFunction) {\n switch (typeof workFunction) {\n case 'function':\n return true;\n case 'string': {\n return isValidFunction(workFunction);\n }\n case 'object':\n return DcpURL.isURL(workFunction);\n default:\n return false;\n }\n }\n\n /**\n * Evaluates to true if the string can be coerced into a javascript\n * function.\n * @param {string} workFunction\n */\n function isValidFunction(workFunction) {\n try {\n const fn = indirectEval(`(${workFunction})`);\n return typeof fn === 'function';\n } catch (e) {\n return false;\n }\n }\n }\n\n /**\n * Form 1.\n * Create a job which will run a work function once\n *\n * @param {function | string | URL | DcpURL} work Work function as string or function literal.\n * @param {object} [workFunctionArgs=[]] optional Array-like object which contains arguments which are passed to the work func\n * @returns {Job}\n * @access public\n * @example\n * const job = compute.do(function (i) {\n * progress('100%')\n * return i*10\n * })\n */\n /**\n * Form 2.\n * Create a job which will run a work function on the values 1..n\n *\n * @param {number} n Number of times to run the work function.\n * @param {function | string | URL | DcpURL} work Work function as string or function literal.\n * @param {object} [workFunctionArgs=[]] optional Array-like object which contains arguments which are passed to the work func\n * @returns {Job}\n * @access public\n * @example\n * const job = compute.do(5, function (i) {\n * progress('100%')\n * return i*10\n * })\n */\n do(...args) {\n let n;\n let work;\n let workFunctionArgs;\n if (typeof args[0] !== 'number') {\n // Form 1.\n n = 1;\n [work, workFunctionArgs] = args;\n } else {\n // Form 2.\n [n, work, workFunctionArgs] = args;\n }\n return this.for(0, n - 1, work, workFunctionArgs);\n }\n\n /** Cancel a running job, by opaque id\n *\n * @param {string} id Address of the job to cancel.\n * @param {wallet.Keystore} ownerKeystore The keystore of the job owner\n * @param {string} reason If provided, will be sent on to client\n * @returns {object} The status of the funds that were released from escrow.\n * @throws when id or ownerKeystore are not provided.\n * @access public\n */\n async cancel (id, ownerKeystore, reason = '') {\n if (typeof id !== 'string') {\n throw new Error(`compute.cancel: Job ID must be a string, not ${typeof id}.`);\n }\n\n if (!(ownerKeystore instanceof wallet.Keystore)) {\n throw new Error('compute.cancel: ownerKeystore must be instance of wallet.Keystore');\n }\n\n if (!this.deployConnection)\n this.openDeployConn();\n\n const response = await this.deployConnection.send('cancelJob', {\n job: id,\n owner: ownerKeystore.address,\n reason,\n }, ownerKeystore);\n\n this.deployConnection.off('close', this.deployConnection)\n await this.deployConnection.close();\n\n return response.payload;\n }\n\n /**\n * Reconnect/resume a job, by opaque id\n *\n * @param {string} id Opaque id of the Job to resume/reconnect.\n * @param {wallet.Keystore} [ownerKeystore=wallet.get()] The keystore of the job owner\n * @returns {Job}\n * @throws when id is not provided.\n * @access public\n */\n async resume (id, ownerKeystore) {\n if (typeof id !== 'string') {\n throw new Error(`compute.resume: Job ID must be a string, not ${typeof id}.`);\n }\n\n if (!(ownerKeystore instanceof wallet.Keystore)) {\n ownerKeystore = await wallet.get();\n }\n\n const response = await this.schedulerConnection.send('fetchJob', {\n job: id, // XXX@TODO change 'address' to 'job' once we port this operation to v4\n owner: ownerKeystore.address,\n }, ownerKeystore);\n\n const job = new Job(response.payload);\n job.setPaymentAccountKeystore(ownerKeystore);\n return job;\n }\n\n /**\n * Form 1. Query the status of a job that has been deployed to the scheduler.\n *\n * @access public\n * @param {string|Job} job A job ID or a Job instance\n * @throws when ownerKeystore is not provided\n * @returns {Promise<object>} A status object describing a given job.\n */\n /**\n * Form 2. Query the status of jobs deployed to the scheduler.\n *\n * @access public\n * @param {object} [opts] Query options\n * @param {number} [opts.startTime] Jobs older than this will be excluded from\n * the results.\n * @param {number} [opts.endTime] Jobs newer than this will be excluded from\n * the results.\n * @throws when ownerKeystore is not provided\n * @returns {Promise<Array<Object>>} An array of status objects corresponding to the status of\n * jobs deployed by the given payment account.\n */\n async status(...args) {\n const requestPayload = {};\n\n const firstArg = args.shift();\n if (typeof firstArg === 'string' || firstArg instanceof Job) {\n // Form 1\n requestPayload.jobAddress = firstArg instanceof Job ? firstArg.id : firstArg;\n requestPayload.statuses = jobStatuses;\n } else if (typeof firstArg === 'object') {\n // Form 2\n if (\n (typeof firstArg.startTime !== 'undefined' &&\n typeof firstArg.startTime !== 'number' &&\n !(firstArg.startTime instanceof Date)) ||\n (typeof firstArg.endTime !== 'undefined' &&\n typeof firstArg.endTime !== 'number' &&\n !(firstArg.endTime instanceof Date))\n ) {\n throw new TypeError(\n 'startTime and or endTime were not specified as instances of Date or milliseconds since epoch',\n );\n }\n\n requestPayload.startDeployTime = firstArg.startTime;\n requestPayload.endDeployTime = firstArg.endTime;\n \n if (typeof firstArg.statuses !== 'undefined'\n && (!(firstArg.statuses instanceof Array)\n || !(firstArg.statuses.every((status) => jobStatuses.includes(status))))\n ) {\n throw new TypeError(`Job statuses must be specified in an Array and can only include ${jobStatuses}`)\n }\n \n requestPayload.statuses = firstArg.statuses || jobStatuses;\n \n } else {\n throw new TypeError(`Unknown first argument type ${typeof firstArg}`);\n }\n\n if (!this.deployConnection)\n this.openDeployConn();\n\n const {\n success,\n payload: responsePayload,\n } = await this.deployConnection.send(\n 'listJobs',\n requestPayload\n );\n\n if (!success) {\n throw new DCPError('Request for job status failed', responsePayload);\n }\n\n this.deployConnection.off('close', this.deployConnection)\n await this.deployConnection.close();\n if (responsePayload.length > 0 && (typeof firstArg === 'string' || firstArg instanceof Job)) {\n return responsePayload[0];\n }\n\n return responsePayload;\n }\n\n /**\n * Returns information for the job with the provided ID. It will use\n * `wallet.getId()` to retrieve the auth keystore.\n *\n * @param {string} jobAddress\n * @returns {Promise<object>} status object describing a given job\n * @access public\n */\n async getJobInfo(jobAddress) {\n this.phemeUses++;\n if (!this.phemeConnection)\n this.openPhemeConn();\n\n const ks = await wallet.getId();\n const { payload: jobResponsePayload } = await this.phemeConnection.send('fetchJobReport', \n {\n jobOwner: ks.address,\n job: jobAddress\n });\n\n const { payload: sliceResponsePayload } = await this.phemeConnection.send(\n 'fetchSliceReport', \n {\n jobOwner: ks.address,\n job: jobAddress\n },\n );\n\n const jobReport = {};\n Object.assign(jobReport, jobResponsePayload, sliceResponsePayload);\n\n if (this.phemeConnection && this.phemeUses === 1)\n {\n this.phemeConnection.off('close', this.openPhemeConn);\n this.phemeConnection.close();\n this.phemeConnection = null;\n }\n this.phemeUses--;\n\n return jobReport;\n }\n\n /**\n * Returns information for the slices of the job with the provided ID. It will\n * use `wallet.getId()` to retrieve the auth keystore.\n *\n * @param {string} jobAddress\n * @returns {Promise<object>} A slice status object for the given job.\n * @access public\n */\n async getSliceInfo(jobAddress) {\n this.phemeUses++;\n if (!this.phemeConnection)\n this.openPhemeConn();\n\n const identityKeystore = await wallet.getId();\n const {\n success,\n payload: responsePayload,\n } = await this.phemeConnection.send(\n 'fetchSliceReport',\n {\n job: jobAddress,\n jobOwner: identityKeystore.address,\n },\n identityKeystore,\n );\n\n if (!success) {\n throw new DCPError('Request for slice info failed', responsePayload);\n }\n\n if (this.phemeConnection && this.phemeUses === 1)\n {\n this.phemeConnection.off('close', this.openPhemeConn)\n this.phemeConnection.close();\n this.phemeConnection = null;\n }\n this.phemeUses--;\n\n return responsePayload;\n }\n}\n\nexports.RemoteDataSet = RemoteDataSet;\nexports.RemoteDataPattern = RemoteDataPattern;\nexports.MARKET_VALUE = MARKET_VALUE;\nexports.compute = new Compute(); /* :( */\nexports.compute.getPublicComputeGroupId = () => (__webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\").computeGroups[\"public\"].id);\nexports.compute.getPublicComputeGroupDescriptor = () => (__webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\").computeGroups[\"public\"]);\nexports.compute.ResultHandle = ResultHandle; // export as compute.ResultHandle for instanceof checking\nexports.compute.Job = Job;\nexports.version = {\n api: '1.5.2',\n provides: '1.0.0' /* dcpConfig.scheduler.compatibility.operations.compute */\n};\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/compute.js?");
|
|
4150
|
+
eval("/**\n * @file Module that implements Compute API\n * @module dcp/compute\n * @access public\n */\n\n/* global dcpConfig */\n\n/**\n * @access private - Will change soon!\n * @typedef {object} JobRequirements\n * @property {object} environment\n * @property {boolean} environment.fdlibm - Workers express capability 'fdlibm' in order for client applications to have confidence that results will be bitwise identical across workers. This library is recommended, but not required, for implementaiton of Google chrome, Mozilla Firefox and the DCP standalone worker use fdlibm but Microsoft Edge and Apple Safari do not use it.\n * @property {boolean} environment.offscreenCanvas When present, this capability indicates that the worker environment has the OffscreenCanvas symbol defined.\n * @property {object} details\n * @property {object} details.offscreenCanvas\n * @property {boolean} details.offscreenCanvas.bigTexture4096 This capability indicates that the worker's WebGL MAX_TEXTURE_SIZE is at least 4096.\n * @property {boolean} details.offscreenCanvas.bigTexture8192 This capability indicates that the worker's WebGL MAX_TEXTURE_SIZE is at least 8192.\n * @property {boolean} details.offscreenCanvas.bigTexture16384 This capability indicates that the worker's WebGL MAX_TEXTURE_SIZE is at least 16384.\n * @property {boolean} details.offscreenCanvas.bigTexture32768 This capability indicates that the worker's WebGL MAX_TEXTURE_SIZE is at least 32768.\n * @property {object} engine\n * @property {boolean} engine.es7 This capability enforces that the worker is running an es7-compliant JavaScript engine.\n * @property {boolean} engine.spidermonkey The capability enforces that the worker will run in the SpiderMonkey JS engine.\n * @property {boolean} browser.chrome The capability to exclude Chrome browsers.\n */\n\n/**\n * This JSDoc 'namespace' is a convenient way to document some globals that are available only in\n * the sandbox environment provided to work functions.\n * @access public\n * @namespace sandboxEnv\n */\n\n/**\n * This function emits a progress event. Progress events should be emitted approximately once per second; a task which fails to emit a \n * progress event for a certain period of time will be cancelled by the supervisor. The argument to this function is interpreted to six \n * significant digits, and must increase for every call. *All work functions must emit at least one progress event* - this requirement \n * will be enforced by the estimator. * The period of time mentioned above will be at least 30 wall-clock seconds and at least 30 \n * benchmark-adjusted seconds\n * @access public\n * @method progress\n * @param {string|number|undefined} n a number between 0 and 1 (inclusive) that represents a best-guess \n * at the completed portion of the task as a ratio of completed work to total work. If the argument is a string ending in the `%` symbol, \n * it will be interpreted as a number in the usual mathematical sense. If the argument is `undefined`, the slice progress is noted but \n * the amount of progress is considered to be indeterminate.\n * @memberof module:dcp/compute~sandboxEnv\n * @returns {boolean} true\n */\n\n/**\n * Emit a `console` event to the `JobHandle` in the client. If the sandbox is running in an environment with a native console object, the native method may also be invoked.\n * \n * @member console\n * @property {function} log fire console event with 'log' as `level` (see {@link Job#event:console|Job#event:console.level}).\n * @property {function} debug fire console event with 'debug' as `level` (see {@link Job#event:console|Job#event:console.level}).\n * @property {function} info fire console event with 'info' as `level` (see {@link Job#event:console|Job#event:console.level}).\n * @property {function} warn fire console event with 'warn' as `level` (see {@link Job#event:console|Job#event:console.level}).\n * @property {function} error fire console event with 'error' as `level` (see {@link Job#event:console|Job#event:console.level}).\n * \n * **Note** some ES environments (Chrome, Firefox) implement C-style print formatting in this method. This is currently not supported.\n * @memberof module:dcp/compute~sandboxEnv\n * @emits Job#console\n * @access public\n */\n\n/**\n * @member work An object that contains the following properties\n * @property {function} emit (eventName, ...) - send arbitrary args with `eventName`. This event is received client-side with those args (see {@link Job#work}).\n * @property {object} job\n * @property {object} job.public Property set on job handle, see {@link Job#public}.\n * @memberof module:dcp/compute~sandboxEnv\n * @access public\n */\n\n/**\n * @member {OffscreenCanvas} OffscreenCanvas As defined in the {@link https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas|HTML Standard}, provides a canvas which can be rendered off-screen. If this interface is not available in a given worker, the worker will not report capability \"offscreenCanvas\".\n * @memberof module:dcp/compute~sandboxEnv\n * @access public\n */\n\nconst protocolV4 = __webpack_require__(/*! dcp/protocol-v4 */ \"./src/protocol-v4/index.js\");\nconst { EventEmitter } = __webpack_require__(/*! dcp/common/dcp-events/event-emitter */ \"./src/common/dcp-events/event-emitter.js\");\nconst { RangeObject, rehydrateRange } = __webpack_require__(/*! dcp/dcp-client/range-object */ \"./src/dcp-client/range-object.js\");\nconst { RemoteDataSet } = __webpack_require__(/*! dcp/dcp-client/remote-data-set */ \"./src/dcp-client/remote-data-set.js\");\nconst { RemoteDataPattern } = __webpack_require__(/*! dcp/dcp-client/remote-data-pattern */ \"./src/dcp-client/remote-data-pattern.js\");\nconst { Job, ResultHandle } = __webpack_require__(/*! dcp/dcp-client/job */ \"./src/dcp-client/job/index.js\");\nconst { jobStatuses } = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst wallet = __webpack_require__(/*! dcp/dcp-client/wallet */ \"./src/dcp-client/wallet/index.js\");\nconst { DcpURL } = __webpack_require__(/*! dcp/common/dcp-url */ \"./src/common/dcp-url.js\");\nconst { disableWorker } = __webpack_require__(/*! dcp/dcp-client/worker */ \"./src/dcp-client/worker/index.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp-client');\n\nconst debug = (...args) => {\n if (debugging()) {\n args.unshift('dcp-client:compute');\n console.debug(...args);\n }\n};\n\nconst MARKET_VALUE = Symbol('Market Value');\nconst indirectEval = eval; // eslint-disable-line no-eval\n\n/**\n * Instance methods of this class are exported as static members of {@link module:dcp/compute|dcp/compute}.\n * @example\n * let job = compute.for(1, 3, function (i) {\n * progress('100%');\n * return i*10;\n * })\n * let results = await job.exec();\n */\nclass Compute extends EventEmitter {\n constructor () {\n super('Compute');\n\n /** @todo move these into some central config */\n this.defaultMaxGPUs = 1;\n this.GPUsAssigned = 0;\n this.allowSlicesToFinishBeforeStopping = false;\n\n this.schedulerURL = undefined; // Will default to dcpConfig.scheduler.location if not overidden\n this.bankURL = undefined; // Will default to dcpConfig.bank.services.bankTeller.location if not overidden\n\n Object.defineProperty(this.marketValue, MARKET_VALUE, {\n value: true,\n writable: false,\n configurable: false,\n enumerable: false,\n });\n\n const ceci = this;\n this.RemoteDataSet = RemoteDataSet;\n this.RemoteDataPattern = RemoteDataPattern;\n // Initialize to null so these properties are recognized for the Job class\n this.phemeConnection = null;\n this.phemeUses = 0;\n this.deployConnection = null;\n this.openPhemeConn = function openPhemeConn()\n {\n ceci.phemeConnection = new protocolV4.Connection(dcpConfig.scheduler.services.pheme);\n ceci.phemeConnection.on('close', ceci.openPhemeConn);\n }\n\n this.openDeployConn = function openDeployConn()\n {\n ceci.deployConnection = new protocolV4.Connection(dcpConfig.scheduler.services.jobSubmit);\n ceci.deployConnection.on('close', ceci.openDeployConn);\n }\n }\n\n get defaultMaxWorkers () {\n return portalWorker.supervisor.maxWorkingSandboxes\n }\n\n set defaultMaxWorkers (value) {\n portalWorker.supervisor.maxWorkingSandboxes = value\n }\n\n get paymentAddress () {\n return portalWorker.supervisor.paymentAddress\n }\n\n set paymentAddress (value) {\n portalWorker.supervisor.paymentAddress = value\n }\n\n get mining () {\n return portalWorker.working\n }\n\n /** Return a special object that represents the current market value\n * @param {number} factor - OPTIONAL multiplier. default: 1.0\n */\n marketValue (factor = 1) {\n return {\n [MARKET_VALUE]: factor\n };\n }\n\n /** Describes the payment required to compute such a slice on a worker\n * or in a market working at the rates described in the WorkValue object.\n * This function does not take into account job-related overhead.\n * \n * @param {object} sliceProfile A SliceProfile-shaped object\n * @param {object} workValue A WorkValue-shaped object\n * @returns {number} Payment value in DCC describing rate to compute slice\n */\n calculateSlicePayment(sliceProfile, workValue) {\n return sliceProfile.cpuHours * workValue.cpuHour\n + sliceProfile.gpuHours * workValue.gpuHour\n + sliceProfile.inputMBytes * workValue.inputMByte\n + sliceProfile.outputMBytes * workValue.outputMByte;\n }\n\n \n /**\n * Deprecated `mine` API. Use `work` instead\n */\n async mine (numberOfCores, paymentAddress = this.paymentAddress) {\n return this.work(numberOfCores, paymentAddress);\n }\n\n /**\n * Instruct the Worker to stop working, then emit workerInactive\n */\n async stopWorking() {\n console.warn('Using the Compute API for controlling workers is deprecated. Use the Worker API instead.')\n await portalWorker.stop(!this.allowSlicesToFinishBeforeStopping)\n }\n\n /**\n * Deprecated `stopMining` API. Use `stopWorking` instead\n */\n stopMining () {\n console.log(\"Warning: used deprecated 'stopMining' API\");\n this.stopWorking();\n }\n \n /**\n * Flag worker as disabled\n */\n disableWorker() {\n disableWorker();\n }\n\n /**\n * Deprecated `disableMiner` API. Use `disableWorker` instead\n */\n disableMiner() {\n this.disableWorker();\n }\n\n /**\n * Experimental replacement for mine() /* wg aug 2019\n * @param {number} [numberOfCores = this.defaultMaxWorkers] - Number of parallel processes to run\n * @param {string} [paymentAddress = this.paymentAddress || wallet.get().address] - The address of the account to work for\n */\n async work(numCores, paymentAddress) {\n console.warn('Using the Compute API for controlling workers is deprecated. Use the Worker API instead.');\n\n portalWorker.supervisor.paymentAddress = paymentAddress || this.paymentAddress || (await wallet.get()).address;\n portalWorker.supervisor.maxWorkingSandboxes = numCores || this.defaultMaxWorkers || 1;\n await portalWorker.start();\n\n return portalWorker;\n }\n\n /**\n * Form 1. Create a Job for the distributed computer by specifying the start,\n * end, and step for a RangeObject with a given work function and arguments.\n *\n * @param {number} start First number in a range (see {@link RangeObject})\n * @param {number} end Last number in a range (see {@link RangeObject})\n * @param {number} [step=1] the space between numbers in a range (see\n * {@link RangeObject})\n * @param {function | string | URL | DcpURL} work Work function as string or\n * function literal.\n * @param {object} [arguments=[]] optional Array-like object which contains\n * arguments that are passed to the work function\n * @returns {Job}\n * @access public\n * @example\n * let job = compute.for(1, 3, function (i) {\n * progress('100%')\n * return i*10\n * })\n */\n /**\n * Form 2.a Create a Job for the distributed computer by specifying the input\n * as a RemoteDataSet with a given work function and arguments.\n *\n * @param {RemoteDataSet} remoteDataSet Array, ES6 function* generator, or any\n * other type of iterable object.\n * \n * Form 2.b Create a Job for the distributed computer by specifying the input\n * as a RemoteDataPattern with a given work function and arguments.\n * @param {RemoteDataPattern} RemoteDataPattern Array, ES6 function* generator, or any\n * other type of iterable object.\n * @param {function | string | URL | DcpURL} work Work function as string or\n * function literal.\n * @param {object} [arguments=[]] optional Array-like object which contains\n * arguments that are passed to the work function\n * @returns {Job}\n * @access public\n * @example\n * const remoteData = ['https://example.com/data'];\n * const job = compute.for(new RemoteDataSet(...remoteData), function(i) {\n * progress(1)\n * return i\n * })\n * \n * * @example\n * const pattern = 'https://example.com/data/{slice}.json';\n * const NumberOfSlices = 5;\n * const job = compute.for(new RemoteDataSet(pattern, NumberOfSlices), function(i) {\n * progress(1)\n * return i\n * })\n */\n /**\n * Form 3. Create a Job for the distributed computer by specifying the input\n * as an Iterable object with a given work function and arguments.\n *\n * @param {Iterable} iterableObject Array, ES6 function* generator, or any\n * other type of iterable object.\n * @param {function | string | URL | DcpURL} work Work function as string or\n * function literal.\n * @param {object} [arguments=[]] optional Array-like object which contains\n * arguments that are passed to the work function\n * @returns {Job}\n * @access public\n * @example\n * let job = compute.for([123,456], function(i) {\n * progress(1)\n * return i\n * })\n */\n /**\n * Form 4. Create a Job for the distributed computer by specifying the input\n * as a RangeObject with a given work function and arguments.\n *\n * @param {RangeObject} rangeObject Either a RangeObject or an object literal\n * which can be used to create a RangeObject (see {@link RangeObject}).\n * @param {function | string | URL | DcpURL} work Work function as string or\n * function literal.\n * @param {object} [arguments=[]] optional Array-like object which contains\n * arguments that are passed to the work function\n * @returns {Job}\n * @access public\n * @example\n * let job = compute.for({start: 10, end: 13, step: 2}, (i) => progress(1) && i)\n */\n /**\n * Form 5. Similar to Form 4, except with a multi range object containing an\n * array of range objects in the key ranges. These are used to create\n * multi-dimensional ranges, like nested loops. If they were written as\n * traditional loops, the outermost loop would be the leftmost range object,\n * and the innermost loop would be the rightmost range object.\n *\n * @param {MultiRangeObject} multiRange Either a MultiRangeObject or any valid\n * argument to the constructor.\n * @param {function | string | URL | DcpURL} work Work function as string or\n * function literal.\n * @param {object} [arguments=[]] optional Array-like object which contains\n * arguments that are passed to the work function\n * @returns {Job}\n * @access public\n * @example\n * let job = compute.for([{start: 1, end: 2}, {start: 3, end: 5}], function(i,j) {\n * return [i, j, i*j]\n * })\n */\n for(...args) {\n // args, or work function\n const lastArg = args[args.length - 1];\n // work function, if args provided\n const secondLastArg = args[args.length - 2];\n let range = null;\n let work;\n let genArgs;\n\n // All forms: extract work and optional args from the end and validate it.\n if (isValidWorkFunctionType(secondLastArg)) {\n // for(..., workFunction, arguments)\n work = secondLastArg\n if (lastArg && Array.from(lastArg).length > 0 || typeof lastArg === 'string') {\n // array or string, use as-is (string should be a URI)\n genArgs = lastArg;\n } else if (lastArg === undefined) {\n genArgs = [];\n } else {\n throw new TypeError(\n 'Work function arguments (the last parameter), should be an array.',\n );\n }\n } else if (isValidWorkFunctionType(lastArg)) {\n // for(..., workFunction)\n work = lastArg;\n genArgs = [];\n }\n\n if (!isWorkFunctionValid(work)) {\n throw new TypeError(\n `Work parameter must be a function or a string that evaluates to a function, or a URL that points to the location of a function.\\nReceived type ${typeof work} (${work})`,\n );\n }\n\n // Extract the proper information from every form.\n const [firstArg, secondArg, thirdArg] = args;\n if (['object', 'string'].includes(typeof firstArg)) {\n // Forms 2 & 3 (data is iterable, eg. an Array or RemoteDataSet)\n // Forms 4 & 5 (data is RangeObject & Friends)\n range = rehydrateRange(firstArg);\n } else if (typeof firstArg === 'number' && typeof secondArg === 'number') {\n // Form 1 (data is start, end[, step])\n const start = firstArg;\n const end = secondArg;\n\n /**\n * If there is an argument between the end and the work fn, use it as step\n * otherwise set step to undefined.\n */\n const step = typeof thirdArg === 'number' ? thirdArg : undefined;\n range = new RangeObject(start, end, step);\n } else {\n throw new TypeError(\n 'Parameters do not match one of the accepted call forms.',\n );\n }\n\n if (range.length === 0) {\n throw new Error('Length of input set must be greater than zero.');\n }\n\n debug('Work:', work);\n debug('Input Set:', (range instanceof RangeObject || range instanceof Function) ? range : (Array.isArray(range) ? range.length + ' elements' : '<yes>'));\n debug('Arguments:', Array.isArray(genArgs) && genArgs.length + ' elements');\n return new Job(work, range, genArgs);\n\n /**\n * @param {any} workFunction\n */\n function isValidWorkFunctionType(workFunction) {\n return (\n typeof workFunction === 'function' ||\n typeof workFunction === 'string' ||\n DcpURL.isURL(workFunction)\n );\n }\n\n /**\n * Checks if the workFunction can be evaluated to a function or if it's\n * protocol is HTTPS,\n * @param {function | string | URL | DcpURL} workFunction\n */\n function isWorkFunctionValid(workFunction) {\n switch (typeof workFunction) {\n case 'function':\n return true;\n case 'string': {\n return isValidFunction(workFunction);\n }\n case 'object':\n return DcpURL.isURL(workFunction);\n default:\n return false;\n }\n }\n\n /**\n * Evaluates to true if the string can be coerced into a javascript\n * function.\n * @param {string} workFunction\n */\n function isValidFunction(workFunction) {\n try {\n const fn = indirectEval(`(${workFunction})`);\n return typeof fn === 'function';\n } catch (e) {\n return false;\n }\n }\n }\n\n /**\n * Form 1.\n * Create a job which will run a work function once\n *\n * @param {function | string | URL | DcpURL} work Work function as string or function literal.\n * @param {object} [workFunctionArgs=[]] optional Array-like object which contains arguments which are passed to the work func\n * @returns {Job}\n * @access public\n * @example\n * const job = compute.do(function (i) {\n * progress('100%')\n * return i*10\n * })\n */\n /**\n * Form 2.\n * Create a job which will run a work function on the values 1..n\n *\n * @param {number} n Number of times to run the work function.\n * @param {function | string | URL | DcpURL} work Work function as string or function literal.\n * @param {object} [workFunctionArgs=[]] optional Array-like object which contains arguments which are passed to the work func\n * @returns {Job}\n * @access public\n * @example\n * const job = compute.do(5, function (i) {\n * progress('100%')\n * return i*10\n * })\n */\n do(...args) {\n let n;\n let work;\n let workFunctionArgs;\n if (typeof args[0] !== 'number') {\n // Form 1.\n n = 1;\n [work, workFunctionArgs] = args;\n } else {\n // Form 2.\n [n, work, workFunctionArgs] = args;\n }\n return this.for(0, n - 1, work, workFunctionArgs);\n }\n\n /** Cancel a running job, by opaque id\n *\n * @param {string} id Address of the job to cancel.\n * @param {wallet.Keystore} ownerKeystore The keystore of the job owner\n * @param {string} reason If provided, will be sent on to client\n * @returns {object} The status of the funds that were released from escrow.\n * @throws when id or ownerKeystore are not provided.\n * @access public\n */\n async cancel (id, ownerKeystore, reason = '') {\n if (typeof id !== 'string') {\n throw new Error(`compute.cancel: Job ID must be a string, not ${typeof id}.`);\n }\n\n if (!(ownerKeystore instanceof wallet.Keystore)) {\n throw new Error('compute.cancel: ownerKeystore must be instance of wallet.Keystore');\n }\n\n if (!this.deployConnection)\n this.openDeployConn();\n\n const response = await this.deployConnection.send('cancelJob', {\n job: id,\n owner: ownerKeystore.address,\n reason,\n }, ownerKeystore);\n\n this.deployConnection.off('close', this.deployConnection)\n await this.deployConnection.close();\n\n return response.payload;\n }\n\n /**\n * Reconnect/resume a job, by opaque id\n *\n * @param {string} id Opaque id of the Job to resume/reconnect.\n * @param {wallet.Keystore} [ownerKeystore=wallet.get()] The keystore of the job owner\n * @returns {Job}\n * @throws when id is not provided.\n * @access public\n */\n async resume (id, ownerKeystore) {\n if (typeof id !== 'string') {\n throw new Error(`compute.resume: Job ID must be a string, not ${typeof id}.`);\n }\n\n if (!(ownerKeystore instanceof wallet.Keystore)) {\n ownerKeystore = await wallet.get();\n }\n\n const response = await this.schedulerConnection.send('fetchJob', {\n job: id, // XXX@TODO change 'address' to 'job' once we port this operation to v4\n owner: ownerKeystore.address,\n }, ownerKeystore);\n\n const job = new Job(response.payload);\n job.setPaymentAccountKeystore(ownerKeystore);\n return job;\n }\n\n /**\n * Form 1. Query the status of a job that has been deployed to the scheduler.\n *\n * @access public\n * @param {string|Job} job A job ID or a Job instance\n * @throws when ownerKeystore is not provided\n * @returns {Promise<object>} A status object describing a given job.\n */\n /**\n * Form 2. Query the status of jobs deployed to the scheduler.\n *\n * @access public\n * @param {object} [opts] Query options\n * @param {number} [opts.startTime] Jobs older than this will be excluded from\n * the results.\n * @param {number} [opts.endTime] Jobs newer than this will be excluded from\n * the results.\n * @throws when ownerKeystore is not provided\n * @returns {Promise<Array<Object>>} An array of status objects corresponding to the status of\n * jobs deployed by the given payment account.\n */\n async status(...args) {\n const requestPayload = {};\n\n const firstArg = args.shift();\n if (typeof firstArg === 'string' || firstArg instanceof Job) {\n // Form 1\n requestPayload.jobAddress = firstArg instanceof Job ? firstArg.id : firstArg;\n requestPayload.statuses = jobStatuses;\n } else if (typeof firstArg === 'object') {\n // Form 2\n if (\n (typeof firstArg.startTime !== 'undefined' &&\n typeof firstArg.startTime !== 'number' &&\n !(firstArg.startTime instanceof Date)) ||\n (typeof firstArg.endTime !== 'undefined' &&\n typeof firstArg.endTime !== 'number' &&\n !(firstArg.endTime instanceof Date))\n ) {\n throw new TypeError(\n 'startTime and or endTime were not specified as instances of Date or milliseconds since epoch',\n );\n }\n\n requestPayload.startDeployTime = firstArg.startTime;\n requestPayload.endDeployTime = firstArg.endTime;\n \n if (typeof firstArg.statuses !== 'undefined'\n && (!(firstArg.statuses instanceof Array)\n || !(firstArg.statuses.every((status) => jobStatuses.includes(status))))\n ) {\n throw new TypeError(`Job statuses must be specified in an Array and can only include ${jobStatuses}`)\n }\n \n requestPayload.statuses = firstArg.statuses || jobStatuses;\n \n } else {\n throw new TypeError(`Unknown first argument type ${typeof firstArg}`);\n }\n\n if (!this.deployConnection)\n this.openDeployConn();\n\n const {\n success,\n payload: responsePayload,\n } = await this.deployConnection.send(\n 'listJobs',\n requestPayload\n );\n\n if (!success) {\n throw new DCPError('Request for job status failed', responsePayload);\n }\n\n this.deployConnection.off('close', this.deployConnection)\n await this.deployConnection.close();\n if (responsePayload.length > 0 && (typeof firstArg === 'string' || firstArg instanceof Job)) {\n return responsePayload[0];\n }\n\n return responsePayload;\n }\n\n /**\n * Returns information for the job with the provided ID. It will use\n * `wallet.getId()` to retrieve the auth keystore.\n *\n * @param {string} jobAddress\n * @returns {Promise<object>} status object describing a given job\n * @access public\n */\n async getJobInfo(jobAddress) {\n this.phemeUses++;\n if (!this.phemeConnection)\n this.openPhemeConn();\n\n const ks = await wallet.getId();\n const { payload: jobResponsePayload } = await this.phemeConnection.send('fetchJobReport', \n {\n jobOwner: ks.address,\n job: jobAddress\n });\n\n const { payload: sliceResponsePayload } = await this.phemeConnection.send(\n 'fetchSliceReport', \n {\n jobOwner: ks.address,\n job: jobAddress\n },\n );\n\n const jobReport = {};\n Object.assign(jobReport, jobResponsePayload, sliceResponsePayload);\n\n if (this.phemeConnection && this.phemeUses === 1)\n {\n this.phemeConnection.off('close', this.openPhemeConn);\n this.phemeConnection.close();\n this.phemeConnection = null;\n }\n this.phemeUses--;\n\n return jobReport;\n }\n\n /**\n * Returns information for the slices of the job with the provided ID. It will\n * use `wallet.getId()` to retrieve the auth keystore.\n *\n * @param {string} jobAddress\n * @returns {Promise<object>} A slice status object for the given job.\n * @access public\n */\n async getSliceInfo(jobAddress) {\n this.phemeUses++;\n if (!this.phemeConnection)\n this.openPhemeConn();\n\n const identityKeystore = await wallet.getId();\n const {\n success,\n payload: responsePayload,\n } = await this.phemeConnection.send(\n 'fetchSliceReport',\n {\n job: jobAddress,\n jobOwner: identityKeystore.address,\n },\n identityKeystore,\n );\n\n if (!success) {\n throw new DCPError('Request for slice info failed', responsePayload);\n }\n\n if (this.phemeConnection && this.phemeUses === 1)\n {\n this.phemeConnection.off('close', this.openPhemeConn)\n this.phemeConnection.close();\n this.phemeConnection = null;\n }\n this.phemeUses--;\n\n return responsePayload;\n }\n}\n\nexports.RemoteDataSet = RemoteDataSet;\nexports.RemoteDataPattern = RemoteDataPattern;\nexports.MARKET_VALUE = MARKET_VALUE;\nexports.compute = new Compute(); /* :( */\nexports.compute.getPublicComputeGroupId = () => (__webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\").computeGroups[\"public\"].id);\nexports.compute.getPublicComputeGroupDescriptor = () => (__webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\").computeGroups[\"public\"]);\nexports.compute.ResultHandle = ResultHandle; // export as compute.ResultHandle for instanceof checking\nexports.compute.Job = Job;\nexports.version = {\n api: '1.5.2',\n provides: '1.0.0' /* dcpConfig.scheduler.compatibility.operations.compute */\n};\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/compute.js?");
|
|
4151
4151
|
|
|
4152
4152
|
/***/ }),
|
|
4153
4153
|
|
|
@@ -4157,7 +4157,7 @@ eval("/**\n * @file Module that implements Compute API\n * @module dcp/comput
|
|
|
4157
4157
|
\*********************************/
|
|
4158
4158
|
/***/ ((module, exports, __webpack_require__) => {
|
|
4159
4159
|
|
|
4160
|
-
eval("/* module decorator */ module = __webpack_require__.nmd(module);\n/**\n * @file dcp-client-bundle-src.js\n * Top-level file which gets webpacked into the bundle consumed by dcp-client 2.5\n * @author Wes Garland, wes@kingsds.network\n * @date July 2019\n */\n\n{\n let thisScript = typeof document !== 'undefined' ? (typeof document.currentScript !== 'undefined' && document.currentScript) || document.getElementById('_dcp_client_bundle') : {}\n let realModuleDeclare\n\n if ( false || typeof module.declare === 'undefined') {\n realModuleDeclare = ( true) ? module.declare : 0\n if (false) {}\n module.declare = function moduleUnWrapper (deps, factory) {\n factory(null, module.exports, module)\n return module.exports\n }\n }\n\n let _debugging = () => false\n dcpConfig.future = (__webpack_require__(/*! ../common/config-future.js */ \"./src/common/config-future.js\").futureFactory)(_debugging, dcpConfig);\n\n /* These modules are official API and must be part of DCP Client */\n let officialApi = {\n 'protocol': __webpack_require__(/*! ../protocol-v4 */ \"./src/protocol-v4/index.js\"),\n 'compute': (__webpack_require__(/*! ./compute */ \"./src/dcp-client/compute.js\").compute),\n 'worker': __webpack_require__(/*! ./worker */ \"./src/dcp-client/worker/index.js\"),\n 'wallet': __webpack_require__(/*! ./wallet */ \"./src/dcp-client/wallet/index.js\"),\n };\n\n /* Allow client programs to use modules which happen to be in the bundle anyhow */\n let conveniencePeers = {\n 'ethereumjs-wallet': (__webpack_require__(/*! ./wallet/keystore */ \"./src/dcp-client/wallet/keystore.js\")._internalEth.wallet),\n 'ethereumjs-util': (__webpack_require__(/*! ./wallet/keystore */ \"./src/dcp-client/wallet/keystore.js\")._internalEth.util),\n 'socket.io-client': __webpack_require__(/*! socket.io-client */ \"./node_modules/socket.io-client/build/cjs/index.js\"),\n 'bignumber.js': __webpack_require__(/*! bignumber.js */ \"./node_modules/bignumber.js/bignumber.js\"),\n 'semver': __webpack_require__(/*! semver */ \"./node_modules/semver/semver.js\"),\n };\n\n /* Some of these modules are API-track. Some of them need to be published to be\n * available for top-level resolution by DCP internals. Those (mostly) should have\n * been written using relative module paths.....\n */\n let modules = Object.assign({\n 'dcp-build': {\"version\":\"b5864d51c1e7a9b3024698306574b60003a91d62\",\"branch\":\"release\",\"dcpClient\":{\"version\":\"4.2.
|
|
4160
|
+
eval("/* module decorator */ module = __webpack_require__.nmd(module);\n/**\n * @file dcp-client-bundle-src.js\n * Top-level file which gets webpacked into the bundle consumed by dcp-client 2.5\n * @author Wes Garland, wes@kingsds.network\n * @date July 2019\n */\n\n{\n let thisScript = typeof document !== 'undefined' ? (typeof document.currentScript !== 'undefined' && document.currentScript) || document.getElementById('_dcp_client_bundle') : {}\n let realModuleDeclare\n\n if ( false || typeof module.declare === 'undefined') {\n realModuleDeclare = ( true) ? module.declare : 0\n if (false) {}\n module.declare = function moduleUnWrapper (deps, factory) {\n factory(null, module.exports, module)\n return module.exports\n }\n }\n\n let _debugging = () => false\n dcpConfig.future = (__webpack_require__(/*! ../common/config-future.js */ \"./src/common/config-future.js\").futureFactory)(_debugging, dcpConfig);\n\n /* These modules are official API and must be part of DCP Client */\n let officialApi = {\n 'protocol': __webpack_require__(/*! ../protocol-v4 */ \"./src/protocol-v4/index.js\"),\n 'compute': (__webpack_require__(/*! ./compute */ \"./src/dcp-client/compute.js\").compute),\n 'worker': __webpack_require__(/*! ./worker */ \"./src/dcp-client/worker/index.js\"),\n 'wallet': __webpack_require__(/*! ./wallet */ \"./src/dcp-client/wallet/index.js\"),\n };\n\n /* Allow client programs to use modules which happen to be in the bundle anyhow */\n let conveniencePeers = {\n 'ethereumjs-wallet': (__webpack_require__(/*! ./wallet/keystore */ \"./src/dcp-client/wallet/keystore.js\")._internalEth.wallet),\n 'ethereumjs-util': (__webpack_require__(/*! ./wallet/keystore */ \"./src/dcp-client/wallet/keystore.js\")._internalEth.util),\n 'socket.io-client': __webpack_require__(/*! socket.io-client */ \"./node_modules/socket.io-client/build/cjs/index.js\"),\n 'bignumber.js': __webpack_require__(/*! bignumber.js */ \"./node_modules/bignumber.js/bignumber.js\"),\n 'semver': __webpack_require__(/*! semver */ \"./node_modules/semver/semver.js\"),\n };\n\n /* Some of these modules are API-track. Some of them need to be published to be\n * available for top-level resolution by DCP internals. Those (mostly) should have\n * been written using relative module paths.....\n */\n let modules = Object.assign({\n 'dcp-build': {\"version\":\"b5864d51c1e7a9b3024698306574b60003a91d62\",\"branch\":\"release\",\"dcpClient\":{\"version\":\"4.2.10\",\"from\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#prod-20220803\",\"resolved\":\"git+ssh://git@gitlab.com/Distributed-Compute-Protocol/dcp-client.git#1555d2f6658646b466c9e1cc2c06f673b25a2e53\"},\"built\":\"Fri Aug 05 2022 09:46:39 GMT-0400 (Eastern Daylight Saving Time)\",\"config\":{\"generated\":\"Fri 05 Aug 2022 09:46:36 AM EDT by erose on lorge\",\"build\":\"debug\"},\"webpack\":\"5.70.0\",\"node\":\"v14.20.0\"},\n 'dcp-xhr': __webpack_require__(/*! ../common/dcp-xhr */ \"./src/common/dcp-xhr.js\"),\n 'dcp-env': __webpack_require__(/*! ../common/dcp-env */ \"./src/common/dcp-env.js\"),\n 'dcp-url': __webpack_require__(/*! ../common/dcp-url */ \"./src/common/dcp-url.js\"),\n 'cli': __webpack_require__(/*! ../common/cli */ \"./src/common/cli.js\"),\n 'dcp-timers': __webpack_require__(/*! ../common/dcp-timers */ \"./src/common/dcp-timers.js\"),\n 'dcp-dot-dir': __webpack_require__(/*! ../common/dcp-dot-dir */ \"./src/common/dcp-dot-dir.js\"),\n 'dcp-assert': __webpack_require__(/*! ../common/dcp-assert */ \"./src/common/dcp-assert.js\"),\n 'dcp-events': __webpack_require__(/*! ../common/dcp-events */ \"./src/common/dcp-events/index.js\"),\n 'utils': __webpack_require__(/*! ../utils */ \"./src/utils/index.js\"),\n 'debugging': __webpack_require__(/*! ../debugging */ \"./src/debugging.js\"),\n 'publish': __webpack_require__(/*! ../common/dcp-publish */ \"./src/common/dcp-publish.js\"),\n 'compute-groups': {\n ...__webpack_require__(/*! ./compute-groups */ \"./src/dcp-client/compute-groups/index.js\"),\n publicGroupOpaqueId: (__webpack_require__(/*! ../common/scheduler-constants */ \"./src/common/scheduler-constants.js\").computeGroups[\"public\"].opaqueId),\n },\n 'bank-util': __webpack_require__(/*! ./bank-util */ \"./src/dcp-client/bank-util.js\"),\n 'protocol-v4': __webpack_require__(/*! ../protocol-v4 */ \"./src/protocol-v4/index.js\"), /* deprecated */\n 'client-modal': __webpack_require__(/*! ./client-modal */ \"./src/dcp-client/client-modal/index.js\"),\n 'legacy-modal': (__webpack_require__(/*! ../../portal/www/js/modal */ \"./portal/www/js/modal.js\").Modal),\n 'eth': __webpack_require__(/*! ./wallet/eth */ \"./src/dcp-client/wallet/eth.js\"),\n 'serialize': __webpack_require__(/*! ../utils/serialize */ \"./src/utils/serialize.js\"),\n 'job': __webpack_require__(/*! ./job */ \"./src/dcp-client/job/index.js\"),\n 'range-object': __webpack_require__(/*! ./range-object */ \"./src/dcp-client/range-object.js\"),\n 'stats-ranges': __webpack_require__(/*! ./stats-ranges */ \"./src/dcp-client/stats-ranges.js\"),\n 'standard-objects': {}\n }, conveniencePeers, officialApi);\n\n /* Export the JS Standard Classes (etc) from the global object of the bundle evaluation context,\n * in case we have code somewhere that needs to use these for instanceof checks.\n */\n ;[ Object, Function, Boolean, Symbol,\n Error, EvalError, RangeError, ReferenceError, SyntaxError, TypeError, URIError,\n Number, Math, Date,\n String, RegExp,\n Array, Int8Array, Uint8Array, Uint8ClampedArray, Int16Array, Uint16Array, Int32Array, Uint32Array, Float32Array, Float64Array,\n Map, Set, WeakMap, WeakSet,\n ArrayBuffer, DataView, JSON,\n Promise, \n Reflect, Proxy, Intl, WebAssembly, __webpack_require__\n ].forEach(function (obj) {\n if (obj.name && (typeof obj === 'function' || typeof obj === 'object'))\n modules['standard-objects'][obj.name] = obj\n })\n\n if (typeof BigInt !== 'undefined')\n modules['standard-objects']['BigInt'] === BigInt;\n if (typeof BigInt64Array !== 'undefined')\n modules['standard-objects']['BigInt64Array'] === BigInt64Array;\n if (typeof BigInt64Array !== 'undefined')\n modules['standard-objects']['BigUint64Array'] === BigUint64Array;\n\n module.declare([], function(require, exports, module) {\n Object.assign(exports, modules)\n exports['dcp-config'] = dcpConfig\n })\n if (realModuleDeclare)\n module.declare = realModuleDeclare\n\n bundleExports = thisScript.exports = exports; /* must be last expression evaluated! */\n}\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/index.js?");
|
|
4161
4161
|
|
|
4162
4162
|
/***/ }),
|
|
4163
4163
|
|
|
@@ -4168,7 +4168,7 @@ eval("/* module decorator */ module = __webpack_require__.nmd(module);\n/**\n *
|
|
|
4168
4168
|
/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
|
|
4169
4169
|
|
|
4170
4170
|
"use strict";
|
|
4171
|
-
eval("/**\n * @file job/index.js\n * @author Eddie Roosenmaallen, eddie@kingsds.network\n * Matthew Palma, mpalma@kingsds.network\n * Wes Garland, wes@kingsds.network\n * Paul, paul@kingsds.network\n * Ryan Saweczko, ryansaweczko@kingsds.network\n * @date November 2018\n * November 2018\n * February 2022\n * May 2022\n * Jun 2022\n *\n * This module implements the Compute API's Job Handle\n *\n */\n/** @typedef {import('dcp/dcp-client/wallet/keystore').Keystore} Keystore */\n\n\nconst { BigNumber } = __webpack_require__(/*! bignumber.js */ \"./node_modules/bignumber.js/bignumber.js\");\nconst { v4: uuidv4 } = __webpack_require__(/*! uuid */ \"./node_modules/uuid/dist/esm-browser/index.js\");\nconst { EventEmitter, PropagatingEventEmitter } = __webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\");\nconst { RangeObject, MultiRangeObject, DistributionRange, SuperRangeObject, SparseRangeObject } = __webpack_require__(/*! dcp/dcp-client/range-object */ \"./src/dcp-client/range-object.js\");\nconst { fetchURI, encodeDataURI, dumpObject } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst { getTextEncoder, createTempFile } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst serialize = __webpack_require__(/*! dcp/utils/serialize */ \"./src/utils/serialize.js\");\nconst wallet = __webpack_require__(/*! dcp/dcp-client/wallet */ \"./src/dcp-client/wallet/index.js\");\nconst protocolV4 = __webpack_require__(/*! dcp/protocol-v4 */ \"./src/protocol-v4/index.js\");\nconst { EventSubscriber } = __webpack_require__(/*! dcp/events/event-subscriber */ \"./src/events/event-subscriber.js\");\nconst { DcpURL } = __webpack_require__(/*! dcp/common/dcp-url */ \"./src/common/dcp-url.js\");\nconst ClientModal = __webpack_require__(/*! dcp/dcp-client/client-modal */ \"./src/dcp-client/client-modal/index.js\");\nconst { Worker } = __webpack_require__(/*! dcp/dcp-client/worker */ \"./src/dcp-client/worker/index.js\");\nconst { RemoteDataSet } = __webpack_require__(/*! dcp/dcp-client/remote-data-set */ \"./src/dcp-client/remote-data-set.js\");\nconst { RemoteDataPattern } = __webpack_require__(/*! dcp/dcp-client/remote-data-pattern */ \"./src/dcp-client/remote-data-pattern.js\");\nconst { ResultHandle } = __webpack_require__(/*! ./result-handle */ \"./src/dcp-client/job/result-handle.js\");\nconst { SlicePaymentOffer } = __webpack_require__(/*! ./slice-payment-offer */ \"./src/dcp-client/job/slice-payment-offer.js\");\nconst { addSlices } = __webpack_require__(/*! ./upload-slices */ \"./src/dcp-client/job/upload-slices.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nconst dcpPublish = __webpack_require__(/*! dcp/common/dcp-publish */ \"./src/common/dcp-publish.js\");\nconst computeGroups = __webpack_require__(/*! dcp/dcp-client/compute-groups */ \"./src/dcp-client/compute-groups/index.js\");\nconst schedulerConstants = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst { sliceStatus } = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst { jobStatus } = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst bankUtil = __webpack_require__(/*! dcp/dcp-client/bank-util */ \"./src/dcp-client/bank-util.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp-client');\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\nlet tunedKvin;\n\nconst log = (...args) => {\n if (debugging('job')) {\n console.debug('dcp-client:job', ...args);\n }\n};\n\nconst ON_BROWSER = DCP_ENV.isBrowserPlatform;\nconst sideloaderModuleIdentifier = 'sideloader-v1';\n\n\nconst DEFAULT_REQUIREMENTS = {\n engine: {\n es7: null,\n spidermonkey: null\n },\n environment: {\n webgpu: null,\n offscreenCanvas: null,\n fdlibm: null\n },\n browser: {\n chrome: null\n },\n details: {\n offscreenCanvas: {\n bigTexture4096: null,\n bigTexture8192: null,\n bigTexture16384: null,\n bigTexture32768: null,\n }\n },\n discrete: null,\n useStrict: null,\n};\n\n/** @typedef {import('../range-object').RangeLike} RangeLike */\n\n/**\n * Ensure input data is an appropriate format\n * @param {RangeObject | DistributionRange | RemoteDataSet | Array | Iterable}\n * inputData - A URI-shaped string, a [Multi]RangeObject-constructing value, or\n * an array of slice data\n * @return {RangeObject | RangeLike | DistributionRange | RemoteDataSet | Array}\n * The coerced input in an appropriate format ([Multi]RangeObject,\n * DistributionRange, RemoteDataSet, or array)\n */\n const wrangleData = (inputData) => {\n\n if (RangeObject.isRangelike(inputData)) { return inputData }\n if (RangeObject.isRangeObject(inputData)) { return inputData }\n if (DistributionRange.isDistribution(inputData)) { return inputData }\n if (inputData instanceof SparseRangeObject) { return inputData }\n if (inputData instanceof MultiRangeObject) { return inputData }\n if (MultiRangeObject.isProtoMultiRangelike(inputData)) { return new MultiRangeObject(inputData) }\n if (RangeObject.isProtoRangelike(inputData)) { return new RangeObject(inputData) }\n if (DistributionRange.isProtoDistribution(inputData)) { return new DistributionRange(inputData) }\n if (RemoteDataSet.isRemoteDataSet(inputData)) { return inputData }\n if (RemoteDataPattern.isRemoteDataPattern(inputData)) { return inputData }\n\n return Array.isArray(inputData) ? inputData : [inputData];\n};\n\n/**\n * @classdesc The Compute API's Job Handle (see {@link https://docs.dcp.dev/specs/compute-api.html#job-handles|Compute API spec})\n * Job handles are objects which correspond to jobs. \n * They are created by some exports of the compute module, such as {@link module:dcp/compute.do|compute.do} and {@link module:dcp/compute.for|compute.for}.\n * @extends module:dcp/dcp-events.PropagatingEventEmitter\n * @hideconstructor\n * @access public\n */\nclass Job extends PropagatingEventEmitter\n{\n /**\n * Fired when the job is accepted by the scheduler on deploy.\n * \n * @event Job#accepted\n * @access public\n * @type {object}\n *//**\n * Fired when the job is cancelled.\n * \n * @event Job#cancel\n * @access public\n *//**\n * Fired when a result is returned.\n * \n * @event Job#result\n * @access public\n * @type {object}\n * @property {string} jobAddress Address of the job\n * @property {string} task ID of the task (slice) the result came from\n * @property {number} sort The index of the slice\n * @property {object} result\n * @property {string} result.request\n * @property {*} result.result The value returned from the work function\n *//**\n * Fired when the result handle is modified, either when a new `result` event is fired or when the results are populated with `results.fetch()`\n * \n * @event Job#resultsUpdated\n * @access public\n *//**\n * Fired when the job has been completed.\n * \n * @event Job#complete\n * @access public\n * @type {ResultHandle}\n *//**\n * Fired when the job's status changes.\n * \n * @event Job#status\n * @access public\n * @type {object}\n * @property {string} jobAddress Address of the job\n * @property {number} total Total number of slices in the job\n * @property {number} distributed Number of slices that have been distributed\n * @property {number} computed Number of slices that have completed execution (returned a result)\n * @property {string} runStatus Current runStatus of the job\n *//**\n * Fired when a slice throws an error.\n * \n * @event Job#error\n * @access public\n * @type {object}\n * @property {string} jobAddress Address of the job\n * @property {number} sliceIndex Index of the slice that threw the error\n * @property {string} message The error message\n * @property {string} stack The error stacktrace\n * @property {string} name The error type name\n *//**\n * Fired when a slice uses one of the console log functions.\n * \n * @event Job#console\n * @access public\n * @type {object}\n * @property {string} jobAddress Address of the job\n * @property {number} sliceIndex The index of the slice that produced this event\n * @property {string} level The log level, one of `debug`, `info`, `log`, `warn`, or `error`\n * @property {string} message The console log message\n *//**\n * Fired when a slice is stopped for not calling progress. Contains information about how long the slice ran for, and about the last reported progress calls.\n * \n * @event Job#noProgress\n * @access public\n * @type {object}\n * @property {string} jobAddress Address of the job\n * @property {number} sliceIndex The index of the slice that failed due to no progress\n * @property {number} timestamp How long the slice ran before failing\n * @property {object} progressReports\n * @property {object} progressReports.last The last progress report received from the worker\n * @property {number} progressReports.last.timestamp Time since the start of the slice\n * @property {number} progressReports.last.progress Progress value reported\n * @property {*} progressReports.last.value The last value that was passed to the progress function\n * @property {number} progressReports.last.throttledReports Number of calls to progress that were throttled since the last report\n * @property {object} progressReports.lastUpdate The last determinate (update to the progress param) progress report received from the worker\n * @property {number} progressReports.lastUpdate.timestamp\n * @property {number} progressReports.lastUpdate.progress\n * @property {*} progressReports.lastUpdate.value\n * @property {number} progressReports.lastUpdate.throttledReports\n *//**\n @todo: is this in the spec? is there a no progress data? should there be?\n * Identical to `noProgress`, except that it also contains the data that the slice was executed with.\n * \n * @event Job#noProgressData\n * @access public\n * @type {object}\n * @property {*} data The data that the slice was executed with\n *//**\n * Fired when the job is paused due to running out of funds. The job can be resumed by escrowing more funds then resuming the job.\n * @todo: is this a thing, should it be a thing (the payload)\n * Event payload is the estimated funds required to complete the job\n * \n * @event Job#nofunds\n * @access public\n * @type {BigNumber}\n *//**\n * Fired when the job cannot be deployed due to no bank account / not enough balance to deploy the job\n * \n * @event Job#ENOFUNDS\n * @access public\n *//**\n * Fired when the job is cancelled due to the work function not calling the `progress` method frequently enough.\n * \n * @event Job#ENOPROGRESS\n * @access public\n *//**\n * The job was cancelled because scheduler has determined that individual tasks in this job exceed the maximum allowable execution time.\n * \n * @event Job#ESLICETOOSLOW\n * @access public\n *//**\n * Fired when the job is cancelled because too many work functions are terminating with uncaught exceptions.\n * \n * @event Job#ETOOMANYERRORS\n * @access public\n */\n\n /**\n * @form1 new Job('application_worker_address'[, data[, arguments]])\n * @form2a new Job('worker source'[, data[, arguments]])\n * @form2b new Job(worker_function[, data[, arguments]])\n */\n constructor ()\n {\n super('Job');\n if (typeof arguments[0] === 'function')\n arguments[0] = arguments[0].toString();\n\n if (typeof arguments[0] === 'string')\n {\n const { encodeDataURI } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\n this.workFunctionURI = encodeDataURI(arguments[0], 'application/javascript');\n } \n else if (DcpURL.isURL(arguments[0]))\n this.workFunctionURI = arguments[0].href;\n\n this.jobInputData = wrangleData(arguments[1] || []);\n this.jobArguments = wrangleData(arguments[2] || []);\n \n log('wrangledInputData:', this.jobInputData);\n log('wrangledArguments:', this.jobArguments);\n\n this.initEventSystems();\n\n /**\n * An object describing the cost the user believes each the average slice will incur, in terms of CPU/GPU and I/O.\n * If defined, this object is used to provide initial scheduling hints and to calculate escrow amounts.\n *\n * @type {object}\n * @access public\n */\n this.initialSliceProfile = undefined;\n \n // The max value that the client is willing to spend to deploy\n // (list on the scheduler, doesn't include compute payment)\n // maxDeployPayment is the max the user is willing to pay to DCP (as a\n // Hold), in addition to the per-slice offer and associated scrape.\n // Currently calculated as `deployCost = costPerKB *\n // (JSON.stringify(job).length / 1024) // 1e-9 per kb`\n // @todo: figure this out / er nov 2018\n this.maxDeployPayment = 1;\n\n /**\n * An object describing the requirements that workers must have to be eligible for this job. See\n * {@link https://docs.dcp.dev/specs/compute-api.html#requirements-objects|Requirements Objects}.\n *\n * @type {object}\n * @access public\n */\n this.requirements = JSON.parse(JSON.stringify(DEFAULT_REQUIREMENTS));\n\n /**\n * A place to store public-facing attributes of the job. Anything stored on this object will be available inside the work \n * function (see {@link module:dcp/compute~sandboxEnv.work}). The properties documented here may be used by workers to display what jobs are currently being \n * worked on.\n * @access public\n * @property {string} name Public-facing name of this job.\n * @property {string} description Public-facing description for this job.\n * @property {string} link Public-facing link to external resource about this job.\n */\n this.public = {\n name: null,\n description: null,\n link: null,\n };\n\n /**\n * A number (can be null, undefined, or infinity) describing the estimationSlicesRemaining in the jpd (dcp-2593)\n * @type {number}\n * @access public\n */\n this.estimationSlices = undefined;\n /**\n * When true, allows a job in estimation to have requestTask return multiple estimation slices.\n * This flag applies independent of infinite estimation, viz., this.estimationSlices === null .\n * @type {boolean}\n * @access public\n */\n this.greedyEstimation = false;\n /**\n * tunable parameters per job\n * @access public\n * @param {object} tuning \n * @param {string} tuning.kvin Encode the TypedArray into a string, trying multiple methods to determine optimum \n * size/performance. The this.tune variable affects the behavior of this code this:\n * @param {boolean} speed If true, only do naive encoding: floats get represented as byte-per-digit strings\n * @param {boolean} size If true, try the naive, ab8, and ab16 encodings; pick the smallest\n * If both are false try the naive encoding if under typedArrayPackThreshold and use if smaller\n * than ab8; otherwise, use ab8\n */\n this.tuning = {\n kvin: {\n size: false,\n speed: false,\n },\n }\n /* For API interface to end-users only */\n Object.defineProperty(this, 'id', {\n get: () => this.address,\n set: (id) => { this.address = id }\n });\n \n this.uuid = uuidv4(); /** @see {@link https://kingsds.atlassian.net/browse/DCP-1475?atlOrigin=eyJpIjoiNzg3NmEzOWE0OWI4NGZkNmI5NjU0MWNmZGY2OTYzZDUiLCJwIjoiaiJ9|Jira Issue} */\n this.dependencies = []; /* dependencies of the work function */\n this.requirePath = []; /* require path for dependencies */\n this.connected = false; /* true when exec or resume called */\n this.results = new ResultHandle(this); /* result handle */\n this.collateResults = true; /* option to receive results as they are computed & ensure all are received on finish */\n this.force100pctCPUDensity = false; /* tell scheduler to assume this job uses 100% cpu density */\n this.workerConsole = false; /* tell workers to log more information about failures in the evaluator this job causes */\n this.address = null; /* job address, created by scheduler during exec call. */\n this.paymentAccountKeystore = null; /* keystore for payment for job to come from */\n this.status = { /* job status details */\n runStatus: null,\n total: null,\n distributed: null,\n computed: null\n };\n // Compute groups. Add to public compute group by default\n this.computeGroups = [ Object.assign({}, schedulerConstants.computeGroups.public) ];\n \n // Update the ready state as we go through job deployment\n this.readyState = sliceStatus.new;\n const that = this;\n this.readyStateChange = function job$$readyStateChange (readyState)\n {\n that.readyState = readyState;\n that.emit('readyStateChange', that.readyState);\n }\n }\n \n /**\n * Initialize the various event systems the job handle requires. These include:\n * - an internal event emitter (this.ee)\n * - an event emitter for any events emitted on `work.emit` within work functions (this.work)\n * - an event subscriber to subscribe (to receive) events from the scheduler (this.eventSubscriber)\n */\n initEventSystems ()\n {\n // Handle the various event-related things required in the constructor\n\n // Internal event emitter for events within job handle\n this.ee = new EventEmitter('Job Internal');\n\n /**\n * An EventEmitter for custom events dispatched by the work function.\n * @type {module:dcp/dcp-events.EventEmitter}\n * @access public\n * @example\n * // in work function\n * work.emit('myEventName', 1, [2], \"three\");\n * // client-side\n * job.work.on('myEventName', (num, arr, string) => { });\n */\n this.work = new EventEmitter('job.work');\n this.listenForCustomEvents = false;\n\n // Event subscriber - receive events from the event router\n this.eventSubscriber = new EventSubscriber(this);\n \n // Some events from the event subscriber can't be emitted immediately upon receipt without having \n // weird/wrong output due to things like serialization. We allow interceptors in the event subscriber\n // to handle this.\n const that = this\n var lastConsoleEv;\n var sameCounter = 1;\n const parseConsole = function deserializeConsoleMessage(ev) {\n if (tunedKvin)\n ev.message = tunedKvin.unmarshal(ev.message);\n else \n ev.message = kvin.unmarshal(ev.message);\n \n if (lastConsoleEv && ev.message[0] === lastConsoleEv.message[0] && ev.sliceNumber === lastConsoleEv.sliceNumber && ev.level === lastConsoleEv.level)\n ev.same = ++sameCounter;\n else\n sameCounter = 1;\n lastConsoleEv = ev;\n \n /* if we have the same message being logged (same sliceNumber, message, log level), the console event object will have the sole property same, nothing else */\n if (ev.same > 1)\n that.emit('console', { same: ev.same });\n else\n {\n delete ev.same;\n that.emit('console', ev);\n }\n }\n\n this.eventIntercepts = {\n result: (ev) => this.handleResult(ev),\n status: (ev) => this.handleStatus(ev),\n cancel: (ev) => this.ee.emit('stopped', ev),\n custom: (ev) => this.work.emit(ev.customEvent, ev),\n console: parseConsole,\n };\n \n this.eventTypes = (__webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\").eventTypes);\n\n this.work.on('newListener', (evt) => {\n this.listenForCustomEvents = true;\n });\n this.desiredEvents = []\n this.on('newListener', (evt) => {\n if (!this.connected && evt !== 'newListener')\n this.desiredEvents.push(evt);\n if (evt === 'cancel')\n this.listeningForCancel = true;\n });\n }\n \n /** \n * Cancel the job\n * @access public\n * @param {string} reason If provided, will be sent to client\n */\n async cancel (reason = undefined)\n {\n const response = await this.useDeployConnection('cancelJob', {\n job: this.address,\n owner: this.paymentAccountKeystore.address,\n reason,\n }, this.paymentAccountKeystore);\n\n return response.payload;\n }\n\n /** \n * Resume this job\n * @access public\n */\n async resume ()\n {\n const response = await this.schedulerConnection.send('resumeJob', {\n job: this.address,\n owner: this.paymentAccountKeystore.address,\n }, this.paymentAccountKeystore);\n\n return response.payload;\n }\n\n /**\n * Helper function for retrieving info about the job. The job must have already been deployed.\n * An alias for {@link module:dcp/compute.getJobInfo}.\n * @access public\n */\n getJobInfo ()\n {\n return (__webpack_require__(/*! ../compute */ \"./src/dcp-client/compute.js\").compute.getJobInfo)(this.address);\n }\n \n /**\n * Helper function for retrieving info about the job's slices. The job must have already been deployed.\n * An alias for {@link module:dcp/compute.getSliceInfo}.\n * @access public\n */\n getSliceInfo ()\n {\n return (__webpack_require__(/*! ../compute */ \"./src/dcp-client/compute.js\").compute.getSliceInfo)(this.address);\n }\n \n /** Escrow additional funds for this job\n * @access public\n * @param {number|BigNumber} fundsRequired - A number or BigNumber instance representing the funds to escrow for this job\n */\n async escrow (fundsRequired)\n {\n if ((typeof fundsRequired !== 'number' && !BigNumber.isBigNumber(fundsRequired))\n || fundsRequired <= 0 || !Number.isFinite(fundsRequired) || Number.isNaN(fundsRequired))\n throw new Error(`Job.escrow: fundsRequired must be a number greater than zero. (not ${fundsRequired})`);\n\n const bankConnection = new protocolV4.Connection(dcpConfig.bank.services.bankTeller);\n\n /*\n * escrow has been broken for an unknown amount of time. `feeStructureId` is not defined anywhere in the job class, and hasn't\n * for a period of time. When fixed, `this[INTERNAL_SYMBOL].payloadDetails.feeStructureId` will likely become just `this.feeStructureId`,\n * but it's being left alone until someone spends the time to fix escrow. / rs Jul 2022\n */\n const response = await bankConnection.send('embiggenFeeStructure', {\n feeStructureAddress: this[INTERNAL_SYMBOL].payloadDetails.feeStructureId,\n additionalEscrow: new BigNumber(fundsRequired),\n fromAddress: this.paymentAccountKeystore.address,\n }, this.paymentAccountKeystore);\n\n bankConnection.close();\n const receipt = response.payload;\n return receipt;\n }\n\n /**\n * create bundles for local dependencies\n */\n _pack ()\n {\n var retval = (__webpack_require__(/*! ./node-modules */ \"./src/dcp-client/job/node-modules.js\").createModuleBundle)(this.dependencies);\n return retval;\n }\n\n /** \n * Collect all of the dependencies together, throw them into a BravoJS\n * module which sideloads them as a side effect of declaration, and transmit\n * them to the package manager. Then we return the package descriptor object,\n * which is guaranteed to have only one file in it.\n *\n * @returns {object} with properties name and files[0]\n */\n async _publishLocalModules ()\n {\n const { tempFile, hash, unresolved } = await this._pack();\n\n if (!tempFile) {\n return { unresolved };\n }\n\n const sideloaderFilename = tempFile.filename;\n const pkg = {\n name: `dcp-pkg-v1-localhost-${hash.toString('hex')}`,\n version: '1.0.0',\n files: {\n [sideloaderFilename]: `${sideloaderModuleIdentifier}.js`,\n },\n }\n\n await dcpPublish.publish(pkg);\n tempFile.remove();\n\n return { pkg, unresolved };\n }\n \n /**\n * This function specifies a module dependency (when the argument is a string)\n * or a list of dependencies (when the argument is an array) of the work\n * function. This function can be invoked multiple times before deployment.\n * @param {string | string[]} modulePaths - A string or array describing one\n * or more dependencies of the job.\n * @access public\n */\n requires (modulePaths)\n {\n if (typeof modulePaths !== 'string' && (!Array.isArray(modulePaths) || modulePaths.some((modulePath) => typeof modulePath !== 'string')))\n throw new TypeError('The argument to dependencies is not a string or an array of strings');\n else if (modulePaths.length === 0)\n throw new RangeError('The argument to dependencies cannot be an empty string or array');\n else if (Array.isArray(modulePaths) && modulePaths.some((modulePath) => modulePath.length === 0))\n throw new RangeError('The argument to dependencies cannot be an array containing an empty string');\n\n if (!Array.isArray(modulePaths))\n modulePaths = [modulePaths];\n\n for (const modulePath of modulePaths)\n {\n if (modulePath[0] !== '.' && modulePath.indexOf('/') !== -1)\n {\n const modulePrefixRegEx = /^(.*)\\/.*?$/;\n const [, modulePrefix] = modulePath.match(modulePrefixRegEx);\n if (modulePrefix && this.requirePath.indexOf(modulePrefix) === -1)\n this.requirePath.push(modulePrefix);\n }\n this.dependencies.push(modulePath);\n }\n }\n \n /** Set the account upon which funds will be drawn to pay for the job.\n * @param {module:dcp/wallet.AuthKeystore} keystore A keystore that representa a bank account.\n * @access public\n */\n setPaymentAccountKeystore (keystore)\n {\n if (this.address)\n {\n if (!keystore.address.eq(this.paymentAccountKeystore))\n {\n let message = 'Cannot change payment account after job has been deployed';\n this.emit('EPERM', message);\n throw new Error(`EPERM: ${message}`);\n }\n }\n \n if (!(keystore instanceof wallet.Keystore))\n throw new Error('Not an instance of Keystore: ' + keystore.toString());\n this.paymentAccountKeystore = keystore;\n }\n \n /** Set the slice payment offer. This is equivalent to the first argument to exec.\n * @param {number} slicePaymentOffer - The number of DCC the user is willing to pay to compute one slice of this job\n */\n setSlicePaymentOffer (slicePaymentOffer)\n {\n this.slicePaymentOffer = new SlicePaymentOffer(slicePaymentOffer);\n }\n \n \n /**\n * @param {URL|DcpURL} locationUrl - A URL object\n * @param {object} postParams - An object with any parameters that a user would like to be passed to a \n * remote result location. This object is capable of carry API keys for S3, \n * DropBox, etc. These parameters are passed as parameters in an \n * application/x-www-form-urlencoded request.\n */\n setResultStorage (locationUrl, postParams)\n {\n if (locationUrl instanceof URL || locationUrl instanceof DcpURL)\n this.resultStorageDetails = locationUrl;\n else\n throw new Error('Not an instance of a DCP URL: ' + locationUrl);\n \n\n // resultStorageParams contains any post params required for off-prem storage\n if (typeof postParams !== 'undefined' && typeof postParams === 'object' )\n this.resultStorageParams = postParams;\n else\n throw new Error('Not an instance of a object: ' + postParams);\n\n // Some type of object here\n this.resultStorageType = 'pattern';\n }\n \n /**\n * This function is identical to exec, except that the job is executed locally\n * in the client.\n * @async\n * @param {number} cores - the number of local cores in which to execute the job.\n * @param {...any} args - The remaining arguments are identical to the arguments of exec\n * @return {Promise<ResultHandle>} - resolves with the results of the job, rejects on an error\n * @access public\n */\n localExec (cores = 1, ...args)\n {\n this.inLocalExec = true;\n this.estimationSlices = 0;\n this.greedyEstimation = false;\n this.isCI = false;\n\n let worker;\n this.on('accepted', () => {\n // Start a worker for this job\n worker = new Worker({\n localExec: true,\n jobAddresses: [this.address],\n allowedOrigins: this.localExecAllowedFiles,\n paymentAddress: this.paymentAccountKeystore.address,\n maxWorkingSandboxes: cores,\n sandboxOptions: {\n ignoreNoProgress: true,\n SandboxConstructor: (DCP_ENV.platform === 'nodejs'\n && (__webpack_require__(/*! ../worker/evaluators */ \"./src/dcp-client/worker/evaluators/index.js\").nodeEvaluatorFactory)())\n },\n });\n\n worker.start().catch((e) => {\n console.error('Failed to start worker for localExec:');\n console.error(e.message);\n });\n });\n\n return this.exec(...args).finally(() => {\n if (worker) {\n setTimeout(() => {\n // stop the worker\n worker.stop(true);\n }, 3000);\n }\n });\n }\n\n /**\n * Deploys the job to the scheduler.\n * @param {number | object} [slicePaymentOffer=compute.marketValue] - Amount\n * in DCC that the user is willing to pay per slice.\n * @param {Keystore} [paymentAccountKeystore=wallet.get] - An instance of the\n * Wallet API Keystore that's used as the payment account when executing the\n * job.\n * @param {object} [initialSliceProfile] - An object describing the cost the\n * user believes the average slice will incur.\n * @access public\n * @emits Job#accepted\n */\n async exec (slicePaymentOffer = (__webpack_require__(/*! ../compute */ \"./src/dcp-client/compute.js\").compute.marketValue), paymentAccountKeystore, initialSliceProfile)\n {\n if (this.connected)\n throw new Error('Exec called twice on the same job handle.');\n \n if (this.estimationSlices === Infinity)\n this.estimationSlices = null;\n else if (this.estimationSlices < 0)\n throw new Error('Incorrect value for estimationSlices; it can be an integer or Infinity!');\n \n if (this.tuning.kvin.speed || this.tuning.kvin.size)\n {\n tunedKvin = new kvin.KVIN();\n tunedKvin.tune = 'size';\n if(this.tuning.kvin.speed)\n tunedKvin.tune = 'speed';\n // If both size and speed are true, kvin will optimize based on speed\n if(this.tuning.kvin.speed && this.tuning.kvin.size)\n console.log('Slices and arguments are being uploaded with speed optimization.');\n }\n \n this.readyStateChange('exec');\n if ((typeof slicePaymentOffer === 'number') || (typeof slicePaymentOffer === 'object')\n || ((this.slicePaymentOffer === null || this.slicePaymentOffer === undefined) && typeof slicePaymentOffer === 'function'))\n this.setSlicePaymentOffer(slicePaymentOffer);\n if (typeof initialSliceProfile !== 'undefined')\n this.initialSliceProfile = initialSliceProfile;\n \n if (typeof paymentAccountKeystore !== 'undefined')\n {\n /** XXX @todo deprecate use of ethereum wallet objects */\n if (typeof paymentAccountKeystore === 'object' && paymentAccountKeystore.hasOwnProperty('_privKey'))\n {\n console.warn('* deprecated API * - job.exec invoked with ethereum wallet object as paymentAccountKeystore') /* /wg oct 2019 */\n paymentAccountKeystore = paymentAccountKeystore._privKey\n }\n /** XXX @todo deprecate use of private keys */\n if (wallet.isPrivateKey(paymentAccountKeystore))\n {\n console.warn('* deprecated API * - job.exec invoked with private key as paymentAccountKeystore') /* /wg dec 2019 */\n paymentAccountKeystore = await new wallet.Keystore(paymentAccountKeystore, '');\n }\n\n this.setPaymentAccountKeystore(paymentAccountKeystore)\n }\n \n if (this.paymentAccountKeystore)\n // Throws if they fail to unlock, we allow this since the keystore was set programmatically. \n await this.paymentAccountKeystore.unlock(undefined, parseFloat(dcpConfig.job.maxDeployTime));\n else\n {\n // If not set programmatically, we keep trying to get an unlocked keystore ... forever.\n let locked = true;\n let safety = 0; // no while loop shall go unguarded\n let ks;\n do\n {\n ks = null;\n // custom message for the browser modal to denote the purpose of keystore submission\n let msg = `This application is requesting a keystore file to execute ${this.public.description || this.public.name || 'this job'}. Please upload the corresponding keystore file. If you upload a keystore file which has been encrypted with a passphrase, the application will not be able to use it until it prompts for a passphrase and you enter it.`;\n try\n {\n ks = await wallet.get({ contextId: this.contextId, jobName: this.public.name, msg});\n }\n catch (e)\n {\n if (e.code !== ClientModal.CancelErrorCode) throw e;\n };\n if (ks)\n {\n try\n {\n await ks.unlock(undefined, parseFloat(dcpConfig.job.maxDeployTime));\n locked = false;\n }\n catch (e)\n {\n // prompt user again if user enters password incorrectly, exit modal otherwise\n if (e.code !== wallet.unlockFailErrorCode) throw e;\n }\n }\n if (safety++ > 1000) throw new Error('EINFINITY: job.exec tried wallet.get more than 1000 times.')\n } while (locked);\n this.setPaymentAccountKeystore(ks)\n }\n \n // We either have a valid keystore + password or we have rejected by this point.\n if (!this.slicePaymentOffer)\n throw new Error('A payment profile must be assigned before executing the job');\n else\n this.feeStructure = this.slicePaymentOffer.toFeeStructure(this.jobInputData.length);\n\n if (!this.address)\n {\n try\n {\n this.readyStateChange('init');\n await this.deployJob();\n const listenersPromise = this.addInitialEvents();\n const computeGroupsPromise = this.joinComputeGroups();\n let uploadSlicePromise;\n // if job data is by value then upload data to the scheduler in a staggered fashion\n if (Array.isArray(this.dataValues))\n {\n this.readyStateChange('uploading');\n uploadSlicePromise = addSlices(this.dataValues, this.address, tunedKvin).then(() => this.close());\n debugging('slice-upload') && uploadSlicePromise.then(() => console.debug('831: x-dbg-uploadComplete'));\n }\n \n // await all promises for operations that can be done after the job is deployed\n await Promise.all([listenersPromise, computeGroupsPromise, uploadSlicePromise]);\n\n this.readyStateChange('deployed');\n this.emit('accepted', { job: this });\n }\n catch (error)\n {\n if (ON_BROWSER)\n await ClientModal.alert(error, { title: 'Failed to deploy job!' });\n throw error;\n }\n }\n else\n {\n // reconnecting to an old job\n await this.addInitialEvents();\n this.readyStateChange('reconnected');\n }\n\n this.connected = true;\n\n return new Promise((resolve, reject) => {\n const onComplete = () => resolve(this.results);\n const onCancel = (event) => {\n /**\n * FIXME(DCP-1150): Remove this since normal cancel event is noisy\n * enough to not need stopped event too.\n */\n if (ON_BROWSER && !this.listeningForCancel)\n ClientModal.alert('More details in console...', { title: 'Job Canceled' });\n this.emit('cancel', event);\n\n let errorMsg = event.reason;\n if (event.error)\n errorMsg = errorMsg +`\\n Recent error message: ${event.error.message}`\n \n reject(new DCPError(errorMsg, event.code));\n };\n\n this.ee.once('stopped', async (stopEvent) => {\n // There is a chance the result submitter will emit finished > 1 time. Only handle it once.\n if (this.receivedStop)\n return;\n this.receivedStop = true;\n this.emit('stopped', stopEvent.runStatus);\n switch (stopEvent.runStatus) {\n case jobStatus.finished:\n if (this.collateResults)\n {\n let report = await this.getJobInfo();\n let allSliceNumbers = Array.from(Array(report.totalSlices)).map((e,i)=>i+1);\n let remainSliceNumbers = allSliceNumbers.filter((e) => !this.results.isAvailable(e));\n\n if (remainSliceNumbers.length)\n {\n const promises = remainSliceNumbers.map(sliceNumber => this.results.fetch([sliceNumber], true));\n await Promise.all(promises);\n }\n }\n\n this.emit('complete', this.results);\n onComplete();\n break;\n case jobStatus.cancelled:\n onCancel(stopEvent);\n break;\n default:\n /**\n * Asserting that we should never be able to reach here. The only\n * scheduler events that should trigger the Job's 'stopped' event\n * are jobStatus.cancelled, jobStatus.finished, and sliceStatus.paused.\n */\n reject(new Error(`Unknown event \"${stopEvent.runStatus}\" caused the job to be stopped.`));\n break;\n }\n });\n\n }).finally(() => {\n const handleErr = (e) => {\n console.error('Error while closing job connection:');\n console.error(e);\n }\n\n // Create an async IIFE to not block the promise chain\n (async () => {\n // delay to let last few events to be received\n await new Promise((resolve) => setTimeout(resolve, 1000));\n \n // close all of the connections so that we don't cause node processes to hang.\n this.closeDeployConnection();\n await this.eventSubscriber.close().catch(handleErr);\n await computeGroups.closeServiceConnection().catch((err) => {\n console.error('Warning: could not close compute groups service connection', err);\n });\n })();\n });\n }\n \n /**\n * job.addListeners(): Private function used to set up event listeners to the scheduler\n * before deploying the job.\n */\n async addInitialEvents ()\n {\n this.readyStateChange('listeners');\n\n // This is important: We need to flush the task queue before adding listeners\n // because we queue pending listeners by listening to the newListener event (in the constructor).\n // If we don't flush here, then the newListener events may fire after this function has run,\n // and the events won't be properly set up.\n await new Promise(resolve => setTimeout(resolve, 0));\n\n // @todo: Listen for an estimated cost, probably emit an \"estimated\" event when it comes in?\n // also @todo: Do the estimation task(s) on the scheduler and send an \"estimated\" event\n\n // Always listen to the stop event. It will resolve the work function promise, so is always needed.\n this.on('stop', (ev) => {this.ee.emit('stopped', ev)});\n\n // Connect listeners that were set up before exec\n if (this.desiredEvents.includes('result'))\n this.listeningForResults = true;\n await this.subscribeNewEvents(this.desiredEvents);\n\n // Connect listeners that are set up after exec\n this.on('newListener', (evt) => {\n if (evt === 'newListener' || this.desiredEvents.includes(evt))\n return;\n this.subscribeNewEvents([evt]);\n });\n \n // automatically add a listener for results if collateResults is on\n if (this.collateResults && !this.listeningForResults)\n this.on('result', () => {});\n\n debugging('dcp-client') && console.debug('subscribedEvents', this.desiredEvents);\n\n // If we have listeners for job.work, subscribe to custom events\n if (this.listenForCustomEvents)\n await this.subscribeCustomEvents();\n // Connect work event listeners that are set up after exec\n else\n this.work.on('newListener', () => this.subscribeCustomEvents());\n }\n \n /**\n * Subscribes to either reliable events or optional events. It is assumed that\n * any call to this function will include only new events.\n * @param {string[]} events \n */\n async subscribeNewEvents (events)\n {\n const reliableEvents = [];\n const optionalEvents = [];\n for (let eventName of events)\n {\n eventName = eventName.toLowerCase();\n if (this.eventTypes[eventName] && this.eventTypes[eventName].reliable)\n reliableEvents.push(eventName);\n else if (this.eventTypes[eventName] && !this.eventTypes[eventName].reliable)\n optionalEvents.push(eventName);\n else\n debugging('dcp-client') && console.debug(`Job handler has listener ${eventName} which isn't an event-router event.`);\n }\n if (debugging('dcp-client'))\n {\n console.debug('reliableEvents', reliableEvents);\n console.debug('optionalEvents', optionalEvents);\n }\n await this.eventSubscriber.subscribeManyEvents(reliableEvents, optionalEvents, { filter: { job: this.address } });\n }\n \n /**\n * Establishes listeners for worker events when requested by the client\n */\n async subscribeCustomEvents ()\n {\n if (!this.listeningForCustomEvents)\n await this.eventSubscriber.subscribeManyEvents([], ['custom'], { filter: { job: this.address } });\n this.listeningForCustomEvents = true\n }\n \n async joinComputeGroups ()\n {\n // localExec jobs are not entered in any compute group.\n if (!this.inLocalExec && this.computeGroups && this.computeGroups.length)\n {\n this.readyStateChange('compute-groups');\n computeGroups.addRef(); // Just in case we're doing a Promise.all on multiple execs.\n\n // Add this job to its currently-defined compute groups (as well as public group, if included)\n let success;\n try\n {\n const cgPayload = await computeGroups.addJobToGroups(this.address, this.computeGroups);\n success = true; // To support older version of CG service where addJobToGroups had void/undefined return.\n if (cgPayload) success = cgPayload.success;\n debugging('dcp-client') && console.debug('job/index: addJobToGroups cgPayload:', cgPayload ? cgPayload : 'cgPayload is not defined; probably from legacy CG service.');\n }\n catch (e)\n {\n debugging('dcp-client') && console.debug('job/index: addJobToGroups threw exception:', e);\n success = false;\n }\n\n computeGroups.closeServiceConnection().catch((err) => {\n console.error('Warning: could not close compute groups service connection', err)\n });\n\n /* Could not put the job in any compute group, even though the user wanted it to run. Cancel the job. */\n if (!success)\n {\n await this.cancel('compute-groups::Unable to join any compute groups');\n throw new DCPError(`Access Denied::Failed to add job ${this.address} to any of the desired compute groups`, 'DCPL-1100');\n }\n }\n }\n \n /**\n * Takes result events as input, stores the result and fires off\n * events on the job handle as required. (result, duplicate-result)\n *\n * @param {object} ev - the event recieved from protocol.listen('/results/0xThisGenAdr')\n */\n async handleResult (ev)\n {\n if (this.results === null)\n // This should never happen - the onResult event should only be established/called\n // in addListeners which should also initialize the internal results array\n throw new Error('Job.onResult was invoked before initializing internal results');\n \n const { result: _result, time } = ev.result;\n debugging('dcp-client') && console.debug('handleResult', _result);\n let result = await fetchURI(_result);\n\n if (this.results.isAvailable(ev.sliceNumber))\n {\n const changed = JSON.stringify(this.results[ev.sliceNumber]) !== JSON.stringify(result);\n this.emit('duplicate-result', { sliceNumber: ev.sliceNumber, changed });\n }\n\n this.results.newResult(result, ev.sliceNumber);\n this.emit('result', { job: this.address, sliceNumber: ev.sliceNumber, result });\n this.emit('resultsUpdated');\n }\n \n /**\n * Receives status events from the scheduler, updates the local status object\n * and emits a 'status' event\n *\n * @param {object} ev - the status event received from\n * protocol.listen('/status/0xThisGenAdr')\n * @param {boolean} emitStatus - value indicating whether or not the status\n * event should be emitted\n */\n handleStatus ({ runStatus, total, distributed, computed }, emitStatus = true)\n {\n Object.assign(this.status, {\n runStatus,\n total,\n distributed,\n computed,\n });\n\n if (emitStatus)\n this.emit('status', { ...this.status, job: this.address });\n }\n \n /**\n * Sends a request to the scheduler to deploy the job.\n */\n async deployJob ()\n {\n var moduleDependencies; \n \n /* Send sideloader bundle to the package server */\n if (DCP_ENV.platform === 'nodejs' && this.dependencies.length)\n {\n try\n {\n let { pkg, unresolved } = await this._publishLocalModules();\n\n moduleDependencies = unresolved;\n if (pkg)\n moduleDependencies.push(pkg.name + '/' + sideloaderModuleIdentifier); \n }\n catch(error)\n {\n throw new DCPError(`Error trying to communicate with package manager server: ${error}`);\n }\n }\n else\n moduleDependencies = this.dependencies;\n \n this.readyStateChange('preauth');\n\n const adhocId = this.uuid.slice(this.uuid.length - 6, this.uuid.length);\n const schedId = await dcpConfig.scheduler.identity;\n // The following check is needed for when using dcp-rtlink and loading the config through source, instead of using the dcp-client bundle\n let schedIdAddress = schedId;\n if(schedId.address)\n schedIdAddress = schedId.address;\n const myId = await wallet.getId();\n const preauthToken = await bankUtil.preAuthorizePayment(schedIdAddress, this.maxDeployPayment, this.paymentAccountKeystore);\n const { dataRange, dataValues, dataPattern, sliceCount } = marshalInputData(this.jobInputData);\n if(dataValues)\n this.dataValues = dataValues;\n\n this.readyStateChange('deploying');\n\n /* Payload format is documented in scheduler-v4/libexec/job-submit/operations/submit.js */\n const submitPayload = {\n owner: myId.address,\n paymentAccount: this.paymentAccountKeystore.address,\n priority: 0, // @nyi\n\n workFunctionURI: this.workFunctionURI,\n uuid: this.uuid,\n mvMultSlicePayment: Number(this.feeStructure.marketValue) || 0, // @todo: improve feeStructure internals to better reflect v4\n absoluteSlicePayment: Number(this.feeStructure.maxPerRequest) || 0,\n requirePath: this.requirePath,\n dependencies: moduleDependencies,\n requirements: this.requirements, /* capex */\n localExec: this.inLocalExec,\n force100pctCPUDensity: this.force100pctCPUDensity,\n estimationSlices: this.estimationSlices,\n greedyEstimation: this.greedyEstimation,\n workerConsole: this.workerConsole,\n isCI: this.isCI,\n\n description: this.public.description || 'Discreetly making the world smarter',\n name: this.public.name || 'Ad-Hoc Job' + adhocId,\n link: this.public.link || '',\n\n preauthToken, // XXXwg/er @todo: validate this after fleshing out the stub(s)\n\n resultStorageType: this.resultStorageType, // @todo: implement other result types\n resultStorageDetails: this.resultStorageDetails, // Content depends on resultStorageType\n resultStorageParams: this.resultStorageParams, // Post params for off-prem storage\n dataRange,\n dataPattern,\n sliceCount\n };\n\n /* Determine thee type of the arguments option and set the submit message payload accordingly. */\n if (Array.isArray(this.jobArguments) && this.jobArguments.length === 1 && this.jobArguments[0] instanceof DcpURL)\n submitPayload.arguments = this.jobArguments[0].href;\n else if (this.jobArguments instanceof RemoteDataSet)\n submitPayload.marshaledArguments = kvinMarshal(this.jobArguments.map(e => new URL(e)))\n else if (this.jobArguments)\n {\n try\n {\n submitPayload.marshaledArguments = kvinMarshal(Array.from(this.jobArguments));\n }\n catch(e)\n {\n throw new Error(`Could not convert job arguments to Array (${e.message})`);\n }\n }\n \n if (this.inLocalExec && !DCP_ENV.isBrowserPlatform)\n {\n const workFunctionFile = createTempFile('dcp-localExec-workFunction-XXXXXXXXX', 'js');\n const argumentsFile = createTempFile('dcp-localExec-arguments-XXXXXXXXX', 'js');\n \n // For allowed origins of the localexec worker. Only allow the origins (files in this case) in this list.\n this.localExecAllowedFiles = [workFunctionFile.filename, argumentsFile.filename];\n\n // get the workFunctionURI string before writing to file to prevent the need to double-decode the work function in the worker.\n const workFunction = await fetchURI(this.workFunctionURI);\n workFunctionFile.writeSync(workFunction);\n \n const workFunctionFileURL = new URL('file://' + workFunctionFile);\n submitPayload.workFunctionURI = workFunctionFileURL.href;\n this.workFunctionURI = workFunctionFileURL.href;\n \n if (submitPayload.marshaledArguments)\n {\n argumentsFile.writeSync(JSON.stringify(submitPayload.marshaledArguments));\n const argumentsFileURL = new URL('file://' + argumentsFile.filename);\n submitPayload.marshaledArguments = kvinMarshal([argumentsFileURL]);\n }\n }\n\n // XXXpfr Excellent tracing.\n if (debugging('dcp-client'))\n {\n dumpObject(submitPayload, 'Submit: Job Index: submitPayload', 256);\n console.debug('Before Deploy', myId);\n }\n\n // Deploy the job! If we get an error, try again a few times until threshold of errors is reached, then actually throw it\n let deployed\n let deployAttempts = 0;\n while (deployAttempts++ < (dcpConfig.job.deployAttempts || 10))\n {\n try\n {\n deployed = await this.useDeployConnection('submit', submitPayload, myId);\n break;\n }\n catch (e)\n {\n if (deployAttempts < 10)\n debugging('dcp-client') && console.debug('Error when trying to deploy job, trying again', e);\n else\n throw e;\n }\n }\n\n if (!deployed.success)\n {\n // close all of the connections so that we don't cause node processes to hang.\n const handleErr = (e) => {\n console.error('Error while closing job connection:');\n console.error(e);\n };\n \n this.closeDeployConnection();\n this.eventSubscriber.close().catch(handleErr);\n computeGroups.closeServiceConnection().catch(handleErr);\n \n // Yes, it is possible for deployed or deployed.payload to be undefined.\n if (deployed.payload)\n {\n if (deployed.payload.code === 'ENOTFOUND')\n throw new DCPError(`Failed to submit job to scheduler. Account: ${submitPayload.paymentAccount} was not found or does not have sufficient balance (${deployed.payload.info.deployCost} DCCs needed to deploy this job)`, deployed.payload); \n throw new DCPError('Failed to submit job to scheduler', deployed.payload);\n }\n throw new DCPError('Failed to submit job to scheduler (no payload)', deployed ? deployed : '');\n }\n\n debugging('dcp-client') && console.debug('After Deploy', JSON.stringify(deployed));\n\n this.address = deployed.payload.job;\n this.deployCost = deployed.payload.deployCost;\n\n if (!this.status)\n this.status = {\n runStatus: null,\n total: 0,\n computed: 0,\n distributed: 0,\n };\n \n this.status.runStatus = deployed.payload.status;\n this.status.total = deployed.payload.lastSliceNumber;\n this.running = true;\n }\n \n /** close an open job to indicate we are done adding data so it is okay to finish\n * the job at the appropriate time\n */\n close ()\n {\n return this.useDeployConnection('closeJob', {\n job: this.id,\n });\n }\n \n /** Use the connection to job submit service. Will open a new connection if one does not exist,\n * and close the connection if it is idle for more than 10 seconds (tuneable).\n */\n useDeployConnection(...args)\n {\n if (!this.deployConnection)\n {\n this.deployConnection = new protocolV4.Connection(dcpConfig.scheduler.services.jobSubmit); \n this.deployConnection.on('close', () => { this.deployConnection = null; });\n }\n if (this.deployConnectionTimeout)\n clearTimeout(this.deployConnectionTimeout);\n \n const deployPromise = this.deployConnection.send(...args);\n \n this.deployConnectionTimeout = setTimeout(() => this.deployConnection.close(), (dcpConfig.job.deployCloseTimeout || 10 * 1000));\n if (!ON_BROWSER)\n this.deployConnectionTimeout.unref();\n \n return deployPromise;\n }\n \n /**\n * Close the connection to the job submit (if it exists), and clear the close timeout (if needed).\n */\n closeDeployConnection()\n {\n if (this.deployConnection)\n this.deployConnection.close();\n if (this.deployConnectionTimeout)\n clearTimeout(this.deployConnectionTimeout);\n }\n}\n\n/**\n * Depending on the shape of the job's data, resolve it into a RangeObject, a\n * Pattern, or a values array, and return it in the appropriate property.\n *\n * @param {any} data Job's input data\n * @return {MarshaledInputData} An object with one of the following properties set:\n * - dataValues: job input is an array of arbitrary values \n * - dataPattern: job input is a URI pattern \n * - dataRange: job input is a RangeObject (and/or friends)\n */\nfunction marshalInputData (data)\n{\n if (!(data instanceof Object || data instanceof SuperRangeObject))\n throw new TypeError(`Invalid job data type: ${typeof data}`);\n\n /**\n * @type MarshaledInputData\n */\n const marshalledInputData = {};\n\n // TODO(wesgarland): Make this more robust.\n if (data instanceof SuperRangeObject ||\n (data.hasOwnProperty('ranges') && data.ranges instanceof MultiRangeObject) ||\n (data.hasOwnProperty('start') && data.hasOwnProperty('end')))\n marshalledInputData.dataRange = data;\n else if (Array.isArray(data))\n marshalledInputData.dataValues = data;\n else if (data instanceof URL || data instanceof DcpURL)\n marshalledInputData.dataPattern = String(data);\n else if(data instanceof RemoteDataSet)\n marshalledInputData.dataValues = data.map(e => new URL(e));\n else if(data instanceof RemoteDataPattern)\n {\n marshalledInputData.dataPattern = data['pattern'];\n marshalledInputData.sliceCount = data['sliceCount'];\n }\n\n log('marshalledInputData:', marshalledInputData);\n return marshalledInputData;\n}\n\n/**\n * marshal the value using kvin or instance of the kvin (tunedKvin)\n * tunedKvin is defined if job.tuning.kvin is specified.\n *\n * @param {any} value \n * @return {object} A marshaled object\n * \n */\nfunction kvinMarshal (value) {\n if (tunedKvin)\n return tunedKvin.marshal(value);\n\n return kvin.marshal(value);\n}\n\n\n\nexports.Job = Job;\nexports.SlicePaymentOffer = SlicePaymentOffer;\nexports.ResultHandle = ResultHandle;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/job/index.js?");
|
|
4171
|
+
eval("/**\n * @file job/index.js\n * @author Eddie Roosenmaallen, eddie@kingsds.network\n * Matthew Palma, mpalma@kingsds.network\n * Wes Garland, wes@kingsds.network\n * Paul, paul@kingsds.network\n * Ryan Saweczko, ryansaweczko@kingsds.network\n * @date November 2018\n * November 2018\n * February 2022\n * May 2022\n * Jun 2022\n *\n * This module implements the Compute API's Job Handle\n *\n */\n/** @typedef {import('dcp/dcp-client/wallet/keystore').Keystore} Keystore */\n\n\nconst { BigNumber } = __webpack_require__(/*! bignumber.js */ \"./node_modules/bignumber.js/bignumber.js\");\nconst { v4: uuidv4 } = __webpack_require__(/*! uuid */ \"./node_modules/uuid/dist/esm-browser/index.js\");\nconst { EventEmitter, PropagatingEventEmitter } = __webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\");\nconst { RangeObject, MultiRangeObject, DistributionRange, SuperRangeObject, SparseRangeObject } = __webpack_require__(/*! dcp/dcp-client/range-object */ \"./src/dcp-client/range-object.js\");\nconst { fetchURI, encodeDataURI, dumpObject } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst { getTextEncoder, createTempFile } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst serialize = __webpack_require__(/*! dcp/utils/serialize */ \"./src/utils/serialize.js\");\nconst wallet = __webpack_require__(/*! dcp/dcp-client/wallet */ \"./src/dcp-client/wallet/index.js\");\nconst protocolV4 = __webpack_require__(/*! dcp/protocol-v4 */ \"./src/protocol-v4/index.js\");\nconst { EventSubscriber } = __webpack_require__(/*! dcp/events/event-subscriber */ \"./src/events/event-subscriber.js\");\nconst { DcpURL } = __webpack_require__(/*! dcp/common/dcp-url */ \"./src/common/dcp-url.js\");\nconst ClientModal = __webpack_require__(/*! dcp/dcp-client/client-modal */ \"./src/dcp-client/client-modal/index.js\");\nconst { Worker } = __webpack_require__(/*! dcp/dcp-client/worker */ \"./src/dcp-client/worker/index.js\");\nconst { RemoteDataSet } = __webpack_require__(/*! dcp/dcp-client/remote-data-set */ \"./src/dcp-client/remote-data-set.js\");\nconst { RemoteDataPattern } = __webpack_require__(/*! dcp/dcp-client/remote-data-pattern */ \"./src/dcp-client/remote-data-pattern.js\");\nconst { ResultHandle } = __webpack_require__(/*! ./result-handle */ \"./src/dcp-client/job/result-handle.js\");\nconst { SlicePaymentOffer } = __webpack_require__(/*! ./slice-payment-offer */ \"./src/dcp-client/job/slice-payment-offer.js\");\nconst { addSlices } = __webpack_require__(/*! ./upload-slices */ \"./src/dcp-client/job/upload-slices.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nconst dcpPublish = __webpack_require__(/*! dcp/common/dcp-publish */ \"./src/common/dcp-publish.js\");\nconst computeGroups = __webpack_require__(/*! dcp/dcp-client/compute-groups */ \"./src/dcp-client/compute-groups/index.js\");\nconst schedulerConstants = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst { sliceStatus } = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst { jobStatus } = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst bankUtil = __webpack_require__(/*! dcp/dcp-client/bank-util */ \"./src/dcp-client/bank-util.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp-client');\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\nlet tunedKvin;\n\nconst log = (...args) => {\n if (debugging('job')) {\n console.debug('dcp-client:job', ...args);\n }\n};\n\nconst ON_BROWSER = DCP_ENV.isBrowserPlatform;\nconst sideloaderModuleIdentifier = 'sideloader-v1';\n\n\nconst DEFAULT_REQUIREMENTS = {\n engine: {\n es7: null,\n spidermonkey: null\n },\n environment: {\n webgpu: null,\n offscreenCanvas: null,\n fdlibm: null\n },\n browser: {\n chrome: null\n },\n details: {\n offscreenCanvas: {\n bigTexture4096: null,\n bigTexture8192: null,\n bigTexture16384: null,\n bigTexture32768: null,\n }\n },\n discrete: null,\n useStrict: null,\n};\n\n/** @typedef {import('../range-object').RangeLike} RangeLike */\n\n/**\n * Ensure input data is an appropriate format\n * @param {RangeObject | DistributionRange | RemoteDataSet | Array | Iterable}\n * inputData - A URI-shaped string, a [Multi]RangeObject-constructing value, or\n * an array of slice data\n * @return {RangeObject | RangeLike | DistributionRange | RemoteDataSet | Array}\n * The coerced input in an appropriate format ([Multi]RangeObject,\n * DistributionRange, RemoteDataSet, or array)\n */\n const wrangleData = (inputData) => {\n\n if (RangeObject.isRangelike(inputData)) { return inputData }\n if (RangeObject.isRangeObject(inputData)) { return inputData }\n if (DistributionRange.isDistribution(inputData)) { return inputData }\n if (inputData instanceof SparseRangeObject) { return inputData }\n if (inputData instanceof MultiRangeObject) { return inputData }\n if (MultiRangeObject.isProtoMultiRangelike(inputData)) { return new MultiRangeObject(inputData) }\n if (RangeObject.isProtoRangelike(inputData)) { return new RangeObject(inputData) }\n if (DistributionRange.isProtoDistribution(inputData)) { return new DistributionRange(inputData) }\n if (RemoteDataSet.isRemoteDataSet(inputData)) { return inputData }\n if (RemoteDataPattern.isRemoteDataPattern(inputData)) { return inputData }\n\n return Array.isArray(inputData) ? inputData : [inputData];\n};\n\n/**\n * @classdesc The Compute API's Job Handle (see {@link https://docs.dcp.dev/specs/compute-api.html#job-handles|Compute API spec})\n * Job handles are objects which correspond to jobs. \n * They are created by some exports of the compute module, such as {@link module:dcp/compute.do|compute.do} and {@link module:dcp/compute.for|compute.for}.\n * @extends module:dcp/dcp-events.PropagatingEventEmitter\n * @hideconstructor\n * @access public\n */\nclass Job extends PropagatingEventEmitter\n{\n /**\n * Fired when the job is accepted by the scheduler on deploy.\n * \n * @event Job#accepted\n * @access public\n * @type {object}\n *//**\n * Fired when the job is cancelled.\n * \n * @event Job#cancel\n * @access public\n *//**\n * Fired when a result is returned.\n * \n * @event Job#result\n * @access public\n * @type {object}\n * @property {string} jobAddress Address of the job\n * @property {string} task ID of the task (slice) the result came from\n * @property {number} sort The index of the slice\n * @property {object} result\n * @property {string} result.request\n * @property {*} result.result The value returned from the work function\n *//**\n * Fired when the result handle is modified, either when a new `result` event is fired or when the results are populated with `results.fetch()`\n * \n * @event Job#resultsUpdated\n * @access public\n *//**\n * Fired when the job has been completed.\n * \n * @event Job#complete\n * @access public\n * @type {ResultHandle}\n *//**\n * Fired when the job's status changes.\n * \n * @event Job#status\n * @access public\n * @type {object}\n * @property {string} jobAddress Address of the job\n * @property {number} total Total number of slices in the job\n * @property {number} distributed Number of slices that have been distributed\n * @property {number} computed Number of slices that have completed execution (returned a result)\n * @property {string} runStatus Current runStatus of the job\n *//**\n * Fired when a slice throws an error.\n * \n * @event Job#error\n * @access public\n * @type {object}\n * @property {string} jobAddress Address of the job\n * @property {number} sliceIndex Index of the slice that threw the error\n * @property {string} message The error message\n * @property {string} stack The error stacktrace\n * @property {string} name The error type name\n *//**\n * Fired when a slice uses one of the console log functions.\n * \n * @event Job#console\n * @access public\n * @type {object}\n * @property {string} jobAddress Address of the job\n * @property {number} sliceIndex The index of the slice that produced this event\n * @property {string} level The log level, one of `debug`, `info`, `log`, `warn`, or `error`\n * @property {string} message The console log message\n *//**\n * Fired when a slice is stopped for not calling progress. Contains information about how long the slice ran for, and about the last reported progress calls.\n * \n * @event Job#noProgress\n * @access public\n * @type {object}\n * @property {string} jobAddress Address of the job\n * @property {number} sliceIndex The index of the slice that failed due to no progress\n * @property {number} timestamp How long the slice ran before failing\n * @property {object} progressReports\n * @property {object} progressReports.last The last progress report received from the worker\n * @property {number} progressReports.last.timestamp Time since the start of the slice\n * @property {number} progressReports.last.progress Progress value reported\n * @property {*} progressReports.last.value The last value that was passed to the progress function\n * @property {number} progressReports.last.throttledReports Number of calls to progress that were throttled since the last report\n * @property {object} progressReports.lastUpdate The last determinate (update to the progress param) progress report received from the worker\n * @property {number} progressReports.lastUpdate.timestamp\n * @property {number} progressReports.lastUpdate.progress\n * @property {*} progressReports.lastUpdate.value\n * @property {number} progressReports.lastUpdate.throttledReports\n *//**\n @todo: is this in the spec? is there a no progress data? should there be?\n * Identical to `noProgress`, except that it also contains the data that the slice was executed with.\n * \n * @event Job#noProgressData\n * @access public\n * @type {object}\n * @property {*} data The data that the slice was executed with\n *//**\n * Fired when the job is paused due to running out of funds. The job can be resumed by escrowing more funds then resuming the job.\n * @todo: is this a thing, should it be a thing (the payload)\n * Event payload is the estimated funds required to complete the job\n * \n * @event Job#nofunds\n * @access public\n * @type {BigNumber}\n *//**\n * Fired when the job cannot be deployed due to no bank account / not enough balance to deploy the job\n * \n * @event Job#ENOFUNDS\n * @access public\n *//**\n * Fired when the job is cancelled due to the work function not calling the `progress` method frequently enough.\n * \n * @event Job#ENOPROGRESS\n * @access public\n *//**\n * The job was cancelled because scheduler has determined that individual tasks in this job exceed the maximum allowable execution time.\n * \n * @event Job#ESLICETOOSLOW\n * @access public\n *//**\n * Fired when the job is cancelled because too many work functions are terminating with uncaught exceptions.\n * \n * @event Job#ETOOMANYERRORS\n * @access public\n */\n\n /**\n * @form1 new Job('application_worker_address'[, data[, arguments]])\n * @form2a new Job('worker source'[, data[, arguments]])\n * @form2b new Job(worker_function[, data[, arguments]])\n */\n constructor ()\n {\n super('Job');\n if (typeof arguments[0] === 'function')\n arguments[0] = arguments[0].toString();\n\n if (typeof arguments[0] === 'string')\n {\n const { encodeDataURI } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\n this.workFunctionURI = encodeDataURI(arguments[0], 'application/javascript');\n } \n else if (DcpURL.isURL(arguments[0]))\n this.workFunctionURI = arguments[0].href;\n\n this.jobInputData = wrangleData(arguments[1] || []);\n this.jobArguments = wrangleData(arguments[2] || []);\n \n log('num wrangledInputData:', this.jobInputData.length);\n log('num wrangledArguments:', this.jobArguments.length);\n\n this.initEventSystems();\n\n /**\n * An object describing the cost the user believes each the average slice will incur, in terms of CPU/GPU and I/O.\n * If defined, this object is used to provide initial scheduling hints and to calculate escrow amounts.\n *\n * @type {object}\n * @access public\n */\n this.initialSliceProfile = undefined;\n \n // The max value that the client is willing to spend to deploy\n // (list on the scheduler, doesn't include compute payment)\n // maxDeployPayment is the max the user is willing to pay to DCP (as a\n // Hold), in addition to the per-slice offer and associated scrape.\n // Currently calculated as `deployCost = costPerKB *\n // (JSON.stringify(job).length / 1024) // 1e-9 per kb`\n // @todo: figure this out / er nov 2018\n this.maxDeployPayment = 1;\n\n /**\n * An object describing the requirements that workers must have to be eligible for this job. See\n * {@link https://docs.dcp.dev/specs/compute-api.html#requirements-objects|Requirements Objects}.\n *\n * @type {object}\n * @access public\n */\n this.requirements = JSON.parse(JSON.stringify(DEFAULT_REQUIREMENTS));\n\n /**\n * A place to store public-facing attributes of the job. Anything stored on this object will be available inside the work \n * function (see {@link module:dcp/compute~sandboxEnv.work}). The properties documented here may be used by workers to display what jobs are currently being \n * worked on.\n * @access public\n * @property {string} name Public-facing name of this job.\n * @property {string} description Public-facing description for this job.\n * @property {string} link Public-facing link to external resource about this job.\n */\n this.public = {\n name: null,\n description: null,\n link: null,\n };\n\n /**\n * A number (can be null, undefined, or infinity) describing the estimationSlicesRemaining in the jpd (dcp-2593)\n * @type {number}\n * @access public\n */\n this.estimationSlices = undefined;\n /**\n * When true, allows a job in estimation to have requestTask return multiple estimation slices.\n * This flag applies independent of infinite estimation, viz., this.estimationSlices === null .\n * @type {boolean}\n * @access public\n */\n this.greedyEstimation = false;\n /**\n * tunable parameters per job\n * @access public\n * @param {object} tuning \n * @param {string} tuning.kvin Encode the TypedArray into a string, trying multiple methods to determine optimum \n * size/performance. The this.tune variable affects the behavior of this code this:\n * @param {boolean} speed If true, only do naive encoding: floats get represented as byte-per-digit strings\n * @param {boolean} size If true, try the naive, ab8, and ab16 encodings; pick the smallest\n * If both are false try the naive encoding if under typedArrayPackThreshold and use if smaller\n * than ab8; otherwise, use ab8\n */\n this.tuning = {\n kvin: {\n size: false,\n speed: false,\n },\n }\n /* For API interface to end-users only */\n Object.defineProperty(this, 'id', {\n get: () => this.address,\n set: (id) => { this.address = id }\n });\n \n this.uuid = uuidv4(); /** @see {@link https://kingsds.atlassian.net/browse/DCP-1475?atlOrigin=eyJpIjoiNzg3NmEzOWE0OWI4NGZkNmI5NjU0MWNmZGY2OTYzZDUiLCJwIjoiaiJ9|Jira Issue} */\n this.dependencies = []; /* dependencies of the work function */\n this.requirePath = []; /* require path for dependencies */\n this.connected = false; /* true when exec or resume called */\n this.results = new ResultHandle(this); /* result handle */\n this.collateResults = true; /* option to receive results as they are computed & ensure all are received on finish */\n this.force100pctCPUDensity = false; /* tell scheduler to assume this job uses 100% cpu density */\n this.workerConsole = false; /* tell workers to log more information about failures in the evaluator this job causes */\n this.address = null; /* job address, created by scheduler during exec call. */\n this.paymentAccountKeystore = null; /* keystore for payment for job to come from */\n this.status = { /* job status details */\n runStatus: null,\n total: null,\n distributed: null,\n computed: null\n };\n // Compute groups. Add to public compute group by default\n this.computeGroups = [ Object.assign({}, schedulerConstants.computeGroups.public) ];\n \n // Update the ready state as we go through job deployment\n this.readyState = sliceStatus.new;\n const that = this;\n this.readyStateChange = function job$$readyStateChange (readyState)\n {\n that.readyState = readyState;\n that.emit('readyStateChange', that.readyState);\n }\n }\n \n /**\n * Initialize the various event systems the job handle requires. These include:\n * - an internal event emitter (this.ee)\n * - an event emitter for any events emitted on `work.emit` within work functions (this.work)\n * - an event subscriber to subscribe (to receive) events from the scheduler (this.eventSubscriber)\n */\n initEventSystems ()\n {\n // Handle the various event-related things required in the constructor\n\n // Internal event emitter for events within job handle\n this.ee = new EventEmitter('Job Internal');\n\n /**\n * An EventEmitter for custom events dispatched by the work function.\n * @type {module:dcp/dcp-events.EventEmitter}\n * @access public\n * @example\n * // in work function\n * work.emit('myEventName', 1, [2], \"three\");\n * // client-side\n * job.work.on('myEventName', (num, arr, string) => { });\n */\n this.work = new EventEmitter('job.work');\n this.listenForCustomEvents = false;\n\n // Event subscriber - receive events from the event router\n this.eventSubscriber = new EventSubscriber(this);\n \n // Some events from the event subscriber can't be emitted immediately upon receipt without having \n // weird/wrong output due to things like serialization. We allow interceptors in the event subscriber\n // to handle this.\n const that = this\n var lastConsoleEv;\n var sameCounter = 1;\n const parseConsole = function deserializeConsoleMessage(ev) {\n if (tunedKvin)\n ev.message = tunedKvin.unmarshal(ev.message);\n else \n ev.message = kvin.unmarshal(ev.message);\n \n if (lastConsoleEv && ev.message[0] === lastConsoleEv.message[0] && ev.sliceNumber === lastConsoleEv.sliceNumber && ev.level === lastConsoleEv.level)\n ev.same = ++sameCounter;\n else\n sameCounter = 1;\n lastConsoleEv = ev;\n \n /* if we have the same message being logged (same sliceNumber, message, log level), the console event object will have the sole property same, nothing else */\n if (ev.same > 1)\n that.emit('console', { same: ev.same });\n else\n {\n delete ev.same;\n that.emit('console', ev);\n }\n }\n\n this.eventIntercepts = {\n result: (ev) => this.handleResult(ev),\n status: (ev) => this.handleStatus(ev),\n cancel: (ev) => this.ee.emit('stopped', ev),\n custom: (ev) => this.work.emit(ev.customEvent, ev),\n console: parseConsole,\n };\n \n this.eventTypes = (__webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\").eventTypes);\n\n this.work.on('newListener', (evt) => {\n this.listenForCustomEvents = true;\n });\n this.desiredEvents = []\n this.on('newListener', (evt) => {\n if (!this.connected && evt !== 'newListener')\n this.desiredEvents.push(evt);\n if (evt === 'cancel')\n this.listeningForCancel = true;\n });\n }\n \n /** \n * Cancel the job\n * @access public\n * @param {string} reason If provided, will be sent to client\n */\n async cancel (reason = undefined)\n {\n const response = await this.useDeployConnection('cancelJob', {\n job: this.address,\n owner: this.paymentAccountKeystore.address,\n reason,\n }, this.paymentAccountKeystore);\n\n return response.payload;\n }\n\n /** \n * Resume this job\n * @access public\n */\n async resume ()\n {\n const response = await this.schedulerConnection.send('resumeJob', {\n job: this.address,\n owner: this.paymentAccountKeystore.address,\n }, this.paymentAccountKeystore);\n\n return response.payload;\n }\n\n /**\n * Helper function for retrieving info about the job. The job must have already been deployed.\n * An alias for {@link module:dcp/compute.getJobInfo}.\n * @access public\n */\n getJobInfo ()\n {\n return (__webpack_require__(/*! ../compute */ \"./src/dcp-client/compute.js\").compute.getJobInfo)(this.address);\n }\n \n /**\n * Helper function for retrieving info about the job's slices. The job must have already been deployed.\n * An alias for {@link module:dcp/compute.getSliceInfo}.\n * @access public\n */\n getSliceInfo ()\n {\n return (__webpack_require__(/*! ../compute */ \"./src/dcp-client/compute.js\").compute.getSliceInfo)(this.address);\n }\n \n /** Escrow additional funds for this job\n * @access public\n * @param {number|BigNumber} fundsRequired - A number or BigNumber instance representing the funds to escrow for this job\n */\n async escrow (fundsRequired)\n {\n if ((typeof fundsRequired !== 'number' && !BigNumber.isBigNumber(fundsRequired))\n || fundsRequired <= 0 || !Number.isFinite(fundsRequired) || Number.isNaN(fundsRequired))\n throw new Error(`Job.escrow: fundsRequired must be a number greater than zero. (not ${fundsRequired})`);\n\n const bankConnection = new protocolV4.Connection(dcpConfig.bank.services.bankTeller);\n\n /*\n * escrow has been broken for an unknown amount of time. `feeStructureId` is not defined anywhere in the job class, and hasn't\n * for a period of time. When fixed, `this[INTERNAL_SYMBOL].payloadDetails.feeStructureId` will likely become just `this.feeStructureId`,\n * but it's being left alone until someone spends the time to fix escrow. / rs Jul 2022\n */\n const response = await bankConnection.send('embiggenFeeStructure', {\n feeStructureAddress: this[INTERNAL_SYMBOL].payloadDetails.feeStructureId,\n additionalEscrow: new BigNumber(fundsRequired),\n fromAddress: this.paymentAccountKeystore.address,\n }, this.paymentAccountKeystore);\n\n bankConnection.close();\n const receipt = response.payload;\n return receipt;\n }\n\n /**\n * create bundles for local dependencies\n */\n _pack ()\n {\n var retval = (__webpack_require__(/*! ./node-modules */ \"./src/dcp-client/job/node-modules.js\").createModuleBundle)(this.dependencies);\n return retval;\n }\n\n /** \n * Collect all of the dependencies together, throw them into a BravoJS\n * module which sideloads them as a side effect of declaration, and transmit\n * them to the package manager. Then we return the package descriptor object,\n * which is guaranteed to have only one file in it.\n *\n * @returns {object} with properties name and files[0]\n */\n async _publishLocalModules ()\n {\n const { tempFile, hash, unresolved } = await this._pack();\n\n if (!tempFile) {\n return { unresolved };\n }\n\n const sideloaderFilename = tempFile.filename;\n const pkg = {\n name: `dcp-pkg-v1-localhost-${hash.toString('hex')}`,\n version: '1.0.0',\n files: {\n [sideloaderFilename]: `${sideloaderModuleIdentifier}.js`,\n },\n }\n\n await dcpPublish.publish(pkg);\n tempFile.remove();\n\n return { pkg, unresolved };\n }\n \n /**\n * This function specifies a module dependency (when the argument is a string)\n * or a list of dependencies (when the argument is an array) of the work\n * function. This function can be invoked multiple times before deployment.\n * @param {string | string[]} modulePaths - A string or array describing one\n * or more dependencies of the job.\n * @access public\n */\n requires (modulePaths)\n {\n if (typeof modulePaths !== 'string' && (!Array.isArray(modulePaths) || modulePaths.some((modulePath) => typeof modulePath !== 'string')))\n throw new TypeError('The argument to dependencies is not a string or an array of strings');\n else if (modulePaths.length === 0)\n throw new RangeError('The argument to dependencies cannot be an empty string or array');\n else if (Array.isArray(modulePaths) && modulePaths.some((modulePath) => modulePath.length === 0))\n throw new RangeError('The argument to dependencies cannot be an array containing an empty string');\n\n if (!Array.isArray(modulePaths))\n modulePaths = [modulePaths];\n\n for (const modulePath of modulePaths)\n {\n if (modulePath[0] !== '.' && modulePath.indexOf('/') !== -1)\n {\n const modulePrefixRegEx = /^(.*)\\/.*?$/;\n const [, modulePrefix] = modulePath.match(modulePrefixRegEx);\n if (modulePrefix && this.requirePath.indexOf(modulePrefix) === -1)\n this.requirePath.push(modulePrefix);\n }\n this.dependencies.push(modulePath);\n }\n }\n \n /** Set the account upon which funds will be drawn to pay for the job.\n * @param {module:dcp/wallet.AuthKeystore} keystore A keystore that representa a bank account.\n * @access public\n */\n setPaymentAccountKeystore (keystore)\n {\n if (this.address)\n {\n if (!keystore.address.eq(this.paymentAccountKeystore))\n {\n let message = 'Cannot change payment account after job has been deployed';\n this.emit('EPERM', message);\n throw new Error(`EPERM: ${message}`);\n }\n }\n \n if (!(keystore instanceof wallet.Keystore))\n throw new Error('Not an instance of Keystore: ' + keystore.toString());\n this.paymentAccountKeystore = keystore;\n }\n \n /** Set the slice payment offer. This is equivalent to the first argument to exec.\n * @param {number} slicePaymentOffer - The number of DCC the user is willing to pay to compute one slice of this job\n */\n setSlicePaymentOffer (slicePaymentOffer)\n {\n this.slicePaymentOffer = new SlicePaymentOffer(slicePaymentOffer);\n }\n \n \n /**\n * @param {URL|DcpURL} locationUrl - A URL object\n * @param {object} postParams - An object with any parameters that a user would like to be passed to a \n * remote result location. This object is capable of carry API keys for S3, \n * DropBox, etc. These parameters are passed as parameters in an \n * application/x-www-form-urlencoded request.\n */\n setResultStorage (locationUrl, postParams)\n {\n if (locationUrl instanceof URL || locationUrl instanceof DcpURL)\n this.resultStorageDetails = locationUrl;\n else\n throw new Error('Not an instance of a DCP URL: ' + locationUrl);\n \n\n // resultStorageParams contains any post params required for off-prem storage\n if (typeof postParams !== 'undefined' && typeof postParams === 'object' )\n this.resultStorageParams = postParams;\n else\n throw new Error('Not an instance of a object: ' + postParams);\n\n // Some type of object here\n this.resultStorageType = 'pattern';\n }\n \n /**\n * This function is identical to exec, except that the job is executed locally\n * in the client.\n * @async\n * @param {number} cores - the number of local cores in which to execute the job.\n * @param {...any} args - The remaining arguments are identical to the arguments of exec\n * @return {Promise<ResultHandle>} - resolves with the results of the job, rejects on an error\n * @access public\n */\n localExec (cores = 1, ...args)\n {\n this.inLocalExec = true;\n this.estimationSlices = 0;\n this.greedyEstimation = false;\n this.isCI = false;\n\n let worker;\n this.on('accepted', () => {\n // Start a worker for this job\n worker = new Worker({\n localExec: true,\n jobAddresses: [this.address],\n allowedOrigins: this.localExecAllowedFiles,\n paymentAddress: this.paymentAccountKeystore.address,\n maxWorkingSandboxes: cores,\n sandboxOptions: {\n ignoreNoProgress: true,\n SandboxConstructor: (DCP_ENV.platform === 'nodejs'\n && (__webpack_require__(/*! ../worker/evaluators */ \"./src/dcp-client/worker/evaluators/index.js\").nodeEvaluatorFactory)())\n },\n });\n\n worker.start().catch((e) => {\n console.error('Failed to start worker for localExec:');\n console.error(e.message);\n });\n });\n\n return this.exec(...args).finally(() => {\n if (worker) {\n setTimeout(() => {\n // stop the worker\n worker.stop(true);\n }, 3000);\n }\n });\n }\n\n /**\n * Deploys the job to the scheduler.\n * @param {number | object} [slicePaymentOffer=compute.marketValue] - Amount\n * in DCC that the user is willing to pay per slice.\n * @param {Keystore} [paymentAccountKeystore=wallet.get] - An instance of the\n * Wallet API Keystore that's used as the payment account when executing the\n * job.\n * @param {object} [initialSliceProfile] - An object describing the cost the\n * user believes the average slice will incur.\n * @access public\n * @emits Job#accepted\n */\n async exec (slicePaymentOffer = (__webpack_require__(/*! ../compute */ \"./src/dcp-client/compute.js\").compute.marketValue), paymentAccountKeystore, initialSliceProfile)\n {\n if (this.connected)\n throw new Error('Exec called twice on the same job handle.');\n \n if (this.estimationSlices === Infinity)\n this.estimationSlices = null;\n else if (this.estimationSlices < 0)\n throw new Error('Incorrect value for estimationSlices; it can be an integer or Infinity!');\n \n if (this.tuning.kvin.speed || this.tuning.kvin.size)\n {\n tunedKvin = new kvin.KVIN();\n tunedKvin.tune = 'size';\n if(this.tuning.kvin.speed)\n tunedKvin.tune = 'speed';\n // If both size and speed are true, kvin will optimize based on speed\n if(this.tuning.kvin.speed && this.tuning.kvin.size)\n console.log('Slices and arguments are being uploaded with speed optimization.');\n }\n \n this.readyStateChange('exec');\n if ((typeof slicePaymentOffer === 'number') || (typeof slicePaymentOffer === 'object')\n || ((this.slicePaymentOffer === null || this.slicePaymentOffer === undefined) && typeof slicePaymentOffer === 'function'))\n this.setSlicePaymentOffer(slicePaymentOffer);\n if (typeof initialSliceProfile !== 'undefined')\n this.initialSliceProfile = initialSliceProfile;\n \n if (typeof paymentAccountKeystore !== 'undefined')\n {\n /** XXX @todo deprecate use of ethereum wallet objects */\n if (typeof paymentAccountKeystore === 'object' && paymentAccountKeystore.hasOwnProperty('_privKey'))\n {\n console.warn('* deprecated API * - job.exec invoked with ethereum wallet object as paymentAccountKeystore') /* /wg oct 2019 */\n paymentAccountKeystore = paymentAccountKeystore._privKey\n }\n /** XXX @todo deprecate use of private keys */\n if (wallet.isPrivateKey(paymentAccountKeystore))\n {\n console.warn('* deprecated API * - job.exec invoked with private key as paymentAccountKeystore') /* /wg dec 2019 */\n paymentAccountKeystore = await new wallet.Keystore(paymentAccountKeystore, '');\n }\n\n this.setPaymentAccountKeystore(paymentAccountKeystore)\n }\n \n if (this.paymentAccountKeystore)\n // Throws if they fail to unlock, we allow this since the keystore was set programmatically. \n await this.paymentAccountKeystore.unlock(undefined, parseFloat(dcpConfig.job.maxDeployTime));\n else\n {\n // If not set programmatically, we keep trying to get an unlocked keystore ... forever.\n let locked = true;\n let safety = 0; // no while loop shall go unguarded\n let ks;\n do\n {\n ks = null;\n // custom message for the browser modal to denote the purpose of keystore submission\n let msg = `This application is requesting a keystore file to execute ${this.public.description || this.public.name || 'this job'}. Please upload the corresponding keystore file. If you upload a keystore file which has been encrypted with a passphrase, the application will not be able to use it until it prompts for a passphrase and you enter it.`;\n try\n {\n ks = await wallet.get({ contextId: this.contextId, jobName: this.public.name, msg});\n }\n catch (e)\n {\n if (e.code !== ClientModal.CancelErrorCode) throw e;\n };\n if (ks)\n {\n try\n {\n await ks.unlock(undefined, parseFloat(dcpConfig.job.maxDeployTime));\n locked = false;\n }\n catch (e)\n {\n // prompt user again if user enters password incorrectly, exit modal otherwise\n if (e.code !== wallet.unlockFailErrorCode) throw e;\n }\n }\n if (safety++ > 1000) throw new Error('EINFINITY: job.exec tried wallet.get more than 1000 times.')\n } while (locked);\n this.setPaymentAccountKeystore(ks)\n }\n \n // We either have a valid keystore + password or we have rejected by this point.\n if (!this.slicePaymentOffer)\n throw new Error('A payment profile must be assigned before executing the job');\n else\n this.feeStructure = this.slicePaymentOffer.toFeeStructure(this.jobInputData.length);\n\n if (!this.address)\n {\n try\n {\n this.readyStateChange('init');\n await this.deployJob();\n const listenersPromise = this.addInitialEvents();\n const computeGroupsPromise = this.joinComputeGroups();\n let uploadSlicePromise;\n // if job data is by value then upload data to the scheduler in a staggered fashion\n if (Array.isArray(this.dataValues))\n {\n this.readyStateChange('uploading');\n if (!this.inLocalExec)\n uploadSlicePromise = addSlices(this.dataValues, this.address, tunedKvin, this.inLocalExec).then(() => this.close());\n }\n \n // await all promises for operations that can be done after the job is deployed\n if (!this.inLocalExec)\n await Promise.all([listenersPromise, computeGroupsPromise, uploadSlicePromise]);\n else\n await Promise.all([listenersPromise, computeGroupsPromise]);\n \n this.readyStateChange('deployed');\n this.emit('accepted', { job: this });\n }\n catch (error)\n {\n if (ON_BROWSER)\n await ClientModal.alert(error, { title: 'Failed to deploy job!' });\n throw error;\n }\n }\n else\n {\n // reconnecting to an old job\n await this.addInitialEvents();\n this.readyStateChange('reconnected');\n }\n\n this.connected = true;\n\n return new Promise((resolve, reject) => {\n const onComplete = () => resolve(this.results);\n const onCancel = (event) => {\n /**\n * FIXME(DCP-1150): Remove this since normal cancel event is noisy\n * enough to not need stopped event too.\n */\n if (ON_BROWSER && !this.listeningForCancel)\n ClientModal.alert('More details in console...', { title: 'Job Canceled' });\n this.emit('cancel', event);\n\n let errorMsg = event.reason;\n if (event.error)\n errorMsg = errorMsg +`\\n Recent error message: ${event.error.message}`\n \n reject(new DCPError(errorMsg, event.code));\n };\n\n this.ee.once('stopped', async (stopEvent) => {\n // There is a chance the result submitter will emit finished > 1 time. Only handle it once.\n if (this.receivedStop)\n return;\n this.receivedStop = true;\n this.emit('stopped', stopEvent.runStatus);\n switch (stopEvent.runStatus) {\n case jobStatus.finished:\n if (this.collateResults)\n {\n let report = await this.getJobInfo();\n let allSliceNumbers = Array.from(Array(report.totalSlices)).map((e,i)=>i+1);\n let remainSliceNumbers = allSliceNumbers.filter((e) => !this.results.isAvailable(e));\n\n if (remainSliceNumbers.length)\n {\n const promises = remainSliceNumbers.map(sliceNumber => this.results.fetch([sliceNumber], true));\n await Promise.all(promises);\n }\n }\n\n this.emit('complete', this.results);\n onComplete();\n break;\n case jobStatus.cancelled:\n onCancel(stopEvent);\n break;\n default:\n /**\n * Asserting that we should never be able to reach here. The only\n * scheduler events that should trigger the Job's 'stopped' event\n * are jobStatus.cancelled, jobStatus.finished, and sliceStatus.paused.\n */\n reject(new Error(`Unknown event \"${stopEvent.runStatus}\" caused the job to be stopped.`));\n break;\n }\n });\n\n }).finally(() => {\n const handleErr = (e) => {\n console.error('Error while closing job connection:');\n console.error(e);\n }\n\n // Create an async IIFE to not block the promise chain\n (async () => {\n // delay to let last few events to be received\n await new Promise((resolve) => setTimeout(resolve, 1000));\n \n // close all of the connections so that we don't cause node processes to hang.\n this.closeDeployConnection();\n await this.eventSubscriber.close().catch(handleErr);\n await computeGroups.closeServiceConnection().catch((err) => {\n console.error('Warning: could not close compute groups service connection', err);\n });\n })();\n });\n }\n \n /**\n * job.addListeners(): Private function used to set up event listeners to the scheduler\n * before deploying the job.\n */\n async addInitialEvents ()\n {\n this.readyStateChange('listeners');\n\n // This is important: We need to flush the task queue before adding listeners\n // because we queue pending listeners by listening to the newListener event (in the constructor).\n // If we don't flush here, then the newListener events may fire after this function has run,\n // and the events won't be properly set up.\n await new Promise(resolve => setTimeout(resolve, 0));\n\n // @todo: Listen for an estimated cost, probably emit an \"estimated\" event when it comes in?\n // also @todo: Do the estimation task(s) on the scheduler and send an \"estimated\" event\n\n // Always listen to the stop event. It will resolve the work function promise, so is always needed.\n this.on('stop', (ev) => {this.ee.emit('stopped', ev)});\n\n // Connect listeners that were set up before exec\n if (this.desiredEvents.includes('result'))\n this.listeningForResults = true;\n await this.subscribeNewEvents(this.desiredEvents);\n\n // Connect listeners that are set up after exec\n this.on('newListener', (evt) => {\n if (evt === 'newListener' || this.desiredEvents.includes(evt))\n return;\n this.subscribeNewEvents([evt]);\n });\n \n // automatically add a listener for results if collateResults is on\n if (this.collateResults && !this.listeningForResults)\n this.on('result', () => {});\n\n debugging('dcp-client') && console.debug('subscribedEvents', this.desiredEvents);\n\n // If we have listeners for job.work, subscribe to custom events\n if (this.listenForCustomEvents)\n await this.subscribeCustomEvents();\n // Connect work event listeners that are set up after exec\n else\n this.work.on('newListener', () => this.subscribeCustomEvents());\n }\n \n /**\n * Subscribes to either reliable events or optional events. It is assumed that\n * any call to this function will include only new events.\n * @param {string[]} events \n */\n async subscribeNewEvents (events)\n {\n const reliableEvents = [];\n const optionalEvents = [];\n for (let eventName of events)\n {\n eventName = eventName.toLowerCase();\n if (this.eventTypes[eventName] && this.eventTypes[eventName].reliable)\n reliableEvents.push(eventName);\n else if (this.eventTypes[eventName] && !this.eventTypes[eventName].reliable)\n optionalEvents.push(eventName);\n else\n debugging('dcp-client') && console.debug(`Job handler has listener ${eventName} which isn't an event-router event.`);\n }\n if (debugging('dcp-client'))\n {\n console.debug('reliableEvents', reliableEvents);\n console.debug('optionalEvents', optionalEvents);\n }\n await this.eventSubscriber.subscribeManyEvents(reliableEvents, optionalEvents, { filter: { job: this.address } });\n }\n \n /**\n * Establishes listeners for worker events when requested by the client\n */\n async subscribeCustomEvents ()\n {\n if (!this.listeningForCustomEvents)\n await this.eventSubscriber.subscribeManyEvents([], ['custom'], { filter: { job: this.address } });\n this.listeningForCustomEvents = true\n }\n \n async joinComputeGroups ()\n {\n // localExec jobs are not entered in any compute group.\n if (!this.inLocalExec && this.computeGroups && this.computeGroups.length)\n {\n this.readyStateChange('compute-groups');\n computeGroups.addRef(); // Just in case we're doing a Promise.all on multiple execs.\n\n // Add this job to its currently-defined compute groups (as well as public group, if included)\n let success;\n try\n {\n const cgPayload = await computeGroups.addJobToGroups(this.address, this.computeGroups);\n success = true; // To support older version of CG service where addJobToGroups had void/undefined return.\n if (cgPayload) success = cgPayload.success;\n debugging('dcp-client') && console.debug('job/index: addJobToGroups cgPayload:', cgPayload ? cgPayload : 'cgPayload is not defined; probably from legacy CG service.');\n }\n catch (e)\n {\n debugging('dcp-client') && console.debug('job/index: addJobToGroups threw exception:', e);\n success = false;\n }\n\n computeGroups.closeServiceConnection().catch((err) => {\n console.error('Warning: could not close compute groups service connection', err)\n });\n\n /* Could not put the job in any compute group, even though the user wanted it to run. Cancel the job. */\n if (!success)\n {\n await this.cancel('compute-groups::Unable to join any compute groups');\n throw new DCPError(`Access Denied::Failed to add job ${this.address} to any of the desired compute groups`, 'DCPL-1100');\n }\n }\n }\n \n /**\n * Takes result events as input, stores the result and fires off\n * events on the job handle as required. (result, duplicate-result)\n *\n * @param {object} ev - the event recieved from protocol.listen('/results/0xThisGenAdr')\n */\n async handleResult (ev)\n {\n if (this.results === null)\n // This should never happen - the onResult event should only be established/called\n // in addListeners which should also initialize the internal results array\n throw new Error('Job.onResult was invoked before initializing internal results');\n \n const { result: _result, time } = ev.result;\n debugging('dcp-client') && console.debug('handleResult', _result);\n let result = await fetchURI(_result);\n\n if (this.results.isAvailable(ev.sliceNumber))\n {\n const changed = JSON.stringify(this.results[ev.sliceNumber]) !== JSON.stringify(result);\n this.emit('duplicate-result', { sliceNumber: ev.sliceNumber, changed });\n }\n\n this.results.newResult(result, ev.sliceNumber);\n }\n \n /**\n * Receives status events from the scheduler, updates the local status object\n * and emits a 'status' event\n *\n * @param {object} ev - the status event received from\n * protocol.listen('/status/0xThisGenAdr')\n * @param {boolean} emitStatus - value indicating whether or not the status\n * event should be emitted\n */\n handleStatus ({ runStatus, total, distributed, computed }, emitStatus = true)\n {\n Object.assign(this.status, {\n runStatus,\n total,\n distributed,\n computed,\n });\n\n if (emitStatus)\n this.emit('status', { ...this.status, job: this.address });\n }\n \n /**\n * Sends a request to the scheduler to deploy the job.\n */\n async deployJob ()\n {\n var moduleDependencies; \n \n /* Send sideloader bundle to the package server */\n if (DCP_ENV.platform === 'nodejs' && this.dependencies.length)\n {\n try\n {\n let { pkg, unresolved } = await this._publishLocalModules();\n\n moduleDependencies = unresolved;\n if (pkg)\n moduleDependencies.push(pkg.name + '/' + sideloaderModuleIdentifier); \n }\n catch(error)\n {\n throw new DCPError(`Error trying to communicate with package manager server: ${error}`);\n }\n }\n else\n moduleDependencies = this.dependencies;\n \n this.readyStateChange('preauth');\n\n const adhocId = this.uuid.slice(this.uuid.length - 6, this.uuid.length);\n const schedId = await dcpConfig.scheduler.identity;\n // The following check is needed for when using dcp-rtlink and loading the config through source, instead of using the dcp-client bundle\n let schedIdAddress = schedId;\n if(schedId.address)\n schedIdAddress = schedId.address;\n const myId = await wallet.getId();\n const preauthToken = await bankUtil.preAuthorizePayment(schedIdAddress, this.maxDeployPayment, this.paymentAccountKeystore);\n const { dataRange, dataValues, dataPattern, sliceCount } = marshalInputData(this.jobInputData);\n if(dataValues)\n this.dataValues = dataValues;\n\n this.readyStateChange('deploying');\n\n /* Payload format is documented in scheduler-v4/libexec/job-submit/operations/submit.js */\n const submitPayload = {\n owner: myId.address,\n paymentAccount: this.paymentAccountKeystore.address,\n priority: 0, // @nyi\n\n workFunctionURI: this.workFunctionURI,\n uuid: this.uuid,\n mvMultSlicePayment: Number(this.feeStructure.marketValue) || 0, // @todo: improve feeStructure internals to better reflect v4\n absoluteSlicePayment: Number(this.feeStructure.maxPerRequest) || 0,\n requirePath: this.requirePath,\n dependencies: moduleDependencies,\n requirements: this.requirements, /* capex */\n localExec: this.inLocalExec,\n force100pctCPUDensity: this.force100pctCPUDensity,\n estimationSlices: this.estimationSlices,\n greedyEstimation: this.greedyEstimation,\n workerConsole: this.workerConsole,\n isCI: this.isCI,\n\n description: this.public.description || 'Discreetly making the world smarter',\n name: this.public.name || 'Ad-Hoc Job' + adhocId,\n link: this.public.link || '',\n\n preauthToken, // XXXwg/er @todo: validate this after fleshing out the stub(s)\n\n resultStorageType: this.resultStorageType, // @todo: implement other result types\n resultStorageDetails: this.resultStorageDetails, // Content depends on resultStorageType\n resultStorageParams: this.resultStorageParams, // Post params for off-prem storage\n dataRange,\n dataPattern,\n sliceCount\n };\n\n /* Determine thee type of the arguments option and set the submit message payload accordingly. */\n if (Array.isArray(this.jobArguments) && this.jobArguments.length === 1 && this.jobArguments[0] instanceof DcpURL)\n submitPayload.arguments = this.jobArguments[0].href;\n else if (this.jobArguments instanceof RemoteDataSet)\n submitPayload.marshaledArguments = kvinMarshal(this.jobArguments.map(e => new URL(e)))\n else if (this.jobArguments)\n {\n try\n {\n submitPayload.marshaledArguments = kvinMarshal(Array.from(this.jobArguments));\n }\n catch(e)\n {\n throw new Error(`Could not convert job arguments to Array (${e.message})`);\n }\n }\n \n if (this.inLocalExec && !DCP_ENV.isBrowserPlatform)\n {\n const workFunctionFile = createTempFile('dcp-localExec-workFunction-XXXXXXXXX', 'js');\n const argumentsFile = createTempFile('dcp-localExec-arguments-XXXXXXXXX', 'js');\n const inputSetFiles = [];\n \n submitPayload.marshaledInputValues = [];\n let inputSetFileURLs = []\n let dataSet;\n \n if (this.dataValues)\n {\n for (let i = 0; i < this.dataValues.length; i++)\n {\n inputSetFiles.push(createTempFile('dcp-localExec-sliceData-XXXXXXXXX', 'js'));\n let marshaledInputSet = kvinMarshal(this.dataValues[i]);\n inputSetFiles[i].writeSync(JSON.stringify(marshaledInputSet));\n inputSetFileURLs.push(new URL('file://' + inputSetFiles[i].filename));\n }\n dataSet = new RemoteDataSet(inputSetFileURLs);\n submitPayload.marshaledInputValues = dataSet;\n }\n if (dataRange)\n {\n submitPayload.rangeLength = dataRange.length;\n inputSetFiles.push(createTempFile('dcp-localExec-sliceData-XXXXXXXXX', 'js'));\n let marshaledInputSet = encodeDataURI(JSON.stringify(dataRange), 'application/json');\n inputSetFiles[0].writeSync(marshaledInputSet)\n inputSetFileURLs.push(new URL('file://' + inputSetFiles[0].filename));\n dataSet = new RemoteDataSet(inputSetFileURLs);\n submitPayload.dataRange = dataSet;\n }\n \n if (dataPattern)\n {\n inputSetFiles.push(createTempFile('dcp-localExec-sliceData-XXXXXXXXX', 'js'));\n let marshaledInputSet = kvinMarshal(dataPattern)\n inputSetFiles[0].writeSync(JSON.stringify(marshaledInputSet))\n inputSetFileURLs.push(new URL('file://' + inputSetFiles[0].filename));\n dataSet = new RemoteDataSet(inputSetFileURLs);\n submitPayload.dataPattern = dataSet[0];\n }\n \n // For allowed origins of the localexec worker. Only allow the origins (files in this case) in this list.\n this.localExecAllowedFiles = [workFunctionFile.filename, argumentsFile.filename];\n for (let i = 0; i < inputSetFiles.length; i++)\n this.localExecAllowedFiles.push(inputSetFiles[i].filename);\n\n // get the workFunctionURI string before writing to file to prevent the need to double-decode the work function in the worker.\n const workFunction = await fetchURI(this.workFunctionURI);\n workFunctionFile.writeSync(workFunction);\n \n const workFunctionFileURL = new URL('file://' + workFunctionFile);\n submitPayload.workFunctionURI = workFunctionFileURL.href;\n this.workFunctionURI = workFunctionFileURL.href;\n \n if (submitPayload.marshaledArguments)\n {\n argumentsFile.writeSync(JSON.stringify(submitPayload.marshaledArguments));\n const argumentsFileURL = new URL('file://' + argumentsFile.filename);\n submitPayload.marshaledArguments = kvinMarshal([argumentsFileURL]);\n }\n }\n\n // XXXpfr Excellent tracing.\n if (debugging('dcp-client'))\n {\n dumpObject(submitPayload, 'Submit: Job Index: submitPayload', 256);\n console.debug('Before Deploy', myId);\n }\n\n // Deploy the job! If we get an error, try again a few times until threshold of errors is reached, then actually throw it\n let deployed\n let deployAttempts = 0;\n while (deployAttempts++ < (dcpConfig.job.deployAttempts || 10))\n {\n try\n {\n deployed = await this.useDeployConnection('submit', submitPayload, myId);\n break;\n }\n catch (e)\n {\n if (deployAttempts < 10)\n debugging('dcp-client') && console.debug('Error when trying to deploy job, trying again', e);\n else\n throw e;\n }\n }\n\n if (!deployed.success)\n {\n // close all of the connections so that we don't cause node processes to hang.\n const handleErr = (e) => {\n console.error('Error while closing job connection:');\n console.error(e);\n };\n \n this.closeDeployConnection();\n this.eventSubscriber.close().catch(handleErr);\n computeGroups.closeServiceConnection().catch(handleErr);\n \n // Yes, it is possible for deployed or deployed.payload to be undefined.\n if (deployed.payload)\n {\n if (deployed.payload.code === 'ENOTFOUND')\n throw new DCPError(`Failed to submit job to scheduler. Account: ${submitPayload.paymentAccount} was not found or does not have sufficient balance (${deployed.payload.info.deployCost} DCCs needed to deploy this job)`, deployed.payload); \n throw new DCPError('Failed to submit job to scheduler', deployed.payload);\n }\n throw new DCPError('Failed to submit job to scheduler (no payload)', deployed ? deployed : '');\n }\n\n debugging('dcp-client') && console.debug('After Deploy', JSON.stringify(deployed));\n\n this.address = deployed.payload.job;\n this.deployCost = deployed.payload.deployCost;\n\n if (!this.status)\n this.status = {\n runStatus: null,\n total: 0,\n computed: 0,\n distributed: 0,\n };\n \n this.status.runStatus = deployed.payload.status;\n this.status.total = deployed.payload.lastSliceNumber;\n this.running = true;\n }\n \n /** close an open job to indicate we are done adding data so it is okay to finish\n * the job at the appropriate time\n */\n close ()\n {\n return this.useDeployConnection('closeJob', {\n job: this.id,\n });\n }\n \n /** Use the connection to job submit service. Will open a new connection if one does not exist,\n * and close the connection if it is idle for more than 10 seconds (tuneable).\n */\n useDeployConnection(...args)\n {\n if (!this.deployConnection)\n {\n this.deployConnection = new protocolV4.Connection(dcpConfig.scheduler.services.jobSubmit); \n this.deployConnection.on('close', () => { this.deployConnection = null; });\n }\n if (this.deployConnectionTimeout)\n clearTimeout(this.deployConnectionTimeout);\n \n const deployPromise = this.deployConnection.send(...args);\n \n this.deployConnectionTimeout = setTimeout(() => {this.deployConnection && this.deployConnection.close()}, (dcpConfig.job.deployCloseTimeout || 10 * 1000));\n if (!ON_BROWSER)\n this.deployConnectionTimeout.unref();\n \n return deployPromise;\n }\n \n /**\n * Close the connection to the job submit (if it exists), and clear the close timeout (if needed).\n */\n closeDeployConnection()\n {\n if (this.deployConnection)\n this.deployConnection.close();\n if (this.deployConnectionTimeout)\n clearTimeout(this.deployConnectionTimeout);\n }\n}\n\n/**\n * Depending on the shape of the job's data, resolve it into a RangeObject, a\n * Pattern, or a values array, and return it in the appropriate property.\n *\n * @param {any} data Job's input data\n * @return {MarshaledInputData} An object with one of the following properties set:\n * - dataValues: job input is an array of arbitrary values \n * - dataPattern: job input is a URI pattern \n * - dataRange: job input is a RangeObject (and/or friends)\n */\nfunction marshalInputData (data)\n{\n if (!(data instanceof Object || data instanceof SuperRangeObject))\n throw new TypeError(`Invalid job data type: ${typeof data}`);\n\n /**\n * @type MarshaledInputData\n */\n const marshalledInputData = {};\n\n // TODO(wesgarland): Make this more robust.\n if (data instanceof SuperRangeObject ||\n (data.hasOwnProperty('ranges') && data.ranges instanceof MultiRangeObject) ||\n (data.hasOwnProperty('start') && data.hasOwnProperty('end')))\n marshalledInputData.dataRange = data;\n else if (Array.isArray(data))\n marshalledInputData.dataValues = data;\n else if (data instanceof URL || data instanceof DcpURL)\n marshalledInputData.dataPattern = String(data);\n else if(data instanceof RemoteDataSet)\n marshalledInputData.dataValues = data.map(e => new URL(e));\n else if(data instanceof RemoteDataPattern)\n {\n marshalledInputData.dataPattern = data['pattern'];\n marshalledInputData.sliceCount = data['sliceCount'];\n }\n\n log('marshalledInputData:', marshalledInputData);\n return marshalledInputData;\n}\n\n/**\n * marshal the value using kvin or instance of the kvin (tunedKvin)\n * tunedKvin is defined if job.tuning.kvin is specified.\n *\n * @param {any} value \n * @return {object} A marshaled object\n * \n */\nfunction kvinMarshal (value) {\n if (tunedKvin)\n return tunedKvin.marshal(value);\n\n return kvin.marshal(value);\n}\n\n\n\nexports.Job = Job;\nexports.SlicePaymentOffer = SlicePaymentOffer;\nexports.ResultHandle = ResultHandle;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/job/index.js?");
|
|
4172
4172
|
|
|
4173
4173
|
/***/ }),
|
|
4174
4174
|
|
|
@@ -4189,7 +4189,7 @@ eval("/**\n * @file node-modules.js Node-specific support for sen
|
|
|
4189
4189
|
\*********************************************/
|
|
4190
4190
|
/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
|
|
4191
4191
|
|
|
4192
|
-
eval("/**\n * @file job/result-handle.js\n * @author Ryan Rossiter, ryan@kingsds.network\n * @date June 2020\n *\n * The ResultHandle acts as a proxy for a job's results, querying\n * internal results when available or the scheduler when the results\n * are not available locally.\n */\n\nconst { rehydrateRange } = __webpack_require__(/*! ../range-object */ \"./src/dcp-client/range-object.js\");\nconst protocolV4 = __webpack_require__(/*! dcp/protocol-v4 */ \"./src/protocol-v4/index.js\");\nconst wallet = __webpack_require__(/*! dcp/dcp-client/wallet */ \"./src/dcp-client/wallet/index.js\");\nconst { fetchURI } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst customInspectSymbol = Symbol.for('nodejs.util.inspect.custom');\n\n// Array methods that don't modify the array, will be applied to the ResultHandle's values object\nconst RH_ARRAY_METHODS = [\n 'slice', 'filter', 'concat', 'find', 'findIndex', 'indexOf', 'map', 'reduce', 'includes', 'toString', 'forEach'\n];\n\n/**\n * This class represents a handle on a job's results. It can be used for accessing the job's results, or for querying the scheduler to fetch results.\n * In addition to the properties and methods of this class, the following standard array methods are also available for accessing the available results: `slice`, `filter`, `concat`, `find`, `findIndex`, `indexOf`, `map`, `reduce`, `includes`, `toString`, and `forEach`.\n * The results can also be accessed by index (`results[i]`), but an error will be thrown if the result for that index is not yet available.\n * @access public\n */\nclass ResultHandle extends Array\n{\n constructor(job)\n {\n super();\n\n const { Job } = __webpack_require__(/*! . */ \"./src/dcp-client/job/index.js\");\n if (!(job instanceof Job))\n throw new TypeError('ResultHandle must be constructed from a Job object');\n\n /**\n * The length of the available results array.\n * @type {number}\n * @access public\n */\n this.length = 0; // overridden by the proxy, here for JSDoc purposes\n\n this.job = job;\n this.realKeys = job.jobInputData;\n this.realValues = [];\n this.valuesAvailable = [];\n\n return new Proxy(this, {\n get: (target, name) => {\n if ((typeof name === 'string' || typeof name === 'number') && Number.isInteger(parseFloat(name)))\n {\n let i = parseFloat(name);\n if (target.isAvailable(i))\n return target.realValues[i];\n else\n throw new Error(`Result ${i} is not available. It has either not been computed or you need to fetch it.`);\n } \n else if (name === 'length')\n return target.getLength();\n else if (name === 'constructor')\n return Array.constructor;\n else if (RH_ARRAY_METHODS.includes(name))\n {\n let values = target.values();\n return values[name].bind(values);\n }\n else\n {\n // only return methods on this class, don't allow access\n // to array methods other than the whitelisted ones\n let p = target.__proto__[name];\n if (typeof p === 'function') {\n return p.bind(target);\n } else {\n return p;\n }\n }\n }\n });\n }\n\n toJSON ()\n {\n return JSON.stringify(this.values());\n }\n\n isAvailable(index)\n {\n return this.valuesAvailable[index];\n }\n\n reset() {\n // quickly empty the realValues array\n this.realValues.length = this.valuesAvailable.length = 0;\n this.realValues.length = this.valuesAvailable.length =this.realKeys.length;\n }\n\n /**\n * Returns an array of input values. Will only return input values that have a completed result available.\n * @access public\n * @returns {Array<*>}\n */\n keys() {\n // Keys can be a RangeObject, faster to iterate over valuesAvailable\n return this.valuesAvailable.reduce((keysList, valueAvailable, sliceNumber) => {\n if (valueAvailable) keysList.push(this.realKeys[sliceNumber - 1]);\n return keysList;\n }, []);\n }\n\n /**\n * Returns an array of results. Will only return results that have been received from the scheduler, if only one result is complete the array will contain one value.\n * @access public\n * @returns {Array<*>}\n */\n values() {\n return this.realValues.filter((v, i) => this.isAvailable(i));\n }\n\n // Aliased as length in proxy, can't use getter because Array.length is not configurable\n getLength() {\n return this.valuesAvailable.reduce((n, v) => (n + (v ? 1 : 0)), 0);\n }\n \n [customInspectSymbol]() /* overrides the output when console.log(result) is called */\n { \n return this.values(); \n }\n\n [Symbol.iterator]() {\n let index = 0;\n let values = this.values(); // use available values\n\n return {\n next: () => ({\n value: values[index++],\n done: index > values.length\n })\n };\n }\n\n /** \n * Returns an array of [input, output] pairs, in sliceNumber order.\n * Return value is undefined if the input is not an ES5 primitive.\n * @access public\n * @returns {Array<*>}\n */\n entries() {\n return this.valuesAvailable.reduce((keyValuePairs, valueAvailable, sliceNumber) => {\n if (valueAvailable) keyValuePairs.push([\n String(this.realKeys[sliceNumber - 1]),\n this.realValues[sliceNumber],\n ]);\n return keyValuePairs;\n }, []);\n }\n\n /** \n * Returns an Object associating input and output values where the inputs are ES5 primitive types.\n * Return value is undefined if the input is not an ES5 primitive.\n * @access public\n * @returns {object}\n */\n fromEntries() {\n return this.entries().reduce((o, [k, v]) => {\n o[k] = v; return o;\n }, {});\n }\n\n /** \n * Return the nth input value/input vector \n * @access public\n * @param {number} n Index in the input set to return the value for.\n * @returns {*} Input set value\n */\n key(n) {\n return this.realKeys[n];\n }\n\n /** \n * Return the value corresponding to the provided key \n * @access public\n * @param {*} key Corresponds to a value in the job's input set.\n * @returns {*} Result corresponding to the input value\n */\n lookupValue(key) {\n // use keys instead of _keys so we only lookup on available results\n let ind = this.keys().indexOf(key);\n if (ind === -1)\n throw new DCPError('Argument passed into the function was not found in input set', 'ERANGE');\n\n return this.values()[ind];\n }\n\n /**\n * Sends request to scheduler to fetch results, the retrieved results will be populated on this object.\n * @param {RangeObject} [rangeObject] - range object to query results\n * @param {string} [emitEvents] - if
|
|
4192
|
+
eval("/**\n * @file job/result-handle.js\n * @author Ryan Rossiter, ryan@kingsds.network\n * @date June 2020\n *\n * The ResultHandle acts as a proxy for a job's results, querying\n * internal results when available or the scheduler when the results\n * are not available locally.\n */\n\nconst { rehydrateRange } = __webpack_require__(/*! ../range-object */ \"./src/dcp-client/range-object.js\");\nconst protocolV4 = __webpack_require__(/*! dcp/protocol-v4 */ \"./src/protocol-v4/index.js\");\nconst wallet = __webpack_require__(/*! dcp/dcp-client/wallet */ \"./src/dcp-client/wallet/index.js\");\nconst { fetchURI } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst customInspectSymbol = Symbol.for('nodejs.util.inspect.custom');\n\n// Array methods that don't modify the array, will be applied to the ResultHandle's values object\nconst RH_ARRAY_METHODS = [\n 'slice', 'filter', 'concat', 'find', 'findIndex', 'indexOf', 'map', 'reduce', 'includes', 'toString', 'forEach'\n];\n\n/**\n * This class represents a handle on a job's results. It can be used for accessing the job's results, or for querying the scheduler to fetch results.\n * In addition to the properties and methods of this class, the following standard array methods are also available for accessing the available results: `slice`, `filter`, `concat`, `find`, `findIndex`, `indexOf`, `map`, `reduce`, `includes`, `toString`, and `forEach`.\n * The results can also be accessed by index (`results[i]`), but an error will be thrown if the result for that index is not yet available.\n * @access public\n */\nclass ResultHandle extends Array\n{\n constructor(job)\n {\n super();\n\n const { Job } = __webpack_require__(/*! . */ \"./src/dcp-client/job/index.js\");\n if (!(job instanceof Job))\n throw new TypeError('ResultHandle must be constructed from a Job object');\n\n /**\n * The length of the available results array.\n * @type {number}\n * @access public\n */\n this.length = 0; // overridden by the proxy, here for JSDoc purposes\n\n this.job = job;\n this.realKeys = job.jobInputData;\n this.realValues = [];\n this.valuesAvailable = [];\n\n return new Proxy(this, {\n get: (target, name) => {\n if ((typeof name === 'string' || typeof name === 'number') && Number.isInteger(parseFloat(name)))\n {\n let i = parseFloat(name);\n if (target.isAvailable(i))\n return target.realValues[i];\n else\n throw new Error(`Result ${i} is not available. It has either not been computed or you need to fetch it.`);\n } \n else if (name === 'length')\n return target.getLength();\n else if (name === 'constructor')\n return Array.constructor;\n else if (RH_ARRAY_METHODS.includes(name))\n {\n let values = target.values();\n return values[name].bind(values);\n }\n else\n {\n // only return methods on this class, don't allow access\n // to array methods other than the whitelisted ones\n let p = target.__proto__[name];\n if (typeof p === 'function') {\n return p.bind(target);\n } else {\n return p;\n }\n }\n }\n });\n }\n\n toJSON ()\n {\n return JSON.stringify(this.values());\n }\n\n isAvailable(index)\n {\n return this.valuesAvailable[index];\n }\n\n reset() {\n // quickly empty the realValues array\n this.realValues.length = this.valuesAvailable.length = 0;\n this.realValues.length = this.valuesAvailable.length =this.realKeys.length;\n }\n\n /**\n * Returns an array of input values. Will only return input values that have a completed result available.\n * @access public\n * @returns {Array<*>}\n */\n keys() {\n // Keys can be a RangeObject, faster to iterate over valuesAvailable\n return this.valuesAvailable.reduce((keysList, valueAvailable, sliceNumber) => {\n if (valueAvailable) keysList.push(this.realKeys[sliceNumber - 1]);\n return keysList;\n }, []);\n }\n\n /**\n * Returns an array of results. Will only return results that have been received from the scheduler, if only one result is complete the array will contain one value.\n * @access public\n * @returns {Array<*>}\n */\n values() {\n return this.realValues.filter((v, i) => this.isAvailable(i));\n }\n\n // Aliased as length in proxy, can't use getter because Array.length is not configurable\n getLength() {\n return this.valuesAvailable.reduce((n, v) => (n + (v ? 1 : 0)), 0);\n }\n \n [customInspectSymbol]() /* overrides the output when console.log(result) is called */\n { \n return this.values(); \n }\n\n [Symbol.iterator]() {\n let index = 0;\n let values = this.values(); // use available values\n\n return {\n next: () => ({\n value: values[index++],\n done: index > values.length\n })\n };\n }\n\n /** \n * Returns an array of [input, output] pairs, in sliceNumber order.\n * Return value is undefined if the input is not an ES5 primitive.\n * @access public\n * @returns {Array<*>}\n */\n entries() {\n return this.valuesAvailable.reduce((keyValuePairs, valueAvailable, sliceNumber) => {\n if (valueAvailable) keyValuePairs.push([\n String(this.realKeys[sliceNumber - 1]),\n this.realValues[sliceNumber],\n ]);\n return keyValuePairs;\n }, []);\n }\n\n /** \n * Returns an Object associating input and output values where the inputs are ES5 primitive types.\n * Return value is undefined if the input is not an ES5 primitive.\n * @access public\n * @returns {object}\n */\n fromEntries() {\n return this.entries().reduce((o, [k, v]) => {\n o[k] = v; return o;\n }, {});\n }\n\n /** \n * Return the nth input value/input vector \n * @access public\n * @param {number} n Index in the input set to return the value for.\n * @returns {*} Input set value\n */\n key(n) {\n return this.realKeys[n];\n }\n\n /** \n * Return the value corresponding to the provided key \n * @access public\n * @param {*} key Corresponds to a value in the job's input set.\n * @returns {*} Result corresponding to the input value\n */\n lookupValue(key) {\n // use keys instead of _keys so we only lookup on available results\n let ind = this.keys().indexOf(key);\n if (ind === -1)\n throw new DCPError('Argument passed into the function was not found in input set', 'ERANGE');\n\n return this.values()[ind];\n }\n\n /**\n * Sends request to scheduler to fetch results, the retrieved results will be populated on this object.\n * @param {RangeObject} [rangeObject] - range object to query results\n * @param {string} [emitEvents] - if truthy, emits a `result` event for new results as they are added to the handle, and if set to 'all' emits 'resultUpdated' for duplicate results \n * @access public\n * @emits Job#resultsUpdated\n */\n async fetch(rangeObject, emitEvents) {\n const range = rangeObject && rehydrateRange(rangeObject);\n\n const job = this.job;\n \n // Fetch any existing results\n let ks = await wallet.getId();\n const conn = new protocolV4.Connection(dcpConfig.scheduler.services.resultSubmitter.location, ks);\n\n const { success, payload } = await conn.send('fetchResult', {\n job: job.id,\n owner: job.paymentAccountKeystore.address,\n range,\n }, job.paymentAccountKeystore);\n\n // Unpack results, using fetchURI to decode/fetch result URIs\n await Promise.all(payload.map(async r => {\n if (!r) {\n console.warn(`ResultHandle.fetch: Received result was not defined (${r}), ignoring...`);\n return;\n }\n\n this.realValues[r.slice] = await fetchURI(decodeURIComponent(r.value), dcpConfig.scheduler.location.origin);\n\n if (emitEvents)\n {\n if (!this.valuesAvailable[r.slice])\n job.emit('result', { sliceNumber: r.slice, result: this.realValues[r.slice] });\n else if (emitEvents === 'all')\n this.job.emit('resultsUpdated', { sliceNumber: r.slice }); \n }\n this.valuesAvailable[r.slice] = true;\n }));\n\n await conn.close(null, true);\n }\n \n /**\n * Handle adding a new result\n */\n newResult(result, index)\n {\n if (this.valuesAvailable[index])\n {\n this.realValues[index] = result;\n this.job.emit('resultsUpdated', { sliceNumber: index }); \n }\n else\n {\n this.valuesAvailable[index] = true;\n this.realValues[index] = result;\n this.job.emit('result', { job: this.job.address, sliceNumber: index, result });\n }\n }\n\n async list(rangeObject) {\n const range = rehydrateRange(rangeObject);\n throw new Error(\"ResultHandle.list not implemented\");\n }\n\n async delete(rangeObject) {\n const range = rehydrateRange(rangeObject);\n throw new Error(\"ResultHandle.delete not implemented\");\n }\n\n async stat(rangeObject) {\n const range = rehydrateRange(rangeObject);\n throw new Error(\"ResultHandle.stat not implemented\");\n }\n}\n\nObject.assign(exports, {\n ResultHandle,\n});\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/job/result-handle.js?");
|
|
4193
4193
|
|
|
4194
4194
|
/***/ }),
|
|
4195
4195
|
|
|
@@ -4209,7 +4209,7 @@ eval("/**\n * @file job/slice-payment-offer.js\n * @author Ryan Ross
|
|
|
4209
4209
|
\*********************************************/
|
|
4210
4210
|
/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
|
|
4211
4211
|
|
|
4212
|
-
eval("/**\n * @file job/upload-slices.js\n * @author Ryan Saweczko, ryansaweczko@kingsds.network\n * Danny Akbarzadeh, danny@kingsds.network\n * \n * @date Jun 2022\n *\n * Implement functions to upload slices to the scheduler after a job has been deployed.\n * This area will have it's own connection to the job submit service which it is responsible\n * for handling.\n */\n\nconst { Connection } = __webpack_require__(/*! dcp/protocol-v4 */ \"./src/protocol-v4/index.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp-client');\n\nlet uploadConnection = null;\nlet uploadRefs = 0;\nlet tunedKvin;\n\nfunction createNewConnection()\n{\n uploadConnection = new Connection(dcpConfig.scheduler.services.jobSubmit);\n debugging('slice-upload') && console.debug('025: opening new uploadConnection', uploadConnection.id);\n uploadConnection.on('close', () => {\n debugging('slice-upload') && console.debug('027: uploadConnection is closing', uploadConnection?.id);\n uploadConnection = null;\n });\n return uploadConnection.connect();\n}\n\n/*\nHelper function: reprepare a message if a connection breaks before error tolerance\n*/\nfunction reprepareMessage(details)\n{\n return uploadConnection.prepare('addSliceData', {\n job: details.job,\n dataValues: kvinMarshal(details.pile),\n });\n}\n\n/**\n * Helper function that tries to upload slicePile to scheduler for the job with the given address\n * If the connection throws, we will continue trying to upload until it has thrown errorTolerance times\n * However, if the upload is unsuccessful, we throw immediately.\n * @param {object} pileMessage \n * @param {object} originalMessageDetails - used if the connection breaks and we need to reprepare the message.\n * @returns payload containing success property (pertaining to success of adding slices to job) as well as lastSliceNumber of job \n */\nasync function safeSliceUpload(pileMessage, originalMessageDetails)\n{\n debugging('slice-upload') && console.debug('055: safeSliceUpload()', {\n slices: originalMessageDetails.pile.length,\n job: originalMessageDetails.job,\n });\n let payload = undefined;\n let errorTolerance = dcpConfig.job.sliceUploadErrorTolerance; // copy number of times we will tolerate non-success when uploading slices directly from config\n\n while (true) // eslint-disable-line no-constant-condition\n {\n try\n {\n const start = Date.now();\n debugging('slice-upload') && console.log('x-dbg-uploadStart', {length:pileMessage.signedMessage.length});\n // If the connection closes after we prepare the message but before our error tolerance is reached, re-prepare and continue trying to send message.\n if (!uploadConnection)\n {\n debugging('slice-upload') && console.log('x-dbg-openNewConnection');\n await createNewConnection();\n pileMessage = await reprepareMessage(originalMessageDetails); /* Assuming preparing a identical message will result in the same length */\n }\n\n debugging('slice-upload') && console.debug('070: sending addSliceData request...');\n payload = await uploadConnection.sendPreparedMessage(pileMessage);\n debugging('slice-upload') && console.debug('072: success=', { success: payload?.success, payload: payload?.payload || payload });\n if (!payload.success)\n {\n debugging('slice-upload') && console.log('x-dbg-uploadBackoff', {length:pileMessage.signedMessage.length});\n throw payload.payload;\n }\n else\n {\n debugging('slice-upload') && console.log('x-dbg-uploadProgress', { el: Date.now() - start});\n break;\n }\n }\n catch (error)\n {\n debugging('slice-upload') && console.debug('080: caught upload error:', error?.code, error?.message || error, `${errorTolerance-1} tries left...`);\n if (uploadConnection) {\n debugging('slice-upload') && console.debug(`089: closing connection`, uploadConnection.id);\n await uploadConnection.close(); // clean up the connection on upload error\n debugging('slice-upload') && console.debug(`091: closed`, uploadConnection?.id);\n }\n\n if (--errorTolerance <= 0)\n {\n debugging('slice-upload') && console.log('x-dbg-uploadError');\n throw error;\n }\n \n if (error.slicesSaved) {\n const newPile = originalMessageDetails.pile.slice(error.slicesSaved);\n\n debugging('slice-upload') && console.debug(`103: recursing into addSlices with a new pile of ${newPile.length}...`)\n // recursively call back into addSlices, with only the unsaved remainder of this pile\n const retry = await addSlices(newPile, originalMessageDetails.job);\n\n // if that worked, then break out of this loop\n if (retry.success) {\n payload = {\n success: retry.success,\n payload: retry,\n };\n break;\n }\n }\n }\n }\n debugging('slice-upload') && console.debug(`120< safeSliceUpload() is returning`, {\n success: payload.success,\n payload: payload.payload,\n });\n return payload;\n}\n\n/**\n* This function contains the actual logic behind staggered slice uploads\n* to the scheduler which makes quicker deployment possible.\n* \n* Note that we pass in mostToTake so that the uploadLogic function can update \n* it to the new value it needs to be, and then pass it back to the wrapper \n* function (addSlices) which actually does the work of picking up slices \n* and thus uses this value\n* @param {Array} pile the actual array of slices being uploaded to scheduler\n* @param {Number} mostToTake number of slices that should be taken by the wrapper function (addSlices) \n* which actually does the work of picking up slices and thus uses this value.\n* We pass in mostToTake so that the uploadLogic function can update it to the \n* new value it needs to be, and then pass it back to the wrapper\n* @param {*} jobAddress Address of job to upload the slices to \n* @returns payload containing success property (pertaining to success of adding slices to job) as well as lastSliceNumber of job\n*/\nasync function sliceUploadLogic(pile, mostToTake, jobAddress)\n{\n debugging('slice-upload') && console.debug(`111: sliceUploadLogic(), ${pile.length} slices, up to ${mostToTake}`);\n if (!uploadConnection)\n await createNewConnection();\n\n const slicesTaken = pile.length;\n let pileMessage = await uploadConnection.prepare('addSliceData', {\n job: jobAddress,\n dataValues: kvinMarshal(pile),\n });\n \n let pileSize = pileMessage.signedMessage.length;\n \n let newMostToTake;\n let uploadedSlices;\n\n // if the pile is larger than the ceiling but we only took one slice, there's no smaller pile we can make\n // so we upload it anyway but we don't try taking more next time cause we were over the ceiling (which \n // is a hard limit on upload sizes)\n if ((pileSize > dcpConfig.job.uploadSlicesCeiling) && (slicesTaken === 1))\n {\n uploadedSlices = await safeSliceUpload(pileMessage, { job: jobAddress, pile });\n newMostToTake = 1;\n }\n \n // if the pile is larger than the target but we only took one slice, there's no smaller pile we can make\n // so we upload it anyway and still try taking more\n else if ((pileSize > dcpConfig.job.uploadSlicesTarget) && (slicesTaken === 1))\n {\n uploadedSlices = await safeSliceUpload(pileMessage, { job: jobAddress, pile });\n newMostToTake = mostToTake * dcpConfig.job.uploadIncreaseFactor;\n }\n \n // otherwise, if the pile is smaller than the soft ceiling, send up the pile anyway (since piles are expensive to make) \n // but remember to include incrementFactor times as many slices in the next pile\n else if (pileSize <= dcpConfig.job.uploadSlicesTarget)\n {\n uploadedSlices = await safeSliceUpload(pileMessage, { job: jobAddress, pile });\n newMostToTake = mostToTake * dcpConfig.job.uploadIncreaseFactor;\n }\n \n // if the pile is over the ceiling then we do not upload and begin reassembling our piles from scratch\n else if (pileSize > dcpConfig.job.uploadSlicesCeiling)\n {\n newMostToTake = -1;\n }\n \n // if the pile is over the target (but implicitly under the ceiling), then upload the pile to scheduler but lower mostToTake\n // by a smaller factor than incrementFactor to allow us to begin \"centering\" sizes of piles around the target\n else if (pileSize > dcpConfig.job.uploadSlicesTarget)\n {\n uploadedSlices = await safeSliceUpload(pileMessage, { job: jobAddress, pile });\n newMostToTake = Math.ceil(mostToTake / ((2 / 3) * dcpConfig.job.uploadIncreaseFactor));\n }\n else\n throw new Error('hopefully impossible code in slice upload logic');\n\n let payload = uploadedSlices ? uploadedSlices.payload : undefined;\n debugging('slice-upload') && console.debug(`198< sliceUploadLogic() is returning`, { payload, newMostToTake });\n return { payload, newMostToTake }; // in case the user needs lastSliceNumber's value\n}\n\n/**\n* Uploads slices to the scheduler in a staggered fashion\n* @param {Array} dataValues actual array of slices being uploaded to scheduler\n* @param {*} jobAddress Address of job these slices are for\n* @param {*} newTunedKvin undefined, or new version of kvin tuned for speed or size specifically. Use if defined.\n* @returns payload containing success property (pertaining to success of adding slices to job) as well as lastSliceNumber of job\n*/\nasync function addSlices(dataValues, jobAddress, newTunedKvin)\n{\n debugging('slice-upload') && console.debug(`178: addSlices:${uploadRefs}()`, {\n slices: dataValues.length,\n job: jobAddress,\n })\n if (newTunedKvin)\n tunedKvin = newTunedKvin;\n\n if (!Array.isArray(dataValues))\n throw new TypeError('Only data-by-value jobs may dynamically add slices');\n\n let mostToTake = dcpConfig.job.uploadInitialNumberOfSlices; // maximum number of slices we could take in per pile\n let payload = undefined; // used in return value\n let slicesTaken = 0; // number of slices in the pile already\n let pile = [];\n uploadRefs++;\n\n for (let slice of dataValues)\n {\n pile.push(slice);\n slicesTaken++;\n if (slicesTaken === mostToTake)\n {\n let total = await sliceUploadLogic(pile, mostToTake, jobAddress);\n payload = total.payload;\n \n if (total.newMostToTake < 0)\n {\n /* if total.newMostToTake == -1 (only non-positive value returned), then the pile was not successfully\n * uploaded because it was over the ceiling and we need to upload the pile *itself* again, recursively\n */\n payload = await addSlices(pile, jobAddress);\n /* and next time, the number of slices we take is the number from this time *divided* by the incrementFactor\n * since we know invariably that number of slices was under the ceiling AND target\n * if you're curious why that's an invariant, this is because mostToTake only ever *increases* by being multiplied by \n * a factor of incrementFactor within sliceUploadLogic, and this only occurs when the pile being uploaded that time\n * was under the target\n */\n mostToTake = mostToTake / dcpConfig.job.uploadIncreaseFactor;\n }\n else\n {\n /* in all other cases (other than the pile size being over the ceiling) the sliceUploadLogic helper \n * determines the number of slices we should pick up next time, so we just use the value it spits out\n */\n mostToTake = total.newMostToTake;\n }\n \n // reset slicesTaken and pile since at this point we know for sure the pile has been uploaded\n pile = [];\n slicesTaken = 0;\n }\n }\n // upload the pile one last time in case we continued off the last slice with a non-empty pile\n if (pile.length !== 0)\n {\n let finalObj = await sliceUploadLogic(pile, mostToTake, jobAddress);\n payload = finalObj.payload;\n mostToTake = finalObj.newMostToTake;\n \n if (mostToTake < 0)\n {\n // if you need documentation on the next two lines, look inside the if (total.newMostToTake < 0) just above\n payload = await addSlices(pile, jobAddress);\n mostToTake = mostToTake / dcpConfig.job.uploadIncreaseFactor;\n }\n }\n\n // and finally assign whatever mostToTake was at the end of this run of the function to be returned \n // as part of the payload in case addSlices was called recursively\n if (!payload) {\n console.warn(`280: payload was somehow not an Object...`, payload);\n payload = {};\n }\n payload.mostToTake = mostToTake;\n\n // cleanup connection when done with this function and all recursive calls\n if (uploadConnection && uploadRefs === 1) {\n debugging('slice-upload') && console.debug(`250: closing uploadConnection`, uploadConnection?.id);\n await uploadConnection.close();\n debugging('slice-upload') && console.debug(`252: closed uploadConnection`, uploadConnection?.id);\n }\n uploadRefs--;\n\n /* contains the job's lastSliceNumber (the only externally-meaningful value returned from \n * the uploading of slices to the scheduler) in case the calling function needs it \n */\n debugging('slice-upload') && console.log(`294< addSlices:${uploadRefs}() is returning:`, payload);\n return payload;\n}\n\n/**\n * marshal the value using kvin or instance of the kvin (tunedKvin)\n * tunedKvin is defined if job.tuning.kvin is specified.\n *\n * @param {any} value \n * @return {object} A marshaled object\n * \n */\nfunction kvinMarshal (value) {\n if (tunedKvin)\n return tunedKvin.marshal(value);\n\n return kvin.marshal(value);\n}\n\n\nexports.addSlices = addSlices;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/job/upload-slices.js?");
|
|
4212
|
+
eval("/**\n * @file job/upload-slices.js\n * @author Ryan Saweczko, ryansaweczko@kingsds.network\n * Danny Akbarzadeh, danny@kingsds.network\n * \n * @date Jun 2022\n *\n * Implement functions to upload slices to the scheduler after a job has been deployed.\n * This area will have it's own connection to the job submit service which it is responsible\n * for handling.\n */\n\nconst { Connection } = __webpack_require__(/*! dcp/protocol-v4 */ \"./src/protocol-v4/index.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp-client');\n\nlet uploadConnection = null;\nlet uploadRefs = 0;\nlet tunedKvin;\n\nfunction createNewConnection()\n{\n uploadConnection = new Connection(dcpConfig.scheduler.services.jobSubmit);\n uploadConnection.on('close', () => { uploadConnection = null; });\n return uploadConnection.connect();\n}\n\n/*\nHelper function: reprepare a message if a connection breaks before error tolerance\n*/\nfunction reprepareMessage(details)\n{\n return uploadConnection.prepare('addSliceData', {\n job: details.job,\n dataValues: kvinMarshal(details.pile),\n });\n}\n\n/**\n * Helper function that tries to upload slicePile to scheduler for the job with the given address\n * If the connection throws, we will continue trying to upload until it has thrown errorTolerance times\n * However, if the upload is unsuccessful, we throw immediately.\n * @param {object} pileMessage \n * @param {object} originalMessageDetails - used if the connection breaks and we need to reprepare the message.\n * @returns payload containing success property (pertaining to success of adding slices to job) as well as lastSliceNumber of job \n */\nasync function safeSliceUpload(pileMessage, originalMessageDetails)\n{\n let payload = undefined;\n let errorTolerance = dcpConfig.job.sliceUploadErrorTolerance; // copy number of times we will tolerate non-success when uploading slices directly from config\n\n while (true) // eslint-disable-line no-constant-condition\n {\n try\n {\n const start = Date.now();\n debugging('slice-upload') && console.log('x-dbg-uploadStart', pileMessage.signedMessage.length);\n // If the connection closes after we prepare the message but before our error tolerance is reached, re-prepare and continue trying to send message.\n if (!uploadConnection)\n {\n debugging('slice-upload') && console.log('x-dbg-openNewConnection');\n await createNewConnection();\n pileMessage = await reprepareMessage(originalMessageDetails); /* Assuming preparing a identical message will result in the same length */\n }\n\n payload = await uploadConnection.sendPreparedMessage(pileMessage);\n if (!payload.success)\n {\n debugging('slice-upload') && console.log('x-dbg-uploadBackoff', {length:pileMessage.signedMessage.length});\n throw payload;\n }\n else\n {\n debugging('slice-upload') && console.log('x-dbg-uploadProgress', Date.now() - start);\n break;\n }\n }\n catch (error)\n {\n if (--errorTolerance <= 0)\n {\n debugging('slice-upload') && console.log('x-dbg-uploadError', error);\n if (uploadConnection)\n uploadConnection.close(); // ensure we clean up the connection\n throw error;\n }\n }\n }\n return payload;\n}\n\n/**\n* This function contains the actual logic behind staggered slice uploads\n* to the scheduler which makes quicker deployment possible.\n* \n* Note that we pass in mostToTake so that the uploadLogic function can update \n* it to the new value it needs to be, and then pass it back to the wrapper \n* function (addSlices) which actually does the work of picking up slices \n* and thus uses this value\n* @param {Array} pile the actual array of slices being uploaded to scheduler\n* @param {Number} mostToTake number of slices that should be taken by the wrapper function (addSlices) \n* which actually does the work of picking up slices and thus uses this value.\n* We pass in mostToTake so that the uploadLogic function can update it to the \n* new value it needs to be, and then pass it back to the wrapper\n* @param {*} jobAddress Address of job to upload the slices to \n* @returns payload containing success property (pertaining to success of adding slices to job) as well as lastSliceNumber of job\n*/\nasync function sliceUploadLogic(pile, mostToTake, jobAddress)\n{\n if (!uploadConnection)\n await createNewConnection();\n\n const slicesTaken = pile.length;\n let pileMessage = await uploadConnection.prepare('addSliceData', {\n job: jobAddress,\n dataValues: kvinMarshal(pile),\n });\n \n let pileSize = pileMessage.signedMessage.length;\n \n let newMostToTake;\n let uploadedSlices;\n\n // if the pile is larger than the ceiling but we only took one slice, there's no smaller pile we can make\n // so we upload it anyway but we don't try taking more next time cause we were over the ceiling (which \n // is a hard limit on upload sizes)\n if ((pileSize > dcpConfig.job.uploadSlicesCeiling) && (slicesTaken === 1))\n {\n uploadedSlices = await safeSliceUpload(pileMessage, { job: jobAddress, pile });\n newMostToTake = 1;\n }\n \n // if the pile is larger than the target but we only took one slice, there's no smaller pile we can make\n // so we upload it anyway and still try taking more\n else if ((pileSize > dcpConfig.job.uploadSlicesTarget) && (slicesTaken === 1))\n {\n uploadedSlices = await safeSliceUpload(pileMessage, { job: jobAddress, pile });\n newMostToTake = mostToTake * dcpConfig.job.uploadIncreaseFactor;\n }\n \n // otherwise, if the pile is smaller than the soft ceiling, send up the pile anyway (since piles are expensive to make) \n // but remember to include incrementFactor times as many slices in the next pile\n else if (pileSize <= dcpConfig.job.uploadSlicesTarget)\n {\n uploadedSlices = await safeSliceUpload(pileMessage, { job: jobAddress, pile });\n newMostToTake = mostToTake * dcpConfig.job.uploadIncreaseFactor;\n }\n \n // if the pile is over the ceiling then we do not upload and begin reassembling our piles from scratch\n else if (pileSize > dcpConfig.job.uploadSlicesCeiling)\n {\n newMostToTake = -1;\n }\n \n // if the pile is over the target (but implicitly under the ceiling), then upload the pile to scheduler but lower mostToTake\n // by a smaller factor than incrementFactor to allow us to begin \"centering\" sizes of piles around the target\n else if (pileSize > dcpConfig.job.uploadSlicesTarget)\n {\n uploadedSlices = await safeSliceUpload(pileMessage, { job: jobAddress, pile });\n newMostToTake = Math.ceil(mostToTake / ((2 / 3) * dcpConfig.job.uploadIncreaseFactor));\n }\n else\n throw new Error('hopefully impossible code in slice upload logic');\n\n let payload = uploadedSlices ? uploadedSlices.payload : undefined;\n return { payload, newMostToTake }; // in case the user needs lastSliceNumber's value\n}\n\n/**\n* Uploads slices to the scheduler in a staggered fashion\n* @param {Array} dataValues actual array of slices being uploaded to scheduler\n* @param {*} jobAddress Address of job these slices are for\n* @param {*} newTunedKvin undefined, or new version of kvin tuned for speed or size specifically. Use if defined.\n* @returns payload containing success property (pertaining to success of adding slices to job) as well as lastSliceNumber of job\n*/\nasync function addSlices(dataValues, jobAddress, newTunedKvin)\n{\n if (newTunedKvin)\n tunedKvin = newTunedKvin;\n\n if (!Array.isArray(dataValues))\n throw new TypeError('Only data-by-value jobs may dynamically add slices');\n\n let mostToTake = dcpConfig.job.uploadInitialNumberOfSlices; // maximum number of slices we could take in per pile\n let payload = undefined; // used in return value\n let slicesTaken = 0; // number of slices in the pile already\n let pile = [];\n uploadRefs++;\n\n for (let slice of dataValues)\n {\n pile.push(slice);\n slicesTaken++;\n if (slicesTaken === mostToTake)\n {\n let total = await sliceUploadLogic(pile, mostToTake, jobAddress);\n payload = total.payload;\n \n if (total.newMostToTake < 0)\n {\n /* if total.newMostToTake == -1 (only non-positive value returned), then the pile was not successfully\n * uploaded because it was over the ceiling and we need to upload the pile *itself* again, recursively\n */\n payload = await addSlices(pile, jobAddress);\n /* and next time, the number of slices we take is the number from this time *divided* by the incrementFactor\n * since we know invariably that number of slices was under the ceiling AND target\n * if you're curious why that's an invariant, this is because mostToTake only ever *increases* by being multiplied by \n * a factor of incrementFactor within sliceUploadLogic, and this only occurs when the pile being uploaded that time\n * was under the target\n */\n mostToTake = mostToTake / dcpConfig.job.uploadIncreaseFactor;\n }\n else\n {\n /* in all other cases (other than the pile size being over the ceiling) the sliceUploadLogic helper \n * determines the number of slices we should pick up next time, so we just use the value it spits out\n */\n mostToTake = total.newMostToTake;\n }\n \n // reset slicesTaken and pile since at this point we know for sure the pile has been uploaded\n pile = [];\n slicesTaken = 0;\n }\n }\n // upload the pile one last time in case we continued off the last slice with a non-empty pile\n if (pile.length !== 0)\n {\n let finalObj = await sliceUploadLogic(pile, mostToTake, jobAddress);\n payload = finalObj.payload;\n mostToTake = finalObj.newMostToTake;\n \n if (mostToTake < 0)\n {\n // if you need documentation on the next two lines, look inside the if (total.newMostToTake < 0) just above\n payload = await addSlices(pile, jobAddress);\n mostToTake = mostToTake / dcpConfig.job.uploadIncreaseFactor;\n }\n }\n\n // and finally assign whatever mostToTake was at the end of this run of the function to be returned \n // as part of the payload in case addSlices was called recursively\n payload.mostToTake = mostToTake;\n\n // cleanup connection when done with this function\n if (uploadConnection && uploadRefs === 1) {\n debugging('slice-upload') && console.debug(`250: closing uploadConnection`, uploadConnection?.id);\n await uploadConnection.close();\n debugging('slice-upload') && console.debug(`252: closed uploadConnection`, uploadConnection);\n }\n uploadRefs--;\n\n /* contains the job's lastSliceNumber (the only externally-meaningful value returned from \n * the uploading of slices to the scheduler) in case the calling function needs it \n */\n return payload;\n}\n\n/**\n * marshal the value using kvin or instance of the kvin (tunedKvin)\n * tunedKvin is defined if job.tuning.kvin is specified.\n *\n * @param {any} value \n * @return {object} A marshaled object\n * \n */\nfunction kvinMarshal (value) {\n if (tunedKvin)\n return tunedKvin.marshal(value);\n\n return kvin.marshal(value);\n}\n\n\nexports.addSlices = addSlices;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/job/upload-slices.js?");
|
|
4213
4213
|
|
|
4214
4214
|
/***/ }),
|
|
4215
4215
|
|
|
@@ -4424,7 +4424,7 @@ eval("/**\n * @file This module implements the Worker API, used to create worker
|
|
|
4424
4424
|
/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
|
|
4425
4425
|
|
|
4426
4426
|
"use strict";
|
|
4427
|
-
eval("// NOTE - need timeout/postmessage function\n/**\n * @file dcp-client/worker/sandbox.js\n *\n * A sandbox that when constructed and assigned can do work for\n * a distributed slice. A sandbox runs for a single slice at a time.\n *\n * Usage:\n * let sandbox = new Sandbox()\n * await sandbox.start()\n * let results = await sandbox.run(slice)\n *\n * Debug flags:\n * Sandbox.debugWork = true // - turns off 30 second timeout to let user debug sandbox innards more easily\n * Sandbox.debugState = true // - logs all state transitions for this sandbox\n * Sandbox.debugEvents = true // - logs all events received from the sandbox\n *\n * @author Matthew Palma, mpalma@kingsds.network\n * Ryan Rossiter, ryan@kingsds.network\n * @date May 2019\n * @module sandbox\n */\n/* global dcpConfig */\n// @ts-check\n\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst { assert, assertEq3 } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nconst { rehydrateRange } = __webpack_require__(/*! dcp/dcp-client/range-object */ \"./src/dcp-client/range-object.js\");\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\nconst scopedKvin = new kvin.KVIN({Object: ({}).constructor,\n Array: ([]).constructor, \n Function: (()=>{}).constructor});\n\nlet timeDilation = 1;\nif (DCP_ENV.platform === 'nodejs') {\n const { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\n /** Make timers 10x slower when running in niim */\n timeDilation = (requireNative('module')._cache.niim instanceof requireNative('module').Module) ? 10 : 1;\n}\n\n/**\n * Wraps console.debug to emulate debug module prefixing messages on npm.\n * @param {...any} args\n */\nconst debug = (...args) => {\n if (debugging()) {\n console.debug('dcp-client:worker:sandbox', ...args);\n }\n};\n\nconst nanoid = (__webpack_require__(/*! nanoid */ \"./node_modules/nanoid/index.browser.js\").nanoid);\n\nconst { EventEmitter } = __webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\");\nconst { fetchURI, encodeDataURI } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\n\n// Sandbox states\nconst UNREADY = 'UNREADY' // No Sandbox (web worker, saworker, etc) has been constructed yet\nconst READYING = 'READYING' // Sandbox is being constructed and environment (bravojs, env) is being set up\nconst READY_FOR_ASSIGN = 'READY_FOR_ASSIGN' // Sandbox is ready to be assigned\nconst ASSIGNING = 'ASSIGNING' // Sandbox is running through assigning steps\nconst ASSIGNED = 'ASSIGNED' // Sandbox is assigned but not working\nconst WORKING = 'WORKING' // Sandbox is working\nconst TERMINATED = 'TERMINATED'\nconst EVAL_RESULT_PREFIX = 'evalResult::';\n\nclass SandboxError extends Error {}\nclass NoProgressError extends SandboxError { constructor(...args) { super(...args); this.errorCode = 'ENOPROGRESS'; } }\nclass SliceTooSlowError extends SandboxError { constructor(...args) { super(...args); this.errorCode = 'ESLICETOOSLOW'; } }\nclass UncaughtExceptionError extends SandboxError { constructor(...args) { super(...args); this.errorCode = 'EUNCAUGHT'; } }\nclass RemoteFetchError extends SandboxError { constructor(...args) { super(...args); this.errorCode = 'EFETCH'; }}\n\n/** @typedef {import('dcp/common/dcp-events').EventEmitter} EventEmitter */\n/** @typedef {import('./slice').Slice} Slice */\n/** @typedef {import('./supervisor-cache').SupervisorCache} SupervisorCache */\n/** @typedef {*} opaqueId */\n\n/**\n * @access public\n * @typedef {object} SandboxOptions\n * @constructor {function} [SandboxConstructor]\n * @property {boolean} [ignoreNoProgress] - When true, the sandbox will not be stopped for not calling progress\n */\n\nclass Sandbox extends EventEmitter {\n /**\n * A Sandbox (i.e. a worker sandbox) which executes distributed slices.\n *\n * @constructor\n * @param {SupervisorCache} cache\n * @param {SandboxOptions} options\n */\n constructor (cache, options, origins) {\n super('Sandbox');\n /** @type {SupervisorCache} */\n this.supervisorCache = cache;\n /** @type {SandboxOptions} */\n this.options = {\n ignoreNoProgress: false,\n ...options,\n SandboxConstructor: options.SandboxConstructor ||\n (__webpack_require__(/*! ./evaluators */ \"./src/dcp-client/worker/evaluators/index.js\").BrowserEvaluator),\n }\n this.allowedOrigins = origins;\n\n /** @type {opaqueId} */\n this.jobAddress = null;\n /** @type {object} */\n this.evaluatorHandle = null;\n /** @type {object} */\n this.capabilities = null;\n /** @type {EventEmitter} */\n this.ee = new EventEmitter('SandboxInternal')\n\n /** @type {string} */\n this._state = UNREADY;\n /** @type {boolean} */\n this.allocated = false;\n /** @type {number?} */\n this.progress = 100;\n /** @type {object} */\n this.progressReports = null;\n /** @type {object} */\n this.progressTimeout = null;\n /** @type {object} */\n this.sliceTimeout = null;\n /** @type {object} */\n this.rejectionData = null;\n /** @type {Slice} */\n this.slice = null;\n\n /** @type {number?} */\n this.started = null;\n /** @type {number?} */\n this.sliceStartTime = null;\n /** @type {boolean} */\n this.requiresGPU = false;\n /** @type {string|URL} */\n this.packageURL = dcpConfig.packageManager.location\n /** @type {number?} */\n this.id = Sandbox.getNewId();\n\n this.ringMessageHandlers = [\n this.handleRing0Message,\n this.handleRing1Message,\n this.handleRing2Message,\n this.handleRing3Message,\n ];\n\n this.resetSliceTimeReport();\n }\n\n get identifier() {\n if (this.allocated)\n return `${this.id}.${this.jobAddress}.${this.state}.${this.allocated}`;\n return `${this.id}.${this.jobAddress}.${this.state}`;\n }\n\n static getNewId() {\n return Sandbox.idCounter++;\n }\n\n get state () {\n return this._state\n }\n\n set state (value) {\n if (Sandbox.debugState) {\n console.debug(`sandbox - changing state of ${this.id}... ${this._state} -> ${value}`)\n }\n\n if (this.state === TERMINATED && value !== TERMINATED) {\n // For safety!\n throw new Error(`Sandbox set state violation, attepted to change state from ${this.state} to ${value}`);\n }\n\n this._state = value;\n }\n\n get isReadyForAssign () {\n return this.state === READY_FOR_ASSIGN;\n }\n\n get isAssigned () {\n return this.state === ASSIGNED;\n }\n\n get isWorking () {\n return this.state === WORKING;\n }\n\n get isTerminated () {\n return this.state === TERMINATED;\n }\n\n changeWorkingToAssigned () {\n if (this.isWorking)\n this.state = ASSIGNED;\n }\n\n setIsAssigning () {\n this.state = ASSIGNING;\n }\n\n /**\n * Readies the sandbox. This will result in the sandbox being ready and not assigned,\n * it will need to be assigned with a job before it is able to do work.\n *\n * @todo maybe preload specific modules or let the cache pass in what modules to load?\n * @throws on failure to ready\n */\n async start(delay = 0) {\n this.started = Date.now();\n this.state = READYING;\n\n if (delay > 0) await new Promise((resolve) => setTimeout(resolve, delay * timeDilation));\n\n try {\n // RING 0\n this.evaluatorHandle = new this.options.SandboxConstructor({\n name: `DCP Sandbox #${this.id}`,\n });\n this.evaluatorHandle.onerror = this.onerror.bind(this);\n\n const messageHandler = this.onmessage.bind(this);\n this.evaluatorHandle.onmessage = function onmessage(event)\n {\n let data;\n if (event.data.serialized)\n {\n data = kvin.parse(event.data.message);\n }\n else\n {\n data = kvin.unmarshal(event.data);\n }\n messageHandler({ data });\n }\n\n const evaluatorPostMessage = this.evaluatorHandle.postMessage.bind(this.evaluatorHandle);\n this.evaluatorHandle.postMessage = function postMessage(message)\n {\n evaluatorPostMessage(scopedKvin.marshal(message));\n }\n\n const ceci = this;\n this.evaluatorHandle.addEventListener('end', () => ceci.terminate(true));\n\n // Now in RING 1\n\n // Now in RING 2\n await this.describe();\n this.state = READY_FOR_ASSIGN;\n this.emit('ready', this);\n } catch (error) {\n console.warn('Failed to start the sandbox -', error.message);\n this.terminate(false);\n throw error;\n }\n }\n\n /**\n * This will assign the sandbox with a job, loading its sandbox code\n * into the sandbox.\n *\n * @param {string} jobAddress The address of the job to assign to\n * @throws on initialization failure\n */\n async assign(jobAddress) {\n this.jobAddress = jobAddress;\n this.job = await this.supervisorCache.fetchJob(jobAddress, this.allowedOrigins);\n\n assertEq3(this.job.address, jobAddress);\n assert(typeof this.job === 'object');\n assert(typeof this.job.requirements === 'object');\n assert(Array.isArray(this.job.dependencies));\n assert(Array.isArray(this.job.requirePath));\n\n // Extract public data from job, with defaults\n this.public = Object.assign({\n name: `Anonymous Job ${this.job.address.slice(0, 6)}`,\n description: 'Discreetly helping make the world smarter.',\n link: 'https://distributed.computer/about',\n }, this.job.public);\n\n // Future: We may want other filename tags for appliances // RR Nov 2019\n\n // Important: The order of applying requirements before loading the sandbox code\n // is important for modules and sandbox code to set globals over the whitelist.\n await this.applySandboxRequirements(this.job.requirements);\n await this.assignEvaluator();\n this.state = ASSIGNED;\n }\n\n async assignEvaluator() {\n debug('Begin assigning job to evaluator');\n const ceci = this;\n\n return new Promise(function sandbox$$assignEvaluatorPromise(resolve, reject) {\n const message = {\n request: 'assign',\n job: ceci.job,\n sandboxConfig: dcpConfig.worker.sandbox,\n };\n\n /* note - onFailListener used for removal. This is necessary due to a bug in ee.once. /wg Feb 2022 */\n \n const onSuccess = (event) => {\n // eslint-disable-next-line no-use-before-define\n ceci.ee.removeListener('reject', onFailListener);\n ceci.emit('assigned', event.jobAddress);\n debug('Job assigned to evaluator');\n resolve();\n };\n\n const onFail = (error) => {\n // eslint-disable-next-line no-use-before-define\n ceci.ee.removeListener('assigned', onSuccessListener);\n reject(error);\n };\n\n const onSuccessListener = ceci.ee.once('assigned', onSuccess);\n const onFailListener = ceci.ee.once('reject', onFail);\n ceci.evaluatorHandle.postMessage(message);\n });\n }\n\n /**\n * Evaluates a string inside the sandbox.\n *\n * @param {string} code - the code to evaluate in the sandbox\n * @param {string} filename - the name of the 'file' to help with debugging,\n * no longer working though?\n * @returns {Promise} - resolves with eval result on success, rejects\n * otherwise\n */\n eval(code, filename) {\n var ceci = this;\n \n return new Promise(function sandbox$$eval$Promise(resolve, reject) {\n let msgId = nanoid();\n let msg = {\n request: 'eval',\n data: code,\n filename,\n msgId, \n }\n\n const eventId = EVAL_RESULT_PREFIX + msgId;\n\n let onSuccess = (event) => {\n ceci.ee.removeListener('reject', onFailListener)\n resolve(event)\n }\n\n let onFail = (event) => {\n ceci.ee.removeListener(eventId, onSuccessListener)\n reject(event)\n }\n\n let onSuccessListener = ceci.ee.once(eventId, onSuccess);\n let onFailListener = ceci.ee.once('reject', onFail)\n\n ceci.evaluatorHandle.postMessage(msg)\n })\n }\n\n /**\n * Resets the state of the bootstrap, without resetting the sandbox function if assigned.\n * Mostly used to reset the progress status before reusing a sandbox on another slice.\n * Must be called after @start.\n *\n * @returns {Promise} - resolves with result on success, rejects otherwise\n */\n resetSandboxState () {\n var ceci = this;\n\n return new Promise(function sandbox$resetSandboxStatePromise(resolve, reject) {\n let successCb, failTimeout;\n let msg = {\n request: 'resetState',\n };\n\n successCb = ceci.ee.once('resetStateDone', function sandbox$resetSandboxState$success () {\n if (failTimeout === false)\n return; /* already rejected */\n clearTimeout(failTimeout);\n failTimeout = false;\n resolve();\n });\n\n failTimeout = setTimeout(function sandbox$resetSandboxState$fail() {\n if (failTimeout === false)\n return; /* already resolved */\n \n ceci.ee.removeListener('resetStateDone', successCb);\n ceci.terminate(false);\n failTimeout = false;\n\n reject(new Error('resetState never received resetStateDone event from sandbox'));\n }, 3000 * timeDilation); /* XXXwg need tuneable */\n\n assert(ceci.evaluatorHandle); // It is possible that ceci.terminate nulls out evaluatorHandle before getting here.\n ceci.evaluatorHandle.postMessage(msg);\n });\n }\n\n /**\n * Clear all timers that are set inside the sandbox (evaluator) environment.\n *\n * @returns {Promise} - resolves with result on success, rejects otherwise\n */\n clearSandboxTimers() {\n var ceci = this;\n \n return new Promise(function sandbox$clearSandboxTimersPromise(resolve, reject) {\n let successCb, failTimeout;\n let msg = {\n request: 'clearTimers',\n };\n\n successCb = ceci.ee.once('clearTimersDone', function sandbox$clearSandboxTimers$success() {\n if (failTimeout === false)\n return; /* already rejected */\n clearTimeout(failTimeout);\n failTimeout = false;\n resolve();\n });\n\n failTimeout = setTimeout(function sanbox$clearSandboxTimers$fail() {\n if (failTimeout === false)\n return; /* already resolved */\n \n ceci.ee.removeListener('clearTimersDone', successCb);\n ceci.terminate(false);\n failTimeout = false;\n \n reject(new Error('clearTimers never received clearTimersDone event from sandbox'));\n }, 3000 * timeDilation); /* XXXwg need tuneable */\n\n if (ceci.evaluatorHandle) // Sometimes ceci.terminate nulls out evaluatorHandle before getting here.\n ceci.evaluatorHandle.postMessage(msg);\n });\n }\n\n /**\n * Sends a post message to describe its capabilities.\n *\n * Side effect: Sets the capabilities property of the current sandbox.\n *\n * @returns {Promise} Resolves with the sandbox's capabilities. Rejects with\n * an error saying a response was not received.\n * @memberof Sandbox\n */\n describe() {\n debug('Beginning to describe evaluator');\n var ceci = this;\n \n return new Promise(function sandbox$describePromise(resolve, reject) {\n if (ceci.evaluatorHandle === null) {\n return reject(new Error('Evaluator has not been initialized.'));\n }\n\n /**\n * Opted to create a flag for the describe response being received so that\n * we don't have to *hoist* the timeout's id to clear it in the response\n * handler.\n */\n let didReceiveDescribeResponse = false;\n const describeResponseHandler = ceci.ee.once('describe', (data) => {\n didReceiveDescribeResponse = true;\n const { capabilities } = data;\n if (typeof capabilities === 'undefined') {\n reject(\n new Error('Did not receive capabilities from describe response.'),\n );\n }\n ceci.capabilities = capabilities;\n\n // Currently only used in tests. May use the event in the future.\n ceci.emit('described', capabilities);\n debug('Evaluator has been described');\n resolve(capabilities);\n });\n const describeResponseFailedHandler = () => {\n if (!didReceiveDescribeResponse) {\n ceci.ee.removeListener('describe', describeResponseHandler);\n ceci.terminate(false);\n reject(\n new Error(\n 'Describe message timed-out. No describe response was received from the describe command.',\n ),\n );\n }\n };\n\n const message = {\n request: 'describe',\n };\n\n // Arbitrarily set the waiting time.\n setTimeout(describeResponseFailedHandler, 6000 * timeDilation); /* XXXwg need tuneable */\n assert(ceci.evaluatorHandle); // It is possible that ceci.terminate nulls out evaluatorHandle before getting here.\n ceci.evaluatorHandle.postMessage(message);\n });\n }\n\n /**\n * Passes the job's requirements object into the sandbox so that the global\n * access lists can be updated accordingly.\n *\n * e.g. disallow access to OffscreenCanvas without\n * environment.offscreenCanvas=true present.\n *\n * Must be called after @start.\n *\n * @returns {Promise} - resolves with result on success, rejects otherwise\n */\n applySandboxRequirements(requirements) {\n var ceci = this;\n \n return new Promise(function sandbox$applySandboxRequirementsPromise(resolve, reject) {\n const message = {\n requirements,\n request: 'applyRequirements',\n };\n let wereRequirementsApplied = false;\n\n const successCb = ceci.ee.once(\n 'applyRequirementsDone',\n function sandbox$applyRequirements$success() {\n wereRequirementsApplied = true;\n resolve();\n },\n );\n\n assert(typeof message.requirements === 'object');\n ceci.evaluatorHandle.postMessage(message);\n\n setTimeout(function sandbox$finishApplySandboxRequirements() {\n if (!wereRequirementsApplied) {\n ceci.ee.removeListener('applyRequirementsDone', successCb);\n ceci.terminate(false);\n reject(\n new Error(\n 'applyRequirements never received applyRequirementsDone response from sandbox',\n ),\n );\n }\n }, 3000 * timeDilation); /* XXXwg needs tunable */\n });\n }\n\n /**\n * Executes a slice received from the supervisor.\n * Must be called after @start.\n *\n * @param {Slice} slice - bare minimum data required for the job/job code to be executed on\n * @param {number} [delay = 0] the delay that this method should wait before beginning work, used to avoid starting all sandboxes at once\n *\n * @returns {Promise} - resolves with result on success, rejects otherwise\n */\n\n async work (slice, delay = 0) {\n var ceci = this;\n\n if (!ceci.isAssigned) {\n throw new Error(\"Sandbox.run: Sandbox is not ready to work, state=\" + ceci.state);\n }\n\n ceci.state = WORKING;\n ceci.slice = slice;\n assert(slice);\n\n // cf. DCP-1720\n this.resetSliceTimeReport();\n \n // Now wait for the delay if provided, prevents many sandboxes starting at once from crashing the supervisor\n if (delay > 0) await new Promise(resolve => setTimeout(resolve, (delay + 1) * timeDilation));\n if (!ceci.isWorking) return; // sandbox.terminate could have been called during the delay timeout\n\n // Prepare the sandbox to begin work\n // will be replaced by `assign` message that should be called before emitting a `work` message\n if (ceci.jobAddress !== slice.jobAddress) {\n throw new Error(`Sandbox.run: Sandbox is already assigned and jobAddress doesn't match previous (${ceci.jobAddress} !== ${slice.jobAddress})`);\n }\n\n let sliceHnd = { job: ceci.public, sandbox: ceci };\n await ceci.resetSandboxState();\n if (!ceci.slice) {\n console.error(`Slice for job ${ceci.jobAddress} vanished during work initialization - aborting`);\n return;\n }\n\n let inputDatum;\n let dataError = false;\n try {\n if (ceci.slice.datumUri)\n inputDatum = await fetchURI(\n ceci.slice.datumUri,\n this.allowedOrigins,\n dcpConfig.worker.allowOrigins.fetchData,\n );\n else {\n let { mro } = await ceci.supervisorCache.fetchJob(\n ceci.jobAddress,\n this.allowedOrigins.concat(dcpConfig.worker.allowOrigins.fetchData)\n );\n const ro = rehydrateRange(mro);\n // -1 to prevent an OBOE since slice numbers start at 1.\n inputDatum = ro[ceci.slice.sliceNumber - 1];\n }\n } catch (err) {\n dataError = err;\n if(err.code === 'EFETCH')\n dataError.errorCode = 'EFETCH'\n else\n dataError.errorCode = 'EUNCAUGHTERROR'\n ceci.emit('workEmit', {\n eventName: 'error',\n payload: {\n message: dataError.message,\n stack:dataError.stack,\n name: ceci.public.name\n }\n });\n }\n\n debugging('sandbox') && debug(`Fetched datum: ${inputDatum}`);\n\n if (!ceci.slice) {\n console.error(`Slice for job ${ceci.jobAddress} vanished after data fetch - aborting`);\n return;\n }\n\n ceci.resetProgressTimeout();\n ceci.resetSliceTimeout();\n\n return new Promise(function sandbox$$workPromise(resolve, reject) {\n let onSuccess, onFail\n\n onSuccess = ceci.ee.once('resolve', function sandbox$$work$success (event) {\n ceci.ee.removeListener('reject', onFail)\n resolve(event)\n }.bind(ceci))\n\n onFail = ceci.ee.once('reject', function sandbox$$work$fail (err) {\n ceci.ee.removeListener('resolve', onSuccess)\n reject(err)\n }.bind(ceci))\n\n ceci.sliceStartTime = Date.now();\n ceci.progress = null;\n ceci.progressReports = {\n last: undefined,\n lastDeterministic: undefined,\n };\n\n ceci.resetProgressTimeout();\n ceci.resetSliceTimeout();\n ceci.emit('start', sliceHnd);\n \n if(dataError){\n ceci.ee.removeListener('resolve', onSuccess);\n ceci.ee.removeListener('reject', onFail);\n setTimeout(() => reject(dataError), 0)\n\n } else {\n ceci.evaluatorHandle.postMessage({\n request: 'main',\n data: inputDatum,\n })\n }\n })\n .then(async function sandbox$$work$then(event) {\n // Ceci is the complete callback when the slice completes\n // prevent any hanging timers from being fired\n await ceci.clearSandboxTimers();\n\n // TODO: Should sliceHnd just be replaced with ceci.public?\n ceci.emit('slice', sliceHnd); /** @todo: decide which event is right */\n ceci.emit('sliceFinish', event);\n ceci.emit('complete', event);\n\n ceci.changeWorkingToAssigned();\n ceci.slice = false;\n return event;\n })\n .catch((err) => { \n if (err.name === 'EWORKREJECT') {\n this.rejectionData = err;\n ceci.evaluatorHandle.postMessage({ request: 'resetAndGetCPUTime' })\n } else { // sandbox termination for rejected work happens in Supervisor.handleRejectedWork\n // Ceci is the reject callback for when the slice throws an error\n ceci.terminate(false);\n }\n\n ceci.emit('error', err, 'slice');\n\n if (err instanceof NoProgressError) {\n ceci.emit('workEmit', {\n eventName: 'noProgress',\n payload: {\n timestamp: Date.now() - ceci.sliceStartTime,\n data: ceci.slice.datumUri,\n progressReports: ceci.progressReports,\n }\n });\n }\n throw err;\n })\n .finally(function sandbox$$work$finally() {\n ceci.emit('end', sliceHnd);\n });\n }\n\n resetProgressTimeout() {\n if (this.progressTimeout) {\n clearTimeout(this.progressTimeout);\n this.progressTimeout = null;\n }\n\n this.progressTimeout = setTimeout(() => {\n if (this.options.ignoreNoProgress) {\n return console.warn(\"ENOPROGRESS silenced by localExec: In a remote worker, this slice would be stopped for not calling progress frequently enough.\");\n }\n\n this.ee.emit('reject', new NoProgressError(`No progress event was received in the last ${dcpConfig.worker.sandbox.progressTimeout / 1000} seconds.`));\n }, +dcpConfig.worker.sandbox.progressTimeout * timeDilation);\n }\n\n resetSliceTimeout() {\n if (this.sliceTimeout) clearTimeout(this.sliceTimeout);\n\n this.sliceTimeout = setTimeout(() => {\n if (Sandbox.debugWork) return console.warn(\"Sandbox.debugWork: Ignoring slice timeout\");\n\n this.ee.emit('reject', new SliceTooSlowError(`Slice took longer than ${dcpConfig.worker.sandbox.sliceTimeout / 1000} seconds.`));\n }, +dcpConfig.worker.sandbox.sliceTimeout * timeDilation);\n }\n \n async handleRing0Message(data) {\n debugging('event:ring-0') && debug('event:ring-0', data);\n //handling a true ring 0 message\n switch (data.request) {\n case 'sandboxLoaded':\n // emit externally\n this.emit('sandboxLoaded', this)\n break;\n\n case 'scriptLoaded':\n // emit externally\n this.emit('scriptLoaded', data);\n \n if(data.result !== \"success\") {\n this.onerror(data);\n }\n break;\n \n case 'clearTimersDone':\n this.ee.emit(data.request, data);\n break;\n case 'totalCPUTime':\n this.updateTime(data);\n if (this.ee.listenerCount('resolve') > 0) {\n this.completeData.timeReport = this.sliceTimeReport;\n this.ee.emit('resolve', this.completeData);\n delete this.completeData;\n } else {\n this.rejectionData.timeReport = this.sliceTimeReport\n this.emit('rejectedWorkMetrics', this.rejectionData) // If there is no internal listener for 'resolve', the slice was rejected and\n delete this.rejectionData; // we need to send the rejected metrics to the supervisor\n } \n break;\n case 'error':\n // Warning: rejecting here with just event.data.error causes issues\n // where the reject handlers modify the object so it interferes with the\n // workEmit event payload, wrapping in an Error instance copies the values\n let e = new Error(\n data.error.message,\n data.error.fileName,\n data.error.lineNumber);\n e.stack = data.error.stack;\n e.name = data.error.name;\n \n if (this.ee.listenerCount('reject') > 0) {\n this.ee.emit('reject', e);\n } else {\n // This will happen if the error is thrown during initialization\n throw e;\n }\n\n break;\n default:\n let error = new Error('Received unhandled request from sandbox: ' + data.request + '\\n\\t' + JSON.stringify(data));\n console.error(error);\n break; \n }\n }\n\n async handleRing1Message(data) {\n switch (data.request) {\n case 'applyRequirementsDone':\n // emit internally\n this.ee.emit(data.request, data)\n break;\n default:\n let error = new Error('Received unhandled request from sandbox ring 1: ' + data.request + '\\n\\t' + JSON.stringify(data));\n console.error(error)\n break; \n }\n }\n\n async handleRing2Message(data) {\n debugging('event:ring-2') && debug('event:ring-2', data);\n switch (data.request) {\n case 'dependency': {\n let moduleData;\n try {\n moduleData = await this.supervisorCache.fetchModule(data.data);\n } catch (error) {\n /*\n * In the event of an error here, we want to let the client know there was a problem in\n * loading their module. However, there hasn't yet been an actual slice assigned to the sandbox.\n * Therefore, we assign 'slice 0' to the sandbox, a slice that will never exist, and is used\n * purely for this purpose. \n */\n this.slice = {\n jobAddress: this.jobAddress,\n sliceNumber: 0,\n };\n\n const payload = {\n name: error.name,\n timestamp: error.timestamp,\n message: error.message,\n };\n\n this.emit('workEmit', {\n eventName: 'error',\n payload,\n });\n this.ee.emit('reject', error);\n break;\n }\n this.evaluatorHandle.postMessage({\n request: 'moduleGroup',\n data: moduleData,\n id: data.id,\n });\n break;\n }\n case 'error':\n /*\n * Ring 2 error messages will only fire for problems inside of the worker that are separate from\n * the work function. In most cases there are other handlers for situations where 'error' may be emitted\n * such as timeouts if the expected message isn't recieved. Thus, we will output the error, but nothing else.\n */\n console.error(data.error);\n break;\n case 'describe':\n case 'evalResult':\n case 'resetStateDone':\n case 'assigned':\n // emit internally\n this.ee.emit(data.request, data);\n break;\n case 'reject':\n // emit internally\n this.ee.emit(data.request, data.error);\n break;\n default: {\n const error = new Error(\n `Received unhandled request from sandbox ring 2. Data: ${JSON.stringify(\n data,\n null,\n 2,\n )}`,\n );\n console.error(error);\n break;\n }\n }\n }\n\n async handleRing3Message(data) {\n switch (data.request) {\n case 'complete':\n clearTimeout(this.progressTimeout);\n clearTimeout(this.sliceTimeout);\n this.progressTimeout = this.sliceTimeout = null;\n\n if (this.progress === null) {\n if (this.options.ignoreNoProgress) {\n console.warn(\"ENOPROGRESS silenced by localExec: Progress was not called during this slice's execution, in a remote sandbox this would cause the slice to be failed\");\n } else {\n // If a progress update was never received (progress === null) then reject\n this.ee.emit('reject', new NoProgressError('Sandbox never emitted a progress event.'));\n break;\n }\n }\n this.evaluatorHandle.postMessage({ request: 'resetAndGetCPUTime' })\n this.progress = 100;\n this.completeData = data;\n // The timing report and resolve will be emitted when the CPU time is received. \n break;\n case 'progress':\n let { progress, indeterminate, throttledReports, value } = data;\n this.progress = progress;\n const progressReport = {\n timestamp: Date.now() - this.sliceStartTime,\n progress,\n value,\n throttledReports,\n }\n this.progressReports.last = progressReport;\n if (!indeterminate) {\n this.progressReports.lastDeterministic = progressReport;\n }\n\n this.resetProgressTimeout();\n\n this.emit('sliceProgress', data);\n break;\n\n case 'noProgress':\n let { message } = data;\n\n this.ee.emit('reject', new NoProgressError(message));\n break;\n case 'console':\n data.payload.message = scopedKvin.marshal(data.payload.message); \n this.emit('workEmit', {\n eventName: 'console',\n payload: data.payload, \n });\n break;\n\n case 'emitEvent':/* ad-hoc event from the sandbox (work.emit) */\n this.emit('workEmit', {\n eventName: 'custom',\n payload: data.payload\n })\n break;\n case 'measurement':\n this.updateTime(data);\n break;\n case 'sandboxError': /* the sandbox itself has an error condition */\n {\n this.emit('sandboxError', data.error);\n break;\n }\n case 'workError': { /* the work function threw/rejected */\n this.emit('workEmit', {\n eventName: 'error',\n payload: data.error,\n });\n\n // Warning: rejecting here with just .data.error causes issues\n // where the reject handlers modify the object so it interferes with the\n // workEmit payload, wrapping in an Error instance copies the values\n const wrappedError = new UncaughtExceptionError(\n data.error.message,\n data.error.fileName,\n data.error.lineNumber,\n );\n wrappedError.stack = data.error.stack;\n wrappedError.name = data.error.name;\n\n if (this.ee.listenerCount('reject') > 0) {\n this.ee.emit('reject', wrappedError);\n } else {\n // This will happen if the error is thrown during initialization\n throw wrappedError;\n }\n break;\n }\n default:\n let error = new Error('Received unhandled request from sandbox ring 3: ' + data.request + '\\n\\t' + JSON.stringify(data));\n console.error(error)\n break; \n }\n }\n\n /**\n * Handles progress and completion events from sandbox.\n * Unless explicitly returned out of this function will re-emit the event\n * on @this.ee where the name of the event is event.data.request.\n *\n * @param {object} event - event received from the sandbox\n */\n async onmessage(event) {\n debugging('event') && debug('event', event);\n if (Sandbox.debugEvents) {\n console.debug('sandbox - eventDebug:', {\n id: this.id,\n state: this.state,\n event: JSON.stringify(event)\n })\n }\n\n const { data } = event;\n const ringLevel = data.ringSource\n\n // Give the data to a handler depending on ring level\n if (ringLevel === -1) {\n console.error('Message sent directly from raw postMessage. Terminating worker...');\n console.debug(event);\n return this.terminate(true);\n } else {\n const handler = this.ringMessageHandlers[ringLevel];\n if (handler) {\n handler.call(this, data.value);\n } else {\n console.warn(`No handler defined for message from ring ${ringLevel}`);\n console.debug(event);\n }\n }\n }\n\n /**\n * Error handler for the internal sandbox.\n * Currently just logs the errors that the sandbox spits out.\n */\n onerror(event) {\n console.error('Sandbox emitted an error:', event);\n this.terminate(true, true);\n }\n\n /**\n * Clears the timeout and terminates the sandbox and sometimes emits a reject event.\n *\n * @param {boolean} [reject = true] - if true emit reject event\n * @param {boolean} [immediate = false] - passed to terminate, used by standaloneWorker to immediately close the connection\n */\n terminate (reject = true, immediate = false) {\n const oldState = this.state;\n this.state = TERMINATED;\n \n clearTimeout(this.progressTimeout);\n clearTimeout(this.sliceTimeout);\n this.progressTimeout = this.sliceTimeout = null;\n \n if (this.evaluatorHandle && typeof this.evaluatorHandle.terminate === 'function') {\n try {\n this.evaluatorHandle.terminate(immediate);\n this.evaluatorHandle = null;\n } catch (e) {\n console.error(`Error terminating sandbox ${this.id} (${oldState}):`, e);\n } finally {\n this.emit('terminate', this);\n }\n }\n\n if (reject) {\n this.ee.emit('reject', new Error(`Sandbox ${this.id} (${oldState}) was terminated.`));\n }\n\n this.emit('terminated');\n }\n\n /**\n * Attempts to stop the sandbox from doing completing its current\n * set of work without terminating the working.\n */\n stop () {\n throw new Error('Sandbox.stop is not yet implemented.')\n }\n\n /**\n * ringNPostMessage can send a `measurement` request and update these\n * totals.\n */\n updateTime (measurementEvent) {\n ['total', 'CPU', 'webGL'].forEach((key) => {\n if (measurementEvent[key]) this.sliceTimeReport[key] += measurementEvent[key];\n })\n }\n\n resetSliceTimeReport () {\n this.sliceTimeReport = {\n total: 0,\n CPU: 0,\n webGL: 0,\n }\n }\n}\n\nSandbox.idCounter = 1;\nSandbox.debugWork = false;\nSandbox.debugState = false;\nSandbox.debugEvents = false;\n\nexports.Sandbox = Sandbox;\nexports.SandboxError = SandboxError;\nexports.NoProgressError = NoProgressError;\nexports.SliceTooSlowError = SliceTooSlowError;\nexports.UncaughtExceptionError = UncaughtExceptionError;\nexports.RemoteFetchError = RemoteFetchError;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/sandbox.js?");
|
|
4427
|
+
eval("// NOTE - need timeout/postmessage function\n/**\n * @file dcp-client/worker/sandbox.js\n *\n * A sandbox that when constructed and assigned can do work for\n * a distributed slice. A sandbox runs for a single slice at a time.\n *\n * Usage:\n * let sandbox = new Sandbox()\n * await sandbox.start()\n * let results = await sandbox.run(slice)\n *\n * Debug flags:\n * Sandbox.debugWork = true // - turns off 30 second timeout to let user debug sandbox innards more easily\n * Sandbox.debugState = true // - logs all state transitions for this sandbox\n * Sandbox.debugEvents = true // - logs all events received from the sandbox\n *\n * @author Matthew Palma, mpalma@kingsds.network\n * Ryan Rossiter, ryan@kingsds.network\n * @date May 2019\n * @module sandbox\n */\n/* global dcpConfig */\n// @ts-check\n\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst { assert, assertEq3 } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nconst { rehydrateRange } = __webpack_require__(/*! dcp/dcp-client/range-object */ \"./src/dcp-client/range-object.js\");\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\nconst scopedKvin = new kvin.KVIN({Object: ({}).constructor,\n Array: ([]).constructor, \n Function: (()=>{}).constructor});\n\nlet timeDilation = 1;\nif (DCP_ENV.platform === 'nodejs') {\n const { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\n /** Make timers 10x slower when running in niim */\n timeDilation = (requireNative('module')._cache.niim instanceof requireNative('module').Module) ? 10 : 1;\n}\n\n/**\n * Wraps console.debug to emulate debug module prefixing messages on npm.\n * @param {...any} args\n */\nconst debug = (...args) => {\n if (debugging()) {\n console.debug('dcp-client:worker:sandbox', ...args);\n }\n};\n\nconst nanoid = (__webpack_require__(/*! nanoid */ \"./node_modules/nanoid/index.browser.js\").nanoid);\n\nconst { EventEmitter } = __webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\");\nconst { fetchURI, encodeDataURI } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\n\n// Sandbox states\nconst UNREADY = 'UNREADY' // No Sandbox (web worker, saworker, etc) has been constructed yet\nconst READYING = 'READYING' // Sandbox is being constructed and environment (bravojs, env) is being set up\nconst READY_FOR_ASSIGN = 'READY_FOR_ASSIGN' // Sandbox is ready to be assigned\nconst ASSIGNING = 'ASSIGNING' // Sandbox is running through assigning steps\nconst ASSIGNED = 'ASSIGNED' // Sandbox is assigned but not working\nconst WORKING = 'WORKING' // Sandbox is working\nconst TERMINATED = 'TERMINATED'\nconst EVAL_RESULT_PREFIX = 'evalResult::';\n\nclass SandboxError extends Error {}\nclass NoProgressError extends SandboxError { constructor(...args) { super(...args); this.errorCode = 'ENOPROGRESS'; } }\nclass SliceTooSlowError extends SandboxError { constructor(...args) { super(...args); this.errorCode = 'ESLICETOOSLOW'; } }\nclass UncaughtExceptionError extends SandboxError { constructor(...args) { super(...args); this.errorCode = 'EUNCAUGHT'; } }\nclass RemoteFetchError extends SandboxError { constructor(...args) { super(...args); this.errorCode = 'EFETCH'; }}\n\n/** @typedef {import('dcp/common/dcp-events').EventEmitter} EventEmitter */\n/** @typedef {import('./slice').Slice} Slice */\n/** @typedef {import('./supervisor-cache').SupervisorCache} SupervisorCache */\n/** @typedef {*} opaqueId */\n\n/**\n * @access public\n * @typedef {object} SandboxOptions\n * @constructor {function} [SandboxConstructor]\n * @property {boolean} [ignoreNoProgress] - When true, the sandbox will not be stopped for not calling progress\n */\n\nclass Sandbox extends EventEmitter {\n /**\n * A Sandbox (i.e. a worker sandbox) which executes distributed slices.\n *\n * @constructor\n * @param {SupervisorCache} cache\n * @param {SandboxOptions} options\n */\n constructor (cache, options, origins) {\n super('Sandbox');\n /** @type {SupervisorCache} */\n this.supervisorCache = cache;\n /** @type {SandboxOptions} */\n this.options = {\n ignoreNoProgress: false,\n ...options,\n SandboxConstructor: options.SandboxConstructor ||\n (__webpack_require__(/*! ./evaluators */ \"./src/dcp-client/worker/evaluators/index.js\").BrowserEvaluator),\n }\n this.allowedOrigins = origins;\n\n /** @type {opaqueId} */\n this.jobAddress = null;\n /** @type {object} */\n this.evaluatorHandle = null;\n /** @type {object} */\n this.capabilities = null;\n /** @type {EventEmitter} */\n this.ee = new EventEmitter('SandboxInternal')\n\n /** @type {string} */\n this._state = UNREADY;\n /** @type {boolean} */\n this.allocated = false;\n /** @type {number?} */\n this.progress = 100;\n /** @type {object} */\n this.progressReports = null;\n /** @type {object} */\n this.progressTimeout = null;\n /** @type {object} */\n this.sliceTimeout = null;\n /** @type {object} */\n this.rejectionData = null;\n /** @type {Slice} */\n this.slice = null;\n\n /** @type {number?} */\n this.started = null;\n /** @type {number?} */\n this.sliceStartTime = null;\n /** @type {boolean} */\n this.requiresGPU = false;\n /** @type {string|URL} */\n this.packageURL = dcpConfig.packageManager.location\n /** @type {number?} */\n this.id = Sandbox.getNewId();\n\n this.ringMessageHandlers = [\n this.handleRing0Message,\n this.handleRing1Message,\n this.handleRing2Message,\n this.handleRing3Message,\n ];\n\n this.resetSliceTimeReport();\n }\n\n get identifier() {\n if (this.allocated)\n return `${this.id}.${this.jobAddress}.${this.state}.${this.allocated}`;\n return `${this.id}.${this.jobAddress}.${this.state}`;\n }\n\n static getNewId() {\n return Sandbox.idCounter++;\n }\n\n get state () {\n return this._state\n }\n\n set state (value) {\n if (Sandbox.debugState) {\n console.debug(`sandbox - changing state of ${this.id}... ${this._state} -> ${value}`)\n }\n\n if (this.state === TERMINATED && value !== TERMINATED) {\n // For safety!\n throw new Error(`Sandbox set state violation, attepted to change state from ${this.state} to ${value}`);\n }\n\n this._state = value;\n }\n\n get isReadyForAssign () {\n return this.state === READY_FOR_ASSIGN;\n }\n\n get isAssigned () {\n return this.state === ASSIGNED;\n }\n\n get isWorking () {\n return this.state === WORKING;\n }\n\n get isTerminated () {\n return this.state === TERMINATED;\n }\n\n changeWorkingToAssigned () {\n if (this.isWorking)\n this.state = ASSIGNED;\n }\n\n setIsAssigning () {\n this.state = ASSIGNING;\n }\n\n /**\n * Readies the sandbox. This will result in the sandbox being ready and not assigned,\n * it will need to be assigned with a job before it is able to do work.\n *\n * @todo maybe preload specific modules or let the cache pass in what modules to load?\n * @throws on failure to ready\n */\n async start(delay = 0) {\n this.started = Date.now();\n this.state = READYING;\n\n if (delay > 0) await new Promise((resolve) => setTimeout(resolve, delay * timeDilation));\n\n try {\n // RING 0\n this.evaluatorHandle = new this.options.SandboxConstructor({\n name: `DCP Sandbox #${this.id}`,\n });\n this.evaluatorHandle.onerror = this.onerror.bind(this);\n\n const messageHandler = this.onmessage.bind(this);\n this.evaluatorHandle.onmessage = function onmessage(event)\n {\n let data;\n if (event.data.serialized)\n {\n data = kvin.parse(event.data.message);\n }\n else\n {\n data = kvin.unmarshal(event.data);\n }\n messageHandler({ data });\n }\n\n const evaluatorPostMessage = this.evaluatorHandle.postMessage.bind(this.evaluatorHandle);\n this.evaluatorHandle.postMessage = function postMessage(message)\n {\n evaluatorPostMessage(scopedKvin.marshal(message));\n }\n\n const ceci = this;\n this.evaluatorHandle.addEventListener('end', () => ceci.terminate(true));\n\n // Now in RING 1\n\n // Now in RING 2\n await this.describe();\n this.state = READY_FOR_ASSIGN;\n this.emit('ready', this);\n } catch (error) {\n console.warn('Failed to start the sandbox -', error.message);\n this.terminate(false);\n throw error;\n }\n }\n\n /**\n * This will assign the sandbox with a job, loading its sandbox code\n * into the sandbox.\n *\n * @param {string} jobAddress The address of the job to assign to\n * @throws on initialization failure\n */\n async assign(jobAddress) {\n this.jobAddress = jobAddress;\n this.job = await this.supervisorCache.fetchJob(jobAddress, this.allowedOrigins);\n\n assertEq3(this.job.address, jobAddress);\n assert(typeof this.job === 'object');\n assert(typeof this.job.requirements === 'object');\n assert(Array.isArray(this.job.dependencies));\n assert(Array.isArray(this.job.requirePath));\n\n // Extract public data from job, with defaults\n this.public = Object.assign({\n name: `Anonymous Job ${this.job.address.slice(0, 6)}`,\n description: 'Discreetly helping make the world smarter.',\n link: 'https://distributed.computer/about',\n }, this.job.public);\n\n // Future: We may want other filename tags for appliances // RR Nov 2019\n\n // Important: The order of applying requirements before loading the sandbox code\n // is important for modules and sandbox code to set globals over the whitelist.\n await this.applySandboxRequirements(this.job.requirements);\n await this.assignEvaluator();\n this.state = ASSIGNED;\n }\n\n async assignEvaluator() {\n debug('Begin assigning job to evaluator');\n const ceci = this;\n\n return new Promise(function sandbox$$assignEvaluatorPromise(resolve, reject) {\n const message = {\n request: 'assign',\n job: ceci.job,\n sandboxConfig: dcpConfig.worker.sandbox,\n };\n\n /* note - onFailListener used for removal. This is necessary due to a bug in ee.once. /wg Feb 2022 */\n \n const onSuccess = (event) => {\n // eslint-disable-next-line no-use-before-define\n ceci.ee.removeListener('reject', onFailListener);\n ceci.emit('assigned', event.jobAddress);\n debug('Job assigned to evaluator');\n resolve();\n };\n\n const onFail = (error) => {\n // eslint-disable-next-line no-use-before-define\n ceci.ee.removeListener('assigned', onSuccessListener);\n reject(error);\n };\n\n const onSuccessListener = ceci.ee.once('assigned', onSuccess);\n const onFailListener = ceci.ee.once('reject', onFail);\n ceci.evaluatorHandle.postMessage(message);\n });\n }\n\n /**\n * Evaluates a string inside the sandbox.\n *\n * @param {string} code - the code to evaluate in the sandbox\n * @param {string} filename - the name of the 'file' to help with debugging,\n * no longer working though?\n * @returns {Promise} - resolves with eval result on success, rejects\n * otherwise\n */\n eval(code, filename) {\n var ceci = this;\n \n return new Promise(function sandbox$$eval$Promise(resolve, reject) {\n let msgId = nanoid();\n let msg = {\n request: 'eval',\n data: code,\n filename,\n msgId, \n }\n\n const eventId = EVAL_RESULT_PREFIX + msgId;\n\n let onSuccess = (event) => {\n ceci.ee.removeListener('reject', onFailListener)\n resolve(event)\n }\n\n let onFail = (event) => {\n ceci.ee.removeListener(eventId, onSuccessListener)\n reject(event)\n }\n\n let onSuccessListener = ceci.ee.once(eventId, onSuccess);\n let onFailListener = ceci.ee.once('reject', onFail)\n\n ceci.evaluatorHandle.postMessage(msg)\n })\n }\n\n /**\n * Resets the state of the bootstrap, without resetting the sandbox function if assigned.\n * Mostly used to reset the progress status before reusing a sandbox on another slice.\n * Must be called after @start.\n *\n * @returns {Promise} - resolves with result on success, rejects otherwise\n */\n resetSandboxState () {\n var ceci = this;\n\n return new Promise(function sandbox$resetSandboxStatePromise(resolve, reject) {\n let successCb, failTimeout;\n let msg = {\n request: 'resetState',\n };\n\n successCb = ceci.ee.once('resetStateDone', function sandbox$resetSandboxState$success () {\n if (failTimeout === false)\n return; /* already rejected */\n clearTimeout(failTimeout);\n failTimeout = false;\n resolve();\n });\n\n failTimeout = setTimeout(function sandbox$resetSandboxState$fail() {\n if (failTimeout === false)\n return; /* already resolved */\n \n ceci.ee.removeListener('resetStateDone', successCb);\n ceci.terminate(false);\n failTimeout = false;\n\n reject(new Error('resetState never received resetStateDone event from sandbox'));\n }, 3000 * timeDilation); /* XXXwg need tuneable */\n\n assert(ceci.evaluatorHandle); // It is possible that ceci.terminate nulls out evaluatorHandle before getting here.\n ceci.evaluatorHandle.postMessage(msg);\n });\n }\n\n /**\n * Clear all timers that are set inside the sandbox (evaluator) environment.\n *\n * @returns {Promise} - resolves with result on success, rejects otherwise\n */\n clearSandboxTimers() {\n var ceci = this;\n \n return new Promise(function sandbox$clearSandboxTimersPromise(resolve, reject) {\n let successCb, failTimeout;\n let msg = {\n request: 'clearTimers',\n };\n\n successCb = ceci.ee.once('clearTimersDone', function sandbox$clearSandboxTimers$success() {\n if (failTimeout === false)\n return; /* already rejected */\n clearTimeout(failTimeout);\n failTimeout = false;\n resolve();\n });\n\n failTimeout = setTimeout(function sanbox$clearSandboxTimers$fail() {\n if (failTimeout === false)\n return; /* already resolved */\n \n ceci.ee.removeListener('clearTimersDone', successCb);\n ceci.terminate(false);\n failTimeout = false;\n \n reject(new Error('clearTimers never received clearTimersDone event from sandbox'));\n }, 3000 * timeDilation); /* XXXwg need tuneable */\n\n if (ceci.evaluatorHandle) // Sometimes ceci.terminate nulls out evaluatorHandle before getting here.\n ceci.evaluatorHandle.postMessage(msg);\n });\n }\n\n /**\n * Sends a post message to describe its capabilities.\n *\n * Side effect: Sets the capabilities property of the current sandbox.\n *\n * @returns {Promise} Resolves with the sandbox's capabilities. Rejects with\n * an error saying a response was not received.\n * @memberof Sandbox\n */\n describe() {\n debug('Beginning to describe evaluator');\n var ceci = this;\n \n return new Promise(function sandbox$describePromise(resolve, reject) {\n if (ceci.evaluatorHandle === null) {\n return reject(new Error('Evaluator has not been initialized.'));\n }\n\n /**\n * Opted to create a flag for the describe response being received so that\n * we don't have to *hoist* the timeout's id to clear it in the response\n * handler.\n */\n let didReceiveDescribeResponse = false;\n const describeResponseHandler = ceci.ee.once('describe', (data) => {\n didReceiveDescribeResponse = true;\n const { capabilities } = data;\n if (typeof capabilities === 'undefined') {\n reject(\n new Error('Did not receive capabilities from describe response.'),\n );\n }\n ceci.capabilities = capabilities;\n\n // Currently only used in tests. May use the event in the future.\n ceci.emit('described', capabilities);\n debug('Evaluator has been described');\n resolve(capabilities);\n });\n const describeResponseFailedHandler = () => {\n if (!didReceiveDescribeResponse) {\n ceci.ee.removeListener('describe', describeResponseHandler);\n ceci.terminate(false);\n reject(\n new Error(\n 'Describe message timed-out. No describe response was received from the describe command.',\n ),\n );\n }\n };\n\n const message = {\n request: 'describe',\n };\n\n // Arbitrarily set the waiting time.\n setTimeout(describeResponseFailedHandler, 6000 * timeDilation); /* XXXwg need tuneable */\n assert(ceci.evaluatorHandle); // It is possible that ceci.terminate nulls out evaluatorHandle before getting here.\n ceci.evaluatorHandle.postMessage(message);\n });\n }\n\n /**\n * Passes the job's requirements object into the sandbox so that the global\n * access lists can be updated accordingly.\n *\n * e.g. disallow access to OffscreenCanvas without\n * environment.offscreenCanvas=true present.\n *\n * Must be called after @start.\n *\n * @returns {Promise} - resolves with result on success, rejects otherwise\n */\n applySandboxRequirements(requirements) {\n var ceci = this;\n \n return new Promise(function sandbox$applySandboxRequirementsPromise(resolve, reject) {\n const message = {\n requirements,\n request: 'applyRequirements',\n };\n let wereRequirementsApplied = false;\n\n const successCb = ceci.ee.once(\n 'applyRequirementsDone',\n function sandbox$applyRequirements$success() {\n wereRequirementsApplied = true;\n resolve();\n },\n );\n\n assert(typeof message.requirements === 'object');\n ceci.evaluatorHandle.postMessage(message);\n\n setTimeout(function sandbox$finishApplySandboxRequirements() {\n if (!wereRequirementsApplied) {\n ceci.ee.removeListener('applyRequirementsDone', successCb);\n ceci.terminate(false);\n reject(\n new Error(\n 'applyRequirements never received applyRequirementsDone response from sandbox',\n ),\n );\n }\n }, 3000 * timeDilation); /* XXXwg needs tunable */\n });\n }\n\n /**\n * Executes a slice received from the supervisor.\n * Must be called after @start.\n *\n * @param {Slice} slice - bare minimum data required for the job/job code to be executed on\n * @param {number} [delay = 0] the delay that this method should wait before beginning work, used to avoid starting all sandboxes at once\n *\n * @returns {Promise} - resolves with result on success, rejects otherwise\n */\n\n async work (slice, delay = 0) {\n var ceci = this;\n\n if (!ceci.isAssigned) {\n throw new Error(\"Sandbox.run: Sandbox is not ready to work, state=\" + ceci.state);\n }\n\n ceci.state = WORKING;\n ceci.slice = slice;\n assert(slice);\n\n // cf. DCP-1720\n this.resetSliceTimeReport();\n \n // Now wait for the delay if provided, prevents many sandboxes starting at once from crashing the supervisor\n if (delay > 0) await new Promise(resolve => setTimeout(resolve, (delay + 1) * timeDilation));\n if (!ceci.isWorking) return; // sandbox.terminate could have been called during the delay timeout\n\n // Prepare the sandbox to begin work\n // will be replaced by `assign` message that should be called before emitting a `work` message\n if (ceci.jobAddress !== slice.jobAddress) {\n throw new Error(`Sandbox.run: Sandbox is already assigned and jobAddress doesn't match previous (${ceci.jobAddress} !== ${slice.jobAddress})`);\n }\n\n let sliceHnd = { job: ceci.public, sandbox: ceci };\n await ceci.resetSandboxState();\n if (!ceci.slice) {\n console.error(`Slice for job ${ceci.jobAddress} vanished during work initialization - aborting`);\n return;\n }\n\n let inputDatum;\n let dataError = false;\n try {\n if (ceci.slice.datumUri && ceci.slice.datumUri.includes('file'))\n inputDatum = scopedKvin.parse(await fetchURI(\n ceci.slice.datumUri,\n this.allowedOrigins,\n dcpConfig.worker.allowOrigins.fetchData,\n ));\n else if (ceci.slice.datumUri)\n {\n inputDatum = await fetchURI(\n ceci.slice.datumUri,\n this.allowedOrigins,\n dcpConfig.worker.allowOrigins.fetchData,\n );\n }\n else {\n let { mro } = await ceci.supervisorCache.fetchJob(\n ceci.jobAddress,\n this.allowedOrigins.concat(dcpConfig.worker.allowOrigins.fetchData)\n );\n const ro = rehydrateRange(mro);\n // -1 to prevent an OBOE since slice numbers start at 1.\n inputDatum = ro[ceci.slice.sliceNumber - 1];\n }\n } catch (err) {\n dataError = err;\n if(err.code === 'EFETCH')\n dataError.errorCode = 'EFETCH'\n else\n dataError.errorCode = 'EUNCAUGHTERROR'\n ceci.emit('workEmit', {\n eventName: 'error',\n payload: {\n message: dataError.message,\n stack:dataError.stack,\n name: ceci.public.name\n }\n });\n }\n\n debugging('sandbox') && debug(`Fetched datum: ${inputDatum}`);\n\n if (!ceci.slice) {\n console.error(`Slice for job ${ceci.jobAddress} vanished after data fetch - aborting`);\n return;\n }\n\n ceci.resetProgressTimeout();\n ceci.resetSliceTimeout();\n\n return new Promise(function sandbox$$workPromise(resolve, reject) {\n let onSuccess, onFail\n\n onSuccess = ceci.ee.once('resolve', function sandbox$$work$success (event) {\n ceci.ee.removeListener('reject', onFail)\n resolve(event)\n }.bind(ceci))\n\n onFail = ceci.ee.once('reject', function sandbox$$work$fail (err) {\n ceci.ee.removeListener('resolve', onSuccess)\n reject(err)\n }.bind(ceci))\n\n ceci.sliceStartTime = Date.now();\n ceci.progress = null;\n ceci.progressReports = {\n last: undefined,\n lastDeterministic: undefined,\n };\n\n ceci.resetProgressTimeout();\n ceci.resetSliceTimeout();\n ceci.emit('start', sliceHnd);\n \n if(dataError){\n ceci.ee.removeListener('resolve', onSuccess);\n ceci.ee.removeListener('reject', onFail);\n setTimeout(() => reject(dataError), 0)\n\n } else {\n ceci.evaluatorHandle.postMessage({\n request: 'main',\n data: inputDatum,\n })\n }\n })\n .then(async function sandbox$$work$then(event) {\n // Ceci is the complete callback when the slice completes\n // prevent any hanging timers from being fired\n await ceci.clearSandboxTimers();\n\n // TODO: Should sliceHnd just be replaced with ceci.public?\n ceci.emit('slice', sliceHnd); /** @todo: decide which event is right */\n ceci.emit('sliceFinish', event);\n ceci.emit('complete', event);\n\n ceci.changeWorkingToAssigned();\n ceci.slice = false;\n return event;\n })\n .catch((err) => { \n if (err.name === 'EWORKREJECT') {\n this.rejectionData = err;\n ceci.evaluatorHandle.postMessage({ request: 'resetAndGetCPUTime' })\n } else { // sandbox termination for rejected work happens in Supervisor.handleRejectedWork\n // Ceci is the reject callback for when the slice throws an error\n ceci.terminate(false);\n }\n\n ceci.emit('error', err, 'slice');\n\n if (err instanceof NoProgressError) {\n ceci.emit('workEmit', {\n eventName: 'noProgress',\n payload: {\n timestamp: Date.now() - ceci.sliceStartTime,\n data: ceci.slice.datumUri,\n progressReports: ceci.progressReports,\n }\n });\n }\n throw err;\n })\n .finally(function sandbox$$work$finally() {\n ceci.emit('end', sliceHnd);\n });\n }\n\n resetProgressTimeout() {\n if (this.progressTimeout) {\n clearTimeout(this.progressTimeout);\n this.progressTimeout = null;\n }\n\n this.progressTimeout = setTimeout(() => {\n if (this.options.ignoreNoProgress) {\n return console.warn(\"ENOPROGRESS silenced by localExec: In a remote worker, this slice would be stopped for not calling progress frequently enough.\");\n }\n\n this.ee.emit('reject', new NoProgressError(`No progress event was received in the last ${dcpConfig.worker.sandbox.progressTimeout / 1000} seconds.`));\n }, +dcpConfig.worker.sandbox.progressTimeout * timeDilation);\n }\n\n resetSliceTimeout() {\n if (this.sliceTimeout) clearTimeout(this.sliceTimeout);\n\n this.sliceTimeout = setTimeout(() => {\n if (Sandbox.debugWork) return console.warn(\"Sandbox.debugWork: Ignoring slice timeout\");\n\n this.ee.emit('reject', new SliceTooSlowError(`Slice took longer than ${dcpConfig.worker.sandbox.sliceTimeout / 1000} seconds.`));\n }, +dcpConfig.worker.sandbox.sliceTimeout * timeDilation);\n }\n \n async handleRing0Message(data) {\n debugging('event:ring-0') && debug('event:ring-0', data);\n //handling a true ring 0 message\n switch (data.request) {\n case 'sandboxLoaded':\n // emit externally\n this.emit('sandboxLoaded', this)\n break;\n\n case 'scriptLoaded':\n // emit externally\n this.emit('scriptLoaded', data);\n \n if(data.result !== \"success\") {\n this.onerror(data);\n }\n break;\n \n case 'clearTimersDone':\n this.ee.emit(data.request, data);\n break;\n case 'totalCPUTime':\n this.updateTime(data);\n if (this.ee.listenerCount('resolve') > 0) {\n this.completeData.timeReport = this.sliceTimeReport;\n this.ee.emit('resolve', this.completeData);\n delete this.completeData;\n } else {\n this.rejectionData.timeReport = this.sliceTimeReport\n this.emit('rejectedWorkMetrics', this.rejectionData) // If there is no internal listener for 'resolve', the slice was rejected and\n delete this.rejectionData; // we need to send the rejected metrics to the supervisor\n } \n break;\n case 'error':\n // Warning: rejecting here with just event.data.error causes issues\n // where the reject handlers modify the object so it interferes with the\n // workEmit event payload, wrapping in an Error instance copies the values\n let e = new Error(\n data.error.message,\n data.error.fileName,\n data.error.lineNumber);\n e.stack = data.error.stack;\n e.name = data.error.name;\n \n if (this.ee.listenerCount('reject') > 0) {\n this.ee.emit('reject', e);\n } else {\n // This will happen if the error is thrown during initialization\n throw e;\n }\n\n break;\n default:\n let error = new Error('Received unhandled request from sandbox: ' + data.request + '\\n\\t' + JSON.stringify(data));\n console.error(error);\n break; \n }\n }\n\n async handleRing1Message(data) {\n switch (data.request) {\n case 'applyRequirementsDone':\n // emit internally\n this.ee.emit(data.request, data)\n break;\n default:\n let error = new Error('Received unhandled request from sandbox ring 1: ' + data.request + '\\n\\t' + JSON.stringify(data));\n console.error(error)\n break; \n }\n }\n\n async handleRing2Message(data) {\n debugging('event:ring-2') && debug('event:ring-2', data);\n switch (data.request) {\n case 'dependency': {\n let moduleData;\n try {\n moduleData = await this.supervisorCache.fetchModule(data.data);\n } catch (error) {\n /*\n * In the event of an error here, we want to let the client know there was a problem in\n * loading their module. However, there hasn't yet been an actual slice assigned to the sandbox.\n * Therefore, we assign 'slice 0' to the sandbox, a slice that will never exist, and is used\n * purely for this purpose. \n */\n this.slice = {\n jobAddress: this.jobAddress,\n sliceNumber: 0,\n };\n\n const payload = {\n name: error.name,\n timestamp: error.timestamp,\n message: error.message,\n };\n\n this.emit('workEmit', {\n eventName: 'error',\n payload,\n });\n this.ee.emit('reject', error);\n break;\n }\n this.evaluatorHandle.postMessage({\n request: 'moduleGroup',\n data: moduleData,\n id: data.id,\n });\n break;\n }\n case 'error':\n /*\n * Ring 2 error messages will only fire for problems inside of the worker that are separate from\n * the work function. In most cases there are other handlers for situations where 'error' may be emitted\n * such as timeouts if the expected message isn't recieved. Thus, we will output the error, but nothing else.\n */\n console.error(data.error);\n break;\n case 'describe':\n case 'evalResult':\n case 'resetStateDone':\n case 'assigned':\n // emit internally\n this.ee.emit(data.request, data);\n break;\n case 'reject':\n // emit internally\n this.ee.emit(data.request, data.error);\n break;\n default: {\n const error = new Error(\n `Received unhandled request from sandbox ring 2. Data: ${JSON.stringify(\n data,\n null,\n 2,\n )}`,\n );\n console.error(error);\n break;\n }\n }\n }\n\n async handleRing3Message(data) {\n switch (data.request) {\n case 'complete':\n clearTimeout(this.progressTimeout);\n clearTimeout(this.sliceTimeout);\n this.progressTimeout = this.sliceTimeout = null;\n\n if (this.progress === null) {\n if (this.options.ignoreNoProgress) {\n console.warn(\"ENOPROGRESS silenced by localExec: Progress was not called during this slice's execution, in a remote sandbox this would cause the slice to be failed\");\n } else {\n // If a progress update was never received (progress === null) then reject\n this.ee.emit('reject', new NoProgressError('Sandbox never emitted a progress event.'));\n break;\n }\n }\n this.evaluatorHandle.postMessage({ request: 'resetAndGetCPUTime' })\n this.progress = 100;\n this.completeData = data;\n // The timing report and resolve will be emitted when the CPU time is received. \n break;\n case 'progress':\n let { progress, indeterminate, throttledReports, value } = data;\n this.progress = progress;\n const progressReport = {\n timestamp: Date.now() - this.sliceStartTime,\n progress,\n value,\n throttledReports,\n }\n this.progressReports.last = progressReport;\n if (!indeterminate) {\n this.progressReports.lastDeterministic = progressReport;\n }\n\n this.resetProgressTimeout();\n\n this.emit('sliceProgress', data);\n break;\n\n case 'noProgress':\n let { message } = data;\n\n this.ee.emit('reject', new NoProgressError(message));\n break;\n case 'console':\n data.payload.message = scopedKvin.marshal(data.payload.message); \n this.emit('workEmit', {\n eventName: 'console',\n payload: data.payload, \n });\n break;\n\n case 'emitEvent':/* ad-hoc event from the sandbox (work.emit) */\n this.emit('workEmit', {\n eventName: 'custom',\n payload: data.payload\n })\n break;\n case 'measurement':\n this.updateTime(data);\n break;\n case 'sandboxError': /* the sandbox itself has an error condition */\n {\n this.emit('sandboxError', data.error);\n break;\n }\n case 'workError': { /* the work function threw/rejected */\n this.emit('workEmit', {\n eventName: 'error',\n payload: data.error,\n });\n\n // Warning: rejecting here with just .data.error causes issues\n // where the reject handlers modify the object so it interferes with the\n // workEmit payload, wrapping in an Error instance copies the values\n const wrappedError = new UncaughtExceptionError(\n data.error.message,\n data.error.fileName,\n data.error.lineNumber,\n );\n wrappedError.stack = data.error.stack;\n wrappedError.name = data.error.name;\n\n if (this.ee.listenerCount('reject') > 0) {\n this.ee.emit('reject', wrappedError);\n } else {\n // This will happen if the error is thrown during initialization\n throw wrappedError;\n }\n break;\n }\n default:\n let error = new Error('Received unhandled request from sandbox ring 3: ' + data.request + '\\n\\t' + JSON.stringify(data));\n console.error(error)\n break; \n }\n }\n\n /**\n * Handles progress and completion events from sandbox.\n * Unless explicitly returned out of this function will re-emit the event\n * on @this.ee where the name of the event is event.data.request.\n *\n * @param {object} event - event received from the sandbox\n */\n async onmessage(event) {\n debugging('event') && debug('event', event);\n if (Sandbox.debugEvents) {\n console.debug('sandbox - eventDebug:', {\n id: this.id,\n state: this.state,\n event: JSON.stringify(event)\n })\n }\n\n const { data } = event;\n const ringLevel = data.ringSource\n\n // Give the data to a handler depending on ring level\n if (ringLevel === -1) {\n console.error('Message sent directly from raw postMessage. Terminating worker...');\n console.debug(event);\n return this.terminate(true);\n } else {\n const handler = this.ringMessageHandlers[ringLevel];\n if (handler) {\n handler.call(this, data.value);\n } else {\n console.warn(`No handler defined for message from ring ${ringLevel}`);\n console.debug(event);\n }\n }\n }\n\n /**\n * Error handler for the internal sandbox.\n * Currently just logs the errors that the sandbox spits out.\n */\n onerror(event) {\n console.error('Sandbox emitted an error:', event);\n this.terminate(true, true);\n }\n\n /**\n * Clears the timeout and terminates the sandbox and sometimes emits a reject event.\n *\n * @param {boolean} [reject = true] - if true emit reject event\n * @param {boolean} [immediate = false] - passed to terminate, used by standaloneWorker to immediately close the connection\n */\n terminate (reject = true, immediate = false) {\n const oldState = this.state;\n this.state = TERMINATED;\n \n clearTimeout(this.progressTimeout);\n clearTimeout(this.sliceTimeout);\n this.progressTimeout = this.sliceTimeout = null;\n \n if (this.evaluatorHandle && typeof this.evaluatorHandle.terminate === 'function') {\n try {\n this.evaluatorHandle.terminate(immediate);\n this.evaluatorHandle = null;\n } catch (e) {\n console.error(`Error terminating sandbox ${this.id} (${oldState}):`, e);\n } finally {\n this.emit('terminate', this);\n }\n }\n\n if (reject) {\n this.ee.emit('reject', new Error(`Sandbox ${this.id} (${oldState}) was terminated.`));\n }\n\n this.emit('terminated');\n }\n\n /**\n * Attempts to stop the sandbox from doing completing its current\n * set of work without terminating the working.\n */\n stop () {\n throw new Error('Sandbox.stop is not yet implemented.')\n }\n\n /**\n * ringNPostMessage can send a `measurement` request and update these\n * totals.\n */\n updateTime (measurementEvent) {\n ['total', 'CPU', 'webGL'].forEach((key) => {\n if (measurementEvent[key]) this.sliceTimeReport[key] += measurementEvent[key];\n })\n }\n\n resetSliceTimeReport () {\n this.sliceTimeReport = {\n total: 0,\n CPU: 0,\n webGL: 0,\n }\n }\n}\n\nSandbox.idCounter = 1;\nSandbox.debugWork = false;\nSandbox.debugState = false;\nSandbox.debugEvents = false;\n\nexports.Sandbox = Sandbox;\nexports.SandboxError = SandboxError;\nexports.NoProgressError = NoProgressError;\nexports.SliceTooSlowError = SliceTooSlowError;\nexports.UncaughtExceptionError = UncaughtExceptionError;\nexports.RemoteFetchError = RemoteFetchError;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/sandbox.js?");
|
|
4428
4428
|
|
|
4429
4429
|
/***/ }),
|
|
4430
4430
|
|
|
@@ -4445,7 +4445,7 @@ eval("/**\n * @file worker/slice.js\n *\n * A wrapper for the slice object retur
|
|
|
4445
4445
|
\***************************************************/
|
|
4446
4446
|
/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
|
|
4447
4447
|
|
|
4448
|
-
eval("/**\n * @file worker/supervisor-cache.js\n *\n * A cache for the supervisor, anything the supervisor\n * may request that is cacheable (the same every time its requested)\n * can be cached in this class.\n *\n * Currently only jobs and modules are cached.\n *\n * @author Matthew Palma, mpalma@kingsds.network\n * @date May 2019\n */\n\n/* global dcpConfig */\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst { justFetch, fetchURI, dumpObject } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst { EventEmitter } = __webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\n\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\nconst scopedKvin = new kvin.KVIN({Object: ({}).constructor,\n Array: ([]).constructor, \n Function: (()=>{}).constructor});\n \nclass SupervisorCache extends EventEmitter {\n constructor (supervisor) {\n \n super('SupervisorCache')\n this.supervisor = supervisor;\n this.cache = {\n job: {},\n module: {},\n dependency: {},\n }\n \n this.promises = {\n job: {},\n module: {},\n dependency: {},\n }\n\n this.lastAccess = {}\n for (let key in this.cache) {\n this.lastAccess[key] = {}\n }\n }\n\n /**\n * Returns an object listing what jobs and modules are currently cached.\n * @returns {object} - in the form: { job: [0xgen1, 0xgen2,...], modules: [modGroup1, modGroup2,...] }\n */\n get cacheDescription () {\n let description = {}\n for (let key in this.cache) {\n description[key] = Object.keys(this.cache[key])\n }\n return description\n }\n\n /**\n * Returns an array of all jobId currently cached\n * @returns {Array} all the jobId's in the cache\n */\n get jobs () {\n return Object.keys(this.cache.job);\n }\n\n /**\n * Attempts to look up an item from the cache.\n * If item is found its last access time is updated.\n *\n * @param {string} key - the cache to look in (job or module)\n * @param {string} id - the items identifier (jobAddress or module group name)\n *\n * @returns {any} The value of the stored item or null if nothing is found\n */\n fetch(key, id) {\n if (typeof this.cache[key] === 'undefined') {\n throw new Error(`${key} does not relate to any cache.`);\n }\n if (this.cache[key][id]) {\n this.lastAccess[key][id] = Date.now();\n return this.cache[key][id];\n }\n return null;\n }\n\n /**\n * Stores a fetched value for one of the caches.\n *\n * @param {string} key - the cache to store the item in\n * @param {string} id - the items identifier (job Address or module group name)\n * @param {any} value - the item to store\n *\n * @returns {any} - @value passed in\n */\n store (key, id, value) {\n if (typeof this.cache[key] === 'undefined') {\n throw new Error(`${key} does not relate to any cache.`)\n }\n this.cache[key][id] = value\n this.lastAccess[key][id] = Date.now()\n return value\n }\n\n /**\n * Removes a job or module group from the cache.\n *\n * @param {string} key - the cache to remove the item from (jobs or module groups)\n * @param {string} id - the items identifier (job Address or module group name)\n *\n * @returns {string} - @id passed in\n */\n remove (key, id) {\n if (typeof this.cache[key] === 'undefined') {\n throw new Error(`${key} does not relate to any cache.`)\n }\n delete this.cache[key][id];\n delete this.lastAccess[key][id];\n return id;\n }\n\n /** \n * Fetch a job from the job cache. If the job has components which are \n * which need to be fetched over the network, they are fetched before the\n * returned promise is resolved.\n *\n * The job cache is initially populated during fetchTask.\n */\n async fetchJob(address, allowedOrigins) {\n let job = this.fetch('job', address);\n\n if (!job) {\n let e = new Error(`No job in supervisor cache with address ${address}`);\n e.code = 'ENOJOB';\n throw e;\n }\n\n // XXXpfr Excellent tracing.\n if (debugging('worker')) {\n dumpObject(job, 'SupervisorCache.fetchJob: job', 128);\n }\n\n if (!job.workFunction) {\n job.workFunction = await fetchURI(job.codeLocation, allowedOrigins, dcpConfig.worker.allowOrigins.fetchWorkFunctions);\n if(job.requirements.useStrict)\n job.useStrict = true;\n delete job.codeLocation;\n }\n if (!job.arguments) {\n let promises = [];\n let uris = job.argumentsLocation;\n if (uris)\n for (let i = 0; i < uris.length; i++)\n promises.push(fetchURI(uris[i].value, allowedOrigins, dcpConfig.worker.allowOrigins.fetchArguments));\n\n job.arguments = await Promise.all(promises);\n \n // node localExec jobs read arguments from a file, so need to ensure they are properly parsed after being read.\n if (this.supervisor.options.localExec && !DCP_ENV.isBrowserPlatform)\n job.arguments = scopedKvin.parse(job.arguments[0]);\n\n delete job.argumentsLocation;\n }\n // if job input data is range object, we send the range object URI to the worker\n if (!job.mro && job.MROLocation) {\n job.mro = await fetchURI(job.MROLocation, allowedOrigins, dcpConfig.worker.allowOrigins.fetchWorkFunctions);\n delete job.MROLocation;\n }\n \n return job;\n }\n\n /**\n * Attempts to fetch a module group from the cache and\n * if it's not found it attempts to fetch then store\n * the module group from the package manager.\n *\n * @param {array} modulesArray - the array of modules requested \n * - (when stringified it's the identifier of the module group)\n *\n * @returns {Promise<object>} - the module group\n * @throws when the module group can not be fetched\n */\n async fetchModule(modulesArray) {\n const cacheKey = JSON.stringify(modulesArray);\n let modules = this.fetch('module', cacheKey);\n if (modules !== null) {\n return modules;\n }\n\n if (this.promises.module[cacheKey]) {\n return this.promises.module[cacheKey];\n }\n\n const {\n success,\n payload: responsePayload,\n } = await this.supervisor.packageManagerConnection.send('fetchModule', {\n modules: modulesArray,\n });\n\n if (!success) {\n /**\n * Preserving the error message by not rewrapping it with DCPError incase\n * we want to let clients know which module couldn't be fetched.\n */\n throw responsePayload;\n }\n\n this.promises.module[cacheKey] = responsePayload;\n modules = await this.promises.module[cacheKey];\n delete this.promises.module[cacheKey];\n return this.store('module', cacheKey, modules);\n }\n\n /**\n * Attempts to fetch a dependency from the cache and\n * if it's not found it attempts to fetch then store\n * the dependency from the package manager.\n *\n * @param {string} dependencyUri - The URI of the dependency\n *\n * @returns {Promise<string>} file contents\n * @throws when the dependency can not be fetched\n */\n async fetchDependency(dependencyUri) {\n let dependency = this.fetch('dependency', dependencyUri);\n if (dependency !== null) {\n return dependency;\n }\n\n if (this.promises.dependency[dependencyUri]) {\n return this.promises.dependency[dependencyUri];\n }\n\n const url = dcpConfig.packageManager.location.resolve(dependencyUri);\n this.promises.dependency[dependencyUri] = justFetch(url, 'string', 'GET', true);\n\n dependency = await this.promises.dependency[dependencyUri];\n\n delete this.promises.dependency[dependencyUri];\n\n return this.store('dependency', dependencyUri, dependency);\n }\n}\n\nexports.SupervisorCache = SupervisorCache;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/supervisor-cache.js?");
|
|
4448
|
+
eval("/**\n * @file worker/supervisor-cache.js\n *\n * A cache for the supervisor, anything the supervisor\n * may request that is cacheable (the same every time its requested)\n * can be cached in this class.\n *\n * Currently only jobs and modules are cached.\n *\n * @author Matthew Palma, mpalma@kingsds.network\n * @date May 2019\n */\n\n/* global dcpConfig */\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst { justFetch, fetchURI, dumpObject } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst { EventEmitter } = __webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\n\nconst kvin = __webpack_require__(/*! kvin */ \"./node_modules/kvin/kvin.js\");\nconst scopedKvin = new kvin.KVIN({Object: ({}).constructor,\n Array: ([]).constructor, \n Function: (()=>{}).constructor});\n \nclass SupervisorCache extends EventEmitter {\n constructor (supervisor) {\n \n super('SupervisorCache')\n this.supervisor = supervisor;\n this.cache = {\n job: {},\n module: {},\n dependency: {},\n }\n \n this.promises = {\n job: {},\n module: {},\n dependency: {},\n }\n\n this.lastAccess = {}\n for (let key in this.cache) {\n this.lastAccess[key] = {}\n }\n }\n\n /**\n * Returns an object listing what jobs and modules are currently cached.\n * @returns {object} - in the form: { job: [0xgen1, 0xgen2,...], modules: [modGroup1, modGroup2,...] }\n */\n get cacheDescription () {\n let description = {}\n for (let key in this.cache) {\n description[key] = Object.keys(this.cache[key])\n }\n return description\n }\n\n /**\n * Returns an array of all jobId currently cached\n * @returns {Array} all the jobId's in the cache\n */\n get jobs () {\n return Object.keys(this.cache.job);\n }\n\n /**\n * Attempts to look up an item from the cache.\n * If item is found its last access time is updated.\n *\n * @param {string} key - the cache to look in (job or module)\n * @param {string} id - the items identifier (jobAddress or module group name)\n *\n * @returns {any} The value of the stored item or null if nothing is found\n */\n fetch(key, id) {\n if (typeof this.cache[key] === 'undefined') {\n throw new Error(`${key} does not relate to any cache.`);\n }\n if (this.cache[key][id]) {\n this.lastAccess[key][id] = Date.now();\n return this.cache[key][id];\n }\n return null;\n }\n\n /**\n * Stores a fetched value for one of the caches.\n *\n * @param {string} key - the cache to store the item in\n * @param {string} id - the items identifier (job Address or module group name)\n * @param {any} value - the item to store\n *\n * @returns {any} - @value passed in\n */\n store (key, id, value) {\n if (typeof this.cache[key] === 'undefined') {\n throw new Error(`${key} does not relate to any cache.`)\n }\n this.cache[key][id] = value\n this.lastAccess[key][id] = Date.now()\n return value\n }\n\n /**\n * Removes a job or module group from the cache.\n *\n * @param {string} key - the cache to remove the item from (jobs or module groups)\n * @param {string} id - the items identifier (job Address or module group name)\n *\n * @returns {string} - @id passed in\n */\n remove (key, id) {\n if (typeof this.cache[key] === 'undefined') {\n throw new Error(`${key} does not relate to any cache.`)\n }\n delete this.cache[key][id];\n delete this.lastAccess[key][id];\n return id;\n }\n\n /** \n * Fetch a job from the job cache. If the job has components which are \n * which need to be fetched over the network, they are fetched before the\n * returned promise is resolved.\n *\n * The job cache is initially populated during fetchTask.\n */\n async fetchJob(address, allowedOrigins) {\n let job = this.fetch('job', address);\n\n if (!job) {\n let e = new Error(`No job in supervisor cache with address ${address}`);\n e.code = 'ENOJOB';\n throw e;\n }\n\n // XXXpfr Excellent tracing.\n if (debugging('worker')) {\n dumpObject(job, 'SupervisorCache.fetchJob: job', 128);\n }\n\n if (!job.workFunction) {\n job.workFunction = await fetchURI(job.codeLocation, allowedOrigins, dcpConfig.worker.allowOrigins.fetchWorkFunctions);\n if(job.requirements.useStrict)\n job.useStrict = true;\n delete job.codeLocation;\n }\n if (!job.arguments) {\n let promises = [];\n let uris = job.argumentsLocation;\n if (uris)\n for (let i = 0; i < uris.length; i++)\n promises.push(fetchURI(uris[i].value, allowedOrigins, dcpConfig.worker.allowOrigins.fetchArguments));\n\n job.arguments = await Promise.all(promises);\n \n // node localExec jobs read arguments from a file, so need to ensure they are properly parsed after being read.\n if (this.supervisor.options.localExec && !DCP_ENV.isBrowserPlatform)\n job.arguments = scopedKvin.parse(job.arguments[0]);\n\n delete job.argumentsLocation;\n }\n // if job input data is range object, we send the range object URI to the worker\n if (!job.mro && job.MROLocation) {\n \n if (this.supervisor.options.localExec && !DCP_ENV.isBrowserPlatform)\n {\n job.MROLocation = (await fetchURI(\n job.MROLocation,\n allowedOrigins,\n dcpConfig.worker.allowOrigins.fetchData,\n ));\n }\n \n job.mro = await fetchURI(job.MROLocation, allowedOrigins, dcpConfig.worker.allowOrigins.fetchWorkFunctions);\n delete job.MROLocation;\n }\n \n return job;\n }\n\n /**\n * Attempts to fetch a module group from the cache and\n * if it's not found it attempts to fetch then store\n * the module group from the package manager.\n *\n * @param {array} modulesArray - the array of modules requested \n * - (when stringified it's the identifier of the module group)\n *\n * @returns {Promise<object>} - the module group\n * @throws when the module group can not be fetched\n */\n async fetchModule(modulesArray) {\n const cacheKey = JSON.stringify(modulesArray);\n let modules = this.fetch('module', cacheKey);\n if (modules !== null) {\n return modules;\n }\n\n if (this.promises.module[cacheKey]) {\n return this.promises.module[cacheKey];\n }\n\n const {\n success,\n payload: responsePayload,\n } = await this.supervisor.packageManagerConnection.send('fetchModule', {\n modules: modulesArray,\n });\n\n if (!success) {\n /**\n * Preserving the error message by not rewrapping it with DCPError incase\n * we want to let clients know which module couldn't be fetched.\n */\n throw responsePayload;\n }\n\n this.promises.module[cacheKey] = responsePayload;\n modules = await this.promises.module[cacheKey];\n delete this.promises.module[cacheKey];\n return this.store('module', cacheKey, modules);\n }\n\n /**\n * Attempts to fetch a dependency from the cache and\n * if it's not found it attempts to fetch then store\n * the dependency from the package manager.\n *\n * @param {string} dependencyUri - The URI of the dependency\n *\n * @returns {Promise<string>} file contents\n * @throws when the dependency can not be fetched\n */\n async fetchDependency(dependencyUri) {\n let dependency = this.fetch('dependency', dependencyUri);\n if (dependency !== null) {\n return dependency;\n }\n\n if (this.promises.dependency[dependencyUri]) {\n return this.promises.dependency[dependencyUri];\n }\n\n const url = dcpConfig.packageManager.location.resolve(dependencyUri);\n this.promises.dependency[dependencyUri] = justFetch(url, 'string', 'GET', true);\n\n dependency = await this.promises.dependency[dependencyUri];\n\n delete this.promises.dependency[dependencyUri];\n\n return this.store('dependency', dependencyUri, dependency);\n }\n}\n\nexports.SupervisorCache = SupervisorCache;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/supervisor-cache.js?");
|
|
4449
4449
|
|
|
4450
4450
|
/***/ }),
|
|
4451
4451
|
|
|
@@ -4456,7 +4456,7 @@ eval("/**\n * @file worker/supervisor-cache.js\n *\n * A cache for the superviso
|
|
|
4456
4456
|
/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
|
|
4457
4457
|
|
|
4458
4458
|
"use strict";
|
|
4459
|
-
eval("/* provided dependency */ var process = __webpack_require__(/*! ./node_modules/process/browser.js */ \"./node_modules/process/browser.js\");\n/**\n * @file worker/supervisor.js\n *\n * The component that controls each of the sandboxes\n * and distributes work to them. Also communicates with the\n * scheduler to fetch said work.\n *\n * The supervisor readies sandboxes before/while fetching slices.\n * This means sometimes there are extra instantiated WebWorkers\n * that are idle (in this.readiedSandboxes). Readied sandboxes can\n * be used for any slice. After a readied sandbox is given a slice\n * it becomes assigned to slice's job and can only do work\n * for that job.\n *\n * After a sandbox completes its work, the sandbox becomes cached\n * and can be reused if another slice with a matching job is fetched.\n *\n * @author Matthew Palma, mpalma@kingsds.network\n * Ryan Rossiter, ryan@kingsds.network\n * @date May 2019\n */\n\n/* global dcpConfig */\n// @ts-check\n\n\nconst constants = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst hash = __webpack_require__(/*! dcp/common/hash */ \"./src/common/hash.js\");\nconst wallet = __webpack_require__(/*! dcp/dcp-client/wallet */ \"./src/dcp-client/wallet/index.js\");\nconst protocolV4 = __webpack_require__(/*! dcp/protocol-v4 */ \"./src/protocol-v4/index.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst { assert } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\nconst { EventEmitter } = __webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\");\nconst { Sandbox, SandboxError } = __webpack_require__(/*! ./sandbox */ \"./src/dcp-client/worker/sandbox.js\");\nconst { Slice, SLICE_STATUS_UNASSIGNED, SLICE_STATUS_FAILED } = __webpack_require__(/*! ./slice */ \"./src/dcp-client/worker/slice.js\");\nconst { SupervisorCache } = __webpack_require__(/*! ./supervisor-cache */ \"./src/dcp-client/worker/supervisor-cache.js\");\nconst { DcpURL } = __webpack_require__(/*! dcp/common/dcp-url */ \"./src/common/dcp-url.js\");\nconst { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\nconst { localStorage } = __webpack_require__(/*! dcp/common/dcp-localstorage */ \"./src/common/dcp-localstorage.js\");\nconst { booley, encodeDataURI, makeDataURI, leafMerge, a$sleepMs, justFetch, compressJobMap, toJobMap,\n compressSandboxes, compressSlices, truncateAddress, dumpSandboxesIfNotUnique, dumpSlicesIfNotUnique, generateOpaqueId } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst { sliceStatus } = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst { calculateJoinHash } = __webpack_require__(/*! dcp/dcp-client/compute-groups */ \"./src/dcp-client/compute-groups/index.js\");\nconst RingBuffer = __webpack_require__(/*! dcp/utils/ringBuffer */ \"./src/utils/ringBuffer.js\");\nconst supervisorTuning = dcpConfig.future('worker.tuning');\nconst tuning = {\n watchdogInterval: 7, /**< seconds - time between fetches when ENOTASK(? /wg nov 2019) */\n minSandboxStartDelay: 0.1, /**< seconds - minimum time between WebWorker starts */\n maxSandboxStartDelay: 0.7, /**< seconds - maximum delay time between WebWorker starts */\n ...supervisorTuning\n};\n\n/** Make timers 10x slower when running in niim */\nlet timeDilation = 1;\nif (DCP_ENV.platform === 'nodejs') {\n /** Make timers 10x slower when running in niim */\n timeDilation = (requireNative('module')._cache.niim instanceof requireNative('module').Module) ? 10 : 1;\n}\n\ndcpConfig.future('worker.sandbox', { progressReportInterval: (5 * 60 * 1000) });\nconst sandboxTuning = dcpConfig.worker.sandbox;\n\n/**\n * @typedef {*} address\n * @typedef {*} opaqueId\n */\n\n/**\n * @typedef {object} SandboxSlice\n * @property {Sandbox} sandbox\n * @property {Slice} slice\n */\n\n/**\n * @typedef {object} Signature\n * @property {Uint8Array} r\n * @property {Uint8Array} s\n * @property {Uint8Array} v\n */\n\n/**\n * @typedef {object} SignedAuthorizationMessageObject\n * @property {object} auth\n * @property {Signature} signature\n * @property {module:dcp/wallet.Address} owner\n */\n\n/** @typedef {import('.').Worker} Worker */\n/** @typedef {import('.').SupervisorOptions} SupervisorOptions */\n\nclass Supervisor extends EventEmitter {\n /**\n * @constructor\n * @param {Worker} worker\n * @param {SupervisorOptions} options\n */\n constructor (worker, options={}) {\n super('Supervisor');\n\n /** @type {Worker} */\n this.worker = worker;\n\n /** @type {Sandbox[]} */\n this.sandboxes = [];\n\n /** @type {Sandbox[]} */\n this.readiedSandboxes = [];\n\n /** @type {Sandbox[]} */\n this.assignedSandboxes = [];\n\n /** @type {Slice[]} */\n this.slices = [];\n\n /** @type {Slice[]} */\n this.queuedSlices = [];\n\n /** @type {Slice[]} */\n this.lostSlices = [];\n\n /** @type {boolean} */\n this.matching = false;\n\n /** @type {boolean} */\n this.isFetchingNewWork = false;\n\n /** @type {number} */\n this.numberOfCoresReserved = 0;\n\n /** @type {number} */\n this.addressTruncationLength = 20; // Set to -1 for no truncation.\n\n /** @type {Object[]} */\n this.rejectedJobs = [];\n this.rejectedJobReasons = [];\n\n if (!options) {\n console.error('Supervisor Options', options, new Error().stack);\n options = {};\n }\n\n /** @type {object} */\n this.options = {\n jobAddresses: options.jobAddresses || [/* all jobs unless priorityOnly */],\n ...options,\n };\n\n const { paymentAddress, identityKeystore } = options;\n if (paymentAddress) {\n if (paymentAddress instanceof wallet.Keystore) {\n this.paymentAddress = paymentAddress.address;\n } else {\n this.paymentAddress = new wallet.Address(paymentAddress);\n }\n } else {\n this.paymentAddress = null;\n }\n\n this._identityKeystore = identityKeystore;\n\n // In localExec, do not allow work function or arguments to come from the 'any' origins\n this.allowedOrigins = []\n if (this.options.localExec)\n {\n dcpConfig.worker.allowOrigins.fetchData = dcpConfig.worker.allowOrigins.fetchData.concat(dcpConfig.worker.allowOrigins.any)\n dcpConfig.worker.allowOrigins.sendResults = dcpConfig.worker.allowOrigins.sendResults.concat(dcpConfig.worker.allowOrigins.any)\n }\n else\n this.allowedOrigins = dcpConfig.worker.allowOrigins.any;\n\n if(options.allowedOrigins && options.allowedOrigins.length > 0)\n this.allowedOrigins = options.allowedOrigins.concat(this.allowedOrigins);\n\n /**\n * Maximum sandboxes allowed to work at a given time.\n * @type {number}\n */\n this.maxWorkingSandboxes = options.maxWorkingSandboxes || 1;\n\n /** @type {number} */\n this.defaultMaxGPUs = 1;\n // this.GPUsAssigned = 0;\n \n // Object.defineProperty(this, 'GPUsAssigned', {\n // get: () => this.allocatedSandboxes.filter(sb => !!sb.requiresGPU).length,\n // enumerable: true,\n // configurable: false,\n // });\n\n /**\n * TODO: Remove this when the supervisor sends all of the sandbox\n * capabilities to the scheduler when fetching work.\n * @type {object}\n */\n this.capabilities = null;\n\n /** @type {number} */\n this.lastProgressReport = 0;\n\n /** \n * An N-slot ring buffer of job addresses. Stores all jobs that have had no more than 1 slice run in the ring buffer.\n * Required for the implementation of discrete jobs \n * @type {RingBuffer} \n */\n this.ringBufferofJobs = new RingBuffer(200); // N = 200 should be more than enough.\n \n // @hack - dcp-env.isBrowserPlatform is not set unless the platform is _explicitly_ set,\n // using the default detected platform doesn't set it.\n // Fixing that causes an error in the wallet module's startup on web platform, which I\n // probably can't fix in a reasonable time this morning.\n // ~ER2020-02-20\n\n if (!options.maxWorkingSandboxes\n && DCP_ENV.browserPlatformList.includes(DCP_ENV.platform)\n && navigator.hardwareConcurrency > 1) {\n this.maxWorkingSandboxes = navigator.hardwareConcurrency - 1;\n if (typeof navigator.userAgent === 'string') {\n if (/(Android).*(Chrome|Chromium)/.exec(navigator.userAgent)) {\n this.maxWorkingSandboxes = 1;\n console.log('Doing work with Chromimum browsers on Android is currently limited to one sandbox');\n }\n }\n }\n\n /** @type {SupervisorCache} */\n this.cache = new SupervisorCache(this);\n /** @type {object} */\n this._connections = {}; /* active DCPv4 connections */\n // Call the watchdog every 7 seconds.\n this.watchdogInterval = setInterval(() => this.watchdog(), tuning.watchdogInterval * 1000);\n if (DCP_ENV.platform === 'nodejs' && this.options.localExec) /* interval helps keep normal worker alive forever, which we don't want in localexec */\n this.watchdogInterval.unref();\n\n const ceci = this;\n\n // Initialize to null so these properties are recognized for the Supervisor class\n this.taskDistributorConnection = null;\n this.eventRouterConnection = null;\n this.resultSubmitterConnection = null;\n this.packageManagerConnection = null;\n this.openTaskDistributorConn = function openTaskDistributorConn()\n {\n let config = dcpConfig.scheduler.services.taskDistributor;\n ceci.taskDistributorConnection = new protocolV4.Connection(config, ceci.identityKeystore, connectionOptions(config.location, 'taskDistributor'));\n ceci.taskDistributorConnection.on('close', ceci.openTaskDistributorConn);\n }\n\n this.openEventRouterConn = function openEventRouterConn()\n {\n let config = dcpConfig.scheduler.services.eventRouter;\n ceci.eventRouterConnection = new protocolV4.Connection(config, ceci.identityKeystore, connectionOptions(config.location, 'eventRouter'));\n ceci.eventRouterConnection.on('close', ceci.openEventRouterConn);\n if (ceci.eventRouterMessageQueue.length)\n ceci.resendRejectedMessages(ceci.eventRouterConnection, ceci.eventRouterMessageQueue);\n }\n this.eventRouterMessageQueue = [];\n \n this.openResultSubmitterConn = function openResultSubmitterConn()\n {\n let config = dcpConfig.scheduler.services.resultSubmitter;\n ceci.resultSubmitterConnection = new protocolV4.Connection(config, ceci.identityKeystore, connectionOptions(config.location, 'resultSubmitter'));\n ceci.resultSubmitterConnection.on('close', ceci.openResultSubmitterConn);\n if (ceci.resultSubmitterMessageQueue.length)\n ceci.resendRejectedMessages(ceci.resultSubmitterConnection, ceci.resultSubmitterMessageQueue);\n }\n this.resultSubmitterMessageQueue = [];\n\n this.openPackageManagerConn = function openPackageManagerConn()\n {\n let config = dcpConfig.packageManager;\n ceci.packageManagerConnection = new protocolV4.Connection(config, ceci.identityKeystore, connectionOptions(config.location, 'packageManager'));\n ceci.packageManagerConnection.on('close', ceci.openPackageManagerConn);\n if (ceci.packageManagerMessageQueue.length)\n ceci.resendRejectedMessages(ceci.packageManagerConnection, ceci.packageManagerMessageQueue);\n }\n this.packageManagerMessageQueue = [];\n }\n\n /**\n * Return worker opaqueId.\n * @type {opaqueId}\n */\n get workerOpaqueId() {\n if (!this._workerOpaqueId)\n this._workerOpaqueId = localStorage.getItem('workerOpaqueId');\n\n if (!this._workerOpaqueId || this._workerOpaqueId.length !== constants.workerIdLength) {\n this._workerOpaqueId = generateOpaqueId();\n localStorage.setItem('workerOpaqueId', this._workerOpaqueId);\n }\n\n return this._workerOpaqueId;\n }\n\n /**\n * This getter is the absolute source-of-truth for what the\n * identity keystore is for this instance of the Supervisor.\n */\n get identityKeystore() {\n assert(this.defaultIdentityKeystore);\n\n return this._identityKeystore || this.defaultIdentityKeystore;\n }\n\n /**\n * Open all connections. Used when supervisor is instantiated or stopped/started\n * to initially open connections.\n */\n instantiateAllConnections() {\n if (!this.taskDistributorConnection)\n this.openTaskDistributorConn();\n \n if (!this.eventRouterConnection)\n this.openEventRouterConn();\n \n if (!this.resultSubmitterConnection)\n this.openResultSubmitterConn();\n\n if (!this.packageManagerConnection)\n this.openPackageManagerConn();\n }\n \n /**\n * Try sending messages that were rejected on an old instance of the given connection.\n */\n async resendRejectedMessages(connection, messageQueue) {\n var message = messageQueue.shift();\n\n do {\n try {\n await connection.send(message.operation, message.data);\n } catch (e) {\n debugging('supervisor') && console.error(`Failed to send message ${message.operation} to scheduler: ${e}. Will try again on a new connection.`);\n messageQueue.unshift(message);\n connection.close();\n break;\n }\n message = messageQueue.shift();\n } while (message)\n }\n\n /** Set the default identity keystore -- needs to happen before anything that talks\n * to the scheduler for work gets called. This is a wart and should be removed by\n * refactoring.\n *\n * The default identity keystore will be used if the Supervisor was not provided\n * with an alternate. This keystore will be located via the Wallet API, and \n * if not found, a randomized default identity will be generated. \n *\n * @param {object} ks An instance of wallet::Keystore -- if undefined, we pick the best default we can.\n * @returns {Promise<void>}\n */\n async setDefaultIdentityKeystore(ks) {\n try {\n if (ks) {\n this.defaultIdentityKeystore = ks;\n return;\n }\n\n if (this.defaultIdentityKeystore)\n return;\n\n try {\n this.defaultIdentityKeystore = await wallet.getId();\n } catch(e) {\n debugging('supervisor') && console.debug('Error generating default identity, try to do it another way.');\n this.defaultIdentityKeystore = await new wallet.IdKeystore(null, '');\n }\n } finally {\n if (this.defaultIdentityKeystore)\n debugging('supervisor') && console.debug('Set default identity =', this.defaultIdentityKeystore.address);\n else\n debugging('supervisor') && console.debug('Failed to set default identity, worker cannot work.');\n }\n }\n\n //\n // What follows is a bunch of utility properties and functions for creating filtered views\n // of the slices and sandboxes array.\n //\n /** XXXpfr @todo Write sort w/o using promises so we can get rid of async on all the compress functions. */\n\n /**\n * @deprecated -- Please do not use this.workingSandboxes; use this.allocatedSandboxes instead.\n * Sandboxes that are in WORKING state.\n *\n * Warning: Do not rely on this information being 100% accurate -- it may change in the next instant.\n * @type {Sandbox[]}\n */\n get workingSandboxes() {\n return this.sandboxes.filter(sandbox => sandbox.isWorking);\n }\n\n /**\n * Use instead of this.workingSandboxes.\n *\n * When a sandbox is paired with a slice, execution is pending and sandbox.allocated=true and\n * sandbox.slice=slice and sandbox.jobAddress=slice.jobAddress. This is what 'allocated' means.\n * Immediately upon the exit of sandbox.work, sandbox.allocated=false is set and if an exception\n * wasn't thrown the sandbox is placed in this.assignedSandboxes.\n * Thus from the pov of supervisor, this.allocatedSandboxes is deterministic and this.workingSandboxes is not.\n * Please try to not use this.workingSandboxes. It is deprecated.\n *\n * Warning: Do not rely on this information being 100% accurate -- it may change in the next instant.\n * @type {Sandbox[]}\n */\n get allocatedSandboxes() {\n return this.sandboxes.filter(sandbox => sandbox.allocated);\n }\n\n /**\n * Slices that are allocated.\n * Warning: Do not rely on this information being 100% accurate -- it may change in the next instant.\n * @type {Slice[]}\n */\n get allocatedSlices() {\n return this.slices.filter(slice => slice.allocated);\n }\n\n /**\n * This property is used as the target number of sandboxes to be associated with slices and start working.\n *\n * It is used in this.watchdog as to prevent a call to this.work when unallocatedSpace <= 0.\n * It is also used in this.distributeQueuedSlices where it is passed as an argument to this.matchSlicesWithSandboxes to indicate how many sandboxes\n * to associate with slices and start working.\n *\n * Warning: Do not rely on this information being 100% accurate -- it may change in the next instant.\n * @type {number}\n */\n get unallocatedSpace() {\n return this.maxWorkingSandboxes - this.allocatedSandboxes.length - this.numberOfCoresReserved;\n }\n \n /**\n * Call acquire(numberOfCoresToReserve) to reserve numberOfCoresToReserve unallocated sandboxes as measured by unallocatedSpace.\n * Call release() to undo the previous acquire.\n * This pseudo-mutex technique helps prevent races in scheduling slices in Supervisor.\n * @param {number} numberOfCoresToReserve\n */\n acquire(numberOfCoresToReserve) { \n this.numberOfCoresReserved = numberOfCoresToReserve; \n }\n release() { \n this.numberOfCoresReserved = 0; \n }\n\n /**\n * Remove from this.slices.\n * @param {Slice} slice\n */\n removeSlice(slice) {\n this.removeElement(this.slices, slice);\n if (Supervisor.debugBuild) {\n if (this.queuedSlices.indexOf(slice) !== -1)\n throw new Error(`removeSlice: slice ${slice.identifier} is in queuedSlices; inconsistent state.`);\n if (this.lostSlices.length > 0) {\n console.warn(`removeSlice: slice ${slice.identifier}, found lostSlices ${this.lostSlices.map(s => s.identifier)}`);\n if (this.lostSlices.indexOf(slice) !== -1)\n throw new Error(`removeSlice: slice ${slice.identifier} is in lostSlices; inconsistent state.`);\n }\n }\n }\n\n /**\n * Remove from this.slices.\n * @param {Slice[]} slices\n */\n removeSlices(slices) {\n this.slices = this.slices.filter(slice => slices.indexOf(slice) === -1);\n }\n\n /**\n * Remove from this.queuedSlices.\n * @param {Slice[]} slices\n */\n removeQueuedSlices(slices) {\n this.queuedSlices = this.queuedSlices.filter(slice => slices.indexOf(slice) === -1);\n }\n\n /**\n * Remove from this.sandboxes, this.assignedSandboxes and this.readiedSandboxes.\n * @param {Sandbox} sandbox\n */\n removeSandbox(sandbox) {\n debugging('scheduler') && console.log(`removeSandbox ${sandbox.identifier}`);\n this.removeElement(this.sandboxes, sandbox);\n this.removeElement(this.assignedSandboxes, sandbox);\n\n // XXXpfr: April 13, 2022\n // I'm trying to understand and control when sandboxes get removed.\n // A sandbox in this.readiedSandboxes should never have returnSandbox/removeSandbox called on it except in stopWork.\n // Because of races and random worker crashes, it is hard to get this right, but I want to try.\n // If I don't fix this is the next 30 days or I forget, please delete this exception.\n if (false)\n {}\n\n this.removeElement(this.readiedSandboxes, sandbox);\n }\n\n /**\n * Remove from this.sandboxes and this.assignedSandboxes .\n * @param {Sandbox[]} sandboxes\n */\n async removeSandboxes(sandboxes) {\n debugging('scheduler') && console.log(`removeSandboxes: Remove ${sandboxes.length} sandboxes ${this.dumpSandboxes(sandboxes)}`);\n this.sandboxes = this.sandboxes.filter(sandbox => sandboxes.indexOf(sandbox) === -1);\n this.assignedSandboxes = this.assignedSandboxes.filter(sandbox => sandboxes.indexOf(sandbox) === -1);\n\n if (Supervisor.debugBuild) {\n const readied = this.readiedSandboxes.filter(sandbox => sandboxes.indexOf(sandbox) !== -1);\n if (readied.length > 0)\n throw new Error(`removeSandboxes: sandboxes ${readied.map(s => s.identifier)} are in readiedSandboxes; inconsistent state.`);\n }\n }\n\n /**\n * Remove element from theArray.\n * @param {Array<*>} theArray\n * @param {object|number} element\n * @param {boolean} [assertExists = true]\n */\n removeElement(theArray, element, assertExists = false) {\n let index = theArray.indexOf(element);\n assert(index !== -1 || !assertExists);\n if (index !== -1) theArray.splice(index, 1);\n }\n\n /**\n * Log sliceArray.\n * @param {Slice[]} sliceArray\n * @param {string} [header]\n * @returns {string}\n */\n dumpSlices(sliceArray, header) {\n if (header) console.log(`\\n${header}`);\n return compressSlices(sliceArray, this.addressTruncationLength);\n }\n\n /**\n * Log sandboxArray.\n * @param {Sandbox[]} sandboxArray\n * @param {string} [header]\n * @returns {string}\n */\n dumpSandboxes(sandboxArray, header) {\n if (header) console.log(`\\n${header}`);\n return compressSandboxes(sandboxArray, this.addressTruncationLength);\n }\n\n /**\n * If the elements of sandboxSliceArray are not unique, log the duplicates and dump the array.\n * @param {SandboxSlice[]} sandboxSliceArray\n * @param {string} header\n */\n dumpSandboxSlicesIfNotUnique(sandboxSliceArray, header) {\n if (!this.isUniqueSandboxSlices(sandboxSliceArray, header))\n console.log(this.dumpSandboxSlices(sandboxSliceArray));\n }\n\n /**\n * Log { sandbox, slice }.\n * @param {Sandbox} sandbox\n * @param {Slice} slice\n * @returns {string}\n */\n dumpSandboxAndSlice(sandbox, slice) {\n return `${sandbox.id}~${slice.sliceNumber}.${this.dumpJobAddress(slice.jobAddress)}`;\n }\n\n /**\n * Log { sandbox, slice } with state/status.\n * @param {Sandbox} sandbox\n * @param {Slice} slice\n * @returns {string}\n */\n dumpStatefulSandboxAndSlice(sandbox, slice) {\n return `${sandbox.id}.${sandbox.state}~${slice.sliceNumber}.${this.dumpJobAddress(slice.jobAddress)}.${slice.status}`;\n }\n\n /**\n * Truncates jobAddress.toString() to this.addressTruncationLength digits.\n * @param {address} jobAddress\n * @returns {string}\n */\n dumpJobAddress(jobAddress) {\n return truncateAddress(jobAddress, this.addressTruncationLength /* digits*/);\n }\n\n /**\n * Dump sandboxSliceArray.\n * @param {SandboxSlice[]} sandboxSliceArray - input array of { sandbox, slice }\n * @param {string} [header] - optional header\n * @param {boolean} [stateFul] - when true, also includes slice.status and sandbox.state.\n * @returns {string}\n */\n dumpSandboxSlices(sandboxSliceArray, header, stateFul=false) {\n if (header) console.log(`\\n${header}`);\n const jobMap = {};\n sandboxSliceArray.forEach(ss => {\n const sss = stateFul ? `${ss.sandbox.id}.${ss.sandbox.state}~${ss.slice.sliceNumber}.${ss.slice.status}` : `${ss.sandbox.id}~${ss.slice.sliceNumber}`;\n if (!jobMap[ss.slice.jobAddress]) jobMap[ss.slice.jobAddress] = sss;\n else jobMap[ss.slice.jobAddress] += `,${sss}`;\n });\n let output = '';\n for (const [jobAddress, sss] of Object.entries(jobMap))\n output += `${this.dumpJobAddress(jobAddress)}:[${sss}]:`;\n return output;\n }\n\n /**\n * Check sandboxSliceArray for duplicates.\n * @param {SandboxSlice[]} sandboxSliceArray\n * @param {string} [header]\n * @param {function} [log]\n * @returns {boolean}\n */\n isUniqueSandboxSlices(sandboxSliceArray, header, log) {\n const result = [], slices = [], sandboxes = [];\n let once = true;\n sandboxSliceArray.forEach(x => {\n const sliceIndex = slices.indexOf(x.slice);\n const sandboxIndex = sandboxes.indexOf(x.sandbox);\n\n if (sandboxIndex >= 0) {\n if (once && header) console.log(`\\n${header}`); once = false;\n log ? log(x.sandbox) : console.log(`\\tWarning: Found duplicate sandbox ${x.sandbox.identifier}.`);\n } else sandboxes.push(x.sandbox);\n\n if (sliceIndex >= 0) {\n if (once && header) console.log(`\\n${header}`); once = false;\n log ? log(x.slice) : console.log(`\\tWarning: Found duplicate slice ${x.slice.identifier}.`);\n } else {\n slices.push(x.slice);\n if (sandboxIndex < 0) result.push(x);\n }\n });\n return sandboxSliceArray.length === result.length;\n }\n\n /**\n * Attempts to create and start a given number of sandboxes.\n * The sandboxes that are created can then be assigned for a\n * specific job at a later time. All created sandboxes\n * get put into the @this.readiedSandboxes array when allocateLocalSandboxes is false.\n *\n * @param {number} numSandboxes - the number of sandboxes to create\n * @param {boolean} [allocateLocalSandboxes=false] - when true, do not place in this.readiedSandboxes\n * @returns {Promise<Sandbox[]>} - resolves with array of created sandboxes, rejects otherwise\n * @throws when given a numSandboxes is not a number or if numSandboxes is Infinity\n */\n async readySandboxes (numSandboxes, allocateLocalSandboxes = false) {\n debugging('supervisor') && console.debug(`readySandboxes: Readying ${numSandboxes} sandboxes, total sandboxes ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n \n if (typeof numSandboxes !== 'number' || Number.isNaN(numSandboxes) || numSandboxes === Infinity) {\n throw new Error(`${numSandboxes} is not a number of sandboxes that can be readied.`);\n }\n if (numSandboxes <= 0) {\n return [];\n }\n\n const sandboxStartPromises = [];\n const sandboxes = [];\n const errors = [];\n for (let i = 0; i < numSandboxes; i++) {\n const sandbox = new Sandbox(this.cache, {\n ...this.options.sandboxOptions,\n }, this.allowedOrigins);\n sandbox.addListener('ready', () => this.emit('sandboxReady', sandbox));\n sandbox.addListener('start', () => {\n this.emit('sandboxStart', sandbox);\n\n // When sliceNumber == 0, result-submitter status skips the slice,\n // so don't send it in the first place.\n // The 'start' event is fired when a worker starts up, hence there's no way\n // to determine whether sandbox has a valid slice without checking.\n if (sandbox.slice) {\n const jobAddress = sandbox.jobAddress;\n const sliceNumber = sandbox.slice.sliceNumber;\n // !authorizationMessage <==> sliceNumber === 0.\n const authorizationMessage = sandbox.slice.getAuthorizationMessage();\n\n if (authorizationMessage) {\n let statusPayload = {\n worker: this.workerOpaqueId,\n slices: [{\n job: jobAddress,\n sliceNumber: sliceNumber,\n status: 'begin',\n authorizationMessage,\n }],\n }\n \n try /* resultSubmitterConnection can be null if worker is stopped */\n {\n this.resultSubmitterConnection.send('status', statusPayload).catch((error) => {\n debugging('supervisor') && console.error(`Error sending 'status' for slice ${sliceNumber} of job ${jobAddress}:\\n ${error}\\nWill try again on a new connection`);\n this.resultSubmitterMessageQueue.push({ operation: 'status', data: statusPayload });\n this.resultSubmitterConnection.close();\n });\n } catch (error)\n {\n debugging('supervisor') && console.error(`Failed to send 'status' for slice ${sliceNumber} of job ${jobAddress}:, no connection to result submitter:`, error);\n }\n }\n }\n });\n sandbox.addListener('workEmit', ({ eventName, payload }) => {\n // Need to check if the sandbox hasn't been assigned a slice yet.\n if (!sandbox.slice) {\n if (Supervisor.debugBuild) {\n console.error(\n `Sandbox not assigned a slice before sending workEmit message to scheduler. 'workEmit' event originates from \"${eventName}\" event`, \n payload,\n );\n }\n }\n else\n {\n const jobAddress = sandbox.slice.jobAddress;\n const sliceNumber = sandbox.slice.sliceNumber;\n // sliceNumber can be zero if it came from a problem with loading modules.\n assert(jobAddress && (sliceNumber || sliceNumber === 0));\n // Send a work emit message from the sandbox to the event router\n // !authorizationMessage <==> sliceNumber === 0.\n let authorizationMessage;\n try {\n // Sometimes a sliceNumber===0 workEmit comes in before the client bundle is properly loaded.\n // Also happens with minor dcp-client version mismatches.\n authorizationMessage = sandbox.slice.getAuthorizationMessage();\n } catch(e) {\n authorizationMessage = null;\n }\n\n if (!authorizationMessage)\n {\n console.warn(`workEmit: missing authorization message for job ${jobAddress}, slice: ${sliceNumber}`);\n return Promise.resolve();\n }\n \n let workEmitPayload = {\n eventName,\n payload,\n job: jobAddress,\n slice: sliceNumber,\n worker: this.workerOpaqueId,\n authorizationMessage,\n }\n \n const workEmitPromise = this.eventRouterConnection.send('workEmit', workEmitPayload).catch(error => {\n debugging('supervisor') && console.warn(`workEmit: unable to send ${eventName} for slice ${sliceNumber} of job ${jobAddress}: ${error.message}.\\nTrying again on a new connection.`);\n this.eventRouterMessageQueue.push({ operation: 'workEmit', data: workEmitPayload })\n this.eventRouterConnection.close();\n if (Supervisor.debugBuild)\n console.error('workEmit error:', error);\n });\n\n if (Supervisor.debugBuild) {\n workEmitPromise.then(result => {\n if (!result || !result.success)\n console.warn('workEmit: event router did not accept event', result);\n });\n }\n }\n });\n\n // When any sbx completes, \n sandbox.addListener('complete', () => {\n this.watchdog();\n });\n\n sandbox.on('sandboxError', (error) => handleSandboxError(this, sandbox, error));\n \n sandbox.on('rejectedWorkMetrics', (data) =>{\n function updateRejectedMetrics(report) {\n ['total', 'CPU', 'webGL'].forEach((key) => {\n if (report[key]) sandbox.slice.rejectedTimeReport[key] += report[key];\n })\n }\n \n // If the slice already has rejected metrics, add this data to it. If not, assign this data to slices rejected metrics property\n if (sandbox.slice) {\n (sandbox.slice.rejectedTimeReport) ? updateRejectedMetrics(data.timeReport) : sandbox.slice.rejectedTimeReport = data.timeReport;\n }\n })\n \n // If the sandbox terminated and we are not shutting down, then should return all work which is currently\n // not being computed if all sandboxes are dead and the attempt to create a new one fails.\n sandbox.on('terminated',async () => {\n if (this.sandboxes.length > 0) {\n let terminatedSandboxes = this.sandboxes.filter(sbx => sbx.isTerminated);\n if (terminatedSandboxes.length === this.sandboxes.length) {\n debugging('supervisor') && console.debug(`readySandboxes: Create 1 sandbox in the sandbox-terminated-handler, total sandboxes ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n await this.readySandboxes(1);\n \n // If we cannot create a new sandbox, that probably means we're on a screensaver worker\n // and the screensaver is down. So return the slices to the scheduler.\n if (this.sandboxes.length !== terminatedSandboxes.length + 1) {\n this.returnSlices(this.queuedSlices).then(() => {\n this.queuedSlices.length = 0;\n });\n }\n }\n }\n })\n\n const delayMs =\n 1000 *\n (tuning.minSandboxStartDelay +\n Math.random() *\n (tuning.maxSandboxStartDelay - tuning.minSandboxStartDelay));\n \n sandboxStartPromises.push(\n sandbox\n .start(delayMs)\n .then(() => {\n if (!allocateLocalSandboxes) this.readiedSandboxes.push(sandbox);\n this.sandboxes.push(sandbox);\n sandboxes.push(sandbox);\n }).catch((err) => {\n errors.push(err);\n this.returnSandbox(sandbox);\n if (err.code === 'ENOWORKER') {\n throw new DCPError(\"Cannot use localExec without dcp-worker installed. Use the command 'npm install dcp-worker' to install the neccessary modules.\", 'ENOWORKER');\n }\n }));\n }\n \n await Promise.all(sandboxStartPromises);\n\n if (errors.length) {\n console.warn(`Failed to ready ${errors.length} of ${numSandboxes} sandboxes.`, errors);\n throw new Error('Failed to ready sandboxes.');\n }\n\n debugging('supervisor') && console.log(`readySandboxes: Readied ${sandboxes.length} sandboxes ${JSON.stringify(sandboxes.map(sandbox => sandbox.id))}`);\n \n return sandboxes;\n }\n\n /**\n * Accepts a sandbox after it has finished working or encounters an error.\n * If the sandbox was terminated or if \"!slice || slice.failed\" then\n * the sandbox will be removed from the sandboxes array and terminated if necessary.\n * Otherwise it will try to distribute a slice to the sandbox immediately.\n *\n * @param {Sandbox} sandbox - the sandbox to return\n * @param {Slice} [slice] - the slice just worked on; !slice => terminate\n * @param {boolean} [verifySandboxIsNotTerminated=true] - if true, check sandbox is not already terminated\n */\n returnSandbox (sandbox, slice, verifySandboxIsNotTerminated=true) {\n if (!slice || slice.failed || sandbox.isTerminated) {\n \n this.removeSandbox(sandbox);\n \n if (!sandbox.isTerminated) {\n debugging('supervisor') && console.log(`Supervisor.returnSandbox: Terminating ${sandbox.identifier}${slice ? `~${slice.identifier}` : ''}, # of sandboxes ${this.sandboxes.length}`);\n sandbox.terminate(false);\n } else {\n debugging('supervisor') && console.log(`Supervisor.returnSandbox: Already terminated ${sandbox.identifier}${slice ? `~${slice.identifier}` : ''}, # of sandboxes ${this.sandboxes.length}`);\n // XXXpfr: April 13, 2022\n // I'm trying to understand and control when sandboxes get terminated.\n // Because of races and random worker crashes, it is impossible to not try to terminate a sandbox more than once.\n // But at some places where returnSandbox is we shouldn't see this behavior, hence this exception.\n // If I don't fix this is the next 30 days or I forget, please delete this exception.\n if (false)\n {}\n }\n }\n }\n\n /**\n * Terminates sandboxes, in order of creation, when the total started sandboxes exceeds the total allowed sandboxes.\n *\n * @returns {Promise<void>}\n */\n pruneSandboxes () {\n let numOver = this.sandboxes.length - (dcpConfig.worker.maxAllowedSandboxes + this.maxWorkingSandboxes);\n if (numOver <= 0) return;\n \n // Don't kill readied sandboxes while creating readied sandboxes.\n for (let index = 0; index < this.readiedSandboxes.length; ) {\n const sandbox = this.readiedSandboxes[index];\n // If the sandbox is allocated, advance to the next one in the list.\n if (sandbox.allocated) {\n index++;\n continue;\n }\n // Otherwise, remove this sandbox but look at the same array index in the next loop.\n debugging('supervisor') && console.log(`pruneSandboxes: Terminating readied sandbox ${sandbox.identifier}`);\n this.readiedSandboxes.splice(index, 1);\n this.returnSandbox(sandbox);\n\n if (--numOver <= 0) break;\n }\n\n if (numOver <= 0) return;\n for (let index = 0; index < this.assignedSandboxes.length; ) {\n const sandbox = this.assignedSandboxes[index];\n // If the sandbox is allocated, advance to the next one in the list.\n if (sandbox.allocated) {\n index++;\n continue;\n }\n // Otherwise, remove this sandbox but look at the same array index in the next loop.\n debugging('supervisor') && console.log(`pruneSandboxes: Terminating assigned sandbox ${sandbox.identifier}`);\n this.assignedSandboxes.splice(index, 1);\n this.returnSandbox(sandbox);\n\n if (--numOver <= 0) break;\n }\n }\n \n /**\n * Basic watch dog to check if there are idle sandboxes and\n * attempts to nudge the supervisor to feed them work.\n *\n * Run in an interval created in @constructor .\n * @returns {Promise<void>}\n */\n async watchdog () {\n if (!this.watchdogState)\n this.watchdogState = {};\n\n // Every 5 minutes, report progress of all working slices to the scheduler\n if (Date.now() > ((this.lastProgressReport || 0) + sandboxTuning.progressReportInterval)) {\n // console.log('454: Assembling progress update...');\n this.lastProgressReport = Date.now();\n\n //\n // Note: this.slices is the disjoint union of:\n // this.allocatedSlices, \n // this.queuedSlices, \n // this.slices.filter(slice => !slice.isUnassigned) .\n // When a slice is not in these 3 arrays, the slice is lost.\n //\n \n const currentLostSlices = this.slices.filter(slice => slice.isUnassigned \n && this.queuedSlices.indexOf(slice) === -1\n && this.allocatedSlices.indexOf(slice) === -1);\n\n if (currentLostSlices.length > 0) {\n this.lostSlices.push(...currentLostSlices);\n // Try to recover.\n // Needs more work and testing.\n // Test when we can come up with a decent lost slice repro case.\n // --> this.queuedSlices.push(...currentLostSlices);\n }\n\n if (this.lostSlices.length > 0) {\n if (true) { // Keep this on for awhile, until we know lost slices aren't happening.\n console.warn('Supervisor.watchdog: Found lost slices!');\n for (const slice of this.lostSlices)\n console.warn('\\t', slice.identifier);\n }\n this.lostSlices = this.lostSlices.filter(slice => slice.isUnassigned);\n }\n\n const slices = [];\n this.queuedSlices.forEach(slice => {\n assert(slice && slice.sliceNumber > 0);\n addToSlicePayload(slices, slice, sliceStatus.scheduled);\n });\n\n this.allocatedSlices.forEach(slice => {\n assert(slice && slice.sliceNumber > 0);\n addToSlicePayload(slices, slice, 'progress'); // Beacon.\n });\n\n if (slices.length) {\n // console.log('471: sending progress update...');\n const progressReportPayload = {\n worker: this.workerOpaqueId,\n slices,\n };\n\n this.resultSubmitterConnection.send('status', progressReportPayload)\n .catch(error => {\n debugging('supervisor') && console.error('479: Failed to send status update:', error/*.message*/);\n this.resultSubmitterMessageQueue.push({ operation: 'status', data: progressReportPayload })\n this.resultSubmitterConnection.close();\n });\n }\n }\n\n if (this.worker.working) {\n if (this.unallocatedSpace > 0) {\n await this.work().catch(err => {\n if (!this.watchdogState[err.code || '0'])\n this.watchdogState[err.code || '0'] = 0;\n if (Date.now() - this.watchdogState[err.code || '0'] > ((dcpConfig.worker.watchdogLogInterval * timeDilation || 120) * 1000))\n console.error('301: Failed to start work:', err);\n this.watchdogState[err.code || '0'] = Date.now();\n });\n }\n\n this.pruneSandboxes();\n }\n }\n\n /**\n * Gets the logical and physical number of cores and also\n * the total number of sandboxes the worker is allowed to run\n *\n */\n getStatisticsCPU() {\n if (DCP_ENV.isBrowserPlatform) {\n return {\n worker: this.workerOpaqueId,\n lCores: window.navigator.hardwareConcurrency,\n pCores: dcpConfig.worker.pCores || window.navigator.hardwareConcurrency,\n sandbox: this.maxWorkingSandboxes\n }\n }\n\n return {\n worker: this.workerOpaqueId,\n lCores: requireNative('os').cpus().length,\n pCores: requireNative('physical-cpu-count'),\n sandbox: this.maxWorkingSandboxes\n }\n }\n\n /**\n * Returns the number of unallocated sandbox slots to send to fetchTask.\n *\n * @returns {number}\n */\n numberOfAvailableSandboxSlots() {\n let numCores;\n if (this.options.priorityOnly && this.options.jobAddresses.length === 0) {\n numCores = 0;\n } else if (this.queuedSlices.length > 1) {\n // We have slices queued, no need to fetch\n numCores = 0;\n } else {\n // The queue is almost empty (there may be 0 or 1 element), fetch a full task.\n // The task is full, in the sense that it will contain slices whose\n // aggregate execution time is this.maxWorkingSandboxes * 5-minutes.\n // However, there can only be this.unallocatedSpace # of long slices.\n // Thus we need to know whether the last slice in this.queuedSlices is long or not.\n // (A long slice has estimated execution time >= 5-minutes.)\n const longSliceCount = (this.queuedSlices.length > 0 && this.queuedSlices[0].isLongSlice) ? 1 : 0;\n numCores = this.unallocatedSpace - longSliceCount;\n }\n return numCores;\n }\n\n /**\n * Call to start doing work on the network.\n * This is the one place where requests to fetch new slices are made.\n * After the initial slices are fetched it calls this.distributeQueuedSlices.\n *\n * @returns {Promise<void>}, unallocatedSpace ${this.unallocatedSpace}\n */\n async work()\n {\n // When inside matchSlicesWithSandboxes, don't reenter Supervisor.work to fetch new work or create new sandboxes.\n if (this.matching) {\n // Interesting and noisy.\n // debugging('supervisor') && console.log(`Supervisor.work: Do not interleave work, fetch or matching slices with sandboxes: queuedSlices ${this.queuedSlices.length}, unallocatedSpace ${this.unallocatedSpace}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n return Promise.resolve();\n }\n\n await this.setDefaultIdentityKeystore();\n\n // Instantiate connections that don't exist.\n this.instantiateAllConnections();\n\n const numCores = this.numberOfAvailableSandboxSlots();\n\n debugging() && console.log(`Supervisor.work: Try to get ${numCores} slices in working sandboxes, unallocatedSpace ${this.unallocatedSpace}, queued slices ${this.queuedSlices.length}, # of sandboxes ${this.sandboxes.length}, matching ${this.matching}, fetching: ${this.isFetchingNewWork}`);\n \n // Fetch a new task if we have no more slices queued, then start workers\n try {\n if (numCores > 0 && !this.isFetchingNewWork) {\n this.isFetchingNewWork = true;\n\n /**\n * This will only ready sandboxes up to a total count of\n * maxWorkingSandboxes (in any state). It is not possible to know the\n * actual number of sandboxes required until we have the slices because we\n * may have sandboxes assigned for the slice's job already.\n *\n * If the evaluator cannot start (ie. if the evalServer is not running),\n * then the while loop will keep retrying until the evalServer comes online\n */\n if (this.maxWorkingSandboxes > this.sandboxes.length) {\n // Note: The old technique had \n // while (this.maxWorkingSandboxes > this.sandboxes.length) {....\n // and sometimes we'd get far too many sandboxes, because it would keep looping while waiting for\n // this.readySandboxes(this.maxWorkingSandboxes - this.sandboxes.length);\n // to construct the rest of the sandboxes. The fix is to only loop when the 1st \n // await this.readySandboxes(1) \n // is failing.\n let needFirstSandbox = true;\n while (needFirstSandbox) {\n debugging('supervisor') && console.log(`Supervisor.work: ready 1 sandbox, # of sandboxes ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n await this.readySandboxes(1)\n .then(() => {\n debugging('supervisor') && console.log(`Supervisor.work: ready ${this.maxWorkingSandboxes - this.sandboxes.length} sandbox(es), # of sandboxes ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n this.readySandboxes(this.maxWorkingSandboxes - this.sandboxes.length);\n needFirstSandbox = false;\n }).catch(error => {\n console.warn('906: failed to ready sandboxes; will retry', error.code, error.message);\n });\n }\n }\n\n /**\n * Temporary change: Assign the capabilities of one of readied sandboxes\n * before fetching slices from the scheduler.\n *\n * TODO: Remove this once fetchTask uses the capabilities of every\n * sandbox to fetch slices.\n */\n if (!this.capabilities) {\n this.capabilities = this.sandboxes[0].capabilities;\n this.emit('capabilitiesCalculated', this.capabilities);\n }\n\n if (DCP_ENV.isBrowserPlatform && this.capabilities.browser)\n this.capabilities.browser.chrome = DCP_ENV.isBrowserChrome;\n\n const fetchTimeout = setTimeout(() => {\n console.warn(`679: Fetch exceeded timeout, will reconnect at next watchdog interval`);\n \n this.taskDistributorConnection.close('Fetch timed out', Math.random() > 0.5).catch(error => {\n console.error(`931: Failed to close task-distributor connection`, error);\n });\n this.resultSubmitterConnection.close('Fetch timed out', Math.random() > 0.5).catch(error => {\n console.error(`920: Failed to close result-submitter connection`, error);\n });\n this.isFetchingNewWork = false;\n this.instantiateAllConnections();\n }, 3 * 60 * 1000); // max out at 3 minutes to fetch\n\n // ensure result submitter connection before fetching tasks\n try\n {\n await this.resultSubmitterConnection.keepalive();\n }\n catch (e)\n {\n console.error('Failed to connect to result submitter, refusing to fetch slices. Will try again at next fetch cycle.')\n debugging('supervisor') && console.log(`Error: ${e}`);\n this.isFetchingNewWork = false; // <-- done in the `finally` block, below\n clearTimeout(fetchTimeout);\n this.taskDistributorConnection.close('Failed to connect to result-submitter', true).catch(error => {\n console.error(`939: Failed to close task-distributor connection`, error);\n });\n this.resultSubmitterConnection.close('Failed to connect to result-submitter', true).catch(error => {\n console.error(`942: Failed to close result-submitter connection`, error);\n });\n return Promise.resolve();\n }\n await this.fetchTask(numCores).finally(() => {\n clearTimeout(fetchTimeout);\n this.isFetchingNewWork = false;\n });\n }\n\n this.distributeQueuedSlices().then(() => debugging('supervisor') && 'supervisor: finished distributeQueuedSlices()').catch((e) => {\n // We should never get here, because distributeQueuedSlices was changed\n // to try to catch everything and return slices and sandboxes.\n // If we do catch here it may mean a slice was lost. \n console.error('Supervisor.work catch handler for distributeQueuedSlices.', e);\n });\n // No catch(), because it will bubble outward to the caller\n } finally {\n }\n }\n\n /**\n * Generate the workerComputeGroups property of the requestTask message. \n * \n * Concatenate the compute groups object from dcpConfig with the list of compute groups\n * from the supervisor, and remove the public group if accidentally present. Finally,\n * we transform joinSecrets/joinHash into joinHashHash for secure transmission.\n *\n * @note computeGroup objects with joinSecrets are mutated to record their hashes. This\n * affects the supervisor options and dcpConfig. Re-adding a joinSecret property\n * to one of these will cause the hash to be recomputed.\n */\n generateWorkerComputeGroups()\n {\n var computeGroups = Object.values(dcpConfig.worker.computeGroups || {});\n if (this.options.computeGroups)\n computeGroups = computeGroups.concat(this.options.computeGroups);\n computeGroups = computeGroups.filter(group => group.id !== constants.computeGroups.public.id);\n const hashedComputeGroups = [];\n for (const group of computeGroups)\n {\n const groupCopy = Object.assign({}, group);\n if ((group.joinSecret || group.joinHash) && (!group.joinHashHash || this.lastDcpsid !== this.taskDistributorConnection.dcpsid))\n {\n let joinHash;\n if (group.joinHash) {\n joinHash = group.joinHash.replace(/\\s+/g, ''); // strip whitespace\n } else {\n joinHash = calculateJoinHash(groupCopy);\n } \n\n groupCopy.joinHashHash = hash.calculate(hash.eh1, joinHash, this.taskDistributorConnection.dcpsid);\n delete groupCopy.joinSecret;\n delete groupCopy.joinHash;\n debugging('computeGroups') && console.debug(`Calculated joinHash=${joinHash} for`, groupCopy);\n }\n hashedComputeGroups.push(groupCopy);\n }\n this.lastDcpsid = this.taskDistributorConnection.dcpsid;\n debugging('computeGroups') && console.debug('Requesting ', computeGroups.length, 'non-public groups for session', this.lastDcpsid);\n return hashedComputeGroups;\n }\n\n /**\n * Remove all unreferenced jobs in `this.cache`.\n *\n * @param {any[]} newJobs - Jobs that should not be removed from\n * `this.cache`.\n */\n cleanJobCache(newJobs = []) {\n /* Delete all jobs in the supervisorCache that are not represented in this newJobs,\n * or in this.queuedSlices, or there is no sandbox assigned to these jobs.\n * Note: There can easily be 200+ places to check; using a lookup structure to maintain O(n).\n */\n if (this.cache.jobs.length > 0) {\n const jobAddressMap = {};\n Object.keys(newJobs).forEach(jobAddress => { jobAddressMap[jobAddress] = 1; });\n this.slices.forEach(slice => { if (!jobAddressMap[slice.jobAddress]) jobAddressMap[slice.jobAddress] = 1; });\n this.cache.jobs.forEach(jobAddress => {\n if (!jobAddressMap[jobAddress]) {\n this.cache.remove('job', jobAddress);\n // Remove and return the corresponding sandboxes from this.sandboxes.\n const deadSandboxes = this.sandboxes.filter(sb => sb.jobAddress === jobAddress);\n if (deadSandboxes.length > 0) {\n deadSandboxes.forEach(sandbox => { this.returnSandbox(sandbox); });\n debugging('supervisor') && console.log(`Supervisor.fetchTask: Deleting job ${jobAddress} from cache and assigned sandboxes ${deadSandboxes.map(s => s.id)}, # of sandboxes ${this.sandboxes.length}.`);\n }\n }\n });\n }\n }\n\n /**\n * Fetches a task, which contains job information and slices for sandboxes and\n * manages events related to fetching tasks so the UI can more clearly display\n * to user what is actually happening.\n * @param {number} numCores\n * @returns {Promise<void>} The requestTask request, resolve on success, rejects otherwise.\n * @emits Supervisor#fetchingTask\n * @emits Supervisor#fetchedTask\n */\n async fetchTask(numCores) {\n\n // Don't reenter\n if (this.matching || numCores <= 0) {\n // Interesting and noisy.\n debugging('supervisor') && console.log(`Supervisor.fetchTask: Do not nest work, fetch or matching slices with sandboxes: queuedSlices ${this.queuedSlices.length}, unallocatedSpace ${this.unallocatedSpace}, matching ${this.matching}, fetching ${this.isFetchingNewWork}, numCores ${numCores}`);\n return Promise.resolve();\n }\n\n //\n // Oversubscription mitigation.\n // Update when there are less available sandbox slots than numCores.\n const checkNumCores = this.numberOfAvailableSandboxSlots();\n if (numCores > checkNumCores) numCores = checkNumCores;\n if (numCores <= 0) return Promise.resolve();\n\n this.emit('fetchingTask');\n debugging('supervisor') && console.debug('supervisor: fetching task');\n const requestPayload = {\n numCores,\n coreStats: this.getStatisticsCPU(),\n numGPUs: this.defaultMaxGPUs,\n capabilities: this.capabilities,\n paymentAddress: this.paymentAddress,\n jobAddresses: this.options.jobAddresses || [], // force array; when set, only fetches slices for these jobs\n localExec: this.options.localExec,\n workerComputeGroups: this.generateWorkerComputeGroups(),\n minimumWage: dcpConfig.worker.minimumWage || this.options.minimumWage,\n readyJobs: [ /* list of jobs addresses XXXwg */ ],\n previouslyWorkedJobs: this.ringBufferofJobs.buf, //Only discrete jobs\n rejectedJobs: this.rejectedJobs,\n };\n // workers should be part of the public compute group by default\n if (!booley(dcpConfig.worker.leavePublicGroup) && !booley(this.options.leavePublicGroup) && (!requestPayload.localExec))\n requestPayload.workerComputeGroups.push(constants.computeGroups.public);\n debugging('computeGroups') && console.log(`Fetching work for ${requestPayload.workerComputeGroups.length} ComputeGroups: `, requestPayload.workerComputeGroups);\n debugging('supervisor') && console.log(`fetchTask wants ${numCores} slice(s), unallocatedSpace ${this.unallocatedSpace}, queuedSlices ${this.queuedSlices.length}`);\n try {\n debugging('requestTask') && console.debug('fetchTask: requestPayload', requestPayload);\n\n let result = await this.taskDistributorConnection.send('requestTask', requestPayload).catch((error) => {\n debugging('supervisor') && console.error(`Unable to request task from scheduler: ${error}. Will try again on a new connection.`);\n this.taskDistributorConnection.close(error, true);\n throw error; /* caught below */\n });\n let responsePayload = result.payload; \n\n if (!result.success) {\n debugging() && console.log('Task fetch failure; request=', requestPayload);\n debugging() && console.log('Task fetch failure; response=', result.payload);\n throw new DCPError('Unable to fetch task for worker', responsePayload);\n }\n\n const sliceCount = responsePayload.body.task.length || 0;\n\n /**\n * The fetchedTask event fires when the supervisor has finished trying to\n * fetch work from the scheduler (task-manager). The data emitted is the\n * number of new slices to work on in the fetched task.\n *\n * @event Supervisor#fetchedTask\n * @type {number}\n */\n this.emit('fetchedTask', sliceCount);\n\n if (sliceCount < 1) {\n return Promise.resolve();\n }\n\n /**\n * DCP-1698 Send auth msg with tasks to worker, then validate authority of worker to send slice info back to scheduler.\n * payload structure: { owner: this.address, signature: signature, auth: messageLightWeight, body: messageBody };\n * messageLightWeight: { workerId: worker, jobSlices, schedulerId, jobCommissions }\n * messageBody: { newJobs: await getNewJobsForTask(dbScheduler, task, request), task }\n */\n const { body, ...authorizationMessage } = responsePayload;\n const { newJobs, task } = body;\n assert(newJobs); // It should not be possible to have !newJobs -- we throw on !success.\n \n /*\n * Ensure all jobs received from the scheduler are:\n * 1. If we have specified specific jobs the worker may work on, the received jobs are in the specified job list\n * 2. If we are in localExec, at most 1 unique job type was received (since localExec workers are designated for only\n * one job)\n * If the received jobs are not within these parameters, stop the worker since the scheduler cannot be trusted at that point.\n */\n if ((this.options.jobAddresses.length && !Object.keys(newJobs).every((ele) => this.options.jobAddresses.includes(ele)))\n || (this.options.localExec && Object.keys(newJobs).length > 1))\n {\n console.error(\"Worker received slices it shouldn't have. Rejecting the work and stopping.\");\n process.exit(1);\n }\n\n debugging() && console.log(`Supervisor.fetchTask: task: ${task.length}/${numCores}, jobs: ${Object.keys(newJobs).length}, authSlices: ${compressJobMap(authorizationMessage.auth.authSlices, true /* skipFirst*/, this.addressTruncationLength /* digits*/)}`);\n // Delete all jobs in the supervisorCache that are not represented in this task,\n // or in this.queuedSlices, or there is no sandbox assigned to these jobs.\n this.cleanJobCache(newJobs);\n\n for (const jobAddress of Object.keys(newJobs))\n if (!this.cache.cache.job[jobAddress])\n this.cache.store('job', jobAddress, newJobs[jobAddress]);\n\n // Memoize authMessage onto the Slice object, this should\n // follow it for its entire life in the worker.\n const tmpQueuedSlices = task.map(taskElement => new Slice(taskElement, authorizationMessage));\n\n // Make sure old stuff is up front.\n // matchSlicesWithSandboxes dequeues this.queuedSlices as follows:\n // slicesToMatch = this.queuedSlices.slice(0, numCores);\n this.slices.push(...tmpQueuedSlices);\n this.queuedSlices.push(...tmpQueuedSlices);\n \n // Populating the ring buffer based on job's discrete property \n Object.values(newJobs).forEach(job => {\n if(job.requirements.discrete && this.ringBufferofJobs.find(element => element === job.address) === undefined) {\n this.ringBufferofJobs.push(job.address);\n }\n });\n \n } catch (error) {\n this.emit('fetchTaskFailed', error);\n debugging('supervisor') && console.debug(`Supervisor.fetchTask failed!: error: ${error}`);\n }\n }\n\n /**\n * For each slice in this.queuedSlices, match with a sandbox in the following order:\n * 1. Try to find an already assigned sandbox in this.assignedSandboxes for the slice's job.\n * 2. Find a ready sandbox in this.readiedSandboxes that is unassigned.\n * 3. Ready a new sandbox and use that.\n *\n * Take great care in assuring sandboxes and slices are uniquely associated, viz.,\n * a given slice cannot be associated with multiple sandboxes and a given sandbox cannot be associated with multiple slices.\n * The lack of such uniqueness has been the root cause of several difficult bugs.\n *\n * Note: When a sandbox is paired with a slice, execution is pending and sandbox.allocated=true and\n * sandbox.slice=slice and sandbox.jobAddress=slice.jobAddress. This is what 'allocated' means.\n * Immediately upon the exit of sandbox.work, sandbox.allocated=false is set and if an exception\n * wasn't thrown, the paired slice is placed in this.assignedSandboxes.\n * Thus from the pov of supervisor, this.allocatedSandboxes is deterministic and this.workingSandboxes is not.\n * Please try to not use this.workingSandboxes. It is deprecated.\n *\n * The input is numCores, this,queuedSlices, this.assignedSandboxes and this.readiedSandboxes.\n * If there are not enough sandboxes, new readied sandboxes will be created using\n * await this.readySandboxes(...)\n * And it is this await boundary that has caused many bugs.\n * We try not to make assumptions about non-local state across the await boundary.\n *\n * @param {number} numCores - The number of available sandbox slots.\n * @param {boolean} [throwExceptions=true] - Whether to throw exceptions when checking for sanity.\n * @returns {Promise<SandboxSlice[]>} Returns SandboxSlice[], may have length zero.\n */\n async matchSlicesWithSandboxes (numCores, throwExceptions = true) {\n\n const sandboxSlices = [];\n if (this.queuedSlices.length === 0 || this.matching || numCores <= 0) {\n // Interesting and noisy.\n // debugging('supervisor') && console.log(`Supervisor.matchSlicesWithSandboxes: Do not nest work, fetch or matching slices with sandboxes: queuedSlices ${this.queuedSlices.length}, unallocatedSpace ${this.unallocatedSpace}, matching ${this.matching}, fetching ${this.isFetchingNewWork}, numCores ${numCores}`);\n return sandboxSlices;\n }\n\n //\n // Oversubscription mitigation.\n // Update when there are less available sandbox slots than numCores.\n // We cannot use this.unallocatedSpace here because its value is artificially low or zero, because in\n // this.distributedQueuedSlices we use the pseudo-mutex trick: this.acquire(howManySandboxSlotsToReserve)/this.release().\n // Note: Do not use this.numberOfCoresReserved outside of a function locked with this.acquire(howManySandboxSlotsToReserve) .\n const checkNumCores = this.numberOfCoresReserved; // # of locked sandbox slots.\n if (numCores > checkNumCores) numCores = checkNumCores;\n if (numCores <= 0) return sandboxSlices;\n\n // Don't ask for more than we have.\n if (numCores > this.queuedSlices.length)\n numCores = this.queuedSlices.length;\n\n debugging('supervisor') && console.log(`matchSlicesWithSandboxes: numCores ${numCores}, queued slices ${this.queuedSlices.length}: assigned ${this.assignedSandboxes.length}, readied ${this.readiedSandboxes.length}, unallocated ${this.unallocatedSpace}, # of sandboxes: ${this.sandboxes.length}`);\n\n if (debugging('supervisor')) {\n dumpSlicesIfNotUnique(this.queuedSlices, 'Warning: this.queuedSlices slices are not unique -- this is ok when slice is rescheduled.');\n dumpSandboxesIfNotUnique(this.readiedSandboxes, 'Warning: this.readiedSandboxes sandboxes are not unique!');\n dumpSandboxesIfNotUnique(this.assignedSandboxes, 'Warning: this.assignedSandboxes sandboxes are not unique!');\n }\n\n // Three functions to validate slice and sandbox.\n function checkSlice(slice, checkAllocated=true) {\n if (!slice.isUnassigned) throw new DCPError(`Slice must be unassigned: ${slice.identifier}`);\n if (checkAllocated && slice.allocated) throw new DCPError(`Slice must not already be allocated: ${slice.identifier}`);\n }\n function checkSandbox(sandbox, isAssigned) {\n if (sandbox.allocated) throw new DCPError(`Assigned sandbox must not be already allocated: ${sandbox.identifier}`);\n if (isAssigned && !sandbox.isAssigned) throw new DCPError(`Assigned sandbox is not marked as assigned: ${sandbox.identifier}`);\n if (!isAssigned && !sandbox.isReadyForAssign) throw new DCPError(`Readied sandbox is not marked as ready for assign: ${sandbox.identifier}`);\n }\n\n // Sanity checks.\n if (throwExceptions) {\n this.assignedSandboxes.forEach(sandbox => { checkSandbox(sandbox, true /* isAssigned*/); });\n this.readiedSandboxes.forEach(sandbox => { checkSandbox(sandbox, false /* isAssigned*/); });\n this.queuedSlices.forEach(slice => { checkSlice(slice); });\n } else {\n this.assignedSandboxes = this.assignedSandboxes.filter(sandbox => !sandbox.allocated && sandbox.isAssigned);\n this.readiedSandboxes = this.readiedSandboxes.filter(sandbox => !sandbox.allocated && sandbox.isReadyForAssign);\n this.queuedSlices = this.queuedSlices.filter(slice => !slice.allocated && slice.isUnassigned);\n }\n\n const sandboxKind = {\n assigned: 0,\n ready: 1,\n new: 2,\n };\n\n const ceci = this;\n /**\n * Auxiliary function to pair a sandbox with a slice and mark the sandbox as allocated.\n * An allocated sandbox is reserved and will not be released until the slice completes execution on the sandbox.\n *\n * @param {Sandbox} sandbox\n * @param {Slice} slice\n * @param {number} kind\n */\n function pair(sandbox, slice, kind) {\n checkSandbox(sandbox, kind === sandboxKind.assigned);\n checkSlice(slice, kind === sandboxKind.assigned);\n slice.allocated = true;\n sandbox.allocated = true;\n sandbox.jobAddress = slice.jobAddress; // So we can know which jobs to not delete from this.cache .\n sandbox.slice = slice;\n sandboxSlices.push({ sandbox, slice });\n if (Supervisor.sliceTiming) slice['pairingDelta'] = Date.now();\n if (debugging('supervisor')) {\n let fragment = 'New readied';\n if (kind === sandboxKind.assigned) fragment = 'Assigned';\n else if (kind === sandboxKind.ready) fragment = 'Readied';\n console.log(`matchSlicesWithSandboxes.pair: ${fragment} sandbox matched ${ceci.dumpSandboxAndSlice(sandbox, slice)}`);\n }\n }\n\n // These three arrays are used to track/store slices and sandboxes,\n // so that when an exception occurs, the following arrays are restored:\n // this.queuedSlices, this.assignedSandboxes, this.realizedSandboxes.\n let slicesToMatch = [];\n let trackAssignedSandboxes = [];\n let trackReadiedSandboxes = [];\n try\n {\n this.matching = true;\n\n let assignedCounter = 0; // How many assigned sandboxes are being used.\n let readyCounter = 0; // How many sandboxes used from the existing this.readiedSandboxes.\n let newCounter = 0; // How many sandboxes that needed to be newly created.\n\n //\n // The Ideas:\n // 1) We match each slice with a sandbox. First we match with assigned sandboxes in the order\n // that they appear in this.queuedSlices. Then we match in-order with existing this.readiedSandboxes\n // Then we match in-order with new new readied sandboxes created through\n // await this.readySandboxes(newCounter, true /* allocateLocalSandboxes*/);\n // This allows us to try different orderings of execution of slices. E.g. Wes suggested\n // trying to execute slices from different jobs with maximal job diversity -- specifically\n // if there are 3 jobs j1,j2,j3, with slices s11, s12 from j1, s21, s22, s23 from j2 and\n // s31, s32 from j3, then we try to schedule, in order s11, s21, s31, s12, s22, s32, s23.\n //\n // 2) Before matching slices with sandboxes, we allocate available assigned and readied sandboxes\n // and if more are needed then we create and allocate new ones.\n //\n // 3) Finally we match slices with sandboxes and return an array of sandboxSlice pairs.\n //\n // Note: The ordering of sandboxSlices only partially corresponds to the order of this.queuedSlices.\n // It's easy to do. When pairing with assigned sandboxes, any slice in this.queuedSlices which doesn't\n // have an assigned sandbox, will add null to the sandboxSlices array. Then when pairing with readied sandboxes,\n // we fill-in the null entries in the sandboxSlices array.\n //\n /** XXXpfr @todo When it is needed, fix the ordering as described above. */\n\n // Get the slices that are being matched.\n slicesToMatch = this.queuedSlices.slice(0, numCores);\n this.queuedSlices = this.queuedSlices.slice(numCores);\n\n debugging('supervisor') && console.log(`matchSlicesWithSandboxes: slicesToMatch ${this.dumpSlices(slicesToMatch)}`);\n\n // Create object map: jobAddress -> sandboxes with sandboxes.jobAddress === jobAddress .\n const jobSandboxMap = toJobMap(this.assignedSandboxes, sandbox => sandbox);\n \n // Create array to hold slices which do not have assigned sandboxes.\n // These slices will need to be paired with existing and possibly new readied sandboxes.\n // Specifically, the sandboxes from existing this.readiedSandboxes and new sandboxes\n // created through await this.readySandboxes(newCounter, true /* allocateLocalSandboxes*/);\n const slicesThatNeedSandboxes = [];\n\n // Pair assigned sandboxes with slices.\n for (const slice of slicesToMatch) {\n const assigned = jobSandboxMap[slice.jobAddress];\n if (assigned && assigned.length > 0) {\n // Pair.\n const sandbox = assigned.pop();\n pair(sandbox, slice, sandboxKind.assigned);\n this.removeElement(this.assignedSandboxes, sandbox);\n // Track.\n trackAssignedSandboxes.push(sandbox);\n assignedCounter++;\n } else {\n // Don't lose track of these slices.\n slice.allocated = true;\n slicesThatNeedSandboxes.push(slice);\n }\n }\n\n // Pair readied sandboxes with slices.\n readyCounter = Math.min(slicesThatNeedSandboxes.length, this.readiedSandboxes.length);\n newCounter = slicesThatNeedSandboxes.length - readyCounter;\n // Track.\n trackReadiedSandboxes = this.readiedSandboxes.slice(0, readyCounter);\n this.readiedSandboxes = this.readiedSandboxes.slice(readyCounter);\n for (const sandbox of trackReadiedSandboxes) {\n // Pair.\n const slice = slicesThatNeedSandboxes.pop();\n pair(sandbox, slice, sandboxKind.ready);\n }\n \n debugging('supervisor') && console.log(`matchSlicesWithSandboxes: assignedCounter ${assignedCounter}, readyCounter ${readyCounter}, newCounter ${newCounter}, numCores ${numCores}`)\n\n // Validate algorithm consistency.\n if (Supervisor.debugBuild && assignedCounter + readyCounter + newCounter !== numCores) {\n // Structured assert.\n throw new DCPError(`matchSlicesWithSandboxes: Algorithm is corrupt ${assignedCounter} + ${readyCounter} + ${newCounter} !== ${numCores}`);\n }\n\n // Here is an await boundary.\n // Accessing non-local data across an await boundary may result in the unexpected.\n\n // Create new readied sandboxes to associate with slicesThatNeedSandboxes.\n if (newCounter > 0) {\n // When allocateLocalSandboxes is true, this.readySandboxes does not place the new sandboxes\n // on this.readiedSandboxes. Hence the new sandboxes are private and nobody else can see them.\n debugging('supervisor') && console.log(`matchSlicesWithSandboxes: creating ${newCounter} new sandboxes, # of sandboxes ${this.sandboxes.length}`);\n const readied = await this.readySandboxes(newCounter, true /* allocateLocalSandboxes*/);\n // Track.\n trackReadiedSandboxes.push(...readied);\n\n for (const sandbox of readied) {\n assert(slicesThatNeedSandboxes.length > 0);\n // Pair\n const slice = slicesThatNeedSandboxes.pop();\n pair(sandbox, slice, sandboxKind.new);\n }\n \n // Put back any extras. There should not be any unless readySandboxes returned less than asked for.\n if (slicesThatNeedSandboxes.length > 0) {\n slicesThatNeedSandboxes.forEach(slice => {\n slice.allocated = false;\n this.queuedSlices.push(slice);\n });\n }\n }\n\n if ( false || debugging()) {\n console.log(`matchSlicesWithSandboxes: Matches: ${ this.dumpSandboxSlices(sandboxSlices) }`);\n this.dumpSandboxSlicesIfNotUnique(sandboxSlices, 'Warning: sandboxSlices; { sandbox, slice } pairs are not unique!');\n }\n } catch (e) {\n // Clear allocations.\n slicesToMatch.forEach(slice => { slice.allocated = false; });\n trackAssignedSandboxes.forEach(sandbox => { sandbox.allocated = false; sandbox.slice = null; });\n trackReadiedSandboxes.forEach(sandbox => { sandbox.allocated = false; sandbox.slice = null; sandbox.jobAddress = null; });\n \n // Filter out redundancies -- there shouldn't be any...\n slicesToMatch = slicesToMatch.filter(slice => this.queuedSlices.indexOf(slice) === -1);\n trackAssignedSandboxes = trackAssignedSandboxes.filter(sb => this.assignedSandboxes.indexOf(sb) === -1);\n trackReadiedSandboxes = trackReadiedSandboxes.filter(sb => this.readiedSandboxes.indexOf(sb) === -1);\n\n // Sanity checks.\n slicesToMatch.forEach(slice => { checkSlice(slice) });\n trackAssignedSandboxes.forEach(sandbox => { checkSandbox(sandbox, true /* isAssigned*/); });\n trackReadiedSandboxes.forEach(sandbox => { checkSandbox(sandbox, false /* isAssigned*/); });\n\n // Restore arrays.\n this.queuedSlices.push(...slicesToMatch);\n this.assignedSandboxes.push(...trackAssignedSandboxes);\n this.readiedSandboxes.push(...trackReadiedSandboxes);\n \n console.error('Error in matchSlicesWithSandboxes: Attempting to recover slices and sandboxes.', e);\n return [];\n } finally {\n this.matching = false;\n }\n\n debugging('supervisor') && console.log(`matchSlicesWithSandboxes: allocated ${sandboxSlices.length} sandboxes, queuedSlices ${this.queuedSlices.length}, unallocatedSpace ${this.unallocatedSpace}, matching ${this.matching}, fetching ${this.isFetchingNewWork}, # of sandboxes: ${this.sandboxes.length}.`);\n\n return sandboxSlices;\n }\n\n disassociateSandboxAndSlice(sandbox, slice) {\n this.returnSandbox(sandbox);\n sandbox.slice = null;\n this.returnSlice(slice);\n }\n\n /**\n * This method will call this.startSandboxWork(sandbox, slice) for each element { sandbox, slice }\n * of the array returned by this.matchSlicesWithSandboxes(availableSandboxes) until all allocated sandboxes\n * are working. It is possible for a sandbox to interleave with calling distributeQueuedSlices and leave a sandbox\n * that is not working. Moreover, this.queuedSlices may be exhausted before all sandboxes are working.\n * @returns {Promise<void>}\n */\n async distributeQueuedSlices () {\n const numCores = this.unallocatedSpace;\n\n // If there's nothing there, or we're reentering, bail out.\n if (this.queuedSlices.length === 0 || numCores <= 0 || this.matching) {\n // Interesting and noisy.\n // debugging('supervisor') && console.log(`Supervisor.distributeQueuedSlices: Do not nest work, fetch or matching slices with sandboxes: queuedSlices ${this.queuedSlices.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}, numCores ${numCores}`);\n return Promise.resolve();\n }\n\n //\n // Use the pseudo-mutex to prevent uncontrolled interleaving with fetchTask,\n // matchSlicesWithSandboxes and distributeQueuedSlices\n let sandboxSlices;\n this.acquire(numCores);\n try {\n sandboxSlices = await this.matchSlicesWithSandboxes(numCores);\n } finally {\n this.release();\n }\n\n debugging('supervisor') && console.log(`distributeQueuedSlices: ${sandboxSlices.length} sandboxSlices ${this.dumpSandboxSlices(sandboxSlices)}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n\n for (let sandboxSlice of sandboxSlices) {\n\n const { sandbox, slice } = sandboxSlice;\n try {\n if (sandbox.isReadyForAssign) {\n try {\n let timeoutMs = Math.floor(Math.min(+Supervisor.lastAssignFailTimerMs || 0, 10 * 60 * 1000 /* 10m */));\n await a$sleepMs(timeoutMs);\n await this.assignJobToSandbox(sandbox, slice.jobAddress);\n } catch (e) {\n console.error(`Supervisor.distributeQueuedSlices: Could not assign slice ${slice.identifier} to sandbox ${sandbox.identifier}.`);\n if (Supervisor.debugBuild) console.error(`...exception`, e);\n Supervisor.lastAssignFailTimerMs = Supervisor.lastAssignFailTimerMs ? +Supervisor.lastAssignFailTimerMs * 1.25 : Math.random() * 200;\n this.disassociateSandboxAndSlice(sandbox, slice);\n continue;\n }\n }\n\n if (!Supervisor.lastAssignFailTimerMs)\n Supervisor.lastAssignFailTimerMs = Math.random() * 200;\n this.startSandboxWork(sandbox, slice);\n Supervisor.lastAssignFailTimerMs = false;\n\n } catch (e) {\n // We should never get here.\n console.error(`Supervisor.distributeQueuedSlices: Failed to execute slice ${slice.identifier} in sandbox ${sandbox.identifier}.`);\n if (Supervisor.debugBuild) console.error('...exception', e);\n this.disassociateSandboxAndSlice(sandbox, slice);\n }\n }\n }\n\n /**\n *\n * @param {Sandbox} sandbox\n * @param {opaqueId} jobAddress\n * @returns {Promise<void>}\n */\n async assignJobToSandbox(sandbox, jobAddress) {\n var ceci = this;\n\n try {\n return sandbox.assign(jobAddress); // Returns Promise.\n } catch(error) {\n // return slice to scheduler, log error\n console.error('Supervisor.assignJobToSandbox: Failed to assign job to sandbox.', {\n jobAddress: jobAddress.substr(0,10),\n error,\n });\n\n ceci.returnSandbox(sandbox);\n\n throw error;\n }\n }\n\n /**\n * Handles reassigning or returning a slice that was rejected by a sandbox.\n * \n * The sandbox will be terminated by this.returnSandbox in finalizeSandboxAndSlice. In this case,\n * if the slice does not have a rejected property already, reassign the slice to a new sandbox\n * and add a rejected property to the slice to indicate it has already rejected once, then set slice = null\n * in the return SandboxSlice so that finalizeSandboxAndSlice won't return slice to scheduler.\n * \n * If the slice rejects with a reason, or has a rejected time stamp (ie. has been rejected once already)\n * then return the slice and all slices from the job to the scheduler and\n * terminate all sandboxes with that jobAddress.\n * @param {Sandbox} sandbox \n * @param {Slice} slice\n * @returns {Promise<SandboxSlice>}\n */\n async handleWorkReject(sandbox, slice, rejectReason) {\n if (!this.rejectedJobReasons[slice.jobAddress])\n this.rejectedJobReasons[slice.jobAddress] = [];\n\n this.rejectedJobReasons[slice.jobAddress].push(rejectReason); // memoize reasons\n\n // First time rejecting without a reason. Try assigning slice to a new sandbox.\n if (rejectReason === 'false' && !slice.rejected) {\n // Set rejected.\n slice.rejected = Date.now();\n // Schedule the slice for execution.\n this.scheduleSlice(slice, true /* placeInTheFrontOfTheQueue*/, false /* noDuplicateExecution*/);\n \n // Null out slice so this.returnSlice will not be called in finalizeSandboxAndSlice.\n // But we still want this.returnSandbox to terminate the sandbox.\n slice = null;\n } else { // Slice has a reason OR rejected without a reason already and got stamped.\n \n // Purge all slices and sandboxes associated with slice.jobAddress .\n this.purgeAllWork(slice.jobAddress);\n // Clear jobAddress from this.cache .\n this.cleanJobCache();\n\n // Add to array of rejected jobs.\n let rejectedJob = {\n address: slice.jobAddress,\n reasons: this.rejectedJobReasons[slice.jobAddress],\n }\n this.rejectedJobs.push(rejectedJob);\n\n // Tell everyone all about it, when allowed.\n if (dcpConfig.worker.allowConsoleAccess || Supervisor.debugBuild)\n {\n console.warn('Supervisor.handleWorkReject: The slice ${slice.identifier} was rejected twice; slice will be returned to the scheduler.');\n console.warn('Supervisor.handleWorkReject: All slices with the same jobAddress returned to the scheduler.');\n console.warn('Supervisor.handleWorkReject: All sandboxes with the same jobAddress are terminated.');\n }\n //\n // this.purgeAllWork(jobAddress) terminates all sandboxes with jobAddress,\n // and it also returns to scheduler all slices with jobAddress.\n // Therefore null out slice and sandbox so finalizeSandboxAndSlice doesn't do anything.\n // \n sandbox = null;\n slice = null;\n }\n return { sandbox, slice };\n }\n\n /**\n * Schedule the slice to be executed.\n * If slice is already executing and noDuplicateExecution is true, return the slice with reason.\n * @param {Slice} slice\n * @param {boolean} [placeInTheFrontOfTheQueue=false]\n * @param {boolean} [noDuplicateExecution=true]\n * @param {string} [reason]\n */\n scheduleSlice(slice, placeInTheFrontOfTheQueue = false, noDuplicateExecution = true, reason) {\n // When noDuplicateExecution, if slice is already executing, do nothing.\n let workingSlices = [];\n if (noDuplicateExecution)\n workingSlices = this.allocatedSlices;\n\n if (!workingSlices.indexOf(slice)) {\n // Reset slice state to allow execution.\n slice.status = SLICE_STATUS_UNASSIGNED;\n // Enqueue in the to-be-executed queue.\n if (placeInTheFrontOfTheQueue) this.queuedSlices.unshift(slice);\n else this.queuedSlices.push(slice);\n }\n }\n\n /**\n * Purge all slices and sandboxes with this jobAddress.\n * @param {address} jobAddress\n * @param {boolean} [onlyPurgeQueuedAndAllocated=false]\n */\n purgeAllWork(jobAddress, onlyPurgeQueuedAndAllocated = false) {\n // Purge all slices and sandboxes associated with jobAddress .\n const deadSandboxes = this.sandboxes.filter(sandbox => sandbox.jobAddress === jobAddress);\n\n if (deadSandboxes.length > 0) {\n debugging('supervisor') && console.log(`purgeAllWork(${this.dumpJobAddress(jobAddress)}): sandboxes purged ${deadSandboxes.map(s => s.id)}, # of sandboxes ${this.sandboxes.length}`);\n deadSandboxes.forEach(sandbox => this.returnSandbox(sandbox));\n }\n\n let deadSlices;\n if (onlyPurgeQueuedAndAllocated) {\n deadSlices = this.queuedSlices.filter(slice => slice.jobAddress === jobAddress);\n if (deadSlices.length > 0 || this.allocatedSlices.length > 0)\n debugging('supervisor') && console.log(`purgeAllWork(${this.dumpJobAddress(jobAddress)}): dead queuedSlices ${deadSlices.map(s => s.sliceNumber)}, dead allocatedSlices ${this.allocatedSlices.map(s => s.sliceNumber)}`);\n deadSlices.push(...this.allocatedSlices);\n } else {\n deadSlices = this.slices.filter(slice => slice.jobAddress === jobAddress);\n }\n\n if (deadSlices.length > 0) {\n debugging('supervisor') && console.log(`purgeAllWork(${this.dumpJobAddress(jobAddress)}): slices purged ${deadSlices.map(s => s.sliceNumber)}, # of sandboxes ${this.sandboxes.length}`);\n this.returnSlices(deadSlices);\n this.removeQueuedSlices(deadSlices);\n }\n debugging('supervisor') && console.log(`purgeAllWork(${this.dumpJobAddress(jobAddress)}): Finished: slices ${this.slices.length}, queuedSlices ${this.queuedSlices.length}, assigned ${this.assignedSandboxes.length}, readied ${this.readiedSandboxes.length}, # of sandboxes ${this.sandboxes.length}`);\n }\n\n /**\n * Gives a slice to a sandbox which begins working. Handles collecting\n * the slice result (complete/fail) from the sandbox and submitting the result to the scheduler.\n * It will also return the sandbox to @this.returnSandbox when completed so the sandbox can be re-assigned.\n *\n * @param {Sandbox} sandbox - the sandbox to give the slice\n * @param {Slice} slice - the slice to distribute\n * @returns {Promise<void>} Promise returned from sandbox.run\n */\n async startSandboxWork (sandbox, slice) {\n var startDelayMs, reason = 'unknown';\n\n try {\n slice.markAsWorking();\n } catch (e) {\n // This will occur when the same slice is distributed twice.\n // It is normal because two sandboxes could finish at the same time and be assigned the\n // same slice before the slice is marked as working.\n debugging() && console.debug('startSandboxWork: slice.markAsWorking exception:', e);\n return Promise.resolve();\n }\n\n // sandbox.requiresGPU = slice.requiresGPU;\n // if (sandbox.requiresGPU) {\n // this.GPUsAssigned++;\n // }\n\n if (Supervisor.startSandboxWork_beenCalled)\n startDelayMs = 1000 * (tuning.minSandboxStartDelay + (Math.random() * (tuning.maxSandboxStartDelay - tuning.minSandboxStartDelay)));\n else {\n startDelayMs = 1000 * tuning.minSandboxStartDelay;\n Supervisor.startSandboxWork_beenCalled = true;\n }\n\n try {\n debugging() && console.log(`startSandboxWork: Started ${this.dumpStatefulSandboxAndSlice(sandbox, slice)}, total sandbox count: ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n if (Supervisor.sliceTiming) {\n slice['pairingDelta'] = Date.now() - slice['pairingDelta'];\n slice['executionDelta'] = Date.now();\n }\n let result;\n try {\n result = await sandbox.work(slice, startDelayMs);\n } finally {\n sandbox.allocated = false;\n slice.allocated = false;\n }\n if (Supervisor.sliceTiming) {\n slice['executionDelta'] = Date.now() - slice['executionDelta'];\n slice['resultDelta'] = Date.now();\n }\n slice.collectResult(result, true);\n // In watchdog, all sandboxes in working state, have their slice status sent to result submitter.\n // However, this can happen after the sandbox/slice has already sent results\n // to result submitter, in which case, the activeSlices table has already removed the row\n // corresponding to slice and hence is incapable of updating status.\n sandbox.changeWorkingToAssigned();\n this.assignedSandboxes.push(sandbox);\n debugging() && console.log(`startSandboxWork: Finished ${this.dumpStatefulSandboxAndSlice(sandbox, slice)}, total sandbox count: ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n } catch(error) {\n let logLevel;\n\n if (error instanceof SandboxError) {\n logLevel = 'warn';\n // The message and stack properties of error objects are not enumerable,\n // so they have to be copied into a plain object this way\n const errorResult = Object.getOwnPropertyNames(error).reduce((o, p) => {\n o[p] = error[p]; return o;\n }, { message: 'Unexpected worker error' });\n slice.collectResult(errorResult, false);\n } else {\n logLevel = 'error';\n // This error was unrelated to the work being done, so just return the slice in the finally block.\n // For extra safety the sandbox is terminated.\n slice.result = null;\n slice.status = SLICE_STATUS_FAILED; /** XXXpfr @todo terminating sandbox? */\n }\n\n let errorString;\n switch (error.errorCode) {\n case 'ENOPROGRESS':\n reason = 'ENOPROGRESS';\n errorString = 'Supervisor.startSandboxWork - No progress error in sandbox.\\n';\n break;\n case 'ESLICETOOSLOW':\n reason = 'ESLICETOOSLOW';\n errorString = 'Supervisor.startSandboxWork - Slice too slow error in sandbox.\\n';\n break;\n case 'EUNCAUGHT':\n reason = 'EUNCAUGHT';\n errorString = `Supervisor.startSandboxWork - Uncaught error in sandbox ${error.message}.\\n`;\n break;\n case 'EFETCH':\n // reason = 'EFETCH'; The status.js processing cannot handle 'EFETCH'\n reason = 'unknown';\n errorString = `Supervisor.startSandboxWork - Could not fetch data: ${error.message}.\\n`;\n break;\n }\n \n // Always display max info under debug builds, otherwise maximal error\n // messages are displayed to the worker, only if both worker and client agree.\n let workerConsole = sandbox.supervisorCache.cache.job[slice.jobAddress].workerConsole;\n const displayMaxInfo = Supervisor.debugBuild || (workerConsole && dcpConfig.worker.allowConsoleAccess);\n\n const errorObject = {\n jobAddress: slice.jobAddress.substr(0,10),\n sliceNumber: slice.sliceNumber,\n sandbox: sandbox.id,\n jobName: sandbox.public ? sandbox.public.name : 'unnamed',\n };\n \n if (error.name === 'EWORKREJECT') {\n error.stack = 'Sandbox was terminated by work.reject()';\n const ss = await this.handleWorkReject(sandbox, slice, error.message);\n sandbox = ss.sandbox; slice = ss.slice;\n }\n\n if (!displayMaxInfo && errorString) {\n console[logLevel](errorString, errorObject);\n } else if (!displayMaxInfo && error.name === 'EWORKREJECT') {\n console[logLevel](`Supervisor.startSandboxWork - Sandbox rejected work: ${error.message}`)\n } else {\n if (displayMaxInfo)\n errorObject.stack += '\\n --------------------\\n' + (error.stack.split('\\n').slice(1).join('\\n'));\n console[logLevel](`Supervisor.startSandboxWork - Sandbox failed: ${error.message}\\n`, errorObject);\n }\n } finally {\n await this.finalizeSandboxAndSlice(sandbox, slice, reason);\n }\n }\n\n /**\n * If slice && slice.result, then call await this.recordResult(slice) and this.returnSandbox(sandbox, slice) will have no effect.\n * If slice && !slice.result, then call this.returnSlice(slice, reason) and then this.returnSandbox(sandbox, slice) which terminates sandbox.\n * If !slice && sandbox, then terminate the sandbox with this.returnSandbox(sandbox, slice) .\n * If !slice && !sandbox, then do nothing.\n * @param {Sandbox} [sandbox]\n * @param {Slice} [slice]\n * @param {string} [reason]\n */\n async finalizeSandboxAndSlice(sandbox, slice, reason) {\n debugging('supervisor') && console.log(`finalizeSandboxAndSlice: sandbox ${sandbox ? sandbox.identifier : 'nade'}, slice ${slice ? slice.identifier : 'nade'}`);\n if (slice) {\n if (slice.result) await this.recordResult(slice);\n else this.returnSlice(slice, reason);\n }\n // It is possible that sandbox is already terminated\n // Because sandbox.allocated=false as soon as sandbox.work(...) completes.\n // But the await at or in finalizeSandboxAndSlice may allow pruneSandboxes to slither in.\n if (sandbox) this.returnSandbox(sandbox, slice, false /* verifySandboxIsNotTerminated*/);\n }\n\n /**\n * Terminates sandboxes and returns slices.\n * Sets the working flag to false, call @this.work to start working again.\n * \n * If forceTerminate is true: Terminates all sandboxes and returns all slices.\n * If forceTerminate is false: Terminates non-allocated sandboxes and returns queued slices.\n *\n * @param {boolean} [forceTerminate = true] - true if you want to stop the sandboxes from completing their current slice.\n * @returns {Promise<void>}\n */\n async stopWork (forceTerminate = true) {\n debugging('supervisor') && console.log('stopWork(${forceTerminate}): terminating sandboxes and returning slices to scheduler.');\n if (forceTerminate) {\n while (this.sandboxes.length) {\n this.returnSandbox(this.sandboxes[0], null, false);\n }\n\n await this.returnSlices(this.slices).then(() => {\n this.queuedSlices.length = 0;\n });\n } else {\n // Only terminate idle sandboxes and return only queued slices\n let idleSandboxes = this.sandboxes.filter(w => !w.allocated);\n for (const sandbox of idleSandboxes) {\n this.returnSandbox(sandbox, null, false /* verifySandboxIsNotTerminated*/);\n }\n\n await this.returnSlices(this.queuedSlices).then(() => {\n this.queuedSlices.length = 0;\n });\n\n await new Promise((resolve, reject) => {\n let sandboxesRemaining = this.allocatedSandboxes.length;\n if (sandboxesRemaining === 0)\n {\n resolve();\n }\n // Resolve and finish work once all sandboxes have finished submitting their results.\n this.on('submitFinished', () => {\n sandboxesRemaining--;\n if (sandboxesRemaining === 0)\n {\n console.log('All sandboxes empty, stopping worker and closing all connections');\n resolve();\n }\n });\n });\n }\n\n if (this.resultSubmitterConnection) {\n this.resultSubmitterConnection.off('close', this.openResultSubmitterConn);\n this.resultSubmitterConnection.close();\n this.resultSubmitterConnection = null;\n }\n\n if (this.taskDistributorConnection) {\n this.taskDistributorConnection.off('close', this.openTaskDistributorConn);\n this.taskDistributorConnection.close();\n this.taskDistributorConnection = null;\n }\n\n if (this.packageManagerConnection) {\n this.packageManagerConnection.off('close', this.openPackageManagerConn);\n this.packageManagerConnection.close();\n this.packageManagerConnection = null;\n }\n\n if (this.eventRouterConnection) {\n this.eventRouterConnection.off('close', this.openEventRouterConn);\n this.eventRouterConnection.close();\n this.eventRouterConnection = null;\n }\n\n this.emit('stop');\n }\n\n /**\n * Takes a slice and returns it to the scheduler to be redistributed.\n * Usually called when an exception is thrown by sandbox.work(slice, startDelayMs) .\n * Or when the supervisor tells it to forcibly stop working.\n *\n * @param {Slice} slice - The slice to return to the scheduler.\n * @param {string} [reason] - Optional reason for the return: 'ENOPROGRESS', 'EUNCAUGHT', 'ESLICETOOSLOW', 'unknown'.\n * @returns {Promise<*>} - Response from the scheduler.\n */\n returnSlice (slice, reason) {\n // When sliceNumber === 0 don't send a status message.\n if (slice.sliceNumber === 0) return Promise.resolve();\n \n debugging() && console.log(`Supervisor.returnSlice: Returning slice ${slice.identifier} with reason ${reason}.`);\n \n const payload = slice.getReturnMessagePayload(this.workerOpaqueId, reason);\n try\n {\n return this.resultSubmitterConnection.send('status', payload) /* resultSubmitterConnection can be null if worker is stopped */\n .then(response => {\n return response;\n }).catch(error => {\n debugging('supervisor') && console.error('Failed to return slice', {\n sliceNumber: slice.sliceNumber,\n jobAddress: slice.jobAddress,\n status: slice.status,\n error,\n }, 'Will try again on a new connection.');\n this.resultSubmitterMessageQueue.push({ operation: 'status', data: payload });\n this.resultSubmitterConnection.close();\n });\n }\n catch (error)\n {\n debugging('supervisor') && console.error(`Failed to return slice ${slice.identifier}, no connection to result submitter:`, error);\n }\n }\n\n /**\n * Bulk-return multiple slices, possibly for assorted jobs.\n * Returns slices to the scheduler to be redistributed.\n * Called in the sandbox terminate handler and purgeAllWork(jobAddress)\n * and stopWork(forceTerminate).\n *\n * @param {Slice[]} slices - The slices to return to the scheduler.\n * @returns {Promise<void>} - Response from the scheduler.\n */\n async returnSlices(slices) {\n if (!slices || !slices.length) return Promise.resolve();\n \n const slicePayload = [];\n slices.forEach(slice => { addToReturnSlicePayload(slicePayload, slice); });\n this.removeSlices(slices);\n\n debugging('supervisor') && console.log(`Supervisor.returnSlices: Returning slices ${this.dumpSlices(slices)}.`);\n\n return this.resultSubmitterConnection.send('status', {\n worker: this.workerOpaqueId,\n slices: slicePayload,\n }).then(response => {\n return response;\n }).catch(error => {\n const errorInfo = slices.map(slice => slice.identifier);\n debugging('supervisor') && console.error('Failed to return slice(s)', { errorInfo, error }, 'Will try again on new connection.');\n this.resultSubmitterMessageQueue.push({ operation: 'status', data: { worker: this.workerOpaqueId, slices: slicePayload } })\n this.resultSubmitterConnection.close();\n // Just in case the caller is expecing a DCP response\n return { success: false, payload: {} };\n });\n }\n\n /**\n * Submits the slice results to the scheduler, either to the\n * work submit or fail endpoints based on the slice status.\n * Then remove the slice from the @this.slices cache.\n *\n * @param {Slice} slice - The slice to submit.\n * @returns {Promise<void>}\n */\n async recordResult (slice) {\n // It is possible for slice.result to be undefined when there are upstream errors.\n if ( !(slice && slice.result))\n throw new Error(`recordResult: slice.result is undefined for slice ${slice.identifier}. This is ok when there are upstream errors.`);\n\n debugging('supervisor') && console.log(`supervisor: recording result for slice ${slice.identifier}.`);\n\n const jobAddress = slice.jobAddress;\n const sliceNumber = slice.sliceNumber;\n const authorizationMessage = slice.getAuthorizationMessage();\n\n /* @see result-submitter::result for full message details */\n const metrics = { GPUTime: 0, CPUTime: 0, CPUDensity: 0, GPUDensity: 0, total: 0 };\n const payloadData = {\n slice: sliceNumber,\n job: jobAddress,\n worker: this.workerOpaqueId,\n paymentAddress: this.paymentAddress,\n metrics,\n authorizationMessage,\n }\n \n const timeReport = slice.timeReport;\n if (timeReport && timeReport.total > 0) {\n metrics.GPUTime = timeReport.webGL;\n metrics.CPUTime = timeReport.CPU;\n metrics.CPUDensity = metrics.CPUTime / timeReport.total;\n metrics.GPUDensity = metrics.GPUTime / timeReport.total;\n metrics.total = timeReport.total;\n metrics.CPUTime = 1 + Math.floor(metrics.CPUTime);\n metrics.GPUTime = 1 + Math.floor(metrics.GPUTime);\n }\n \n this.emit('submittingResult');\n\n if (!slice.isFinished)\n throw new Error('Cannot record result for slice that is not finished');\n\n if (slice.resultStorageType === 'pattern') { /* This is a remote-storage slice. */\n const remoteResult = await this.sendResultToRemote(slice);\n payloadData.result = encodeDataURI(JSON.stringify(remoteResult));\n } else {\n payloadData.result = encodeDataURI(slice.result.result); /* XXXwg - result.result is awful */\n }\n debugging('supervisor') && console.log('Supervisor.recordResult: payloadData.result', payloadData.result.slice(0, 512));\n\n try {\n if (slice.completed) {\n\n /* work function returned a result */\n let resp = await this.resultSubmitterConnection.send(\n 'result',\n payloadData,\n ).catch((error) => {\n debugging('supervisor') && console.error(`Failed to submit result to scheduler for slice ${payloadData.slice} of job ${payloadData.job}:\\n ${error} \\nWill try again on new connection.`);\n this.resultSubmitterMessageQueue.push({ operation: 'result', data: payloadData });\n this.resultSubmitterConnection.close();\n error.handled = true;\n throw error; /* Caught in catch block below */\n });\n \n if (!resp.success)\n throw resp.payload;\n\n if (false) {}\n\n const receipt = {\n accepted: true,\n payment: resp.payload.slicePaymentAmount,\n };\n this.emit('submittedResult', resp.payload);\n this.emit('dccCredit', receipt);\n } else {\n /* slice did not complete for some reason */\n \n // If the slice from a job never completes and the job address exists in the ringBufferofJobs, \n // then we remove it to allow for another slice (from the same job) to be obtained by fetchTask\n this.ringBufferofJobs.buf = this.ringBufferofJobs.filter(element => element !== jobAddress);\n \n await this.returnSlice(slice);\n }\n } catch(error) {\n console.info(`1014: Failed to submit results for slice ${payloadData.slice} of job ${payloadData.job}`, error);\n this.emit('submitSliceFailed', error);\n } finally {\n this.emit('submitFinished');\n // Remove the slice from the slices array.\n this.removeSlice(slice);\n if (Supervisor.sliceTiming) {\n slice['resultDelta'] = Date.now() - slice['resultDelta'];\n console.log(`recordResult(${slice['pairingDelta']}, ${slice['executionDelta']}, ${slice['resultDelta']}): Completed slice ${slice.identifier}.`);\n } else\n debugging('supervisor') && console.log(`recordResult: Completed slice ${slice.identifier}.`);\n }\n }\n\n /**\n * Send a work function's result to a server that speaks our DCP Remote Data Server protocol.\n * The data server dcp-rds is been implemented in https://gitlab.com/Distributed-Compute-Protocol/dcp-rds .\n *\n * @param {Slice} slice - Slice object whose result we are sending.\n * @returns {Promise<object>} - Object of the form { success: true, href: 'http://127.0.0.1:3521/methods/download/jobs/34/result/10' } .\n * @throws When HTTP status not in the 2xx range.\n */\n async sendResultToRemote(slice) {\n const postParams = {\n ...slice.resultStorageParams\n };\n\n const sliceResultUri = makeDataURI('pattern', slice.resultStorageDetails, {\n slice: slice.sliceNumber,\n job: slice.jobAddress,\n });\n\n debugging() && console.log('sendResultToRemote sliceResultUri: ', sliceResultUri);\n\n const url = new DcpURL(sliceResultUri);\n\n // Note: sendResultToRemote was made a member function of class Supervisor to enable access to this.alowedOrigins .\n if (this.allowedOrigins.indexOf(url.origin) === -1 &&\n dcpConfig.worker.allowOrigins.sendResults.indexOf(url.origin) === -1) {\n throw new Error(`Invalid origin for remote result storage: '${url.origin}'`);\n }\n\n postParams.element = slice.sliceNumber;\n postParams.contentType = 'application/json'; // Currently data will be outputed as a JSON object, @todo: Support file upload.\n\n debugging() && console.log('sendResultToRemote: postParams: ', postParams);\n\n let result = slice.result.result;\n if (result) {\n postParams.content = JSON.stringify(result);\n } else {\n postParams.error = JSON.stringify(slice.error);\n }\n\n debugging('supervisor') && console.log('sendResultToRemote: content: ', (result ? postParams.content : postParams.error).slice(0, 512));\n\n //\n // Notes:\n // 1) In recordResults the response from justFetch is JSON serialized and encodeDataURI is called.\n // payloadData.result = await this.sendResultToRemote(slice);\n // payloadData.result = encodeDataURI(JSON.stringify(payloadData.result));\n // 2) We do further processing after the call to sendResultToRemote in recordResult, because\n // if we did it here there would be a perf hit. When the return value is a promise, it gets\n // folded into sendResultToRemote's main promise. If justFetch's promise wasn't a return value then\n // justFetch would be separately added to the micro-task-queue.\n return await justFetch(url, 'JSON', 'POST', false, postParams);\n }\n}\n\n/**\n * Sandbox has had an error which is not from the work function: kill it\n * and try to redo the slice.\n */\nfunction handleSandboxError(supervisor, sandbox, error) {\n const slice = sandbox.slice;\n\n slice.sandboxErrorCount = (slice.sandboxErrorCount || 0) + 1;\n sandbox.slice = null;\n supervisor.returnSandbox(sandbox); /* terminate the sandbox */\n slice.status = SLICE_STATUS_UNASSIGNED; /* ToT */\n console.warn(`Supervisor.handleSandboxError: Sandbox ${sandbox.identifier}...(${sandbox.public.name}/${slice.sandboxErrorCount}) with slice ${slice.identifier} had error.`, error);\n\n if (slice.sandboxErrorCount < dcpConfig.worker.maxSandboxErrorsPerSlice)\n supervisor.queuedSlices.push(slice);\n else {\n slice.error = error;\n supervisor.returnSlice(slice);\n }\n}\n\n/**\n * Add a slice to the slice payload being built. If a sliceList already exists for the\n * job-status-authMessage tuple, then the slice will be added to that, otherwise a new\n * sliceList will be added to the payload.\n *\n * @param {Object[]} slicePayload - Slice payload being built. Will be mutated in place.\n * @param {Slice} slice - The slice.\n * @param {String} status - Status update, eg. progress or scheduled.\n *\n * @returns {Object[]} mutated slicePayload array\n */\nfunction addToSlicePayload(slicePayload, slice, status) {\n // getAuthorizationMessage helps enforces the equivalence\n // !authorizationMessage <==> sliceNumber === 0\n const authorizationMessage = slice.getAuthorizationMessage();\n if (!authorizationMessage) return;\n\n // Try to find a sliceList in the payload which matches the job, status, and auth message\n let sliceList = slicePayload.find(desc => {\n return desc.job === slice.jobAddress\n && desc.status === status\n && desc.authorizationMessage === authorizationMessage;\n });\n\n // If we didn't find a sliceList, start a new one and add it to the payload\n if (!sliceList) {\n sliceList = {\n job: slice.jobAddress,\n sliceNumbers: [],\n status,\n authorizationMessage,\n };\n slicePayload.push(sliceList);\n }\n\n sliceList.sliceNumbers.push(slice.sliceNumber);\n\n return slicePayload;\n}\n\n/**\n * Add a slice to the returnSlice payload being built. If a sliceList already exists for the\n * job-isEstimation-authMessage-reason tuple, then the slice will be added to that, otherwise a new\n * sliceList will be added to the payload.\n *\n * @param {Object[]} slicePayload - Slice payload being built. Will be mutated in place.\n * @param {Slice} slice - The slice.\n * @param {String} [reason] - Optional reason to further characterize status; e.g. 'ENOPROGRESS', 'EUNCAUGHT', 'ESLICETOOSLOW', 'unknown'.\n *\n * @returns {Object[]} mutated slicePayload array\n */\nfunction addToReturnSlicePayload(slicePayload, slice, reason) {\n // getAuthorizationMessage helps enforces the equivalence\n // !authorizationMessage <==> sliceNumber === 0\n const authorizationMessage = slice.getAuthorizationMessage();\n if (!authorizationMessage) return;\n\n if (!reason) reason = slice.error ? 'EUNCAUGHT' : 'unknown';\n\n // Try to find a sliceList in the payload which matches the job, status, and auth message\n let sliceList = slicePayload.find(desc => {\n return desc.job === slice.jobAddress\n && desc.isEstimationSlice === slice.isEstimationSlice\n && desc.authorizationMessage === authorizationMessage\n && desc.reason === reason;\n });\n\n // If we didn't find a sliceList, start a new one and add it to the payload\n if (!sliceList) {\n sliceList = {\n job: slice.jobAddress,\n sliceNumbers: [],\n status: 'return',\n isEstimationSlice: slice.isEstimationSlice,\n authorizationMessage,\n reason,\n };\n slicePayload.push(sliceList);\n }\n\n sliceList.sliceNumbers.push(slice.sliceNumber);\n\n return slicePayload;\n}\n\n/**\n * Return DCPv4-specific connection options, composed of type-specific, URL-specific, \n * and worker-specific options, any/all of which can override the dcpConfig.dcp.connectOptions.\n * The order of precedence is the order of specificity.\n */\nfunction connectionOptions(url, label) {\n return leafMerge(/* ordered from most to least specific */\n dcpConfig.worker.dcp.connectionOptions.default,\n dcpConfig.worker.dcp.connectionOptions[label],\n dcpConfig.worker.dcp.connectionOptions[url.href]);\n}\n\n/** @type {number | boolean} */\nSupervisor.lastAssignFailTimerMs = false;\n/** @type {boolean} */\nSupervisor.startSandboxWork_beenCalled = false;\n/** @type {boolean} */\nSupervisor.debugBuild = ((__webpack_require__(/*! dcp/common/dcp-build */ \"./src/common/dcp-build.js\").build) === 'debug');\n/**\n * When Supervisor.sliceTiming is set to be true, it displays the timings of a every slice\n * slice['pairingDelta'] = timespan of when slice is paired with sandbox until execution starts\n * slice['executionDelta'] = timespan of execution in sandbox\n * slice['resultDelta'] = timespan of when sandbox finishes executing until recordResult completes.\n * @type {boolean}\n */\nSupervisor.sliceTiming = false;\n\nexports.Supervisor = Supervisor;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/supervisor.js?");
|
|
4459
|
+
eval("/* provided dependency */ var process = __webpack_require__(/*! ./node_modules/process/browser.js */ \"./node_modules/process/browser.js\");\n/**\n * @file worker/supervisor.js\n *\n * The component that controls each of the sandboxes\n * and distributes work to them. Also communicates with the\n * scheduler to fetch said work.\n *\n * The supervisor readies sandboxes before/while fetching slices.\n * This means sometimes there are extra instantiated WebWorkers\n * that are idle (in this.readiedSandboxes). Readied sandboxes can\n * be used for any slice. After a readied sandbox is given a slice\n * it becomes assigned to slice's job and can only do work\n * for that job.\n *\n * After a sandbox completes its work, the sandbox becomes cached\n * and can be reused if another slice with a matching job is fetched.\n *\n * @author Matthew Palma, mpalma@kingsds.network\n * Ryan Rossiter, ryan@kingsds.network\n * @date May 2019\n */\n\n/* global dcpConfig */\n// @ts-check\n\n\nconst constants = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst hash = __webpack_require__(/*! dcp/common/hash */ \"./src/common/hash.js\");\nconst wallet = __webpack_require__(/*! dcp/dcp-client/wallet */ \"./src/dcp-client/wallet/index.js\");\nconst protocolV4 = __webpack_require__(/*! dcp/protocol-v4 */ \"./src/protocol-v4/index.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('worker');\nconst { assert } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\nconst { EventEmitter } = __webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\");\nconst { Sandbox, SandboxError } = __webpack_require__(/*! ./sandbox */ \"./src/dcp-client/worker/sandbox.js\");\nconst { Slice, SLICE_STATUS_UNASSIGNED, SLICE_STATUS_FAILED } = __webpack_require__(/*! ./slice */ \"./src/dcp-client/worker/slice.js\");\nconst { SupervisorCache } = __webpack_require__(/*! ./supervisor-cache */ \"./src/dcp-client/worker/supervisor-cache.js\");\nconst { DcpURL } = __webpack_require__(/*! dcp/common/dcp-url */ \"./src/common/dcp-url.js\");\nconst { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\nconst { localStorage } = __webpack_require__(/*! dcp/common/dcp-localstorage */ \"./src/common/dcp-localstorage.js\");\nconst { booley, encodeDataURI, makeDataURI, leafMerge, a$sleepMs, justFetch, compressJobMap, toJobMap,\n compressSandboxes, compressSlices, truncateAddress, dumpSandboxesIfNotUnique, dumpSlicesIfNotUnique, generateOpaqueId } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst { sliceStatus } = __webpack_require__(/*! dcp/common/scheduler-constants */ \"./src/common/scheduler-constants.js\");\nconst { calculateJoinHash } = __webpack_require__(/*! dcp/dcp-client/compute-groups */ \"./src/dcp-client/compute-groups/index.js\");\nconst RingBuffer = __webpack_require__(/*! dcp/utils/ringBuffer */ \"./src/utils/ringBuffer.js\");\nconst supervisorTuning = dcpConfig.future('worker.tuning');\nconst tuning = {\n watchdogInterval: 7, /**< seconds - time between fetches when ENOTASK(? /wg nov 2019) */\n minSandboxStartDelay: 0.1, /**< seconds - minimum time between WebWorker starts */\n maxSandboxStartDelay: 0.7, /**< seconds - maximum delay time between WebWorker starts */\n ...supervisorTuning\n};\n\n/** Make timers 10x slower when running in niim */\nlet timeDilation = 1;\nif (DCP_ENV.platform === 'nodejs') {\n /** Make timers 10x slower when running in niim */\n timeDilation = (requireNative('module')._cache.niim instanceof requireNative('module').Module) ? 10 : 1;\n}\n\ndcpConfig.future('worker.sandbox', { progressReportInterval: (5 * 60 * 1000) });\nconst sandboxTuning = dcpConfig.worker.sandbox;\n\n/**\n * @typedef {*} address\n * @typedef {*} opaqueId\n */\n\n/**\n * @typedef {object} SandboxSlice\n * @property {Sandbox} sandbox\n * @property {Slice} slice\n */\n\n/**\n * @typedef {object} Signature\n * @property {Uint8Array} r\n * @property {Uint8Array} s\n * @property {Uint8Array} v\n */\n\n/**\n * @typedef {object} SignedAuthorizationMessageObject\n * @property {object} auth\n * @property {Signature} signature\n * @property {module:dcp/wallet.Address} owner\n */\n\n/** @typedef {import('.').Worker} Worker */\n/** @typedef {import('.').SupervisorOptions} SupervisorOptions */\n\nclass Supervisor extends EventEmitter {\n /**\n * @constructor\n * @param {Worker} worker\n * @param {SupervisorOptions} options\n */\n constructor (worker, options={}) {\n super('Supervisor');\n\n /** @type {Worker} */\n this.worker = worker;\n\n /** @type {Sandbox[]} */\n this.sandboxes = [];\n\n /** @type {Sandbox[]} */\n this.readiedSandboxes = [];\n\n /** @type {Sandbox[]} */\n this.assignedSandboxes = [];\n\n /** @type {Slice[]} */\n this.slices = [];\n\n /** @type {Slice[]} */\n this.queuedSlices = [];\n\n /** @type {Slice[]} */\n this.lostSlices = [];\n\n /** @type {boolean} */\n this.matching = false;\n\n /** @type {boolean} */\n this.isFetchingNewWork = false;\n\n /** @type {number} */\n this.numberOfCoresReserved = 0;\n\n /** @type {number} */\n this.addressTruncationLength = 20; // Set to -1 for no truncation.\n\n /** @type {Object[]} */\n this.rejectedJobs = [];\n this.rejectedJobReasons = [];\n\n if (!options) {\n console.error('Supervisor Options', options, new Error().stack);\n options = {};\n }\n\n /** @type {object} */\n this.options = {\n jobAddresses: options.jobAddresses || [/* all jobs unless priorityOnly */],\n ...options,\n };\n\n const { paymentAddress, identityKeystore } = options;\n if (paymentAddress) {\n if (paymentAddress instanceof wallet.Keystore) {\n this.paymentAddress = paymentAddress.address;\n } else {\n this.paymentAddress = new wallet.Address(paymentAddress);\n }\n } else {\n this.paymentAddress = null;\n }\n\n this._identityKeystore = identityKeystore;\n\n // In localExec, do not allow work function or arguments to come from the 'any' origins\n this.allowedOrigins = []\n if (this.options.localExec)\n dcpConfig.worker.allowOrigins.sendResults = dcpConfig.worker.allowOrigins.sendResults.concat(dcpConfig.worker.allowOrigins.any)\n else\n this.allowedOrigins = dcpConfig.worker.allowOrigins.any;\n\n if(options.allowedOrigins && options.allowedOrigins.length > 0)\n this.allowedOrigins = options.allowedOrigins.concat(this.allowedOrigins);\n\n /**\n * Maximum sandboxes allowed to work at a given time.\n * @type {number}\n */\n this.maxWorkingSandboxes = options.maxWorkingSandboxes || 1;\n\n /** @type {number} */\n this.defaultMaxGPUs = 1;\n // this.GPUsAssigned = 0;\n \n // Object.defineProperty(this, 'GPUsAssigned', {\n // get: () => this.allocatedSandboxes.filter(sb => !!sb.requiresGPU).length,\n // enumerable: true,\n // configurable: false,\n // });\n\n /**\n * TODO: Remove this when the supervisor sends all of the sandbox\n * capabilities to the scheduler when fetching work.\n * @type {object}\n */\n this.capabilities = null;\n\n /** @type {number} */\n this.lastProgressReport = 0;\n\n /** \n * An N-slot ring buffer of job addresses. Stores all jobs that have had no more than 1 slice run in the ring buffer.\n * Required for the implementation of discrete jobs \n * @type {RingBuffer} \n */\n this.ringBufferofJobs = new RingBuffer(200); // N = 200 should be more than enough.\n \n // @hack - dcp-env.isBrowserPlatform is not set unless the platform is _explicitly_ set,\n // using the default detected platform doesn't set it.\n // Fixing that causes an error in the wallet module's startup on web platform, which I\n // probably can't fix in a reasonable time this morning.\n // ~ER2020-02-20\n\n if (!options.maxWorkingSandboxes\n && DCP_ENV.browserPlatformList.includes(DCP_ENV.platform)\n && navigator.hardwareConcurrency > 1) {\n this.maxWorkingSandboxes = navigator.hardwareConcurrency - 1;\n if (typeof navigator.userAgent === 'string') {\n if (/(Android).*(Chrome|Chromium)/.exec(navigator.userAgent)) {\n this.maxWorkingSandboxes = 1;\n console.log('Doing work with Chromimum browsers on Android is currently limited to one sandbox');\n }\n }\n }\n\n /** @type {SupervisorCache} */\n this.cache = new SupervisorCache(this);\n /** @type {object} */\n this._connections = {}; /* active DCPv4 connections */\n // Call the watchdog every 7 seconds.\n this.watchdogInterval = setInterval(() => this.watchdog(), tuning.watchdogInterval * 1000);\n if (DCP_ENV.platform === 'nodejs' && this.options.localExec) /* interval helps keep normal worker alive forever, which we don't want in localexec */\n this.watchdogInterval.unref();\n\n const ceci = this;\n\n // Initialize to null so these properties are recognized for the Supervisor class\n this.taskDistributorConnection = null;\n this.eventRouterConnection = null;\n this.resultSubmitterConnection = null;\n this.packageManagerConnection = null;\n this.openTaskDistributorConn = function openTaskDistributorConn()\n {\n let config = dcpConfig.scheduler.services.taskDistributor;\n ceci.taskDistributorConnection = new protocolV4.Connection(config, ceci.identityKeystore, connectionOptions(config.location, 'taskDistributor'));\n ceci.taskDistributorConnection.on('close', ceci.openTaskDistributorConn);\n }\n\n this.openEventRouterConn = function openEventRouterConn()\n {\n let config = dcpConfig.scheduler.services.eventRouter;\n ceci.eventRouterConnection = new protocolV4.Connection(config, ceci.identityKeystore, connectionOptions(config.location, 'eventRouter'));\n ceci.eventRouterConnection.on('close', ceci.openEventRouterConn);\n if (ceci.eventRouterMessageQueue.length)\n ceci.resendRejectedMessages(ceci.eventRouterConnection, ceci.eventRouterMessageQueue);\n }\n this.eventRouterMessageQueue = [];\n \n this.openResultSubmitterConn = function openResultSubmitterConn()\n {\n let config = dcpConfig.scheduler.services.resultSubmitter;\n ceci.resultSubmitterConnection = new protocolV4.Connection(config, ceci.identityKeystore, connectionOptions(config.location, 'resultSubmitter'));\n ceci.resultSubmitterConnection.on('close', ceci.openResultSubmitterConn);\n if (ceci.resultSubmitterMessageQueue.length)\n ceci.resendRejectedMessages(ceci.resultSubmitterConnection, ceci.resultSubmitterMessageQueue);\n }\n this.resultSubmitterMessageQueue = [];\n\n this.openPackageManagerConn = function openPackageManagerConn()\n {\n let config = dcpConfig.packageManager;\n ceci.packageManagerConnection = new protocolV4.Connection(config, ceci.identityKeystore, connectionOptions(config.location, 'packageManager'));\n ceci.packageManagerConnection.on('close', ceci.openPackageManagerConn);\n if (ceci.packageManagerMessageQueue.length)\n ceci.resendRejectedMessages(ceci.packageManagerConnection, ceci.packageManagerMessageQueue);\n }\n this.packageManagerMessageQueue = [];\n }\n\n /**\n * Return worker opaqueId.\n * @type {opaqueId}\n */\n get workerOpaqueId() {\n if (!this._workerOpaqueId)\n this._workerOpaqueId = localStorage.getItem('workerOpaqueId');\n\n if (!this._workerOpaqueId || this._workerOpaqueId.length !== constants.workerIdLength) {\n this._workerOpaqueId = generateOpaqueId();\n localStorage.setItem('workerOpaqueId', this._workerOpaqueId);\n }\n\n return this._workerOpaqueId;\n }\n\n /**\n * This getter is the absolute source-of-truth for what the\n * identity keystore is for this instance of the Supervisor.\n */\n get identityKeystore() {\n assert(this.defaultIdentityKeystore);\n\n return this._identityKeystore || this.defaultIdentityKeystore;\n }\n\n /**\n * Open all connections. Used when supervisor is instantiated or stopped/started\n * to initially open connections.\n */\n instantiateAllConnections() {\n if (!this.taskDistributorConnection)\n this.openTaskDistributorConn();\n \n if (!this.eventRouterConnection)\n this.openEventRouterConn();\n \n if (!this.resultSubmitterConnection)\n this.openResultSubmitterConn();\n\n if (!this.packageManagerConnection)\n this.openPackageManagerConn();\n }\n \n /**\n * Try sending messages that were rejected on an old instance of the given connection.\n */\n async resendRejectedMessages(connection, messageQueue) {\n var message = messageQueue.shift();\n\n do {\n try {\n await connection.send(message.operation, message.data);\n } catch (e) {\n debugging('supervisor') && console.error(`Failed to send message ${message.operation} to scheduler: ${e}. Will try again on a new connection.`);\n messageQueue.unshift(message);\n connection.close();\n break;\n }\n message = messageQueue.shift();\n } while (message)\n }\n\n /** Set the default identity keystore -- needs to happen before anything that talks\n * to the scheduler for work gets called. This is a wart and should be removed by\n * refactoring.\n *\n * The default identity keystore will be used if the Supervisor was not provided\n * with an alternate. This keystore will be located via the Wallet API, and \n * if not found, a randomized default identity will be generated. \n *\n * @param {object} ks An instance of wallet::Keystore -- if undefined, we pick the best default we can.\n * @returns {Promise<void>}\n */\n async setDefaultIdentityKeystore(ks) {\n try {\n if (ks) {\n this.defaultIdentityKeystore = ks;\n return;\n }\n\n if (this.defaultIdentityKeystore)\n return;\n\n try {\n this.defaultIdentityKeystore = await wallet.getId();\n } catch(e) {\n debugging('supervisor') && console.debug('Error generating default identity, try to do it another way.');\n this.defaultIdentityKeystore = await new wallet.IdKeystore(null, '');\n }\n } finally {\n if (this.defaultIdentityKeystore)\n debugging('supervisor') && console.debug('Set default identity =', this.defaultIdentityKeystore.address);\n else\n debugging('supervisor') && console.debug('Failed to set default identity, worker cannot work.');\n }\n }\n\n //\n // What follows is a bunch of utility properties and functions for creating filtered views\n // of the slices and sandboxes array.\n //\n /** XXXpfr @todo Write sort w/o using promises so we can get rid of async on all the compress functions. */\n\n /**\n * @deprecated -- Please do not use this.workingSandboxes; use this.allocatedSandboxes instead.\n * Sandboxes that are in WORKING state.\n *\n * Warning: Do not rely on this information being 100% accurate -- it may change in the next instant.\n * @type {Sandbox[]}\n */\n get workingSandboxes() {\n return this.sandboxes.filter(sandbox => sandbox.isWorking);\n }\n\n /**\n * Use instead of this.workingSandboxes.\n *\n * When a sandbox is paired with a slice, execution is pending and sandbox.allocated=true and\n * sandbox.slice=slice and sandbox.jobAddress=slice.jobAddress. This is what 'allocated' means.\n * Immediately upon the exit of sandbox.work, sandbox.allocated=false is set and if an exception\n * wasn't thrown the sandbox is placed in this.assignedSandboxes.\n * Thus from the pov of supervisor, this.allocatedSandboxes is deterministic and this.workingSandboxes is not.\n * Please try to not use this.workingSandboxes. It is deprecated.\n *\n * Warning: Do not rely on this information being 100% accurate -- it may change in the next instant.\n * @type {Sandbox[]}\n */\n get allocatedSandboxes() {\n return this.sandboxes.filter(sandbox => sandbox.allocated);\n }\n\n /**\n * Slices that are allocated.\n * Warning: Do not rely on this information being 100% accurate -- it may change in the next instant.\n * @type {Slice[]}\n */\n get allocatedSlices() {\n return this.slices.filter(slice => slice.allocated);\n }\n\n /**\n * This property is used as the target number of sandboxes to be associated with slices and start working.\n *\n * It is used in this.watchdog as to prevent a call to this.work when unallocatedSpace <= 0.\n * It is also used in this.distributeQueuedSlices where it is passed as an argument to this.matchSlicesWithSandboxes to indicate how many sandboxes\n * to associate with slices and start working.\n *\n * Warning: Do not rely on this information being 100% accurate -- it may change in the next instant.\n * @type {number}\n */\n get unallocatedSpace() {\n return this.maxWorkingSandboxes - this.allocatedSandboxes.length - this.numberOfCoresReserved;\n }\n \n /**\n * Call acquire(numberOfCoresToReserve) to reserve numberOfCoresToReserve unallocated sandboxes as measured by unallocatedSpace.\n * Call release() to undo the previous acquire.\n * This pseudo-mutex technique helps prevent races in scheduling slices in Supervisor.\n * @param {number} numberOfCoresToReserve\n */\n acquire(numberOfCoresToReserve) { \n this.numberOfCoresReserved = numberOfCoresToReserve; \n }\n release() { \n this.numberOfCoresReserved = 0; \n }\n\n /**\n * Remove from this.slices.\n * @param {Slice} slice\n */\n removeSlice(slice) {\n this.removeElement(this.slices, slice);\n if (Supervisor.debugBuild) {\n if (this.queuedSlices.indexOf(slice) !== -1)\n throw new Error(`removeSlice: slice ${slice.identifier} is in queuedSlices; inconsistent state.`);\n if (this.lostSlices.length > 0) {\n console.warn(`removeSlice: slice ${slice.identifier}, found lostSlices ${this.lostSlices.map(s => s.identifier)}`);\n if (this.lostSlices.indexOf(slice) !== -1)\n throw new Error(`removeSlice: slice ${slice.identifier} is in lostSlices; inconsistent state.`);\n }\n }\n }\n\n /**\n * Remove from this.slices.\n * @param {Slice[]} slices\n */\n removeSlices(slices) {\n this.slices = this.slices.filter(slice => slices.indexOf(slice) === -1);\n }\n\n /**\n * Remove from this.queuedSlices.\n * @param {Slice[]} slices\n */\n removeQueuedSlices(slices) {\n this.queuedSlices = this.queuedSlices.filter(slice => slices.indexOf(slice) === -1);\n }\n\n /**\n * Remove from this.sandboxes, this.assignedSandboxes and this.readiedSandboxes.\n * @param {Sandbox} sandbox\n */\n removeSandbox(sandbox) {\n debugging('scheduler') && console.log(`removeSandbox ${sandbox.identifier}`);\n this.removeElement(this.sandboxes, sandbox);\n this.removeElement(this.assignedSandboxes, sandbox);\n\n // XXXpfr: April 13, 2022\n // I'm trying to understand and control when sandboxes get removed.\n // A sandbox in this.readiedSandboxes should never have returnSandbox/removeSandbox called on it except in stopWork.\n // Because of races and random worker crashes, it is hard to get this right, but I want to try.\n // If I don't fix this is the next 30 days or I forget, please delete this exception.\n if (false)\n {}\n\n this.removeElement(this.readiedSandboxes, sandbox);\n }\n\n /**\n * Remove from this.sandboxes and this.assignedSandboxes .\n * @param {Sandbox[]} sandboxes\n */\n async removeSandboxes(sandboxes) {\n debugging('scheduler') && console.log(`removeSandboxes: Remove ${sandboxes.length} sandboxes ${this.dumpSandboxes(sandboxes)}`);\n this.sandboxes = this.sandboxes.filter(sandbox => sandboxes.indexOf(sandbox) === -1);\n this.assignedSandboxes = this.assignedSandboxes.filter(sandbox => sandboxes.indexOf(sandbox) === -1);\n\n if (Supervisor.debugBuild) {\n const readied = this.readiedSandboxes.filter(sandbox => sandboxes.indexOf(sandbox) !== -1);\n if (readied.length > 0)\n throw new Error(`removeSandboxes: sandboxes ${readied.map(s => s.identifier)} are in readiedSandboxes; inconsistent state.`);\n }\n }\n\n /**\n * Remove element from theArray.\n * @param {Array<*>} theArray\n * @param {object|number} element\n * @param {boolean} [assertExists = true]\n */\n removeElement(theArray, element, assertExists = false) {\n let index = theArray.indexOf(element);\n assert(index !== -1 || !assertExists);\n if (index !== -1) theArray.splice(index, 1);\n }\n\n /**\n * Log sliceArray.\n * @param {Slice[]} sliceArray\n * @param {string} [header]\n * @returns {string}\n */\n dumpSlices(sliceArray, header) {\n if (header) console.log(`\\n${header}`);\n return compressSlices(sliceArray, this.addressTruncationLength);\n }\n\n /**\n * Log sandboxArray.\n * @param {Sandbox[]} sandboxArray\n * @param {string} [header]\n * @returns {string}\n */\n dumpSandboxes(sandboxArray, header) {\n if (header) console.log(`\\n${header}`);\n return compressSandboxes(sandboxArray, this.addressTruncationLength);\n }\n\n /**\n * If the elements of sandboxSliceArray are not unique, log the duplicates and dump the array.\n * @param {SandboxSlice[]} sandboxSliceArray\n * @param {string} header\n */\n dumpSandboxSlicesIfNotUnique(sandboxSliceArray, header) {\n if (!this.isUniqueSandboxSlices(sandboxSliceArray, header))\n console.log(this.dumpSandboxSlices(sandboxSliceArray));\n }\n\n /**\n * Log { sandbox, slice }.\n * @param {Sandbox} sandbox\n * @param {Slice} slice\n * @returns {string}\n */\n dumpSandboxAndSlice(sandbox, slice) {\n return `${sandbox.id}~${slice.sliceNumber}.${this.dumpJobAddress(slice.jobAddress)}`;\n }\n\n /**\n * Log { sandbox, slice } with state/status.\n * @param {Sandbox} sandbox\n * @param {Slice} slice\n * @returns {string}\n */\n dumpStatefulSandboxAndSlice(sandbox, slice) {\n return `${sandbox.id}.${sandbox.state}~${slice.sliceNumber}.${this.dumpJobAddress(slice.jobAddress)}.${slice.status}`;\n }\n\n /**\n * Truncates jobAddress.toString() to this.addressTruncationLength digits.\n * @param {address} jobAddress\n * @returns {string}\n */\n dumpJobAddress(jobAddress) {\n return truncateAddress(jobAddress, this.addressTruncationLength /* digits*/);\n }\n\n /**\n * Dump sandboxSliceArray.\n * @param {SandboxSlice[]} sandboxSliceArray - input array of { sandbox, slice }\n * @param {string} [header] - optional header\n * @param {boolean} [stateFul] - when true, also includes slice.status and sandbox.state.\n * @returns {string}\n */\n dumpSandboxSlices(sandboxSliceArray, header, stateFul=false) {\n if (header) console.log(`\\n${header}`);\n const jobMap = {};\n sandboxSliceArray.forEach(ss => {\n const sss = stateFul ? `${ss.sandbox.id}.${ss.sandbox.state}~${ss.slice.sliceNumber}.${ss.slice.status}` : `${ss.sandbox.id}~${ss.slice.sliceNumber}`;\n if (!jobMap[ss.slice.jobAddress]) jobMap[ss.slice.jobAddress] = sss;\n else jobMap[ss.slice.jobAddress] += `,${sss}`;\n });\n let output = '';\n for (const [jobAddress, sss] of Object.entries(jobMap))\n output += `${this.dumpJobAddress(jobAddress)}:[${sss}]:`;\n return output;\n }\n\n /**\n * Check sandboxSliceArray for duplicates.\n * @param {SandboxSlice[]} sandboxSliceArray\n * @param {string} [header]\n * @param {function} [log]\n * @returns {boolean}\n */\n isUniqueSandboxSlices(sandboxSliceArray, header, log) {\n const result = [], slices = [], sandboxes = [];\n let once = true;\n sandboxSliceArray.forEach(x => {\n const sliceIndex = slices.indexOf(x.slice);\n const sandboxIndex = sandboxes.indexOf(x.sandbox);\n\n if (sandboxIndex >= 0) {\n if (once && header) console.log(`\\n${header}`); once = false;\n log ? log(x.sandbox) : console.log(`\\tWarning: Found duplicate sandbox ${x.sandbox.identifier}.`);\n } else sandboxes.push(x.sandbox);\n\n if (sliceIndex >= 0) {\n if (once && header) console.log(`\\n${header}`); once = false;\n log ? log(x.slice) : console.log(`\\tWarning: Found duplicate slice ${x.slice.identifier}.`);\n } else {\n slices.push(x.slice);\n if (sandboxIndex < 0) result.push(x);\n }\n });\n return sandboxSliceArray.length === result.length;\n }\n\n /**\n * Attempts to create and start a given number of sandboxes.\n * The sandboxes that are created can then be assigned for a\n * specific job at a later time. All created sandboxes\n * get put into the @this.readiedSandboxes array when allocateLocalSandboxes is false.\n *\n * @param {number} numSandboxes - the number of sandboxes to create\n * @param {boolean} [allocateLocalSandboxes=false] - when true, do not place in this.readiedSandboxes\n * @returns {Promise<Sandbox[]>} - resolves with array of created sandboxes, rejects otherwise\n * @throws when given a numSandboxes is not a number or if numSandboxes is Infinity\n */\n async readySandboxes (numSandboxes, allocateLocalSandboxes = false) {\n debugging('supervisor') && console.debug(`readySandboxes: Readying ${numSandboxes} sandboxes, total sandboxes ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n \n if (typeof numSandboxes !== 'number' || Number.isNaN(numSandboxes) || numSandboxes === Infinity) {\n throw new Error(`${numSandboxes} is not a number of sandboxes that can be readied.`);\n }\n if (numSandboxes <= 0) {\n return [];\n }\n\n const sandboxStartPromises = [];\n const sandboxes = [];\n const errors = [];\n for (let i = 0; i < numSandboxes; i++) {\n const sandbox = new Sandbox(this.cache, {\n ...this.options.sandboxOptions,\n }, this.allowedOrigins);\n sandbox.addListener('ready', () => this.emit('sandboxReady', sandbox));\n sandbox.addListener('start', () => {\n this.emit('sandboxStart', sandbox);\n\n // When sliceNumber == 0, result-submitter status skips the slice,\n // so don't send it in the first place.\n // The 'start' event is fired when a worker starts up, hence there's no way\n // to determine whether sandbox has a valid slice without checking.\n if (sandbox.slice) {\n const jobAddress = sandbox.jobAddress;\n const sliceNumber = sandbox.slice.sliceNumber;\n // !authorizationMessage <==> sliceNumber === 0.\n const authorizationMessage = sandbox.slice.getAuthorizationMessage();\n\n if (authorizationMessage) {\n let statusPayload = {\n worker: this.workerOpaqueId,\n slices: [{\n job: jobAddress,\n sliceNumber: sliceNumber,\n status: 'begin',\n authorizationMessage,\n }],\n }\n \n try /* resultSubmitterConnection can be null if worker is stopped */\n {\n this.resultSubmitterConnection.send('status', statusPayload).catch((error) => {\n debugging('supervisor') && console.error(`Error sending 'status' for slice ${sliceNumber} of job ${jobAddress}:\\n ${error}\\nWill try again on a new connection`);\n this.resultSubmitterMessageQueue.push({ operation: 'status', data: statusPayload });\n this.resultSubmitterConnection.close();\n });\n } catch (error)\n {\n debugging('supervisor') && console.error(`Failed to send 'status' for slice ${sliceNumber} of job ${jobAddress}:, no connection to result submitter:`, error);\n }\n }\n }\n });\n sandbox.addListener('workEmit', ({ eventName, payload }) => {\n // Need to check if the sandbox hasn't been assigned a slice yet.\n if (!sandbox.slice) {\n if (Supervisor.debugBuild) {\n console.error(\n `Sandbox not assigned a slice before sending workEmit message to scheduler. 'workEmit' event originates from \"${eventName}\" event`, \n payload,\n );\n }\n }\n else\n {\n const jobAddress = sandbox.slice.jobAddress;\n const sliceNumber = sandbox.slice.sliceNumber;\n // sliceNumber can be zero if it came from a problem with loading modules.\n assert(jobAddress && (sliceNumber || sliceNumber === 0));\n // Send a work emit message from the sandbox to the event router\n // !authorizationMessage <==> sliceNumber === 0.\n let authorizationMessage;\n try {\n // Sometimes a sliceNumber===0 workEmit comes in before the client bundle is properly loaded.\n // Also happens with minor dcp-client version mismatches.\n authorizationMessage = sandbox.slice.getAuthorizationMessage();\n } catch(e) {\n authorizationMessage = null;\n }\n\n if (!authorizationMessage)\n {\n console.warn(`workEmit: missing authorization message for job ${jobAddress}, slice: ${sliceNumber}`);\n return Promise.resolve();\n }\n \n let workEmitPayload = {\n eventName,\n payload,\n job: jobAddress,\n slice: sliceNumber,\n worker: this.workerOpaqueId,\n authorizationMessage,\n }\n \n const workEmitPromise = this.eventRouterConnection.send('workEmit', workEmitPayload).catch(error => {\n debugging('supervisor') && console.warn(`workEmit: unable to send ${eventName} for slice ${sliceNumber} of job ${jobAddress}: ${error.message}.\\nTrying again on a new connection.`);\n this.eventRouterMessageQueue.push({ operation: 'workEmit', data: workEmitPayload })\n this.eventRouterConnection.close();\n if (Supervisor.debugBuild)\n console.error('workEmit error:', error);\n });\n\n if (Supervisor.debugBuild) {\n workEmitPromise.then(result => {\n if (!result || !result.success)\n console.warn('workEmit: event router did not accept event', result);\n });\n }\n }\n });\n\n // When any sbx completes, \n sandbox.addListener('complete', () => {\n this.watchdog();\n });\n\n sandbox.on('sandboxError', (error) => handleSandboxError(this, sandbox, error));\n \n sandbox.on('rejectedWorkMetrics', (data) =>{\n function updateRejectedMetrics(report) {\n ['total', 'CPU', 'webGL'].forEach((key) => {\n if (report[key]) sandbox.slice.rejectedTimeReport[key] += report[key];\n })\n }\n \n // If the slice already has rejected metrics, add this data to it. If not, assign this data to slices rejected metrics property\n if (sandbox.slice) {\n (sandbox.slice.rejectedTimeReport) ? updateRejectedMetrics(data.timeReport) : sandbox.slice.rejectedTimeReport = data.timeReport;\n }\n })\n \n // If the sandbox terminated and we are not shutting down, then should return all work which is currently\n // not being computed if all sandboxes are dead and the attempt to create a new one fails.\n sandbox.on('terminated',async () => {\n if (this.sandboxes.length > 0) {\n let terminatedSandboxes = this.sandboxes.filter(sbx => sbx.isTerminated);\n if (terminatedSandboxes.length === this.sandboxes.length) {\n debugging('supervisor') && console.debug(`readySandboxes: Create 1 sandbox in the sandbox-terminated-handler, total sandboxes ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n await this.readySandboxes(1);\n \n // If we cannot create a new sandbox, that probably means we're on a screensaver worker\n // and the screensaver is down. So return the slices to the scheduler.\n if (this.sandboxes.length !== terminatedSandboxes.length + 1) {\n this.returnSlices(this.queuedSlices).then(() => {\n this.queuedSlices.length = 0;\n });\n }\n }\n }\n })\n\n const delayMs =\n 1000 *\n (tuning.minSandboxStartDelay +\n Math.random() *\n (tuning.maxSandboxStartDelay - tuning.minSandboxStartDelay));\n \n sandboxStartPromises.push(\n sandbox\n .start(delayMs)\n .then(() => {\n if (!allocateLocalSandboxes) this.readiedSandboxes.push(sandbox);\n this.sandboxes.push(sandbox);\n sandboxes.push(sandbox);\n }).catch((err) => {\n errors.push(err);\n this.returnSandbox(sandbox);\n if (err.code === 'ENOWORKER') {\n throw new DCPError(\"Cannot use localExec without dcp-worker installed. Use the command 'npm install dcp-worker' to install the neccessary modules.\", 'ENOWORKER');\n }\n }));\n }\n \n await Promise.all(sandboxStartPromises);\n\n if (errors.length) {\n console.warn(`Failed to ready ${errors.length} of ${numSandboxes} sandboxes.`, errors);\n throw new Error('Failed to ready sandboxes.');\n }\n\n debugging('supervisor') && console.log(`readySandboxes: Readied ${sandboxes.length} sandboxes ${JSON.stringify(sandboxes.map(sandbox => sandbox.id))}`);\n \n return sandboxes;\n }\n\n /**\n * Accepts a sandbox after it has finished working or encounters an error.\n * If the sandbox was terminated or if \"!slice || slice.failed\" then\n * the sandbox will be removed from the sandboxes array and terminated if necessary.\n * Otherwise it will try to distribute a slice to the sandbox immediately.\n *\n * @param {Sandbox} sandbox - the sandbox to return\n * @param {Slice} [slice] - the slice just worked on; !slice => terminate\n * @param {boolean} [verifySandboxIsNotTerminated=true] - if true, check sandbox is not already terminated\n */\n returnSandbox (sandbox, slice, verifySandboxIsNotTerminated=true) {\n if (!slice || slice.failed || sandbox.isTerminated) {\n \n this.removeSandbox(sandbox);\n \n if (!sandbox.isTerminated) {\n debugging('supervisor') && console.log(`Supervisor.returnSandbox: Terminating ${sandbox.identifier}${slice ? `~${slice.identifier}` : ''}, # of sandboxes ${this.sandboxes.length}`);\n sandbox.terminate(false);\n } else {\n debugging('supervisor') && console.log(`Supervisor.returnSandbox: Already terminated ${sandbox.identifier}${slice ? `~${slice.identifier}` : ''}, # of sandboxes ${this.sandboxes.length}`);\n // XXXpfr: April 13, 2022\n // I'm trying to understand and control when sandboxes get terminated.\n // Because of races and random worker crashes, it is impossible to not try to terminate a sandbox more than once.\n // But at some places where returnSandbox is we shouldn't see this behavior, hence this exception.\n // If I don't fix this is the next 30 days or I forget, please delete this exception.\n if (false)\n {}\n }\n }\n }\n\n /**\n * Terminates sandboxes, in order of creation, when the total started sandboxes exceeds the total allowed sandboxes.\n *\n * @returns {Promise<void>}\n */\n pruneSandboxes () {\n let numOver = this.sandboxes.length - (dcpConfig.worker.maxAllowedSandboxes + this.maxWorkingSandboxes);\n if (numOver <= 0) return;\n \n // Don't kill readied sandboxes while creating readied sandboxes.\n for (let index = 0; index < this.readiedSandboxes.length; ) {\n const sandbox = this.readiedSandboxes[index];\n // If the sandbox is allocated, advance to the next one in the list.\n if (sandbox.allocated) {\n index++;\n continue;\n }\n // Otherwise, remove this sandbox but look at the same array index in the next loop.\n debugging('supervisor') && console.log(`pruneSandboxes: Terminating readied sandbox ${sandbox.identifier}`);\n this.readiedSandboxes.splice(index, 1);\n this.returnSandbox(sandbox);\n\n if (--numOver <= 0) break;\n }\n\n if (numOver <= 0) return;\n for (let index = 0; index < this.assignedSandboxes.length; ) {\n const sandbox = this.assignedSandboxes[index];\n // If the sandbox is allocated, advance to the next one in the list.\n if (sandbox.allocated) {\n index++;\n continue;\n }\n // Otherwise, remove this sandbox but look at the same array index in the next loop.\n debugging('supervisor') && console.log(`pruneSandboxes: Terminating assigned sandbox ${sandbox.identifier}`);\n this.assignedSandboxes.splice(index, 1);\n this.returnSandbox(sandbox);\n\n if (--numOver <= 0) break;\n }\n }\n \n /**\n * Basic watch dog to check if there are idle sandboxes and\n * attempts to nudge the supervisor to feed them work.\n *\n * Run in an interval created in @constructor .\n * @returns {Promise<void>}\n */\n async watchdog () {\n if (!this.watchdogState)\n this.watchdogState = {};\n\n // Every 5 minutes, report progress of all working slices to the scheduler\n if (Date.now() > ((this.lastProgressReport || 0) + sandboxTuning.progressReportInterval)) {\n // console.log('454: Assembling progress update...');\n this.lastProgressReport = Date.now();\n\n //\n // Note: this.slices is the disjoint union of:\n // this.allocatedSlices, \n // this.queuedSlices, \n // this.slices.filter(slice => !slice.isUnassigned) .\n // When a slice is not in these 3 arrays, the slice is lost.\n //\n \n const currentLostSlices = this.slices.filter(slice => slice.isUnassigned \n && this.queuedSlices.indexOf(slice) === -1\n && this.allocatedSlices.indexOf(slice) === -1);\n\n if (currentLostSlices.length > 0) {\n this.lostSlices.push(...currentLostSlices);\n // Try to recover.\n // Needs more work and testing.\n // Test when we can come up with a decent lost slice repro case.\n // --> this.queuedSlices.push(...currentLostSlices);\n }\n\n if (this.lostSlices.length > 0) {\n if (true) { // Keep this on for awhile, until we know lost slices aren't happening.\n console.warn('Supervisor.watchdog: Found lost slices!');\n for (const slice of this.lostSlices)\n console.warn('\\t', slice.identifier);\n }\n this.lostSlices = this.lostSlices.filter(slice => slice.isUnassigned);\n }\n\n const slices = [];\n this.queuedSlices.forEach(slice => {\n assert(slice && slice.sliceNumber > 0);\n addToSlicePayload(slices, slice, sliceStatus.scheduled);\n });\n\n this.allocatedSlices.forEach(slice => {\n assert(slice && slice.sliceNumber > 0);\n addToSlicePayload(slices, slice, 'progress'); // Beacon.\n });\n\n if (slices.length) {\n // console.log('471: sending progress update...');\n const progressReportPayload = {\n worker: this.workerOpaqueId,\n slices,\n };\n\n this.resultSubmitterConnection.send('status', progressReportPayload)\n .catch(error => {\n debugging('supervisor') && console.error('479: Failed to send status update:', error/*.message*/);\n this.resultSubmitterMessageQueue.push({ operation: 'status', data: progressReportPayload })\n this.resultSubmitterConnection.close();\n });\n }\n }\n\n if (this.worker.working) {\n if (this.unallocatedSpace > 0) {\n await this.work().catch(err => {\n if (!this.watchdogState[err.code || '0'])\n this.watchdogState[err.code || '0'] = 0;\n if (Date.now() - this.watchdogState[err.code || '0'] > ((dcpConfig.worker.watchdogLogInterval * timeDilation || 120) * 1000))\n console.error('301: Failed to start work:', err);\n this.watchdogState[err.code || '0'] = Date.now();\n });\n }\n\n this.pruneSandboxes();\n }\n }\n\n /**\n * Gets the logical and physical number of cores and also\n * the total number of sandboxes the worker is allowed to run\n *\n */\n getStatisticsCPU() {\n if (DCP_ENV.isBrowserPlatform) {\n return {\n worker: this.workerOpaqueId,\n lCores: window.navigator.hardwareConcurrency,\n pCores: dcpConfig.worker.pCores || window.navigator.hardwareConcurrency,\n sandbox: this.maxWorkingSandboxes\n }\n }\n\n return {\n worker: this.workerOpaqueId,\n lCores: requireNative('os').cpus().length,\n pCores: requireNative('physical-cpu-count'),\n sandbox: this.maxWorkingSandboxes\n }\n }\n\n /**\n * Returns the number of unallocated sandbox slots to send to fetchTask.\n *\n * @returns {number}\n */\n numberOfAvailableSandboxSlots() {\n let numCores;\n if (this.options.priorityOnly && this.options.jobAddresses.length === 0) {\n numCores = 0;\n } else if (this.queuedSlices.length > 1) {\n // We have slices queued, no need to fetch\n numCores = 0;\n } else {\n // The queue is almost empty (there may be 0 or 1 element), fetch a full task.\n // The task is full, in the sense that it will contain slices whose\n // aggregate execution time is this.maxWorkingSandboxes * 5-minutes.\n // However, there can only be this.unallocatedSpace # of long slices.\n // Thus we need to know whether the last slice in this.queuedSlices is long or not.\n // (A long slice has estimated execution time >= 5-minutes.)\n const longSliceCount = (this.queuedSlices.length > 0 && this.queuedSlices[0].isLongSlice) ? 1 : 0;\n numCores = this.unallocatedSpace - longSliceCount;\n }\n return numCores;\n }\n\n /**\n * Call to start doing work on the network.\n * This is the one place where requests to fetch new slices are made.\n * After the initial slices are fetched it calls this.distributeQueuedSlices.\n *\n * @returns {Promise<void>}, unallocatedSpace ${this.unallocatedSpace}\n */\n async work()\n {\n // When inside matchSlicesWithSandboxes, don't reenter Supervisor.work to fetch new work or create new sandboxes.\n if (this.matching) {\n // Interesting and noisy.\n // debugging('supervisor') && console.log(`Supervisor.work: Do not interleave work, fetch or matching slices with sandboxes: queuedSlices ${this.queuedSlices.length}, unallocatedSpace ${this.unallocatedSpace}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n return Promise.resolve();\n }\n\n await this.setDefaultIdentityKeystore();\n\n // Instantiate connections that don't exist.\n this.instantiateAllConnections();\n\n const numCores = this.numberOfAvailableSandboxSlots();\n\n debugging() && console.log(`Supervisor.work: Try to get ${numCores} slices in working sandboxes, unallocatedSpace ${this.unallocatedSpace}, queued slices ${this.queuedSlices.length}, # of sandboxes ${this.sandboxes.length}, matching ${this.matching}, fetching: ${this.isFetchingNewWork}`);\n \n // Fetch a new task if we have no more slices queued, then start workers\n try {\n if (numCores > 0 && !this.isFetchingNewWork) {\n this.isFetchingNewWork = true;\n\n /**\n * This will only ready sandboxes up to a total count of\n * maxWorkingSandboxes (in any state). It is not possible to know the\n * actual number of sandboxes required until we have the slices because we\n * may have sandboxes assigned for the slice's job already.\n *\n * If the evaluator cannot start (ie. if the evalServer is not running),\n * then the while loop will keep retrying until the evalServer comes online\n */\n if (this.maxWorkingSandboxes > this.sandboxes.length) {\n // Note: The old technique had \n // while (this.maxWorkingSandboxes > this.sandboxes.length) {....\n // and sometimes we'd get far too many sandboxes, because it would keep looping while waiting for\n // this.readySandboxes(this.maxWorkingSandboxes - this.sandboxes.length);\n // to construct the rest of the sandboxes. The fix is to only loop when the 1st \n // await this.readySandboxes(1) \n // is failing.\n let needFirstSandbox = true;\n while (needFirstSandbox) {\n debugging('supervisor') && console.log(`Supervisor.work: ready 1 sandbox, # of sandboxes ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n await this.readySandboxes(1)\n .then(() => {\n debugging('supervisor') && console.log(`Supervisor.work: ready ${this.maxWorkingSandboxes - this.sandboxes.length} sandbox(es), # of sandboxes ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n this.readySandboxes(this.maxWorkingSandboxes - this.sandboxes.length);\n needFirstSandbox = false;\n }).catch(error => {\n console.warn('906: failed to ready sandboxes; will retry', error.code, error.message);\n });\n }\n }\n\n /**\n * Temporary change: Assign the capabilities of one of readied sandboxes\n * before fetching slices from the scheduler.\n *\n * TODO: Remove this once fetchTask uses the capabilities of every\n * sandbox to fetch slices.\n */\n if (!this.capabilities) {\n this.capabilities = this.sandboxes[0].capabilities;\n this.emit('capabilitiesCalculated', this.capabilities);\n }\n\n if (DCP_ENV.isBrowserPlatform && this.capabilities.browser)\n this.capabilities.browser.chrome = DCP_ENV.isBrowserChrome;\n\n const fetchTimeout = setTimeout(() => {\n console.warn(`679: Fetch exceeded timeout, will reconnect at next watchdog interval`);\n \n this.taskDistributorConnection.close('Fetch timed out', Math.random() > 0.5).catch(error => {\n console.error(`931: Failed to close task-distributor connection`, error);\n });\n this.resultSubmitterConnection.close('Fetch timed out', Math.random() > 0.5).catch(error => {\n console.error(`920: Failed to close result-submitter connection`, error);\n });\n this.isFetchingNewWork = false;\n this.instantiateAllConnections();\n }, 3 * 60 * 1000); // max out at 3 minutes to fetch\n\n // ensure result submitter connection before fetching tasks\n try\n {\n await this.resultSubmitterConnection.keepalive();\n }\n catch (e)\n {\n console.error('Failed to connect to result submitter, refusing to fetch slices. Will try again at next fetch cycle.')\n debugging('supervisor') && console.log(`Error: ${e}`);\n this.isFetchingNewWork = false; // <-- done in the `finally` block, below\n clearTimeout(fetchTimeout);\n this.taskDistributorConnection.close('Failed to connect to result-submitter', true).catch(error => {\n console.error(`939: Failed to close task-distributor connection`, error);\n });\n this.resultSubmitterConnection.close('Failed to connect to result-submitter', true).catch(error => {\n console.error(`942: Failed to close result-submitter connection`, error);\n });\n return Promise.resolve();\n }\n await this.fetchTask(numCores).finally(() => {\n clearTimeout(fetchTimeout);\n this.isFetchingNewWork = false;\n });\n }\n\n this.distributeQueuedSlices().then(() => debugging('supervisor') && 'supervisor: finished distributeQueuedSlices()').catch((e) => {\n // We should never get here, because distributeQueuedSlices was changed\n // to try to catch everything and return slices and sandboxes.\n // If we do catch here it may mean a slice was lost. \n console.error('Supervisor.work catch handler for distributeQueuedSlices.', e);\n });\n // No catch(), because it will bubble outward to the caller\n } finally {\n }\n }\n\n /**\n * Generate the workerComputeGroups property of the requestTask message. \n * \n * Concatenate the compute groups object from dcpConfig with the list of compute groups\n * from the supervisor, and remove the public group if accidentally present. Finally,\n * we transform joinSecrets/joinHash into joinHashHash for secure transmission.\n *\n * @note computeGroup objects with joinSecrets are mutated to record their hashes. This\n * affects the supervisor options and dcpConfig. Re-adding a joinSecret property\n * to one of these will cause the hash to be recomputed.\n */\n generateWorkerComputeGroups()\n {\n var computeGroups = Object.values(dcpConfig.worker.computeGroups || {});\n if (this.options.computeGroups)\n computeGroups = computeGroups.concat(this.options.computeGroups);\n computeGroups = computeGroups.filter(group => group.id !== constants.computeGroups.public.id);\n const hashedComputeGroups = [];\n for (const group of computeGroups)\n {\n const groupCopy = Object.assign({}, group);\n if ((group.joinSecret || group.joinHash) && (!group.joinHashHash || this.lastDcpsid !== this.taskDistributorConnection.dcpsid))\n {\n let joinHash;\n if (group.joinHash) {\n joinHash = group.joinHash.replace(/\\s+/g, ''); // strip whitespace\n } else {\n joinHash = calculateJoinHash(groupCopy);\n } \n\n groupCopy.joinHashHash = hash.calculate(hash.eh1, joinHash, this.taskDistributorConnection.dcpsid);\n delete groupCopy.joinSecret;\n delete groupCopy.joinHash;\n debugging('computeGroups') && console.debug(`Calculated joinHash=${joinHash} for`, groupCopy);\n }\n hashedComputeGroups.push(groupCopy);\n }\n this.lastDcpsid = this.taskDistributorConnection.dcpsid;\n debugging('computeGroups') && console.debug('Requesting ', computeGroups.length, 'non-public groups for session', this.lastDcpsid);\n return hashedComputeGroups;\n }\n\n /**\n * Remove all unreferenced jobs in `this.cache`.\n *\n * @param {any[]} newJobs - Jobs that should not be removed from\n * `this.cache`.\n */\n cleanJobCache(newJobs = []) {\n /* Delete all jobs in the supervisorCache that are not represented in this newJobs,\n * or in this.queuedSlices, or there is no sandbox assigned to these jobs.\n * Note: There can easily be 200+ places to check; using a lookup structure to maintain O(n).\n */\n if (this.cache.jobs.length > 0) {\n const jobAddressMap = {};\n Object.keys(newJobs).forEach(jobAddress => { jobAddressMap[jobAddress] = 1; });\n this.slices.forEach(slice => { if (!jobAddressMap[slice.jobAddress]) jobAddressMap[slice.jobAddress] = 1; });\n this.cache.jobs.forEach(jobAddress => {\n if (!jobAddressMap[jobAddress]) {\n this.cache.remove('job', jobAddress);\n // Remove and return the corresponding sandboxes from this.sandboxes.\n const deadSandboxes = this.sandboxes.filter(sb => sb.jobAddress === jobAddress);\n if (deadSandboxes.length > 0) {\n deadSandboxes.forEach(sandbox => { this.returnSandbox(sandbox); });\n debugging('supervisor') && console.log(`Supervisor.fetchTask: Deleting job ${jobAddress} from cache and assigned sandboxes ${deadSandboxes.map(s => s.id)}, # of sandboxes ${this.sandboxes.length}.`);\n }\n }\n });\n }\n }\n\n /**\n * Fetches a task, which contains job information and slices for sandboxes and\n * manages events related to fetching tasks so the UI can more clearly display\n * to user what is actually happening.\n * @param {number} numCores\n * @returns {Promise<void>} The requestTask request, resolve on success, rejects otherwise.\n * @emits Supervisor#fetchingTask\n * @emits Supervisor#fetchedTask\n */\n async fetchTask(numCores) {\n\n // Don't reenter\n if (this.matching || numCores <= 0) {\n // Interesting and noisy.\n debugging('supervisor') && console.log(`Supervisor.fetchTask: Do not nest work, fetch or matching slices with sandboxes: queuedSlices ${this.queuedSlices.length}, unallocatedSpace ${this.unallocatedSpace}, matching ${this.matching}, fetching ${this.isFetchingNewWork}, numCores ${numCores}`);\n return Promise.resolve();\n }\n\n //\n // Oversubscription mitigation.\n // Update when there are less available sandbox slots than numCores.\n const checkNumCores = this.numberOfAvailableSandboxSlots();\n if (numCores > checkNumCores) numCores = checkNumCores;\n if (numCores <= 0) return Promise.resolve();\n\n this.emit('fetchingTask');\n debugging('supervisor') && console.debug('supervisor: fetching task');\n const requestPayload = {\n numCores,\n coreStats: this.getStatisticsCPU(),\n numGPUs: this.defaultMaxGPUs,\n capabilities: this.capabilities,\n paymentAddress: this.paymentAddress,\n jobAddresses: this.options.jobAddresses || [], // force array; when set, only fetches slices for these jobs\n localExec: this.options.localExec,\n workerComputeGroups: this.generateWorkerComputeGroups(),\n minimumWage: dcpConfig.worker.minimumWage || this.options.minimumWage,\n readyJobs: [ /* list of jobs addresses XXXwg */ ],\n previouslyWorkedJobs: this.ringBufferofJobs.buf, //Only discrete jobs\n rejectedJobs: this.rejectedJobs,\n };\n // workers should be part of the public compute group by default\n if (!booley(dcpConfig.worker.leavePublicGroup) && !booley(this.options.leavePublicGroup) && (!requestPayload.localExec))\n requestPayload.workerComputeGroups.push(constants.computeGroups.public);\n debugging('computeGroups') && console.log(`Fetching work for ${requestPayload.workerComputeGroups.length} ComputeGroups: `, requestPayload.workerComputeGroups);\n debugging('supervisor') && console.log(`fetchTask wants ${numCores} slice(s), unallocatedSpace ${this.unallocatedSpace}, queuedSlices ${this.queuedSlices.length}`);\n try {\n debugging('requestTask') && console.debug('fetchTask: requestPayload', requestPayload);\n\n let result = await this.taskDistributorConnection.send('requestTask', requestPayload).catch((error) => {\n debugging('supervisor') && console.error(`Unable to request task from scheduler: ${error}. Will try again on a new connection.`);\n this.taskDistributorConnection.close(error, true);\n throw error; /* caught below */\n });\n let responsePayload = result.payload; \n\n if (!result.success) {\n debugging() && console.log('Task fetch failure; request=', requestPayload);\n debugging() && console.log('Task fetch failure; response=', result.payload);\n throw new DCPError('Unable to fetch task for worker', responsePayload);\n }\n\n const sliceCount = responsePayload.body.task.length || 0;\n\n /**\n * The fetchedTask event fires when the supervisor has finished trying to\n * fetch work from the scheduler (task-manager). The data emitted is the\n * number of new slices to work on in the fetched task.\n *\n * @event Supervisor#fetchedTask\n * @type {number}\n */\n this.emit('fetchedTask', sliceCount);\n\n if (sliceCount < 1) {\n return Promise.resolve();\n }\n\n /**\n * DCP-1698 Send auth msg with tasks to worker, then validate authority of worker to send slice info back to scheduler.\n * payload structure: { owner: this.address, signature: signature, auth: messageLightWeight, body: messageBody };\n * messageLightWeight: { workerId: worker, jobSlices, schedulerId, jobCommissions }\n * messageBody: { newJobs: await getNewJobsForTask(dbScheduler, task, request), task }\n */\n const { body, ...authorizationMessage } = responsePayload;\n const { newJobs, task } = body;\n assert(newJobs); // It should not be possible to have !newJobs -- we throw on !success.\n \n /*\n * Ensure all jobs received from the scheduler are:\n * 1. If we have specified specific jobs the worker may work on, the received jobs are in the specified job list\n * 2. If we are in localExec, at most 1 unique job type was received (since localExec workers are designated for only\n * one job)\n * If the received jobs are not within these parameters, stop the worker since the scheduler cannot be trusted at that point.\n */\n if ((this.options.jobAddresses.length && !Object.keys(newJobs).every((ele) => this.options.jobAddresses.includes(ele)))\n || (this.options.localExec && Object.keys(newJobs).length > 1))\n {\n console.error(\"Worker received slices it shouldn't have. Rejecting the work and stopping.\");\n process.exit(1);\n }\n\n debugging() && console.log(`Supervisor.fetchTask: task: ${task.length}/${numCores}, jobs: ${Object.keys(newJobs).length}, authSlices: ${compressJobMap(authorizationMessage.auth.authSlices, true /* skipFirst*/, this.addressTruncationLength /* digits*/)}`);\n // Delete all jobs in the supervisorCache that are not represented in this task,\n // or in this.queuedSlices, or there is no sandbox assigned to these jobs.\n this.cleanJobCache(newJobs);\n\n for (const jobAddress of Object.keys(newJobs))\n if (!this.cache.cache.job[jobAddress])\n this.cache.store('job', jobAddress, newJobs[jobAddress]);\n\n // Memoize authMessage onto the Slice object, this should\n // follow it for its entire life in the worker.\n const tmpQueuedSlices = task.map(taskElement => new Slice(taskElement, authorizationMessage));\n\n // Make sure old stuff is up front.\n // matchSlicesWithSandboxes dequeues this.queuedSlices as follows:\n // slicesToMatch = this.queuedSlices.slice(0, numCores);\n this.slices.push(...tmpQueuedSlices);\n this.queuedSlices.push(...tmpQueuedSlices);\n \n // Populating the ring buffer based on job's discrete property \n Object.values(newJobs).forEach(job => {\n if(job.requirements.discrete && this.ringBufferofJobs.find(element => element === job.address) === undefined) {\n this.ringBufferofJobs.push(job.address);\n }\n });\n \n } catch (error) {\n this.emit('fetchTaskFailed', error);\n debugging('supervisor') && console.debug(`Supervisor.fetchTask failed!: error: ${error}`);\n }\n }\n\n /**\n * For each slice in this.queuedSlices, match with a sandbox in the following order:\n * 1. Try to find an already assigned sandbox in this.assignedSandboxes for the slice's job.\n * 2. Find a ready sandbox in this.readiedSandboxes that is unassigned.\n * 3. Ready a new sandbox and use that.\n *\n * Take great care in assuring sandboxes and slices are uniquely associated, viz.,\n * a given slice cannot be associated with multiple sandboxes and a given sandbox cannot be associated with multiple slices.\n * The lack of such uniqueness has been the root cause of several difficult bugs.\n *\n * Note: When a sandbox is paired with a slice, execution is pending and sandbox.allocated=true and\n * sandbox.slice=slice and sandbox.jobAddress=slice.jobAddress. This is what 'allocated' means.\n * Immediately upon the exit of sandbox.work, sandbox.allocated=false is set and if an exception\n * wasn't thrown, the paired slice is placed in this.assignedSandboxes.\n * Thus from the pov of supervisor, this.allocatedSandboxes is deterministic and this.workingSandboxes is not.\n * Please try to not use this.workingSandboxes. It is deprecated.\n *\n * The input is numCores, this,queuedSlices, this.assignedSandboxes and this.readiedSandboxes.\n * If there are not enough sandboxes, new readied sandboxes will be created using\n * await this.readySandboxes(...)\n * And it is this await boundary that has caused many bugs.\n * We try not to make assumptions about non-local state across the await boundary.\n *\n * @param {number} numCores - The number of available sandbox slots.\n * @param {boolean} [throwExceptions=true] - Whether to throw exceptions when checking for sanity.\n * @returns {Promise<SandboxSlice[]>} Returns SandboxSlice[], may have length zero.\n */\n async matchSlicesWithSandboxes (numCores, throwExceptions = true) {\n\n const sandboxSlices = [];\n if (this.queuedSlices.length === 0 || this.matching || numCores <= 0) {\n // Interesting and noisy.\n // debugging('supervisor') && console.log(`Supervisor.matchSlicesWithSandboxes: Do not nest work, fetch or matching slices with sandboxes: queuedSlices ${this.queuedSlices.length}, unallocatedSpace ${this.unallocatedSpace}, matching ${this.matching}, fetching ${this.isFetchingNewWork}, numCores ${numCores}`);\n return sandboxSlices;\n }\n\n //\n // Oversubscription mitigation.\n // Update when there are less available sandbox slots than numCores.\n // We cannot use this.unallocatedSpace here because its value is artificially low or zero, because in\n // this.distributedQueuedSlices we use the pseudo-mutex trick: this.acquire(howManySandboxSlotsToReserve)/this.release().\n // Note: Do not use this.numberOfCoresReserved outside of a function locked with this.acquire(howManySandboxSlotsToReserve) .\n const checkNumCores = this.numberOfCoresReserved; // # of locked sandbox slots.\n if (numCores > checkNumCores) numCores = checkNumCores;\n if (numCores <= 0) return sandboxSlices;\n\n // Don't ask for more than we have.\n if (numCores > this.queuedSlices.length)\n numCores = this.queuedSlices.length;\n\n debugging('supervisor') && console.log(`matchSlicesWithSandboxes: numCores ${numCores}, queued slices ${this.queuedSlices.length}: assigned ${this.assignedSandboxes.length}, readied ${this.readiedSandboxes.length}, unallocated ${this.unallocatedSpace}, # of sandboxes: ${this.sandboxes.length}`);\n\n if (debugging('supervisor')) {\n dumpSlicesIfNotUnique(this.queuedSlices, 'Warning: this.queuedSlices slices are not unique -- this is ok when slice is rescheduled.');\n dumpSandboxesIfNotUnique(this.readiedSandboxes, 'Warning: this.readiedSandboxes sandboxes are not unique!');\n dumpSandboxesIfNotUnique(this.assignedSandboxes, 'Warning: this.assignedSandboxes sandboxes are not unique!');\n }\n\n // Three functions to validate slice and sandbox.\n function checkSlice(slice, checkAllocated=true) {\n if (!slice.isUnassigned) throw new DCPError(`Slice must be unassigned: ${slice.identifier}`);\n if (checkAllocated && slice.allocated) throw new DCPError(`Slice must not already be allocated: ${slice.identifier}`);\n }\n function checkSandbox(sandbox, isAssigned) {\n if (sandbox.allocated) throw new DCPError(`Assigned sandbox must not be already allocated: ${sandbox.identifier}`);\n if (isAssigned && !sandbox.isAssigned) throw new DCPError(`Assigned sandbox is not marked as assigned: ${sandbox.identifier}`);\n if (!isAssigned && !sandbox.isReadyForAssign) throw new DCPError(`Readied sandbox is not marked as ready for assign: ${sandbox.identifier}`);\n }\n\n // Sanity checks.\n if (throwExceptions) {\n this.assignedSandboxes.forEach(sandbox => { checkSandbox(sandbox, true /* isAssigned*/); });\n this.readiedSandboxes.forEach(sandbox => { checkSandbox(sandbox, false /* isAssigned*/); });\n this.queuedSlices.forEach(slice => { checkSlice(slice); });\n } else {\n this.assignedSandboxes = this.assignedSandboxes.filter(sandbox => !sandbox.allocated && sandbox.isAssigned);\n this.readiedSandboxes = this.readiedSandboxes.filter(sandbox => !sandbox.allocated && sandbox.isReadyForAssign);\n this.queuedSlices = this.queuedSlices.filter(slice => !slice.allocated && slice.isUnassigned);\n }\n\n const sandboxKind = {\n assigned: 0,\n ready: 1,\n new: 2,\n };\n\n const ceci = this;\n /**\n * Auxiliary function to pair a sandbox with a slice and mark the sandbox as allocated.\n * An allocated sandbox is reserved and will not be released until the slice completes execution on the sandbox.\n *\n * @param {Sandbox} sandbox\n * @param {Slice} slice\n * @param {number} kind\n */\n function pair(sandbox, slice, kind) {\n checkSandbox(sandbox, kind === sandboxKind.assigned);\n checkSlice(slice, kind === sandboxKind.assigned);\n slice.allocated = true;\n sandbox.allocated = true;\n sandbox.jobAddress = slice.jobAddress; // So we can know which jobs to not delete from this.cache .\n sandbox.slice = slice;\n sandboxSlices.push({ sandbox, slice });\n if (Supervisor.sliceTiming) slice['pairingDelta'] = Date.now();\n if (debugging('supervisor')) {\n let fragment = 'New readied';\n if (kind === sandboxKind.assigned) fragment = 'Assigned';\n else if (kind === sandboxKind.ready) fragment = 'Readied';\n console.log(`matchSlicesWithSandboxes.pair: ${fragment} sandbox matched ${ceci.dumpSandboxAndSlice(sandbox, slice)}`);\n }\n }\n\n // These three arrays are used to track/store slices and sandboxes,\n // so that when an exception occurs, the following arrays are restored:\n // this.queuedSlices, this.assignedSandboxes, this.realizedSandboxes.\n let slicesToMatch = [];\n let trackAssignedSandboxes = [];\n let trackReadiedSandboxes = [];\n try\n {\n this.matching = true;\n\n let assignedCounter = 0; // How many assigned sandboxes are being used.\n let readyCounter = 0; // How many sandboxes used from the existing this.readiedSandboxes.\n let newCounter = 0; // How many sandboxes that needed to be newly created.\n\n //\n // The Ideas:\n // 1) We match each slice with a sandbox. First we match with assigned sandboxes in the order\n // that they appear in this.queuedSlices. Then we match in-order with existing this.readiedSandboxes\n // Then we match in-order with new new readied sandboxes created through\n // await this.readySandboxes(newCounter, true /* allocateLocalSandboxes*/);\n // This allows us to try different orderings of execution of slices. E.g. Wes suggested\n // trying to execute slices from different jobs with maximal job diversity -- specifically\n // if there are 3 jobs j1,j2,j3, with slices s11, s12 from j1, s21, s22, s23 from j2 and\n // s31, s32 from j3, then we try to schedule, in order s11, s21, s31, s12, s22, s32, s23.\n //\n // 2) Before matching slices with sandboxes, we allocate available assigned and readied sandboxes\n // and if more are needed then we create and allocate new ones.\n //\n // 3) Finally we match slices with sandboxes and return an array of sandboxSlice pairs.\n //\n // Note: The ordering of sandboxSlices only partially corresponds to the order of this.queuedSlices.\n // It's easy to do. When pairing with assigned sandboxes, any slice in this.queuedSlices which doesn't\n // have an assigned sandbox, will add null to the sandboxSlices array. Then when pairing with readied sandboxes,\n // we fill-in the null entries in the sandboxSlices array.\n //\n /** XXXpfr @todo When it is needed, fix the ordering as described above. */\n\n // Get the slices that are being matched.\n slicesToMatch = this.queuedSlices.slice(0, numCores);\n this.queuedSlices = this.queuedSlices.slice(numCores);\n\n debugging('supervisor') && console.log(`matchSlicesWithSandboxes: slicesToMatch ${this.dumpSlices(slicesToMatch)}`);\n\n // Create object map: jobAddress -> sandboxes with sandboxes.jobAddress === jobAddress .\n const jobSandboxMap = toJobMap(this.assignedSandboxes, sandbox => sandbox);\n \n // Create array to hold slices which do not have assigned sandboxes.\n // These slices will need to be paired with existing and possibly new readied sandboxes.\n // Specifically, the sandboxes from existing this.readiedSandboxes and new sandboxes\n // created through await this.readySandboxes(newCounter, true /* allocateLocalSandboxes*/);\n const slicesThatNeedSandboxes = [];\n\n // Pair assigned sandboxes with slices.\n for (const slice of slicesToMatch) {\n const assigned = jobSandboxMap[slice.jobAddress];\n if (assigned && assigned.length > 0) {\n // Pair.\n const sandbox = assigned.pop();\n pair(sandbox, slice, sandboxKind.assigned);\n this.removeElement(this.assignedSandboxes, sandbox);\n // Track.\n trackAssignedSandboxes.push(sandbox);\n assignedCounter++;\n } else {\n // Don't lose track of these slices.\n slice.allocated = true;\n slicesThatNeedSandboxes.push(slice);\n }\n }\n\n // Pair readied sandboxes with slices.\n readyCounter = Math.min(slicesThatNeedSandboxes.length, this.readiedSandboxes.length);\n newCounter = slicesThatNeedSandboxes.length - readyCounter;\n // Track.\n trackReadiedSandboxes = this.readiedSandboxes.slice(0, readyCounter);\n this.readiedSandboxes = this.readiedSandboxes.slice(readyCounter);\n for (const sandbox of trackReadiedSandboxes) {\n // Pair.\n const slice = slicesThatNeedSandboxes.pop();\n pair(sandbox, slice, sandboxKind.ready);\n }\n \n debugging('supervisor') && console.log(`matchSlicesWithSandboxes: assignedCounter ${assignedCounter}, readyCounter ${readyCounter}, newCounter ${newCounter}, numCores ${numCores}`)\n\n // Validate algorithm consistency.\n if (Supervisor.debugBuild && assignedCounter + readyCounter + newCounter !== numCores) {\n // Structured assert.\n throw new DCPError(`matchSlicesWithSandboxes: Algorithm is corrupt ${assignedCounter} + ${readyCounter} + ${newCounter} !== ${numCores}`);\n }\n\n // Here is an await boundary.\n // Accessing non-local data across an await boundary may result in the unexpected.\n\n // Create new readied sandboxes to associate with slicesThatNeedSandboxes.\n if (newCounter > 0) {\n // When allocateLocalSandboxes is true, this.readySandboxes does not place the new sandboxes\n // on this.readiedSandboxes. Hence the new sandboxes are private and nobody else can see them.\n debugging('supervisor') && console.log(`matchSlicesWithSandboxes: creating ${newCounter} new sandboxes, # of sandboxes ${this.sandboxes.length}`);\n const readied = await this.readySandboxes(newCounter, true /* allocateLocalSandboxes*/);\n // Track.\n trackReadiedSandboxes.push(...readied);\n\n for (const sandbox of readied) {\n assert(slicesThatNeedSandboxes.length > 0);\n // Pair\n const slice = slicesThatNeedSandboxes.pop();\n pair(sandbox, slice, sandboxKind.new);\n }\n \n // Put back any extras. There should not be any unless readySandboxes returned less than asked for.\n if (slicesThatNeedSandboxes.length > 0) {\n slicesThatNeedSandboxes.forEach(slice => {\n slice.allocated = false;\n this.queuedSlices.push(slice);\n });\n }\n }\n\n if ( false || debugging()) {\n console.log(`matchSlicesWithSandboxes: Matches: ${ this.dumpSandboxSlices(sandboxSlices) }`);\n this.dumpSandboxSlicesIfNotUnique(sandboxSlices, 'Warning: sandboxSlices; { sandbox, slice } pairs are not unique!');\n }\n } catch (e) {\n // Clear allocations.\n slicesToMatch.forEach(slice => { slice.allocated = false; });\n trackAssignedSandboxes.forEach(sandbox => { sandbox.allocated = false; sandbox.slice = null; });\n trackReadiedSandboxes.forEach(sandbox => { sandbox.allocated = false; sandbox.slice = null; sandbox.jobAddress = null; });\n \n // Filter out redundancies -- there shouldn't be any...\n slicesToMatch = slicesToMatch.filter(slice => this.queuedSlices.indexOf(slice) === -1);\n trackAssignedSandboxes = trackAssignedSandboxes.filter(sb => this.assignedSandboxes.indexOf(sb) === -1);\n trackReadiedSandboxes = trackReadiedSandboxes.filter(sb => this.readiedSandboxes.indexOf(sb) === -1);\n\n // Sanity checks.\n slicesToMatch.forEach(slice => { checkSlice(slice) });\n trackAssignedSandboxes.forEach(sandbox => { checkSandbox(sandbox, true /* isAssigned*/); });\n trackReadiedSandboxes.forEach(sandbox => { checkSandbox(sandbox, false /* isAssigned*/); });\n\n // Restore arrays.\n this.queuedSlices.push(...slicesToMatch);\n this.assignedSandboxes.push(...trackAssignedSandboxes);\n this.readiedSandboxes.push(...trackReadiedSandboxes);\n \n console.error('Error in matchSlicesWithSandboxes: Attempting to recover slices and sandboxes.', e);\n return [];\n } finally {\n this.matching = false;\n }\n\n debugging('supervisor') && console.log(`matchSlicesWithSandboxes: allocated ${sandboxSlices.length} sandboxes, queuedSlices ${this.queuedSlices.length}, unallocatedSpace ${this.unallocatedSpace}, matching ${this.matching}, fetching ${this.isFetchingNewWork}, # of sandboxes: ${this.sandboxes.length}.`);\n\n return sandboxSlices;\n }\n\n disassociateSandboxAndSlice(sandbox, slice) {\n this.returnSandbox(sandbox);\n sandbox.slice = null;\n this.returnSlice(slice);\n }\n\n /**\n * This method will call this.startSandboxWork(sandbox, slice) for each element { sandbox, slice }\n * of the array returned by this.matchSlicesWithSandboxes(availableSandboxes) until all allocated sandboxes\n * are working. It is possible for a sandbox to interleave with calling distributeQueuedSlices and leave a sandbox\n * that is not working. Moreover, this.queuedSlices may be exhausted before all sandboxes are working.\n * @returns {Promise<void>}\n */\n async distributeQueuedSlices () {\n const numCores = this.unallocatedSpace;\n\n // If there's nothing there, or we're reentering, bail out.\n if (this.queuedSlices.length === 0 || numCores <= 0 || this.matching) {\n // Interesting and noisy.\n // debugging('supervisor') && console.log(`Supervisor.distributeQueuedSlices: Do not nest work, fetch or matching slices with sandboxes: queuedSlices ${this.queuedSlices.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}, numCores ${numCores}`);\n return Promise.resolve();\n }\n\n //\n // Use the pseudo-mutex to prevent uncontrolled interleaving with fetchTask,\n // matchSlicesWithSandboxes and distributeQueuedSlices\n let sandboxSlices;\n this.acquire(numCores);\n try {\n sandboxSlices = await this.matchSlicesWithSandboxes(numCores);\n } finally {\n this.release();\n }\n\n debugging('supervisor') && console.log(`distributeQueuedSlices: ${sandboxSlices.length} sandboxSlices ${this.dumpSandboxSlices(sandboxSlices)}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n\n for (let sandboxSlice of sandboxSlices) {\n\n const { sandbox, slice } = sandboxSlice;\n try {\n if (sandbox.isReadyForAssign) {\n try {\n let timeoutMs = Math.floor(Math.min(+Supervisor.lastAssignFailTimerMs || 0, 10 * 60 * 1000 /* 10m */));\n await a$sleepMs(timeoutMs);\n await this.assignJobToSandbox(sandbox, slice.jobAddress);\n } catch (e) {\n console.error(`Supervisor.distributeQueuedSlices: Could not assign slice ${slice.identifier} to sandbox ${sandbox.identifier}.`);\n if (Supervisor.debugBuild) console.error(`...exception`, e);\n Supervisor.lastAssignFailTimerMs = Supervisor.lastAssignFailTimerMs ? +Supervisor.lastAssignFailTimerMs * 1.25 : Math.random() * 200;\n this.disassociateSandboxAndSlice(sandbox, slice);\n continue;\n }\n }\n\n if (!Supervisor.lastAssignFailTimerMs)\n Supervisor.lastAssignFailTimerMs = Math.random() * 200;\n this.startSandboxWork(sandbox, slice);\n Supervisor.lastAssignFailTimerMs = false;\n\n } catch (e) {\n // We should never get here.\n console.error(`Supervisor.distributeQueuedSlices: Failed to execute slice ${slice.identifier} in sandbox ${sandbox.identifier}.`);\n if (Supervisor.debugBuild) console.error('...exception', e);\n this.disassociateSandboxAndSlice(sandbox, slice);\n }\n }\n }\n\n /**\n *\n * @param {Sandbox} sandbox\n * @param {opaqueId} jobAddress\n * @returns {Promise<void>}\n */\n async assignJobToSandbox(sandbox, jobAddress) {\n var ceci = this;\n\n try {\n return sandbox.assign(jobAddress); // Returns Promise.\n } catch(error) {\n // return slice to scheduler, log error\n console.error('Supervisor.assignJobToSandbox: Failed to assign job to sandbox.', {\n jobAddress: jobAddress.substr(0,10),\n error,\n });\n\n ceci.returnSandbox(sandbox);\n\n throw error;\n }\n }\n\n /**\n * Handles reassigning or returning a slice that was rejected by a sandbox.\n * \n * The sandbox will be terminated by this.returnSandbox in finalizeSandboxAndSlice. In this case,\n * if the slice does not have a rejected property already, reassign the slice to a new sandbox\n * and add a rejected property to the slice to indicate it has already rejected once, then set slice = null\n * in the return SandboxSlice so that finalizeSandboxAndSlice won't return slice to scheduler.\n * \n * If the slice rejects with a reason, or has a rejected time stamp (ie. has been rejected once already)\n * then return the slice and all slices from the job to the scheduler and\n * terminate all sandboxes with that jobAddress.\n * @param {Sandbox} sandbox \n * @param {Slice} slice\n * @returns {Promise<SandboxSlice>}\n */\n async handleWorkReject(sandbox, slice, rejectReason) {\n if (!this.rejectedJobReasons[slice.jobAddress])\n this.rejectedJobReasons[slice.jobAddress] = [];\n\n this.rejectedJobReasons[slice.jobAddress].push(rejectReason); // memoize reasons\n\n // First time rejecting without a reason. Try assigning slice to a new sandbox.\n if (rejectReason === 'false' && !slice.rejected) {\n // Set rejected.\n slice.rejected = Date.now();\n // Schedule the slice for execution.\n this.scheduleSlice(slice, true /* placeInTheFrontOfTheQueue*/, false /* noDuplicateExecution*/);\n \n // Null out slice so this.returnSlice will not be called in finalizeSandboxAndSlice.\n // But we still want this.returnSandbox to terminate the sandbox.\n slice = null;\n } else { // Slice has a reason OR rejected without a reason already and got stamped.\n \n // Purge all slices and sandboxes associated with slice.jobAddress .\n this.purgeAllWork(slice.jobAddress);\n // Clear jobAddress from this.cache .\n this.cleanJobCache();\n\n // Add to array of rejected jobs.\n let rejectedJob = {\n address: slice.jobAddress,\n reasons: this.rejectedJobReasons[slice.jobAddress],\n }\n this.rejectedJobs.push(rejectedJob);\n\n // Tell everyone all about it, when allowed.\n if (dcpConfig.worker.allowConsoleAccess || Supervisor.debugBuild)\n {\n console.warn('Supervisor.handleWorkReject: The slice ${slice.identifier} was rejected twice; slice will be returned to the scheduler.');\n console.warn('Supervisor.handleWorkReject: All slices with the same jobAddress returned to the scheduler.');\n console.warn('Supervisor.handleWorkReject: All sandboxes with the same jobAddress are terminated.');\n }\n //\n // this.purgeAllWork(jobAddress) terminates all sandboxes with jobAddress,\n // and it also returns to scheduler all slices with jobAddress.\n // Therefore null out slice and sandbox so finalizeSandboxAndSlice doesn't do anything.\n // \n sandbox = null;\n slice = null;\n }\n return { sandbox, slice };\n }\n\n /**\n * Schedule the slice to be executed.\n * If slice is already executing and noDuplicateExecution is true, return the slice with reason.\n * @param {Slice} slice\n * @param {boolean} [placeInTheFrontOfTheQueue=false]\n * @param {boolean} [noDuplicateExecution=true]\n * @param {string} [reason]\n */\n scheduleSlice(slice, placeInTheFrontOfTheQueue = false, noDuplicateExecution = true, reason) {\n // When noDuplicateExecution, if slice is already executing, do nothing.\n let workingSlices = [];\n if (noDuplicateExecution)\n workingSlices = this.allocatedSlices;\n\n if (!workingSlices.indexOf(slice)) {\n // Reset slice state to allow execution.\n slice.status = SLICE_STATUS_UNASSIGNED;\n // Enqueue in the to-be-executed queue.\n if (placeInTheFrontOfTheQueue) this.queuedSlices.unshift(slice);\n else this.queuedSlices.push(slice);\n }\n }\n\n /**\n * Purge all slices and sandboxes with this jobAddress.\n * @param {address} jobAddress\n * @param {boolean} [onlyPurgeQueuedAndAllocated=false]\n */\n purgeAllWork(jobAddress, onlyPurgeQueuedAndAllocated = false) {\n // Purge all slices and sandboxes associated with jobAddress .\n const deadSandboxes = this.sandboxes.filter(sandbox => sandbox.jobAddress === jobAddress);\n\n if (deadSandboxes.length > 0) {\n debugging('supervisor') && console.log(`purgeAllWork(${this.dumpJobAddress(jobAddress)}): sandboxes purged ${deadSandboxes.map(s => s.id)}, # of sandboxes ${this.sandboxes.length}`);\n deadSandboxes.forEach(sandbox => this.returnSandbox(sandbox));\n }\n\n let deadSlices;\n if (onlyPurgeQueuedAndAllocated) {\n deadSlices = this.queuedSlices.filter(slice => slice.jobAddress === jobAddress);\n if (deadSlices.length > 0 || this.allocatedSlices.length > 0)\n debugging('supervisor') && console.log(`purgeAllWork(${this.dumpJobAddress(jobAddress)}): dead queuedSlices ${deadSlices.map(s => s.sliceNumber)}, dead allocatedSlices ${this.allocatedSlices.map(s => s.sliceNumber)}`);\n deadSlices.push(...this.allocatedSlices);\n } else {\n deadSlices = this.slices.filter(slice => slice.jobAddress === jobAddress);\n }\n\n if (deadSlices.length > 0) {\n debugging('supervisor') && console.log(`purgeAllWork(${this.dumpJobAddress(jobAddress)}): slices purged ${deadSlices.map(s => s.sliceNumber)}, # of sandboxes ${this.sandboxes.length}`);\n this.returnSlices(deadSlices);\n this.removeQueuedSlices(deadSlices);\n }\n debugging('supervisor') && console.log(`purgeAllWork(${this.dumpJobAddress(jobAddress)}): Finished: slices ${this.slices.length}, queuedSlices ${this.queuedSlices.length}, assigned ${this.assignedSandboxes.length}, readied ${this.readiedSandboxes.length}, # of sandboxes ${this.sandboxes.length}`);\n }\n\n /**\n * Gives a slice to a sandbox which begins working. Handles collecting\n * the slice result (complete/fail) from the sandbox and submitting the result to the scheduler.\n * It will also return the sandbox to @this.returnSandbox when completed so the sandbox can be re-assigned.\n *\n * @param {Sandbox} sandbox - the sandbox to give the slice\n * @param {Slice} slice - the slice to distribute\n * @returns {Promise<void>} Promise returned from sandbox.run\n */\n async startSandboxWork (sandbox, slice) {\n var startDelayMs, reason = 'unknown';\n\n try {\n slice.markAsWorking();\n } catch (e) {\n // This will occur when the same slice is distributed twice.\n // It is normal because two sandboxes could finish at the same time and be assigned the\n // same slice before the slice is marked as working.\n debugging() && console.debug('startSandboxWork: slice.markAsWorking exception:', e);\n return Promise.resolve();\n }\n\n // sandbox.requiresGPU = slice.requiresGPU;\n // if (sandbox.requiresGPU) {\n // this.GPUsAssigned++;\n // }\n\n if (Supervisor.startSandboxWork_beenCalled)\n startDelayMs = 1000 * (tuning.minSandboxStartDelay + (Math.random() * (tuning.maxSandboxStartDelay - tuning.minSandboxStartDelay)));\n else {\n startDelayMs = 1000 * tuning.minSandboxStartDelay;\n Supervisor.startSandboxWork_beenCalled = true;\n }\n\n try {\n debugging() && console.log(`startSandboxWork: Started ${this.dumpStatefulSandboxAndSlice(sandbox, slice)}, total sandbox count: ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n if (Supervisor.sliceTiming) {\n slice['pairingDelta'] = Date.now() - slice['pairingDelta'];\n slice['executionDelta'] = Date.now();\n }\n let result;\n try {\n result = await sandbox.work(slice, startDelayMs);\n } finally {\n sandbox.allocated = false;\n slice.allocated = false;\n }\n if (Supervisor.sliceTiming) {\n slice['executionDelta'] = Date.now() - slice['executionDelta'];\n slice['resultDelta'] = Date.now();\n }\n slice.collectResult(result, true);\n // In watchdog, all sandboxes in working state, have their slice status sent to result submitter.\n // However, this can happen after the sandbox/slice has already sent results\n // to result submitter, in which case, the activeSlices table has already removed the row\n // corresponding to slice and hence is incapable of updating status.\n sandbox.changeWorkingToAssigned();\n this.assignedSandboxes.push(sandbox);\n debugging() && console.log(`startSandboxWork: Finished ${this.dumpStatefulSandboxAndSlice(sandbox, slice)}, total sandbox count: ${this.sandboxes.length}, matching ${this.matching}, fetching ${this.isFetchingNewWork}`);\n } catch(error) {\n let logLevel;\n\n if (error instanceof SandboxError) {\n logLevel = 'warn';\n // The message and stack properties of error objects are not enumerable,\n // so they have to be copied into a plain object this way\n const errorResult = Object.getOwnPropertyNames(error).reduce((o, p) => {\n o[p] = error[p]; return o;\n }, { message: 'Unexpected worker error' });\n slice.collectResult(errorResult, false);\n } else {\n logLevel = 'error';\n // This error was unrelated to the work being done, so just return the slice in the finally block.\n // For extra safety the sandbox is terminated.\n slice.result = null;\n slice.status = SLICE_STATUS_FAILED; /** XXXpfr @todo terminating sandbox? */\n }\n\n let errorString;\n switch (error.errorCode) {\n case 'ENOPROGRESS':\n reason = 'ENOPROGRESS';\n errorString = 'Supervisor.startSandboxWork - No progress error in sandbox.\\n';\n break;\n case 'ESLICETOOSLOW':\n reason = 'ESLICETOOSLOW';\n errorString = 'Supervisor.startSandboxWork - Slice too slow error in sandbox.\\n';\n break;\n case 'EUNCAUGHT':\n reason = 'EUNCAUGHT';\n errorString = `Supervisor.startSandboxWork - Uncaught error in sandbox ${error.message}.\\n`;\n break;\n case 'EFETCH':\n // reason = 'EFETCH'; The status.js processing cannot handle 'EFETCH'\n reason = 'unknown';\n errorString = `Supervisor.startSandboxWork - Could not fetch data: ${error.message}.\\n`;\n break;\n }\n \n // Always display max info under debug builds, otherwise maximal error\n // messages are displayed to the worker, only if both worker and client agree.\n let workerConsole = sandbox.supervisorCache.cache.job[slice.jobAddress].workerConsole;\n const displayMaxInfo = Supervisor.debugBuild || (workerConsole && dcpConfig.worker.allowConsoleAccess);\n\n const errorObject = {\n jobAddress: slice.jobAddress.substr(0,10),\n sliceNumber: slice.sliceNumber,\n sandbox: sandbox.id,\n jobName: sandbox.public ? sandbox.public.name : 'unnamed',\n };\n \n if (error.name === 'EWORKREJECT') {\n error.stack = 'Sandbox was terminated by work.reject()';\n const ss = await this.handleWorkReject(sandbox, slice, error.message);\n sandbox = ss.sandbox; slice = ss.slice;\n }\n\n if (!displayMaxInfo && errorString) {\n console[logLevel](errorString, errorObject);\n } else if (!displayMaxInfo && error.name === 'EWORKREJECT') {\n console[logLevel](`Supervisor.startSandboxWork - Sandbox rejected work: ${error.message}`)\n } else {\n if (displayMaxInfo)\n errorObject.stack += '\\n --------------------\\n' + (error.stack.split('\\n').slice(1).join('\\n'));\n console[logLevel](`Supervisor.startSandboxWork - Sandbox failed: ${error.message}\\n`, errorObject);\n }\n } finally {\n await this.finalizeSandboxAndSlice(sandbox, slice, reason);\n }\n }\n\n /**\n * If slice && slice.result, then call await this.recordResult(slice) and this.returnSandbox(sandbox, slice) will have no effect.\n * If slice && !slice.result, then call this.returnSlice(slice, reason) and then this.returnSandbox(sandbox, slice) which terminates sandbox.\n * If !slice && sandbox, then terminate the sandbox with this.returnSandbox(sandbox, slice) .\n * If !slice && !sandbox, then do nothing.\n * @param {Sandbox} [sandbox]\n * @param {Slice} [slice]\n * @param {string} [reason]\n */\n async finalizeSandboxAndSlice(sandbox, slice, reason) {\n debugging('supervisor') && console.log(`finalizeSandboxAndSlice: sandbox ${sandbox ? sandbox.identifier : 'nade'}, slice ${slice ? slice.identifier : 'nade'}`);\n if (slice) {\n if (slice.result) await this.recordResult(slice);\n else this.returnSlice(slice, reason);\n }\n // It is possible that sandbox is already terminated\n // Because sandbox.allocated=false as soon as sandbox.work(...) completes.\n // But the await at or in finalizeSandboxAndSlice may allow pruneSandboxes to slither in.\n if (sandbox) this.returnSandbox(sandbox, slice, false /* verifySandboxIsNotTerminated*/);\n }\n\n /**\n * Terminates sandboxes and returns slices.\n * Sets the working flag to false, call @this.work to start working again.\n * \n * If forceTerminate is true: Terminates all sandboxes and returns all slices.\n * If forceTerminate is false: Terminates non-allocated sandboxes and returns queued slices.\n *\n * @param {boolean} [forceTerminate = true] - true if you want to stop the sandboxes from completing their current slice.\n * @returns {Promise<void>}\n */\n async stopWork (forceTerminate = true) {\n debugging('supervisor') && console.log('stopWork(${forceTerminate}): terminating sandboxes and returning slices to scheduler.');\n if (forceTerminate) {\n while (this.sandboxes.length) {\n this.returnSandbox(this.sandboxes[0], null, false);\n }\n\n await this.returnSlices(this.slices).then(() => {\n this.queuedSlices.length = 0;\n });\n } else {\n // Only terminate idle sandboxes and return only queued slices\n let idleSandboxes = this.sandboxes.filter(w => !w.allocated);\n for (const sandbox of idleSandboxes) {\n this.returnSandbox(sandbox, null, false /* verifySandboxIsNotTerminated*/);\n }\n\n await this.returnSlices(this.queuedSlices).then(() => {\n this.queuedSlices.length = 0;\n });\n\n await new Promise((resolve, reject) => {\n let sandboxesRemaining = this.allocatedSandboxes.length;\n if (sandboxesRemaining === 0)\n {\n resolve();\n }\n // Resolve and finish work once all sandboxes have finished submitting their results.\n this.on('submitFinished', () => {\n sandboxesRemaining--;\n if (sandboxesRemaining === 0)\n {\n console.log('All sandboxes empty, stopping worker and closing all connections');\n resolve();\n }\n });\n });\n }\n\n if (this.resultSubmitterConnection) {\n this.resultSubmitterConnection.off('close', this.openResultSubmitterConn);\n this.resultSubmitterConnection.close();\n this.resultSubmitterConnection = null;\n }\n\n if (this.taskDistributorConnection) {\n this.taskDistributorConnection.off('close', this.openTaskDistributorConn);\n this.taskDistributorConnection.close();\n this.taskDistributorConnection = null;\n }\n\n if (this.packageManagerConnection) {\n this.packageManagerConnection.off('close', this.openPackageManagerConn);\n this.packageManagerConnection.close();\n this.packageManagerConnection = null;\n }\n\n if (this.eventRouterConnection) {\n this.eventRouterConnection.off('close', this.openEventRouterConn);\n this.eventRouterConnection.close();\n this.eventRouterConnection = null;\n }\n\n this.emit('stop');\n }\n\n /**\n * Takes a slice and returns it to the scheduler to be redistributed.\n * Usually called when an exception is thrown by sandbox.work(slice, startDelayMs) .\n * Or when the supervisor tells it to forcibly stop working.\n *\n * @param {Slice} slice - The slice to return to the scheduler.\n * @param {string} [reason] - Optional reason for the return: 'ENOPROGRESS', 'EUNCAUGHT', 'ESLICETOOSLOW', 'unknown'.\n * @returns {Promise<*>} - Response from the scheduler.\n */\n returnSlice (slice, reason) {\n // When sliceNumber === 0 don't send a status message.\n if (slice.sliceNumber === 0) return Promise.resolve();\n \n debugging() && console.log(`Supervisor.returnSlice: Returning slice ${slice.identifier} with reason ${reason}.`);\n \n const payload = slice.getReturnMessagePayload(this.workerOpaqueId, reason);\n try\n {\n return this.resultSubmitterConnection.send('status', payload) /* resultSubmitterConnection can be null if worker is stopped */\n .then(response => {\n return response;\n }).catch(error => {\n debugging('supervisor') && console.error('Failed to return slice', {\n sliceNumber: slice.sliceNumber,\n jobAddress: slice.jobAddress,\n status: slice.status,\n error,\n }, 'Will try again on a new connection.');\n this.resultSubmitterMessageQueue.push({ operation: 'status', data: payload });\n this.resultSubmitterConnection.close();\n });\n }\n catch (error)\n {\n debugging('supervisor') && console.error(`Failed to return slice ${slice.identifier}, no connection to result submitter:`, error);\n }\n }\n\n /**\n * Bulk-return multiple slices, possibly for assorted jobs.\n * Returns slices to the scheduler to be redistributed.\n * Called in the sandbox terminate handler and purgeAllWork(jobAddress)\n * and stopWork(forceTerminate).\n *\n * @param {Slice[]} slices - The slices to return to the scheduler.\n * @returns {Promise<void>} - Response from the scheduler.\n */\n async returnSlices(slices) {\n if (!slices || !slices.length) return Promise.resolve();\n \n const slicePayload = [];\n slices.forEach(slice => { addToReturnSlicePayload(slicePayload, slice); });\n this.removeSlices(slices);\n\n debugging('supervisor') && console.log(`Supervisor.returnSlices: Returning slices ${this.dumpSlices(slices)}.`);\n\n return this.resultSubmitterConnection.send('status', {\n worker: this.workerOpaqueId,\n slices: slicePayload,\n }).then(response => {\n return response;\n }).catch(error => {\n const errorInfo = slices.map(slice => slice.identifier);\n debugging('supervisor') && console.error('Failed to return slice(s)', { errorInfo, error }, 'Will try again on new connection.');\n this.resultSubmitterMessageQueue.push({ operation: 'status', data: { worker: this.workerOpaqueId, slices: slicePayload } })\n this.resultSubmitterConnection.close();\n // Just in case the caller is expecing a DCP response\n return { success: false, payload: {} };\n });\n }\n\n /**\n * Submits the slice results to the scheduler, either to the\n * work submit or fail endpoints based on the slice status.\n * Then remove the slice from the @this.slices cache.\n *\n * @param {Slice} slice - The slice to submit.\n * @returns {Promise<void>}\n */\n async recordResult (slice) {\n // It is possible for slice.result to be undefined when there are upstream errors.\n if ( !(slice && slice.result))\n throw new Error(`recordResult: slice.result is undefined for slice ${slice.identifier}. This is ok when there are upstream errors.`);\n\n debugging('supervisor') && console.log(`supervisor: recording result for slice ${slice.identifier}.`);\n\n const jobAddress = slice.jobAddress;\n const sliceNumber = slice.sliceNumber;\n const authorizationMessage = slice.getAuthorizationMessage();\n\n /* @see result-submitter::result for full message details */\n const metrics = { GPUTime: 0, CPUTime: 0, CPUDensity: 0, GPUDensity: 0, total: 0 };\n const payloadData = {\n slice: sliceNumber,\n job: jobAddress,\n worker: this.workerOpaqueId,\n paymentAddress: this.paymentAddress,\n metrics,\n authorizationMessage,\n }\n \n const timeReport = slice.timeReport;\n if (timeReport && timeReport.total > 0) {\n metrics.GPUTime = timeReport.webGL;\n metrics.CPUTime = timeReport.CPU;\n metrics.CPUDensity = metrics.CPUTime / timeReport.total;\n metrics.GPUDensity = metrics.GPUTime / timeReport.total;\n metrics.total = timeReport.total;\n metrics.CPUTime = 1 + Math.floor(metrics.CPUTime);\n metrics.GPUTime = 1 + Math.floor(metrics.GPUTime);\n }\n \n this.emit('submittingResult');\n\n if (!slice.isFinished)\n throw new Error('Cannot record result for slice that is not finished');\n\n if (slice.resultStorageType === 'pattern') { /* This is a remote-storage slice. */\n const remoteResult = await this.sendResultToRemote(slice);\n payloadData.result = encodeDataURI(JSON.stringify(remoteResult));\n } else {\n payloadData.result = encodeDataURI(slice.result.result); /* XXXwg - result.result is awful */\n }\n debugging('supervisor') && console.log('Supervisor.recordResult: payloadData.result', payloadData.result.slice(0, 512));\n\n try {\n if (slice.completed) {\n\n /* work function returned a result */\n let resp = await this.resultSubmitterConnection.send(\n 'result',\n payloadData,\n ).catch((error) => {\n debugging('supervisor') && console.error(`Failed to submit result to scheduler for slice ${payloadData.slice} of job ${payloadData.job}:\\n ${error} \\nWill try again on new connection.`);\n this.resultSubmitterMessageQueue.push({ operation: 'result', data: payloadData });\n this.resultSubmitterConnection.close();\n error.handled = true;\n throw error; /* Caught in catch block below */\n });\n \n if (!resp.success)\n throw resp.payload;\n\n if (false) {}\n\n const receipt = {\n accepted: true,\n payment: resp.payload.slicePaymentAmount,\n };\n this.emit('submittedResult', resp.payload);\n this.emit('dccCredit', receipt);\n } else {\n /* slice did not complete for some reason */\n \n // If the slice from a job never completes and the job address exists in the ringBufferofJobs, \n // then we remove it to allow for another slice (from the same job) to be obtained by fetchTask\n this.ringBufferofJobs.buf = this.ringBufferofJobs.filter(element => element !== jobAddress);\n \n await this.returnSlice(slice);\n }\n } catch(error) {\n console.info(`1014: Failed to submit results for slice ${payloadData.slice} of job ${payloadData.job}`, error);\n this.emit('submitSliceFailed', error);\n } finally {\n this.emit('submitFinished');\n // Remove the slice from the slices array.\n this.removeSlice(slice);\n if (Supervisor.sliceTiming) {\n slice['resultDelta'] = Date.now() - slice['resultDelta'];\n console.log(`recordResult(${slice['pairingDelta']}, ${slice['executionDelta']}, ${slice['resultDelta']}): Completed slice ${slice.identifier}.`);\n } else\n debugging('supervisor') && console.log(`recordResult: Completed slice ${slice.identifier}.`);\n }\n }\n\n /**\n * Send a work function's result to a server that speaks our DCP Remote Data Server protocol.\n * The data server dcp-rds is been implemented in https://gitlab.com/Distributed-Compute-Protocol/dcp-rds .\n *\n * @param {Slice} slice - Slice object whose result we are sending.\n * @returns {Promise<object>} - Object of the form { success: true, href: 'http://127.0.0.1:3521/methods/download/jobs/34/result/10' } .\n * @throws When HTTP status not in the 2xx range.\n */\n async sendResultToRemote(slice) {\n const postParams = {\n ...slice.resultStorageParams\n };\n\n const sliceResultUri = makeDataURI('pattern', slice.resultStorageDetails, {\n slice: slice.sliceNumber,\n job: slice.jobAddress,\n });\n\n debugging() && console.log('sendResultToRemote sliceResultUri: ', sliceResultUri);\n\n const url = new DcpURL(sliceResultUri);\n\n // Note: sendResultToRemote was made a member function of class Supervisor to enable access to this.alowedOrigins .\n if (this.allowedOrigins.indexOf(url.origin) === -1 &&\n dcpConfig.worker.allowOrigins.sendResults.indexOf(url.origin) === -1) {\n throw new Error(`Invalid origin for remote result storage: '${url.origin}'`);\n }\n\n postParams.element = slice.sliceNumber;\n postParams.contentType = 'application/json'; // Currently data will be outputed as a JSON object, @todo: Support file upload.\n\n debugging() && console.log('sendResultToRemote: postParams: ', postParams);\n\n let result = slice.result.result;\n if (result) {\n postParams.content = JSON.stringify(result);\n } else {\n postParams.error = JSON.stringify(slice.error);\n }\n\n debugging('supervisor') && console.log('sendResultToRemote: content: ', (result ? postParams.content : postParams.error).slice(0, 512));\n\n //\n // Notes:\n // 1) In recordResults the response from justFetch is JSON serialized and encodeDataURI is called.\n // payloadData.result = await this.sendResultToRemote(slice);\n // payloadData.result = encodeDataURI(JSON.stringify(payloadData.result));\n // 2) We do further processing after the call to sendResultToRemote in recordResult, because\n // if we did it here there would be a perf hit. When the return value is a promise, it gets\n // folded into sendResultToRemote's main promise. If justFetch's promise wasn't a return value then\n // justFetch would be separately added to the micro-task-queue.\n return await justFetch(url, 'JSON', 'POST', false, postParams);\n }\n}\n\n/**\n * Sandbox has had an error which is not from the work function: kill it\n * and try to redo the slice.\n */\nfunction handleSandboxError(supervisor, sandbox, error) {\n const slice = sandbox.slice;\n\n slice.sandboxErrorCount = (slice.sandboxErrorCount || 0) + 1;\n sandbox.slice = null;\n supervisor.returnSandbox(sandbox); /* terminate the sandbox */\n slice.status = SLICE_STATUS_UNASSIGNED; /* ToT */\n console.warn(`Supervisor.handleSandboxError: Sandbox ${sandbox.identifier}...(${sandbox.public.name}/${slice.sandboxErrorCount}) with slice ${slice.identifier} had error.`, error);\n\n if (slice.sandboxErrorCount < dcpConfig.worker.maxSandboxErrorsPerSlice)\n supervisor.queuedSlices.push(slice);\n else {\n slice.error = error;\n supervisor.returnSlice(slice);\n }\n}\n\n/**\n * Add a slice to the slice payload being built. If a sliceList already exists for the\n * job-status-authMessage tuple, then the slice will be added to that, otherwise a new\n * sliceList will be added to the payload.\n *\n * @param {Object[]} slicePayload - Slice payload being built. Will be mutated in place.\n * @param {Slice} slice - The slice.\n * @param {String} status - Status update, eg. progress or scheduled.\n *\n * @returns {Object[]} mutated slicePayload array\n */\nfunction addToSlicePayload(slicePayload, slice, status) {\n // getAuthorizationMessage helps enforces the equivalence\n // !authorizationMessage <==> sliceNumber === 0\n const authorizationMessage = slice.getAuthorizationMessage();\n if (!authorizationMessage) return;\n\n // Try to find a sliceList in the payload which matches the job, status, and auth message\n let sliceList = slicePayload.find(desc => {\n return desc.job === slice.jobAddress\n && desc.status === status\n && desc.authorizationMessage === authorizationMessage;\n });\n\n // If we didn't find a sliceList, start a new one and add it to the payload\n if (!sliceList) {\n sliceList = {\n job: slice.jobAddress,\n sliceNumbers: [],\n status,\n authorizationMessage,\n };\n slicePayload.push(sliceList);\n }\n\n sliceList.sliceNumbers.push(slice.sliceNumber);\n\n return slicePayload;\n}\n\n/**\n * Add a slice to the returnSlice payload being built. If a sliceList already exists for the\n * job-isEstimation-authMessage-reason tuple, then the slice will be added to that, otherwise a new\n * sliceList will be added to the payload.\n *\n * @param {Object[]} slicePayload - Slice payload being built. Will be mutated in place.\n * @param {Slice} slice - The slice.\n * @param {String} [reason] - Optional reason to further characterize status; e.g. 'ENOPROGRESS', 'EUNCAUGHT', 'ESLICETOOSLOW', 'unknown'.\n *\n * @returns {Object[]} mutated slicePayload array\n */\nfunction addToReturnSlicePayload(slicePayload, slice, reason) {\n // getAuthorizationMessage helps enforces the equivalence\n // !authorizationMessage <==> sliceNumber === 0\n const authorizationMessage = slice.getAuthorizationMessage();\n if (!authorizationMessage) return;\n\n if (!reason) reason = slice.error ? 'EUNCAUGHT' : 'unknown';\n\n // Try to find a sliceList in the payload which matches the job, status, and auth message\n let sliceList = slicePayload.find(desc => {\n return desc.job === slice.jobAddress\n && desc.isEstimationSlice === slice.isEstimationSlice\n && desc.authorizationMessage === authorizationMessage\n && desc.reason === reason;\n });\n\n // If we didn't find a sliceList, start a new one and add it to the payload\n if (!sliceList) {\n sliceList = {\n job: slice.jobAddress,\n sliceNumbers: [],\n status: 'return',\n isEstimationSlice: slice.isEstimationSlice,\n authorizationMessage,\n reason,\n };\n slicePayload.push(sliceList);\n }\n\n sliceList.sliceNumbers.push(slice.sliceNumber);\n\n return slicePayload;\n}\n\n/**\n * Return DCPv4-specific connection options, composed of type-specific, URL-specific, \n * and worker-specific options, any/all of which can override the dcpConfig.dcp.connectOptions.\n * The order of precedence is the order of specificity.\n */\nfunction connectionOptions(url, label) {\n return leafMerge(/* ordered from most to least specific */\n dcpConfig.worker.dcp.connectionOptions.default,\n dcpConfig.worker.dcp.connectionOptions[label],\n dcpConfig.worker.dcp.connectionOptions[url.href]);\n}\n\n/** @type {number | boolean} */\nSupervisor.lastAssignFailTimerMs = false;\n/** @type {boolean} */\nSupervisor.startSandboxWork_beenCalled = false;\n/** @type {boolean} */\nSupervisor.debugBuild = ((__webpack_require__(/*! dcp/common/dcp-build */ \"./src/common/dcp-build.js\").build) === 'debug');\n/**\n * When Supervisor.sliceTiming is set to be true, it displays the timings of a every slice\n * slice['pairingDelta'] = timespan of when slice is paired with sandbox until execution starts\n * slice['executionDelta'] = timespan of execution in sandbox\n * slice['resultDelta'] = timespan of when sandbox finishes executing until recordResult completes.\n * @type {boolean}\n */\nSupervisor.sliceTiming = false;\n\nexports.Supervisor = Supervisor;\n\n\n//# sourceURL=webpack://dcp/./src/dcp-client/worker/supervisor.js?");
|
|
4460
4460
|
|
|
4461
4461
|
/***/ }),
|
|
4462
4462
|
|
|
@@ -4628,7 +4628,7 @@ eval("/**\n * @file protocol/connection/response.js\n * @author Ryan
|
|
|
4628
4628
|
/***/ ((module, __unused_webpack_exports, __webpack_require__) => {
|
|
4629
4629
|
|
|
4630
4630
|
"use strict";
|
|
4631
|
-
eval("/**\n * @file protocol/connection/sender.js\n * @author Ryan Rossiter, ryan@kingsds.network\n * @date January 2020\n *\n * The sender class is responsible for accepting Connection.Message instances,\n * and sending them to the peer via the provided transport instance.\n * Messages are queued in an array, and are sent in FIFO order - with the exception\n * of requests being skipped until there is a nonce available to send them with.\n */\n\n\nconst semver = __webpack_require__(/*! semver */ \"./node_modules/semver/semver.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nconst { assert } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\nconst { a$sleepMs } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp');\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst { setImmediate } = __webpack_require__(/*! dcp/common/dcp-timers */ \"./src/common/dcp-timers.js\");\nconst { role } = __webpack_require__(/*! ./connection-constants */ \"./src/protocol-v4/connection/connection-constants.js\");\n\nlet nanoid;\nif (DCP_ENV.platform === 'nodejs') {\n const { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\n nanoid = requireNative('nanoid').nanoid;\n} else {\n nanoid = (__webpack_require__(/*! nanoid */ \"./node_modules/nanoid/index.browser.js\").nanoid);\n}\n\nclass Sender {\n constructor(connection) {\n this.connection = connection;\n\n this.queue = [];\n this.servicingQueue = false;\n this.inFlight = null;\n this.nonce = null;\n this._ackTokenId = 0;\n \n this.debugLabel = this.connection.role === role.initiator ? `sender(i#${this.connection.id}):` : `sender(t#${this.connection.id}):`;\n }\n\n /**\n * We generate a unique token with each message we send so that \n * the peer can quickly ack the message and prove identity/uniqueness\n */\n makeAckToken() {\n return `${this.connection.id}-${this._ackTokenId++}-${nanoid()}`;\n }\n\n /**\n * Initiates the session by sending the 'connect' operation.\n * Once the response has been received, the session is established.\n * Lots of one-off code in here since it's difficult to re-use the \n * same methods meant for normal messages.\n * \n * @returns {Object} Session info\n */\n async establish() {\n assert(this.connection.transport);\n const halfSid = nanoid();\n const initiatorVersion = this.connection.constructor.VERSION;\n const connectRequest = new this.connection.Request('connect', {\n version: initiatorVersion,\n sid: halfSid,\n });\n connectRequest.id = `${this.connection.id}-connect-${nanoid()}`;\n debugging('sender') && console.debug(this.debugLabel, 'sending initial connect message');\n const resp = await this.specialFirstSend(connectRequest);\n if (typeof resp.payload !== 'object')\n throw new DCPError('Target responded to establish message without payload', 'DCPC-1018');\n if (resp.payload instanceof this.connection.ErrorPayload) {\n assert(resp.payload.type === 'protocol');\n throw new DCPError(resp.payload.message, resp.payload.code);\n }\n const targetVersion = resp.payload.version;\n const initiatorCompatibility = this.connection.constructor.VERSION_COMPATIBILITY;\n const versionCompatible = semver.satisfies(targetVersion, initiatorCompatibility);\n if (!versionCompatible) {\n debugging('sender') && console.log(this.debugLabel, `Target's version (${targetVersion}) is not compatible (must meet ${initiatorCompatibility})`);\n throw new DCPError(`Target's version (${targetVersion}) is not compatible (must meet ${initiatorCompatibility})`, 'DCPC-ETARGETVERSION');\n }\n\n // Memoize the peer version onto the Connection\n this.connection.peerVersion = targetVersion;\n if (this.connection.transport)\n this.connection.transport.peerVersion = targetVersion;\n\n // verify that our dcpsid half is still there, followed by a string\n // provided by the target that is at least the same length.\n // escape special chars:\n // eslint-disable-next-line no-useless-escape\n const regexSafeSid = halfSid.replace(/[-\\/\\\\^$*+?.()|[\\\\]{}]/g, '\\\\$&');\n const sidRegex = new RegExp(`^${regexSafeSid}.{${halfSid.length},}$`);\n if (typeof resp.dcpsid !== 'string' || !resp.dcpsid.match(sidRegex)) {\n throw new DCPError(`Target responded with invalid DCPSID: ${resp.dcpsid}`, 'DCPC-1008');\n }\n debugging('sender') && console.debug(this.debugLabel, 'connection established.');\n return {\n dcpsid: resp.dcpsid,\n peerAddress: resp.owner,\n };\n }\n\n /**\n * Invoked when the connection class becomes aware that a transport is available upon which\n * we can deliver traffic, this method either re-sends the current in-flight message, services\n * the queue if there are messages pending, or sends a keepalive to pump the queue if there's nothing\n * to send.\n */\n notifyTransportReady()\n {\n debugging('sender') && console.debug(this.debugLabel, `Notified transport is ready. inflight ${!!this.inFlight}, state ${this.connection.state}`)\n if (this.inFlight)\n this.sendInFlightMessage();\n else if (this.queue.length > 0)\n this.requestQueueService();\n else\n this.connection.keepalive().catch((error) => \n {\n if (error.code === 'ENODCPSID')\n {\n /* If our target crashed and restarted with no memory of previous connections, it will reject our message\n with the above code. Our connection will already be closing, so swallow and log this error. */\n debugging('connection') && console.debug(`${this.debugLabel} Target connection memo's corrupted. Keepalive rejected with ${error.code} `); \n }\n });\n }\n\n /**\n * We cannot use the normal enqueue logic for the first message\n * and we need many of the same bits of logic (but not all) so\n * this is the place for that kind of one-off logic.\n * We finish preparing the 'connect' request, put it in flight and send it out.\n * @param {Message} message \n */\n async specialFirstSend(message) { /* XXXwg - special first send *will* cause double dcpsid if invoked twice */\n message.ackToken = this.makeAckToken();\n let signedMessage = await message.sign();\n const messageSentPromise = this.connection.messageLedger.addMessage(message);\n \n this.inFlight = { message: message, signedMessage: signedMessage };\n if (message instanceof this.connection.Response) {\n this.connection.registerConnectResponse(this.inFlight.message); /* XXXwg todo - audit this.inFlight */\n }\n this.sendInFlightMessage();\n return messageSentPromise;\n }\n\n /**\n * Places message into `queue` but only schedules queue to be serviced\n * if connection is ready to send. Otherwise other tools will have to\n * handle making it ready to send again.\n * @param {Connection.Request|Connection.Response} message\n * @returns {Promise<Connection.Response>} from messageLedger that resolves when message response received.\n */\n enqueue(message) {\n if (message.payload && message.payload.operation)\n debugging('connection') && console.debug(this.debugLabel, `enqueueing ${message.type} message for operation:`, message.payload.operation);\n else\n debugging('connection') && console.debug(this.debugLabel, `enqueueing ${message.type} message`);\n \n this.queue.push(message);\n \n if (this.connection.state.in(['initial', 'disconnected']))\n this.connection.connect();\n\n setImmediate(() => this.requestQueueService());\n return this.connection.messageLedger.addMessage(message);\n }\n \n /**\n * Checks if we're able to send a message in our current state, and\n * services the queue if so.\n */\n requestQueueService()\n {\n if (this.inFlight || !this.nonce || this.servicingQueue)\n {\n debugging('sender') && debugging('verbose') && console.debug(\n `${this.debugLabel} request for queue service denied. inFlight: ${this.inFlight}, nonce: ${this.nonce}, queue running: ${this.servicingQueue}`\n )\n return;\n }\n else\n {\n debugging('sender') && console.debug(`${this.debugLabel} request for queue service accepted.`)\n this.serviceQueue();\n }\n }\n /**\n * Pulls a message from the queue. If the message is batchable, pauses for 30ms to allow other batchable\n * messages to enter the queue, and then continues dequeuing messages. \n * If the message is not batchable, ie. is here just to finish being prepared, then sign the message and emit\n * `message.id ready` to resolve the `Connection.prepareMessage` promise for the dequeued message.\n * When a message is signed, our nonce is deleted so that it may not be used in another message.\n */\n serviceQueue ()\n {\n if (!this.inFlight && this.connection.state.in(['established', 'closing', 'close-wait']))\n {\n debugging('sender') && console.debug(`${this.debugLabel} serviceQueue: checking queue`)\n this.servicingQueue = true;\n let nextMessage = this.queue.shift();\n if (!nextMessage)\n {\n debugging('sender') && console.debug(`${this.debugLabel} serviceQueue: nothing to send`)\n this.servicingQueue = false;\n return;\n }\n \n if (nextMessage.batchable)\n {\n debugging('sender') && console.debug(`${this.debugLabel} serviceQueue: pulled batchable message, allowing more messages to enter queue`)\n // Buffer the dequeuing of batchable messages to emulate the intention of Nagle's\n // algorithm, used to reduce the number of packets being sent over the network.\n a$sleepMs(this.connection.options.messageBatchingBuffer * 1000)\n .then(() =>\n {\n let rawBatch = [];\n do\n {\n rawBatch.push(nextMessage);\n nextMessage = this.queue.shift();\n } while (nextMessage && nextMessage.batchable && rawBatch.length < this.maxBatchSize)\n // if dequeuing a prepared message stopped the loop, put the message back in front \n if (nextMessage) this.queue.unshift(nextMessage);\n \n this.servicingQueue = false;\n debugging('sender') && console.debug(`${this.debugLabel} serviceQueue: pulled ${rawBatch.length} messages for batching. done servicing queue`)\n this.createAndSend(rawBatch);\n });\n }\n else\n {\n nextMessage.nonce = this.nonce;\n delete this.nonce;\n \n this.connection.emit(`${nextMessage.id} ready`, nextMessage);\n this.servicingQueue = false;\n debugging('sender') && console.debug(`${this.debugLabel} serviceQueue: pulled non-batchable message, finished its preparation. done servicing queue`)\n }\n \n }\n else\n {\n debugging('connection') && console.debug(this.debugLabel, `ignoring call to service queue. inFlight=${!!this.inFlight} state=${this.connection.state.valueOf()}`);\n }\n }\n\n /**\n * Takes an array of messages and batches them if there's more than one. Signs the resultant message and puts it in flight.\n * @param {Array<Connection.Message>} [messages] Array of messages to batch together for a transmission. \n * if not provided, draws from queue.\n *\n * @returns a Promise which resolves to a number which represents the number of messages sent. This is \n * either 0, 1, or the number of messages in a batch (possibly 1).\n */\n async createAndSend(messages) {\n assert(!this.inFlight);\n \n // By default, let `message` be first entry in input array.\n // If more than one entry in array, reassign `message` as batch object.\n let message = messages[0];\n if (messages.length > 1)\n message = new this.connection.Batch(messages, this.makeAckToken()); \n\n message.nonce = this.nonce;\n delete this.nonce;\n \n if (message instanceof this.connection.Batch) this.connection.messageLedger.addBatch(message);\n \n this.inFlight = { message: message };\n this.inFlight.signedMessage = await message.sign();\n \n debugging('connection') && console.debug(this.debugLabel, `sending a signed ${message.type} of ${messages.length}`);\n this.sendInFlightMessage();\n }\n \n /**\n * Sends the message stored in the `inFlight` var over the transport.\n * `clearFlightDeck` is the only method that should be resetting this.inFlight\n * and thus closing the loop.\n */\n sendInFlightMessage()\n {\n assert(this.inFlight);\n\n debugging('sender') && console.debug(this.debugLabel, `sending in-flight message ${this.inFlight.message.id}`);\n\n try\n {\n const message = this.inFlight.message;\n let type;\n \n /** XXXwg todo - figure out why Request/Response have name=Message and get rid of tests */\n if (message instanceof this.connection.Request)\n type = 'Request';\n else if (message instanceof this.connection.Response)\n type = 'Response';\n else \n type = message.constructor.name;\n\n if (!this.connection.transport)\n {\n debugging('sender') && console.debug(this.debugLabel, `no transport for ${type}; not sending`, this.inFlight.message.id,\n `(${this.inFlight.signedMessage.length} bytes)`);\n\n return;\n }\n \n \n debugging('sender') && console.debug(this.debugLabel, `sending ${type}`, this.inFlight.message.id,\n `(${this.inFlight.signedMessage.length} bytes)`);\n\n if (type === 'Batch')\n type += ':' + (this.inFlight.message.messages.length);\n debugging('wire') && console.debug(this.connection.debugLabel, `sendInFlightMessage: ${String(type).padEnd(10, ' ')} -> ${this.connection.loggableDest}`);\n this.connection.transport.send(this.inFlight.signedMessage);\n this.connection.emit('send', this.inFlight);\n }\n catch (error)\n {\n console.error(`Error while sending message ${this.inFlight.message.id} to ${this.connection.loggableDest}:`, error);\n debugging('sender') && console.debug(this.debugLabel, 'call stack:', new Error().stack);\n }\n }\n\n /**\n * Clear a message from the flight deck. For the foreseeable future the deck\n * only holds one message at a time, so we just assert that it matches.\n * @param {Connection.Message} message message that can be cleared from the flight deck\n */\n clearFlightDeck(message, nonce) {\n if (this.inFlight !== null) {\n debugging('sender') && console.debug(this.debugLabel, 'clearing flight deck. nonce =', nonce);\n assert(message === this.inFlight.message);\n this.inFlight = null;\n this.nonce = nonce;\n this.requestQueueService();\n }\n }\n\n /**\n * When a connection is closed the sender needs to cancel its send efforts.\n */\n shutdown() {\n debugging('sender') && console.debug(this.debugLabel, 'shutting down.');\n this.inFlight = null;\n this.nonce = null;\n this.queue = [];\n }\n\n // When allowBatch=false, set the batchSize to 1 so each\n // message gets sent individually (not in a batch)\n get maxBatchSize() {\n if (this._maxBatchSize) return this._maxBatchSize;\n const maxBatchSize = Math.max(this.connection.options.maxMessagesPerBatch, 1);\n this._maxBatchSize = this.connection.options.allowBatch ? maxBatchSize : 1;\n return this._maxBatchSize;\n }\n}\n\nObject.assign(module.exports, {\n Sender,\n});\n\n\n//# sourceURL=webpack://dcp/./src/protocol-v4/connection/sender.js?");
|
|
4631
|
+
eval("/**\n * @file protocol/connection/sender.js\n * @author Ryan Rossiter, ryan@kingsds.network\n * @date January 2020\n *\n * The sender class is responsible for accepting Connection.Message instances,\n * and sending them to the peer via the provided transport instance.\n * Messages are queued in an array, and are sent in FIFO order - with the exception\n * of requests being skipped until there is a nonce available to send them with.\n */\n\n\nconst semver = __webpack_require__(/*! semver */ \"./node_modules/semver/semver.js\");\nconst DCP_ENV = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nconst { assert } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\nconst { a$sleepMs } = __webpack_require__(/*! dcp/utils */ \"./src/utils/index.js\");\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp');\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst { setImmediate } = __webpack_require__(/*! dcp/common/dcp-timers */ \"./src/common/dcp-timers.js\");\nconst { role } = __webpack_require__(/*! ./connection-constants */ \"./src/protocol-v4/connection/connection-constants.js\");\n\nlet nanoid;\nif (DCP_ENV.platform === 'nodejs') {\n const { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\n nanoid = requireNative('nanoid').nanoid;\n} else {\n nanoid = (__webpack_require__(/*! nanoid */ \"./node_modules/nanoid/index.browser.js\").nanoid);\n}\n\nclass Sender {\n constructor(connection) {\n this.connection = connection;\n\n this.queue = [];\n this.servicingQueue = false;\n this.inFlight = null;\n this.nonce = null;\n this._ackTokenId = 0;\n \n this.debugLabel = this.connection.role === role.initiator ? `sender(i#${this.connection.id}):` : `sender(t#${this.connection.id}):`;\n }\n\n /**\n * We generate a unique token with each message we send so that \n * the peer can quickly ack the message and prove identity/uniqueness\n */\n makeAckToken() {\n return `${this.connection.id}-${this._ackTokenId++}-${nanoid()}`;\n }\n\n /**\n * Initiates the session by sending the 'connect' operation.\n * Once the response has been received, the session is established.\n * Lots of one-off code in here since it's difficult to re-use the \n * same methods meant for normal messages.\n * \n * @returns {Object} Session info\n */\n async establish() {\n assert(this.connection.transport);\n const halfSid = nanoid();\n const initiatorVersion = this.connection.constructor.VERSION;\n const connectRequest = new this.connection.Request('connect', {\n version: initiatorVersion,\n sid: halfSid,\n });\n connectRequest.id = `${this.connection.id}-connect-${nanoid()}`;\n debugging('sender') && console.debug(this.debugLabel, 'sending initial connect message');\n const resp = await this.specialFirstSend(connectRequest);\n if (typeof resp.payload !== 'object')\n throw new DCPError('Target responded to establish message without payload', 'DCPC-1018');\n if (resp.payload instanceof this.connection.ErrorPayload) {\n assert(resp.payload.type === 'protocol');\n throw new DCPError(resp.payload.message, resp.payload.code);\n }\n const targetVersion = resp.payload.version;\n const initiatorCompatibility = this.connection.constructor.VERSION_COMPATIBILITY;\n const versionCompatible = semver.satisfies(targetVersion, initiatorCompatibility);\n if (!versionCompatible) {\n debugging('sender') && console.log(this.debugLabel, `Target's version (${targetVersion}) is not compatible (must meet ${initiatorCompatibility})`);\n throw new DCPError(`Target's version (${targetVersion}) is not compatible (must meet ${initiatorCompatibility})`, 'DCPC-ETARGETVERSION');\n }\n\n // Memoize the peer version onto the Connection\n this.connection.peerVersion = targetVersion;\n if (this.connection.transport)\n this.connection.transport.peerVersion = targetVersion;\n\n // verify that our dcpsid half is still there, followed by a string\n // provided by the target that is at least the same length.\n // escape special chars:\n // eslint-disable-next-line no-useless-escape\n const regexSafeSid = halfSid.replace(/[-\\/\\\\^$*+?.()|[\\\\]{}]/g, '\\\\$&');\n const sidRegex = new RegExp(`^${regexSafeSid}.{${halfSid.length},}$`);\n if (typeof resp.dcpsid !== 'string' || !resp.dcpsid.match(sidRegex)) {\n throw new DCPError(`Target responded with invalid DCPSID: ${resp.dcpsid}`, 'DCPC-1008');\n }\n debugging('sender') && console.debug(this.debugLabel, 'connection established.');\n return {\n dcpsid: resp.dcpsid,\n peerAddress: resp.owner,\n };\n }\n\n /**\n * Invoked when the connection class becomes aware that a transport is available upon which\n * we can deliver traffic, this method either re-sends the current in-flight message, services\n * the queue if there are messages pending, or sends a keepalive to pump the queue if there's nothing\n * to send.\n */\n notifyTransportReady()\n {\n debugging('sender') && console.debug(this.debugLabel, `Notified transport is ready. inflight ${!!this.inFlight}, state ${this.connection.state}`)\n if (this.inFlight)\n this.sendInFlightMessage();\n else if (this.queue.length > 0)\n this.requestQueueService();\n else\n this.connection.keepalive().catch((error) => \n {\n if (error.code === 'ENODCPSID')\n {\n /* If our target crashed and restarted with no memory of previous connections, it will reject our message\n with the above code. Our connection will already be closing, so swallow and log this error. */\n debugging('connection') && console.debug(`${this.debugLabel} Target connection memo's corrupted. Keepalive rejected with ${error.code} `); \n }\n });\n }\n\n /**\n * We cannot use the normal enqueue logic for the first message\n * and we need many of the same bits of logic (but not all) so\n * this is the place for that kind of one-off logic.\n * We finish preparing the 'connect' request, put it in flight and send it out.\n * @param {Message} message \n */\n async specialFirstSend(message) { /* XXXwg - special first send *will* cause double dcpsid if invoked twice */\n message.ackToken = this.makeAckToken();\n let signedMessage = await message.sign();\n const messageSentPromise = this.connection.messageLedger.addMessage(message);\n \n this.inFlight = { message: message, signedMessage: signedMessage };\n if (message instanceof this.connection.Response) {\n this.connection.registerConnectResponse(this.inFlight.message); /* XXXwg todo - audit this.inFlight */\n }\n this.sendInFlightMessage();\n return messageSentPromise;\n }\n\n /**\n * Places message into `queue` but only schedules queue to be serviced\n * if connection is ready to send. Otherwise other tools will have to\n * handle making it ready to send again.\n * @param {Connection.Request|Connection.Response} message\n * @returns {Promise<Connection.Response>} from messageLedger that resolves when message response received.\n */\n enqueue(message) {\n if (message.payload && message.payload.operation)\n debugging('connection') && console.debug(this.debugLabel, `enqueueing ${message.type} message for operation:`, message.payload.operation);\n else\n debugging('connection') && console.debug(this.debugLabel, `enqueueing ${message.type} message`);\n \n this.queue.push(message);\n \n if (this.connection.state.in(['initial', 'disconnected']))\n this.connection.connect();\n\n setImmediate(() => this.requestQueueService());\n return this.connection.messageLedger.addMessage(message);\n }\n \n /**\n * Checks if we're able to send a message in our current state, and\n * services the queue if so.\n */\n requestQueueService()\n {\n if (this.inFlight || !this.nonce || this.servicingQueue)\n {\n debugging('sender') && debugging('verbose') && console.debug(\n `${this.debugLabel} request for queue service denied.`,\n `inFlight: ${this.inFlight && this.inFlight.message && this.inFlight.message.id},`,\n `nonce: ${this.nonce}, queue running: ${this.servicingQueue}`\n );\n return;\n }\n else\n {\n debugging('sender') && console.debug(`${this.debugLabel} request for queue service accepted.`)\n this.serviceQueue();\n }\n }\n /**\n * Pulls a message from the queue. If the message is batchable, pauses for 30ms to allow other batchable\n * messages to enter the queue, and then continues dequeuing messages. \n * If the message is not batchable, ie. is here just to finish being prepared, then sign the message and emit\n * `message.id ready` to resolve the `Connection.prepareMessage` promise for the dequeued message.\n * When a message is signed, our nonce is deleted so that it may not be used in another message.\n */\n serviceQueue ()\n {\n if (!this.inFlight && this.connection.state.in(['established', 'closing', 'close-wait']))\n {\n debugging('sender') && console.debug(`${this.debugLabel} serviceQueue: checking queue`)\n this.servicingQueue = true;\n let nextMessage = this.queue.shift();\n if (!nextMessage)\n {\n debugging('sender') && console.debug(`${this.debugLabel} serviceQueue: nothing to send`)\n this.servicingQueue = false;\n return;\n }\n \n if (nextMessage.batchable)\n {\n debugging('sender') && console.debug(`${this.debugLabel} serviceQueue: pulled batchable message, allowing more messages to enter queue`)\n // Buffer the dequeuing of batchable messages to emulate the intention of Nagle's\n // algorithm, used to reduce the number of packets being sent over the network.\n a$sleepMs(this.connection.options.messageBatchingBuffer * 1000)\n .then(() =>\n {\n let rawBatch = [];\n do\n {\n rawBatch.push(nextMessage);\n nextMessage = this.queue.shift();\n } while (nextMessage && nextMessage.batchable && rawBatch.length < this.maxBatchSize)\n // if dequeuing a prepared message stopped the loop, put the message back in front \n if (nextMessage) this.queue.unshift(nextMessage);\n \n this.servicingQueue = false;\n debugging('sender') && console.debug(`${this.debugLabel} serviceQueue: pulled ${rawBatch.length} messages for batching. done servicing queue`)\n this.createAndSend(rawBatch);\n });\n }\n else\n {\n nextMessage.nonce = this.nonce;\n delete this.nonce;\n \n this.connection.emit(`${nextMessage.id} ready`, nextMessage);\n this.servicingQueue = false;\n debugging('sender') && console.debug(`${this.debugLabel} serviceQueue: pulled non-batchable message, finished its preparation. done servicing queue`)\n }\n \n }\n else\n {\n debugging('connection') && console.debug(this.debugLabel, `ignoring call to service queue. inFlight=${!!this.inFlight} state=${this.connection.state.valueOf()}`);\n }\n }\n\n /**\n * Takes an array of messages and batches them if there's more than one. Signs the resultant message and puts it in flight.\n * @param {Array<Connection.Message>} [messages] Array of messages to batch together for a transmission. \n * if not provided, draws from queue.\n *\n * @returns a Promise which resolves to a number which represents the number of messages sent. This is \n * either 0, 1, or the number of messages in a batch (possibly 1).\n */\n async createAndSend(messages) {\n assert(!this.inFlight);\n \n // By default, let `message` be first entry in input array.\n // If more than one entry in array, reassign `message` as batch object.\n let message = messages[0];\n if (messages.length > 1)\n message = new this.connection.Batch(messages, this.makeAckToken()); \n\n message.nonce = this.nonce;\n delete this.nonce;\n \n if (message instanceof this.connection.Batch) this.connection.messageLedger.addBatch(message);\n \n this.inFlight = { message: message };\n this.inFlight.signedMessage = await message.sign();\n \n debugging('connection') && console.debug(this.debugLabel, `sending a signed ${message.type} of ${messages.length}`);\n this.sendInFlightMessage();\n }\n \n /**\n * Sends the message stored in the `inFlight` var over the transport.\n * `clearFlightDeck` is the only method that should be resetting this.inFlight\n * and thus closing the loop.\n */\n sendInFlightMessage()\n {\n assert(this.inFlight);\n\n debugging('sender') && console.debug(this.debugLabel, `sending in-flight message ${this.inFlight.message.id}`);\n\n try\n {\n const message = this.inFlight.message;\n let type;\n \n /** XXXwg todo - figure out why Request/Response have name=Message and get rid of tests */\n if (message instanceof this.connection.Request)\n type = 'Request';\n else if (message instanceof this.connection.Response)\n type = 'Response';\n else \n type = message.constructor.name;\n\n if (!this.connection.transport)\n {\n debugging('sender') && console.debug(this.debugLabel, `no transport for ${type}; not sending`, this.inFlight.message.id,\n `(${this.inFlight.signedMessage.length} bytes)`);\n\n return;\n }\n \n \n debugging('sender') && console.debug(this.debugLabel, `sending ${type}`, this.inFlight.message.id,\n `(${this.inFlight.signedMessage.length} bytes)`);\n\n if (type === 'Batch')\n type += ':' + (this.inFlight.message.messages.length);\n debugging('wire') && console.debug(this.connection.debugLabel, `sendInFlightMessage: ${String(type).padEnd(10, ' ')} -> ${this.connection.loggableDest}`);\n this.connection.transport.send(this.inFlight.signedMessage);\n this.connection.emit('send', this.inFlight);\n }\n catch (error)\n {\n console.error(`Error while sending message ${this.inFlight.message.id} to ${this.connection.loggableDest}:`, error);\n debugging('sender') && console.debug(this.debugLabel, 'call stack:', new Error().stack);\n }\n }\n\n /**\n * Clear a message from the flight deck. For the foreseeable future the deck\n * only holds one message at a time, so we just assert that it matches.\n * @param {Connection.Message} message message that can be cleared from the flight deck\n */\n clearFlightDeck(message, nonce) {\n if (this.inFlight !== null) {\n debugging('sender') && console.debug(this.debugLabel, 'clearing flight deck. nonce =', nonce);\n assert(message === this.inFlight.message);\n this.inFlight = null;\n this.nonce = nonce;\n this.requestQueueService();\n }\n }\n\n /**\n * When a connection is closed the sender needs to cancel its send efforts.\n */\n shutdown() {\n debugging('sender') && console.debug(this.debugLabel, 'shutting down.');\n this.inFlight = null;\n this.nonce = null;\n this.queue = [];\n }\n\n // When allowBatch=false, set the batchSize to 1 so each\n // message gets sent individually (not in a batch)\n get maxBatchSize() {\n if (this._maxBatchSize) return this._maxBatchSize;\n const maxBatchSize = Math.max(this.connection.options.maxMessagesPerBatch, 1);\n this._maxBatchSize = this.connection.options.allowBatch ? maxBatchSize : 1;\n return this._maxBatchSize;\n }\n}\n\nObject.assign(module.exports, {\n Sender,\n});\n\n\n//# sourceURL=webpack://dcp/./src/protocol-v4/connection/sender.js?");
|
|
4632
4632
|
|
|
4633
4633
|
/***/ }),
|
|
4634
4634
|
|
|
@@ -4682,7 +4682,7 @@ eval("/**\n * @file connection/target.js\n * @author Wes Garland, we
|
|
|
4682
4682
|
/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
|
|
4683
4683
|
|
|
4684
4684
|
"use strict";
|
|
4685
|
-
eval("/**\n * @file http-bearer.js\n * Utility code for transports working with http (or https) bearers\n *\n * @author Wes Garland, wes@kingsds.network\n * @date Apr 2022\n */\n\nconst { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp');\n\n/**\n * Returns true if friendLocation should work in place of location from this host.\n * This allows us to transparently configure inter-daemon communication that uses\n * local LAN IPs instead of bouncing off the firewall for NAT.\n */\nexports.a$isFriendlyUrl = async function httpBearer$$isFriendlyUrl(url)\n{\n var remoteIp, dnsA;\n var ifaces;\n \n if (!url)\n return false;\n \n if (url.hostname === 'localhost')\n return true;\n\n switch(url.protocol)\n {\n case 'http:':\n case 'https:':\n case 'ws:':\n case 'tcp:':\n case 'udp:':\n case 'dcpsaw:':\n break;\n default:\n return false;\n }\n\n /* Consider same-origin match friendly */\n if ((__webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\").isBrowserPlatform))\n return url.origin === window.location.origin;\n\n /* Convert an IP address to a 32-bit int in network order */\n function i32(addr)\n {\n var ret = 0;\n var octets = addr.split('.');\n\n ret |= octets[0] << 24; /* Note: JS ints are signed 32, but that doesn't matter for masking */\n ret |= octets[1] << 16;\n ret |= octets[2] << 8;\n ret |= octets[3] << 0;\n\n return ret;\n }\n \n /* Consider machines in same IPv4 subnet friendly */\n dnsA = await requireNative('dns').promises.lookup(url.hostname, { family: 4 });\n if (!dnsA)\n return false;\n remoteIp = i32(dnsA.address);\n ifaces = requireNative('os').networkInterfaces();\n for (let ifaceName of Object.keys(ifaces))\n {\n for (let alias of ifaces[ifaceName])\n {\n if (alias.family !== 'IPv4')\n continue;\n\n let i32_addr = i32(alias.address);\n let i32_mask = i32(alias.netmask);\n\n if ((i32_addr & i32_mask) === (remoteIp & i32_mask))\n return true;\n }\n }\n\n return false;\n}\n\n/**\n * Factory which create instances of Server for targets/listeners, suitable for binding to\n * transports which use an underlying http (or https) bearer.\n *\n * @param {object} config the configuration object for the listener; must have\n * either a location (or ideally) a listen property which\n * is an instance of URL or DcpURL.\n * @param {function} listeningHandler [optional] a function which is invoked as soon as the\n * underlying server is listening for new connections. It\n * receives as its sole argument an instance of Server.\n */\nexports.httpServerFactory = function(config, listeningHandler)\n{\n var bearer;\n var listen = config.listen || config.location;\n\n const listenOpts = {\n host: listen.hostname,\n port: listen.port\n };\n\n switch (listen.protocol)\n {\n case 'http:':\n bearer = requireNative('http');\n listenOpts.port = listenOpts.port || 80;\n break;\n case 'https:':\n bearer = requireNative('https');\n listenOpts.port = listenOpts.port || 443;\n break;\n default:\n throw new Error(`unknown bearer: '${listen.protocol}'`)\n }\n\n if (listenOpts.host === 'inaddr_any' || !listenOpts.host)\n delete listenOpts.host;\n \n const server = bearer.createServer();\n server.listen(listenOpts, () => {\n debugging() && console.debug('http server listening on', (listenOpts.host || 'any/0') + ':' + listenOpts.port);\n listeningHandler(server);\n });\n\n return server;\n}\n\n\n//# sourceURL=webpack://dcp/./src/protocol-v4/transport/http-bearer.js?");
|
|
4685
|
+
eval("/**\n * @file http-bearer.js\n * Utility code for transports working with http (or https) bearers\n *\n * @author Wes Garland, wes@kingsds.network\n * @date Apr 2022\n */\n\nconst { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp');\n\n/**\n * Returns true if friendLocation should work in place of location from this host.\n * This allows us to transparently configure inter-daemon communication that uses\n * local LAN IPs instead of bouncing off the firewall for NAT.\n */\nexports.a$isFriendlyUrl = async function httpBearer$$isFriendlyUrl(url)\n{\n var remoteIp, dnsA;\n var ifaces;\n \n if (!url)\n return false;\n \n if (url.hostname === 'localhost')\n return true;\n\n switch(url.protocol)\n {\n case 'http:':\n case 'https:':\n case 'ws:':\n case 'tcp:':\n case 'udp:':\n case 'dcpsaw:':\n break;\n default:\n return false;\n }\n\n /* Consider same-origin match friendly */\n if ((__webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\").isBrowserPlatform))\n return url.origin === window.location.origin;\n\n /* Convert an IP address to a 32-bit int in network order */\n function i32(addr)\n {\n var ret = 0;\n var octets = addr.split('.');\n\n ret |= octets[0] << 24; /* Note: JS ints are signed 32, but that doesn't matter for masking */\n ret |= octets[1] << 16;\n ret |= octets[2] << 8;\n ret |= octets[3] << 0;\n\n return ret;\n }\n \n /* Consider machines in same IPv4 subnet friendly */\n dnsA = await requireNative('dns').promises.lookup(url.hostname, { family: 4 });\n if (!dnsA)\n return false;\n remoteIp = i32(dnsA.address);\n ifaces = requireNative('os').networkInterfaces();\n for (let ifaceName of Object.keys(ifaces))\n {\n for (let alias of ifaces[ifaceName])\n {\n if (alias.family !== 'IPv4')\n continue;\n\n let i32_addr = i32(alias.address);\n let i32_mask = i32(alias.netmask);\n\n if ((i32_addr & i32_mask) === (remoteIp & i32_mask))\n return true;\n }\n }\n\n return false;\n}\n\n/**\n * Factory which create instances of Server for targets/listeners, suitable for binding to\n * transports which use an underlying http (or https) bearer.\n *\n * @param {object} config the configuration object for the listener; must have\n * either a location (or ideally) a listen property which\n * is an instance of URL or DcpURL.\n * @param {function} listeningHandler [optional] a function which is invoked as soon as the\n * underlying server is listening for new connections. It\n * receives as its sole argument an instance of Server.\n */\nexports.httpServerFactory = function(config, listeningHandler)\n{\n var bearer;\n var listen = config.listen || config.location;\n\n const listenOpts = {\n host: listen.hostname,\n port: listen.port\n };\n\n switch (listen.protocol)\n {\n case 'http:':\n bearer = requireNative('http');\n listenOpts.port = listenOpts.port || 80;\n break;\n case 'https:':\n bearer = requireNative('https');\n listenOpts.port = listenOpts.port || 443;\n break;\n default:\n throw new Error(`unknown bearer: '${listen.protocol}'`)\n }\n\n if (listenOpts.host === 'inaddr_any' || !listenOpts.host)\n delete listenOpts.host;\n if (listenOpts.host === 'localhost')\n listenOpts.host = '::'; /* transparently support IPv4 and IPv6 link-local addresses */\n \n const server = bearer.createServer();\n server.listen(listenOpts, () => {\n debugging() && console.debug('http server listening on', (listenOpts.host || 'any/0') + ':' + listenOpts.port);\n listeningHandler(server);\n });\n\n return server;\n}\n\n\n//# sourceURL=webpack://dcp/./src/protocol-v4/transport/http-bearer.js?");
|
|
4686
4686
|
|
|
4687
4687
|
/***/ }),
|
|
4688
4688
|
|
|
@@ -4715,7 +4715,7 @@ eval("/**\n * @file listener.js\n * Generic API for transpor
|
|
|
4715
4715
|
/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
|
|
4716
4716
|
|
|
4717
4717
|
"use strict";
|
|
4718
|
-
eval("/* provided dependency */ var process = __webpack_require__(/*! ./node_modules/process/browser.js */ \"./node_modules/process/browser.js\");\n/**\n * @file transport/socketio.js\n * @author Ryan Rossiter, ryan@kingsds.network\n * @author Wes Garland, wes@kingsds.network \n * @date January 2020, March 2022\n *\n * This module implements the SocketIO Transport that is\n * used by the protocol to connect and communicate with peers using\n * SocketIO connections.\n *\n * Transport can operate in either ack mode or non-ack-mode, depending on the value of this.ackMode.\n * Ack mode sends an extra socketio-layer packet back after each message is received, and theoretically\n * this can be use tod meter or measure bandwidth or potentially second-guess socketio's ability to\n * interleave its heartbeat messages OOB from the main traffic. This mode could potentially be enabled\n * on-demand as well, although it appears that it is not necessary to get the desired behaviour at this\n * point, so the feature is off-by-default (although reasonably well-tested).\n */\n\n\nconst { Transport } = __webpack_require__(/*! . */ \"./src/protocol-v4/transport/index.js\");\nconst { EventEmitter } = __webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\");\nconst { leafMerge } = __webpack_require__(/*! dcp/utils/obj-merge */ \"./src/utils/obj-merge.js\");\nconst { DcpURL } = __webpack_require__(/*! dcp/common/dcp-url */ \"./src/common/dcp-url.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst { setImmediateN } = __webpack_require__(/*! dcp/common/dcp-timers */ \"./src/common/dcp-timers.js\");\nconst { setImmediate } = __webpack_require__(/*! dcp/common/dcp-timers */ \"./src/common/dcp-timers.js\");\nconst { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\nconst { assert } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\n\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp');\nconst dcpEnv = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nconst bearer = __webpack_require__(/*! ./http-bearer */ \"./src/protocol-v4/transport/http-bearer.js\");\nconst semver = __webpack_require__(/*! semver */ \"./node_modules/semver/semver.js\");\n\nvar listenerSeq = 0;\nvar initiatorSeq = 0;\n\nif (debugging() && !process.env.DEBUG && dcpConfig.build === 'debug')\n{\n if (process.env.DEBUG)\n process.env.DEBUG += ',engine';\n else\n process.env.DEBUG = 'engine';\n\n if (debugging('socketio'))\n {\n /* DCP_DEBUG=dcp:socketio,dcp:verbose to see everything */\n process.env.DEBUG += ',socket.io-client:socket,socket.io:client,socket.io:server';\n if (debugging('verbose') && !debugging('all'))\n process.env.DEBUG += ',socket.io*,engine.io*'\n }\n}\n\nclass SocketIOTransport extends Transport\n{\n /**\n * @constructor\n * @param {socket} newSocket Target: an instance of socket.io Socket that represents\n * a new connection.\n * Initiator: unused\n */\n /**\n * @constructor create new instance of a socketio-flavoured Transport.\n *\n * @param {object} config dcpConfig fragment\n * @param {Url} url Initiator: has .location and optionally .friendLocation\n * Target: has .listen and .location\n *\n * @param {object} options The `socketio` property of a protocol-v4 connectionOptions \n * object, which was built from a combination of defaults and \n * various dcpConfig components.\n *\n * @returns instance suitable for target or initiator, depending on mode, but does not manage connections.\n */\n constructor(config, options={}, newSocket)\n {\n super('Protocol SocketIO Transport');\n const that = this;\n \n this.name = 'socketio';\n this.url = config.location;\n this.msgSequence = 0;\n this.isClosed = false; /* true => connection finalization begun */\n this.isFinal = false; /* true => connection finalization completed */\n this.hasConnected = false; /* true => one 'connect' event has been processed */\n this.initTimeout = false; /* timeout when connecting to allow target to reply with their mode message + buffer size, crash if they don't reply */\n this.modeSet = false; /* true => received response mode message */\n this.bufferSizeSet = false; /* true => received target's buffer size */\n this.rxFragments = {}; /* messages we are receiving in fragments */\n this.rxPending = []; /* strings which are messages that are complete and ready to emit, or objects that in this.rxFragments */\n this.txPending = []; /* messages we are receiving in fragments */\n this.txNumInFlight = 0; /* number of txPending sent but not acknowleged */\n this.txMaxInFlight = 2; /* less than 2 causes bad perf when mode.ack */\n this.debugInfo = { remoteLabel: '<unknown>' };\n this.mode = {\n ack: false,\n concat: false,\n };\n\n assert(DcpURL.isURL(this.url));\n assert(typeof options === 'object');\n\n this.options = this.buildOptions(options);\n\n if (!newSocket) /* Initiator */\n {\n this.debugLabel = `socketio(i:${++initiatorSeq}):`;\n this.rxReady = true; /* relax, said the night man */\n this.debugInfo.remoteLabel = config.location.href;\n if (this.options.disableModeAnnouncement !== false)\n this.sendModeList();\n this.sendMaxMessageInfo();\n\n bearer.a$isFriendlyUrl(config.friendLocation).then((useFriendLocation) => {\n const connectUrl = useFriendLocation ? config.friendLocation : config.location;\n\n debugging('socketio') && console.log(this.debugLabel, 'connecting to', connectUrl.href);\n this.socket = (__webpack_require__(/*! socket.io-client */ \"./node_modules/socket.io-client/build/cjs/index.js\").connect)(connectUrl.origin, this.options);\n this.socket.on('connect_error', (error) => !this.isClosed && this.handleConnectErrorEvent(error));\n this.socket.on('connect', () => !this.isClosed && this.handleConnectEvent());\n\n useSocket();\n }).catch((error) => {\n debugging() && console.error(this.debugLabel, error);\n this.emit('error', error);\n });\n }\n else /* Target - receives socketio instance from SocketIOListener.handleConnectionEvent */\n {\n if (dcpEnv.platform !== 'nodejs')\n throw new Error('socketio: target mode only supported in nodejs');\n\n this.debugLabel = 'socketio(t:<new>):';\n\n this.socket = newSocket;\n this.debugInfo.remoteLabel = this.socket.handshake.address.replace(/^::ffff:/,'');\n debugging('socketio') && console.debug(this.debugLabel, `new socket from ${this.debugInfo.remoteLabel}`\n + ` on ${this.socket.handshake && this.socket.handshake.url}, ${this.socket.id}`);\n\n useSocket();\n }\n\n function useSocket()\n {\n that.socket.compress(dcpConfig.build !== 'debug'); /* try to keep traffic sniffable in debug */\n that.socket.on('message', (msg) => !that.isFinal && that.handleMessageEvent(msg));\n that.socket.on('disconnect', (reason) => !that.isClosed && that.handleDisconnectEvent(reason));\n }\n }\n\n /**\n * API - determine if a transport is usable or not.\n * @returns true if the transport is not closed.\n */\n ready()\n {\n return !this.isClosed;\n }\n \n /**\n * Handle the socket.io disconnect event, which is fired upon disconnection.\n * For some reasons (explicit disconnection), the socket.io code will not try to reconnect, but\n * in all other cases, the socket.io client will normally wait for a small random delay and then \n * try to reconnect, but in our case, we simply tear the transport down and let the Connection\n * class handle reconnects, since it might prefer a different transport at this point anyhow.\n *\n * One tricky case here is ping timeout, since the ping timeout includes the time to transmit the\n * packet before it was sent.\n *\n * @param {string} reason Possible reasons for disconnection.\n */\n handleDisconnectEvent(reason)\n {\n debugging('socketio') && console.debug(this.debugLabel, `disconnected from ${this.debugInfo.remoteLabel}; reason=${reason}`);\n\n if (this.isClosed !== true)\n setImmediate(() => this.#close(reason));\n\n switch(reason)\n {\n case 'io client disconnect':\n return; /* we called socket.disconnect() from another \"thread\" */\n case 'io server disconnect':\n case 'ping timeout':\n case 'transport close':\n case 'transport error':\n this.emit('end', reason);\n /* fallthrough */\n default:\n }\n }\n\n /**\n * Handle a socketio message from the peer.\n *\n * Most of the time, a socketio message contains a DCP message, however we support other message\n * types as well, with a mechanism inspired by 3GPP TS 23.040 (GSM 03.40) message concatenation.\n */\n handleMessageEvent(msg)\n {\n if (typeof msg !== 'string')\n {\n debugging('socketio') && console.debug(this.debugLabel, `received ${typeof msg} message from peer`, this.debugInfo.remoteLabel);\n return;\n }\n\n try\n {\n switch (msg[0])\n {\n case '{': /* all DCP Messages are JSON */\n this.processMessage(msg);\n break;\n case 'E':\n this.processExtendedMessage(msg);\n break;\n case 'A':\n this.processAcknowledgement(msg);\n break;\n case 'T':\n debugging('socketio') && console.debug('Received debug message:', msg);\n break;\n default:\n /* adhoc messages not supported in this transport */\n throw new DCPError(this.debugLabel, `Unrecognized message type indicator from ${this.debugInfo.remoteLabel} (${msg.charCodeAt(0)})`, 'DCPC-1102');\n }\n }\n catch(error)\n {\n debugging('socketio') && console.debug(this.debugLabel, 'handleMessageEvent: transport error, closing\\n ', error);\n this.emit('error', error);\n setImmediate(() => this.#close(error.message));\n }\n }\n\n /**\n * Process an ordinary message. This is just a hunk of JSON that gets passed up to the connection.\n * @param {string} msg\n */\n processMessage(msg)\n {\n this.rxPending.push(msg);\n this.emitReadyMessages();\n if (this.mode.ack)\n this.socket.send('A0000-ACK'); /* Ack w/o header = normal message */\n }\n\n /**\n * Remote has acknowledged the receipt of a message fragment. When in ack mode, this\n * triggers us to send more messages from the txPending queue. We always ack event when \n * not in ack-mode. See processExtendedMessage() comment for header description.\n *\n * Future: this is where maxInFlight might be tuned, based on how fast we get here\n *\n * @param {string} _msg\n */\n processAcknowledgement(_msg)\n {\n if (this.txNumInFlight > 0)\n this.txNumInFlight--;\n else\n {\n if (this.options.strict || !this.mode.ack)\n throw new DCPError(this.debugLabel, `Synchronization error: received unexpected acknowledgement message from ${this.debugInfo.remoteLabel}`, 'DCPC-1109');\n else\n this.mode.ack = true;\n }\n \n this.drainTxPending_soon();\n }\n\n /**\n * Extended messages have the following format:\n * Octet(s) Description\n * 0 E (69) or A (65)\n * 1..4 Header size (N) in hexadecimal\n * 5 Header Type\n * C (67) => concatenation\n * 6..6 + N: Header\n *\n * Upon receipt, extended messages are acknowledged by retransmitting their header, but\n * with the first character changed from E to A.\n *\n * @param {string} msg\n */\n processExtendedMessage(msg)\n {\n const headerSize = parseInt(msg.slice(1,5), 16);\n const headerType = msg[5];\n const header = msg.slice(6, 6 + headerSize);\n\n switch(headerType)\n {\n case 'M':\n this.processExtendedMessage_mode(header, msg.slice(6 + headerSize));\n break;\n case 'B':\n this.processExtendedMessage_maxBufferSize(msg.slice(6 + headerSize));\n break;\n case 'C':\n this.processExtendedMessage_concatenated(header, msg.slice(6 + headerSize));\n break;\n default:\n if (this.options.strict)\n throw new DCPError(`Unrecognized extended message header type indicator (${msg.charCodeAt(5)})`, 'DCPC-1101');\n }\n\n if (this.mode.ack)\n this.socket.send('A' + msg.slice(1, 6 + headerSize));\n }\n\n /**\n * We have received a fragment of a concatenation extended message. Memoize the fragment, and\n * if this is the last fragment, transform the fragments list into a pending message string and\n * then emits all ready messages (pending message strings).\n *\n * This code is capable of handling messages whose fragments arrive out of order. Complete messages \n * are emitted in the order that their first parts are received. Socketio is supposed to be total\n * order-aware but I wanted a safety belt for intermingled messages without occupying the entire\n * socketio network buffer on the first pass of the event loop. Because the first message fragment\n * is emitted to socketio on the same event loop pass as their Connection::send calls, I believe\n * that are no cases where messages will be emitted out of order at the other end, even when \n * intermingled on the wire, and the Nth message is much longer than any (N+k)th messages.\n *\n * @param {object} header the extended message header\n * @param {string} payload the payload of the extended message, i.e. a message fragment\n */\n processExtendedMessage_concatenated(header, payload)\n {\n try\n {\n header = JSON.parse(header);\n }\n catch(e)\n {\n throw new DCPError(`invalid extended message header '${header}'`, 'DCPC-1103');\n }\n\n if (!this.options.strict && !this.mode.concat)\n this.mode.concat = true;\n \n if (!(header.total <= this.options.maxFragmentCount))\n throw new DCPError('excessive message fragmentation', 'DCPC-1107'); /* make it more difficult for attacker to force OOM us */\n\n if (!(header.seq < header.total))\n throw new DCPError(`corrupt header for part ${header.seq + 1} of ${header.total} of message ${header.msgId}`, 'DCPC-1108');\n \n /* First fragment received, initial data structures and memoize the message's arrival order via pending list */\n if (!this.rxFragments[header.msgId])\n {\n const fragments = [];\n this.rxFragments[header.msgId] = fragments;\n this.rxPending.push(fragments);\n }\n\n this.rxFragments[header.msgId][header.seq] = payload;\n debugging('socketio') && console.debug(this.debugLabel, 'received part', header.seq + 1, 'of', header.total, 'for message', header.msgId);\n\n if (header.total !== this.rxFragments[header.msgId].filter(el => el !== undefined).length)\n return;\n\n debugging('socketio') && console.debug(this.debugLabel, 'received all parts of message', header.msgId);\n \n const idx = this.rxPending.indexOf(this.rxFragments[header.msgId])\n this.rxPending[idx] = this.rxFragments[header.msgId].join('');\n delete this.rxFragments[header.msgId];\n this.emitReadyMessages();\n }\n\n processExtendedMessage_mode(_header, payload)\n {\n var modeList;\n \n try\n {\n modeList = JSON.parse(payload);\n }\n catch(e)\n {\n throw new DCPError(`invalid mode message payload '${payload}'`, 'DCPC-1105');\n }\n\n for (let mode of modeList)\n {\n debugging('socketio') && console.debug(this.debugLabel, ` - remote supports ${mode} mode`); \n switch(mode)\n {\n case 'ack':\n this.mode.ack = true;\n break;\n case 'concat':\n this.mode.concat = true;\n break;\n default:\n debugging('socketio') && console.debug(this.debugLabel, ' - remote requested unrecognized mode:', mode);\n break;\n }\n }\n\n // true if we are connecting a new transport, expecting the initial mode reply, and have received the target's buffer size\n if (this.initTimeout && this.bufferSizeSet)\n {\n clearTimeout(this.initTimeout);\n delete this.initTimeout;\n this.emit('connect');\n }\n this.modeSet = true;\n this.sendModeList();\n }\n \n processExtendedMessage_maxBufferSize(payload)\n {\n var receivedOptions = JSON.parse(payload); \n \n this.options.txMaxFragmentSize = receivedOptions.maxFragmentSize;\n this.options.txMaxFragmentCount = receivedOptions.maxFragmentCount;\n \n if (receivedOptions.txMaxInFlight < this.txMaxInFlight)\n this.txMaxInFlight = receivedOptions.txMaxInFlight;\n if (this.options.maxFragmentSize < 0)\n throw new DCPError(`maxFragmentSize was set to < 1, (${this.options.txMaxFragmentSize}). Server requested bad fragment size, cannot connect.`, 'DCPC-1111');\n if (this.txMaxInFlight < 1)\n throw new DCPError(`txMaxInFlight was set to < 1, (${this.txMaxInFlight}). Cannot send messages with fewer than 1 in flight.`, 'DCPC-1111');\n \n // true if we are connecting a new transport, expecting the initial buffer reply, and have received the target's mode designation\n if (this.initTimeout && this.modeSet)\n {\n clearTimeout(this.initTimeout);\n delete this.initTimeout;\n this.emit('connect');\n }\n this.bufferSizeSet = true;\n\n this.sendMaxMessageInfo(); \n }\n\n /* Internal method to tell peer what we can receive */\n sendModeList()\n {\n var rawMessage;\n var modeList = [];\n\n if (this.sentModeList)\n return;\n\n this.sentModeList = true;\n this.options.enableModeAck == true && modeList.push('ack');\n this.options.enableModeConcat !== false && modeList.push('concat');\n rawMessage = 'E0000M' + JSON.stringify(modeList);\n\n debugging('socketio') && console.debug(`${this.debugLabel} sending mode list to ${this.debugInfo.remoteLabel}`)\n if (this.socket)\n this.socket.send(rawMessage);\n else\n this.txPending.unshift({ message: rawMessage, msgSequence: ++this.msgSequence});\n }\n \n /* Internal method to exchange desired fragment size/max inflight messages/max fragments per message */\n sendMaxMessageInfo()\n {\n if (this.sentMessageInfo)\n return;\n\n /* Calculate the optimal maxFragmentSize based on maxHttpBufferSize.\n * Currently unsure of exactly what happens when socketio serializes our serialized\n * JSON. I suspect we may double-encode UTF-8 > 0x80. The largest UTF-8 character is\n * 5 bytes...so we ensure the maxFragmentSize is (maxHttpBufferSize / 5) - (fragmentOverhead * 5)\n * to allow for worst-case scenario\n */\n const fragmentOverhead = 1000 + 64 * 1024; /* derived from concatenation algorithm */\n\n let desiredMaxFragmentSize;\n if (this.options.maxHttpBufferSize)\n desiredMaxFragmentSize = (this.options.maxHttpBufferSize / 5) - (fragmentOverhead * 5);\n else\n desiredMaxFragmentSize = this.options.maxFragmentSize;\n const options = {\n maxFragmentSize: desiredMaxFragmentSize,\n maxFragments: this.options.maxFragmentCount,\n txMaxInFlight: this.txMaxInFlight,\n };\n\n const message = 'E0000B' + JSON.stringify(options);\n\n debugging('socketio') && console.debug(`${this.debugLabel} sending max http buffer size to ${this.debugInfo.remoteLabel}`);\n if (this.socket)\n this.socket.send(message);\n else\n this.txPending.push({ message: message, msgSequence: ++this.msgSequence });\n \n this.sentMessageInfo = true;\n }\n\n /**\n * Emit message events so that the connection can process them. The pending property is an\n * array which is used to order messages, so that they are emitted in the order they were\n * sent. There are edge cases in the concatenation code where a sender could theoretically\n * interleave multiple messages, and have a second, shorter, message fully received before \n * the first is fully transmitted.\n *\n * The pending property stores only strings or objects; objects are effectively positional\n * memoes which are turned into strings when they are ready to be emitted to the connection\n * for processing.\n */\n emitReadyMessages()\n {\n while (this.rxReady && this.rxPending.length && typeof this.rxPending[0] === 'string' && this.listenerCount('message') > 0)\n this.emit('message', this.rxPending.shift());\n }\n \n /** \n * Close the current instance of Socket.\n *\n * This function effectively finalizes the socket so that it can't be used any more and\n * (hopefully) does not entrain garbage. A 'close' event is emitted at the end of socket\n * finalization and should be hooked by things that might entrain this transport instance to free\n * up memory, like maybe a list of active transports, etc.\n *\n * We add few extra hops on/off the event loop here to try and clean up any messages halfway in/out\n * the door that we might want to process. That happens between the setting of the isClosed and \n * isFinal flags.\n *\n * @param {string} reason\n */\n #close(reason)\n {\n const that = this;\n debugging('socketio') && console.debug(this.debugLabel, 'closing connection' + (reason ? ` (${reason})` : ''));\n\n function finalize(socket)\n {\n if (that.isFinal)\n return;\n\n if (socket)\n socket.removeAllListeners();\n that.isFinal = true;\n\n /* Free up memory that might be huge in case something entrains us */\n that.rxPending.length = 0;\n that.txPending.length = 0;\n for (let msgId in that.rxFragments)\n that.rxFragments[msgId].length = 0;\n }\n\n function closeSocket()\n {\n const socket = that.socket;\n \n try\n {\n if (socket)\n socket.disconnect();\n that.isClosed = true;\n that.emit('close', reason);\n delete that.socket;\n }\n finally\n {\n finalize(socket);\n }\n }\n\n if (!this.isClosed)\n {\n /* If we called close, drain any pending messages and then disconnect */\n if (reason === 'api-call')\n {\n this.drainTxPending();\n setImmediate(()=>closeSocket());\n }\n else\n closeSocket();\n }\n }\n\n /* API - close the transport, free up memory. */\n close()\n {\n assert(arguments.length === 0)\n this.#close('api-call');\n }\n\n /**\n * Handle the socketio connect_error event, which is fired when:\n * - the low-level connection cannot be established\n * - the connection is denied by the server in a middleware function\n *\n * In the first case, socket.io will automatically try to reconnect, after a delay,\n * except that will cancel that reconnection attempt here by killing the transport\n * instance so that the Connection class can try failing over to an alternate transport.\n *\n * @param {Error} error optional instance of error describing the underlying reason for the\n * connect failure. If the underlying reason is expressible by an\n * HTTP status code, this code will be placed as a Number in the\n * description property.\n */\n handleConnectErrorEvent(error)\n {\n debugging('socketio') && console.debug(this.debugLabel, `unable to connect to ${this.url}`, error);\n if (error.type === 'TransportError' && typeof error.description === 'number')\n error.httpStatus = Number(error.description);\n\n this.emit('connect-failed', error);\n this.#close(error.message);\n }\n\n /** \n * Handle the socketio connect event, which is fired by the Socket instance upon initiator\n * connection and reconnection to a target.\n */\n handleConnectEvent()\n {\n if (this.hasConnected === true)\n {\n /* this transport is not supposed to reuse connections, and we call .close() on disconnect \n * to prevent this. However, the API does not guarantee that will be race-free.\n */\n debugging('socketio') && console.debug(this.debugLabel, `*** reconnected to ${this.debugInfo.remoteLabel} (ignoring)`);\n return;\n }\n\n /* initial connection */\n this.hasConnected = true;\n debugging('socketio') && console.debug(this.debugLabel, 'connected to', this.debugInfo.remoteLabel);\n\n this.initTimeout = setTimeout(() => this.#close('no mode message received'), this.options.establishTimeout) /* give target 10 second to reply with capabilities before assuming bad connection and crashing */\n if (dcpEnv.platform === 'nodejs')\n this.initTimeout.unref();\n \n this.drainTxPending();\n }\n\n /**\n * API: Send a message to the transport instance on the other side of this connection.\n *\n * @param {string} message the message to send\n */\n send(message)\n {\n const msgSequence = ++this.msgSequence;\n\n debugging('socketio') && console.debug(this.debugLabel, `sending message ${msgSequence} to`, this.debugInfo.remoteLabel);\n debugging('socketio') && debugging('verbose') && !debugging('all') && console.debug(this.debugLabel, message);\n\n if (!this.socket)\n throw new DCPError('SocketIOTransport.send: Not connected', 'DCPC-1110');\n \n if ( false\n || !(message.length > this.options.txMaxFragmentSize)\n || !this.mode.concat)\n this.txPending.push({ message, msgSequence });\n else\n { /* This is a large message. Use message concatenation to send in fragments. A failure of any\n * fragment is a failure of the entire message and will be handled at the connection layer.\n */\n let total = Math.ceil(message.length / this.options.txMaxFragmentSize);\n \n if (total > this.options.txMaxFragmentCount)\n throw new DCPError('extended message has more fragments than the peer will allow', 'DCP-1112');\n \n for (let seq=0; seq < total; seq++)\n {\n let start = seq * this.options.txMaxFragmentSize;\n let fragment = message.slice(start, start + this.options.txMaxFragmentSize);\n let header = JSON.stringify({ msgId: msgSequence, seq, total });\n let extMessage = 'E' + header.length.toString(16).padStart(4, '0') + 'C' + header + fragment;\n\n this.txPending.push({ message: extMessage, msgSequence, seq, total });\n\n /* This should be impossible, but it also effectively kills the connection. */\n if (header.length > 0xFFFF)\n throw new DCPError('extended message header too long', 'DCPC-1104');\n }\n }\n\n this.drainTxPending_soon();\n }\n\n drainTxPending_soon()\n {\n if (this.drainingSoon)\n return;\n this.drainingSoon = true;\n\n setImmediate(() => this.drainTxPending());\n }\n \n /**\n * Drain messages from the txPending queue out to socketio's network buffer. We throw away traffic\n * when it can't be sent; this is no different from packets already on the wire when the other end\n * has changed IP number, etc.\n *\n * In ack mode, this drain is controlled by acknowledgement messages so that socketio can have the\n * opportunity to insert ping/pong messages in the data stream while sending a message large enough\n * to need to be sent via concatenation. [ NOTE - it appears to send ping/pong OOB in version 4.4, \n * so currently ack mode is off by default / wg Mar 2022 ]\n *\n * The reason we put non-concat messages through here is to avoid special casing a bunch of nasty\n * things for the sake of a little tiny bit of performance. Sending message<-ACK and queueing through\n * the drain code is necessary if we wind up interleaving multiple message sends in some interesting\n * edge cases; notably, fast-teardown, but a future Connection class that ran without nonces (or multiple\n * nonces) would need this, too.\n *\n * In non-ack mode, the drain is mostly uncontrolled, but we throw each message on a separate pass of\n * the event loop, to try and keep huge message blasts one connection from overwhelming a busy server,\n * as well allowing socketio the chance to send its heartbeat messages and make it less likely that it\n * will hit its http buffer size limit, because that causes an immediate disconnect.\n *\n * Non-ack-mode is compatible with previous dcp5 versions and probably a little faster.\n */\n drainTxPending()\n {\n const that = this;\n\n if (this.txPending.length)\n debugging('socketio') && console.debug(this.debugLabel, `drain tx pending queue, ${this.txPending.length} ready`\n + (this.mode.ack ? `; ${this.txMaxInFlight - this.txNumInFlight} slots available` : ''));\n delete this.drainingSoon;\n\n if (this.isClosed)\n return;\n \n for (let i=0;\n !this.isClosed && this.txPending.length && (this.txNumInFlight < this.txMaxInFlight || !this.mode.ack);\n i++)\n {\n const pendingElement = this.txPending.shift();\n\n if (i === 0)\n writeToSocket(pendingElement);\n else\n setImmediateN(() => writeToSocket(pendingElement), i);\n\n if (this.mode.ack)\n this.txNumInFlight++;\n\n if (this.txPending.length === 0) /* just sent the last message */\n this.emit('drain');\n }\n\n if (this.txPending.length)\n this.drainTxPending_soon();\n \n function writeToSocket(pendingElement)\n {\n if (that.isClosed)\n return;\n\n const { message, msgSequence, seq, total } = pendingElement;\n that.socket.send(message);\n\n if (typeof seq !== 'undefined')\n debugging('socketio') && console.debug(that.debugLabel, `sent fragment ${seq + 1}/${total} of message ${msgSequence} (${message.length} bytes)`);\n else\n debugging('socketio') && console.debug(that.debugLabel, `sent message ${msgSequence} (${message.length} bytes)`);\n }\n }\n}\nSocketIOTransport.prototype.buildOptions = buildOptions;\n\n/** \n * Build the socketio options object, based on the passed-in options, dcp debug state, and internal\n * object state. Method shared between SocketIOTransport and SocketIOListener.\n */\nfunction buildOptions(options)\n{\n options = leafMerge(\n /* Baked-in Defaults */\n ({\n autoUnref: dcpEnv.platform === 'nodejs' ? true : undefined, /* socket.io + webpack 5 { node:global } work-around */\n perMessageDeflate: dcpConfig.build !== 'debug',\n maxFragmentCount: 1000, /* used to limit memory on receiver */\n maxFragmentSize: 1e6, /* bytes */\n txMaxFragmentCount:1000, /* peer's limit for fragments */\n txMaxFragmentSize: 1e6, /* peer's limit for fragment size */\n maxHttpBufferSize: false, /* bytes; false = auto */\n establishTimeout: 10 * 1000, /* 10 seconds */\n pingTimeout: 30 * 1e3, /* s */\n pingInterval: 90 * 1e3, /* s */\n transports: ['polling', 'websocket'],\n upgrade: true,\n rememberUpgrade: true,\n autoConnect: true,\n cors: {\n origin: '*',\n methods: ['GET', 'POST']\n },\n }),\n options,\n /* Absolutely mandatory */\n ({\n path: this.url.pathname // eslint-disable-line no-invalid-this\n }));\n\n /* draw out errors quickly in dev */\n if ((process.env.DCP_NETWORK_CONFIG_BUILD || dcpConfig.build) === 'debug')\n {\n /* short timeouts and debuggers don't get along well */\n if (dcpEnv.platform === 'nodejs' && !(requireNative('module')._cache.niim instanceof requireNative('module').Module))\n {\n options.pingTimeout /= 5;\n options.pingInterval /= 5;\n }\n }\n\n return options;\n}\n \n/**\n * @constructor\n * API - Factory which creates a new socketio listener, or throws. Emits 'listening' if the server \n * parameter is not provided and we create it, once it is ready for connections.\n * \n * @bug It is possible for a connection event to sneak in before the 'listening' event is fired, due to \n * the two-step event trigger. The right fix is to proxy the event, but dcp-events::event-emitter.proxy\n * itself currently has the same bug. For now, we just have callbacks in API. /wg apr 2022\n * \n * @param {object} config dcpConfig fragment - has .listen and .location\n * @param {object} target the instance of Target to bind to\n * @param {object} options [optional] A DCP config options variable (like dcpConfig.dcp.listenOptions)\n */\nfunction SocketIOListener(config, target, options)\n{\n var matchRegex;\n \n this.url = config.location;\n this.options = this.buildOptions(options);\n this.connSequence = 0;\n this.seq = ++listenerSeq;\n this.debugLabel = `socketio(l:${this.seq}):`;\n this.isShutdown = false;\n\n if (dcpEnv.platform !== 'nodejs')\n throw new Error('target mode only supported in nodejs');\n\n if (target.httpServer.listening)\n setImmediate(() => this.emit('listening', target.httpServer)); /* setImmediate to allow event listeners to be added before emitting */\n else\n target.httpServer.on('listening', () => this.emit('listening', target.httpServer));\n\n matchRegex = '^' + this.url.pathname;\n if (matchRegex.slice(-1) !== '/')\n matchRegex += '\\\\/';\n else\n matchRegex = matchRegex.slice(0, -1) + \"\\\\/\"\n\n matchRegex += '\\\\?EIO=[0-9]';\n debugging('socketio') && console.debug(this.debugLabel, `expecting requests to match regex ${matchRegex}`);\n\n target.all(new RegExp(matchRegex), (request) => {\n /* Socket.io does this intrinsically because it plays with the server's events directly,\n * but we \"handle\" it explicitly here so that we don't trip the 404 handler. \n */\n debugging('socketio') && console.debug(this.debugLabel, 'Handling', request.url);\n }, true);\n\n /* While we normally want to inherit options from parent to child socket via this.options,\n * we don't want this to happen for maxHttpBufferSize due to the way it is used and calculated\n * in the transport constructor. A buffer that is too small (eg false) will cause us to not\n * recognized socket.io SIDs.\n */\n \n this.options.maxHttpBufferSize = 1e7;\n \n const socketOptions = this.options;\n\n this.socketServer = requireNative('socket.io')(target.httpServer, socketOptions);\n this.socketServer.on('connection', (socket) => !this.isShutdown && this.handleConnectionEvent(config, socket));\n debugging() && console.debug(this.debugLabel, 'Socketio listener initialized for path', this.options.path);\n}\nSocketIOListener.prototype = new EventEmitter('socketio(l)');\nSocketIOListener.prototype.buildOptions = buildOptions;\n\nSocketIOListener.prototype.close = function SocketIOListener$$close()\n{\n if (!this.isShutdown)\n this.shutdown();\n this.socketServer.close(() => this.emit('close'));\n}\n\n/** Stop accepting new connections */\nSocketIOListener.prototype.shutdown = function SocketIOListener$$shutdown()\n{\n this.isShutdown = true;\n}\n\n/** Remove any event loop references used by this transport instance */\nSocketIOListener.prototype.unref = function SocketIOListener$$unref()\n{\n void false;\n}\n\n/**\n * Handle the socketio connection event, which is fired upon a connection from client.\n * Used by Target->SocketIOListener to create transport instances.\n *\n * It is expected that code hooking the 'connection' event will hook the 'message' in\n * the same pass of the event loop as it is invoked; otherwise, messages may be missed.\n *\n * @param {object} socket a new connection emitted by the socket.io server\n */\nSocketIOListener.prototype.handleConnectionEvent = function SocketIOListener$$handleConnectionEvent(config, socket)\n{\n var transport = new SocketIOTransport(config, this.options, socket);\n\n this.url = config.location || config.listen;\n this.connSequence++;\n transport.debugLabel = `socketio(t:${this.seq}.${this.connSequence}):`\n\n /* Ensure we don't emit any message events until the application or Connection class\n * has had the chance to set up message handlers via the connection event.\n */\n transport.rxReady = false;\n this.on('connection', () => setImmediate(() => {\n transport.rxReady = true;\n transport.emitReadyMessages();\n }));\n\n this.emit('connection', transport);\n}\n\n/* Define API */\nexports.TransportClass = SocketIOTransport;\nexports.Listener = SocketIOListener;\n\n\n//# sourceURL=webpack://dcp/./src/protocol-v4/transport/socketio.js?");
|
|
4718
|
+
eval("/* provided dependency */ var process = __webpack_require__(/*! ./node_modules/process/browser.js */ \"./node_modules/process/browser.js\");\n/**\n * @file transport/socketio.js\n * @author Ryan Rossiter, ryan@kingsds.network\n * @author Wes Garland, wes@kingsds.network \n * @date January 2020, March 2022\n *\n * This module implements the SocketIO Transport that is\n * used by the protocol to connect and communicate with peers using\n * SocketIO connections.\n *\n * Transport can operate in either ack mode or non-ack-mode, depending on the value of this.ackMode.\n * Ack mode sends an extra socketio-layer packet back after each message is received, and theoretically\n * this can be use tod meter or measure bandwidth or potentially second-guess socketio's ability to\n * interleave its heartbeat messages OOB from the main traffic. This mode could potentially be enabled\n * on-demand as well, although it appears that it is not necessary to get the desired behaviour at this\n * point, so the feature is off-by-default (although reasonably well-tested).\n */\n\n\nconst { Transport } = __webpack_require__(/*! . */ \"./src/protocol-v4/transport/index.js\");\nconst { EventEmitter } = __webpack_require__(/*! dcp/common/dcp-events */ \"./src/common/dcp-events/index.js\");\nconst { leafMerge } = __webpack_require__(/*! dcp/utils/obj-merge */ \"./src/utils/obj-merge.js\");\nconst { DcpURL } = __webpack_require__(/*! dcp/common/dcp-url */ \"./src/common/dcp-url.js\");\nconst { DCPError } = __webpack_require__(/*! dcp/common/dcp-error */ \"./src/common/dcp-error.js\");\nconst { setImmediateN } = __webpack_require__(/*! dcp/common/dcp-timers */ \"./src/common/dcp-timers.js\");\nconst { setImmediate } = __webpack_require__(/*! dcp/common/dcp-timers */ \"./src/common/dcp-timers.js\");\nconst { requireNative } = __webpack_require__(/*! dcp/dcp-client/webpack-native-bridge */ \"./src/dcp-client/webpack-native-bridge.js\");\nconst { assert } = __webpack_require__(/*! dcp/common/dcp-assert */ \"./src/common/dcp-assert.js\");\n\n// Note: trying to `require('process')` in the browser prevents the portal\n// from loading. ~ER 20220803\n// const process = requireNative ? requireNative('process') : require('process');\nconst debugging = (__webpack_require__(/*! dcp/debugging */ \"./src/debugging.js\").scope)('dcp');\nconst dcpEnv = __webpack_require__(/*! dcp/common/dcp-env */ \"./src/common/dcp-env.js\");\nconst bearer = __webpack_require__(/*! ./http-bearer */ \"./src/protocol-v4/transport/http-bearer.js\");\nconst semver = __webpack_require__(/*! semver */ \"./node_modules/semver/semver.js\");\n\nvar listenerSeq = 0;\nvar initiatorSeq = 0;\n\nif (debugging() && !process.env.DEBUG && dcpConfig.build === 'debug')\n{\n\n if (process.env.DEBUG)\n process.env.DEBUG += ',engine';\n else\n process.env.DEBUG = 'engine';\n\n if (debugging('socketio'))\n {\n /* DCP_DEBUG=dcp:socketio,dcp:verbose to see everything */\n process.env.DEBUG += ',socket.io-client:socket,socket.io:client,socket.io:server';\n if (debugging('verbose') && !debugging('all'))\n process.env.DEBUG += ',socket.io*,engine.io*'\n }\n}\n\nclass SocketIOTransport extends Transport\n{\n /**\n * @constructor\n * @param {socket} newSocket Target: an instance of socket.io Socket that represents\n * a new connection.\n * Initiator: unused\n */\n /**\n * @constructor create new instance of a socketio-flavoured Transport.\n *\n * @param {object} config dcpConfig fragment\n * @param {Url} url Initiator: has .location and optionally .friendLocation\n * Target: has .listen and .location\n *\n * @param {object} options The `socketio` property of a protocol-v4 connectionOptions \n * object, which was built from a combination of defaults and \n * various dcpConfig components.\n *\n * @returns instance suitable for target or initiator, depending on mode, but does not manage connections.\n */\n constructor(config, options={}, newSocket)\n {\n super('Protocol SocketIO Transport');\n const that = this;\n \n this.name = 'socketio';\n this.url = config.location;\n this.msgSequence = 0;\n this.isClosed = false; /* true => connection finalization begun */\n this.isFinal = false; /* true => connection finalization completed */\n this.hasConnected = false; /* true => one 'connect' event has been processed */\n this.initTimeout = false; /* timeout when connecting to allow target to reply with their mode message + buffer size, crash if they don't reply */\n this.modeSet = false; /* true => received response mode message */\n this.bufferSizeSet = false; /* true => received target's buffer size */\n this.rxFragments = {}; /* messages we are receiving in fragments */\n this.rxPending = []; /* strings which are messages that are complete and ready to emit, or objects that in this.rxFragments */\n this.txPending = []; /* messages we are receiving in fragments */\n this.txNumInFlight = 0; /* number of txPending sent but not acknowleged */\n this.txMaxInFlight = 2; /* less than 2 causes bad perf when mode.ack */\n this.debugInfo = { remoteLabel: '<unknown>' };\n this.mode = {\n ack: false,\n concat: false,\n };\n\n assert(DcpURL.isURL(this.url));\n assert(typeof options === 'object');\n\n this.options = this.buildOptions(options);\n\n if (!newSocket) /* Initiator */\n {\n this.debugLabel = `socketio(i:${++initiatorSeq}):`;\n this.rxReady = true; /* relax, said the night man */\n this.debugInfo.remoteLabel = config.location.href;\n if (this.options.disableModeAnnouncement !== false)\n this.sendModeList();\n this.sendMaxMessageInfo();\n\n bearer.a$isFriendlyUrl(config.friendLocation).then((useFriendLocation) => {\n const connectUrl = useFriendLocation ? config.friendLocation : config.location;\n\n debugging('socketio') && console.log(this.debugLabel, 'connecting to', connectUrl.href);\n this.socket = (__webpack_require__(/*! socket.io-client */ \"./node_modules/socket.io-client/build/cjs/index.js\").connect)(connectUrl.origin, this.options);\n this.socket.on('connect_error', (error) => !this.isClosed && this.handleConnectErrorEvent(error));\n this.socket.on('connect', () => !this.isClosed && this.handleConnectEvent());\n\n useSocket();\n }).catch((error) => {\n debugging() && console.error(this.debugLabel, error);\n this.emit('error', error);\n });\n }\n else /* Target - receives socketio instance from SocketIOListener.handleConnectionEvent */\n {\n if (dcpEnv.platform !== 'nodejs')\n throw new Error('socketio: target mode only supported in nodejs');\n\n this.debugLabel = 'socketio(t:<new>):';\n\n this.socket = newSocket;\n this.debugInfo.remoteLabel = this.socket.handshake.address.replace(/^::ffff:/,'');\n debugging('socketio') && console.debug(this.debugLabel, `new socket from ${this.debugInfo.remoteLabel}`\n + ` on ${this.socket.handshake && this.socket.handshake.url}, ${this.socket.id}`);\n\n useSocket();\n }\n\n function useSocket()\n {\n that.socket.compress(dcpConfig.build !== 'debug'); /* try to keep traffic sniffable in debug */\n that.socket.on('message', (msg) => !that.isFinal && that.handleMessageEvent(msg));\n that.socket.on('disconnect', (reason) => !that.isClosed && that.handleDisconnectEvent(reason));\n }\n }\n\n /**\n * API - determine if a transport is usable or not.\n * @returns true if the transport is not closed.\n */\n ready()\n {\n return !this.isClosed;\n }\n \n /**\n * Handle the socket.io disconnect event, which is fired upon disconnection.\n * For some reasons (explicit disconnection), the socket.io code will not try to reconnect, but\n * in all other cases, the socket.io client will normally wait for a small random delay and then \n * try to reconnect, but in our case, we simply tear the transport down and let the Connection\n * class handle reconnects, since it might prefer a different transport at this point anyhow.\n *\n * One tricky case here is ping timeout, since the ping timeout includes the time to transmit the\n * packet before it was sent.\n *\n * @param {string} reason Possible reasons for disconnection.\n */\n handleDisconnectEvent(reason)\n {\n debugging('socketio') && console.debug(this.debugLabel, `disconnected from ${this.debugInfo.remoteLabel}; reason=${reason}`);\n\n if (this.isClosed !== true)\n setImmediate(() => this.#close(reason));\n\n switch(reason)\n {\n case 'io client disconnect':\n return; /* we called socket.disconnect() from another \"thread\" */\n case 'io server disconnect':\n case 'ping timeout':\n case 'transport close':\n case 'transport error':\n this.emit('end', reason);\n /* fallthrough */\n default:\n }\n }\n\n /**\n * Handle a socketio message from the peer.\n *\n * Most of the time, a socketio message contains a DCP message, however we support other message\n * types as well, with a mechanism inspired by 3GPP TS 23.040 (GSM 03.40) message concatenation.\n */\n handleMessageEvent(msg)\n {\n if (typeof msg !== 'string')\n {\n debugging('socketio') && console.debug(this.debugLabel, `received ${typeof msg} message from peer`, this.debugInfo.remoteLabel);\n return;\n }\n\n try\n {\n switch (msg[0])\n {\n case '{': /* all DCP Messages are JSON */\n this.processMessage(msg);\n break;\n case 'E':\n this.processExtendedMessage(msg);\n break;\n case 'A':\n this.processAcknowledgement(msg);\n break;\n case 'T':\n debugging('socketio') && console.debug(this.debugLabel, 'Received debug message:', msg);\n break;\n default:\n /* adhoc messages not supported in this transport */\n throw new DCPError(this.debugLabel, `Unrecognized message type indicator from ${this.debugInfo.remoteLabel} (${msg.charCodeAt(0)})`, 'DCPC-1102');\n }\n }\n catch(error)\n {\n debugging('socketio') && console.debug(this.debugLabel, 'handleMessageEvent: transport error, closing\\n ', error);\n this.emit('error', error);\n setImmediate(() => this.#close(error.message));\n }\n }\n\n /**\n * Process an ordinary message. This is just a hunk of JSON that gets passed up to the connection.\n * @param {string} msg\n */\n processMessage(msg)\n {\n this.rxPending.push(msg);\n this.emitReadyMessages();\n if (this.mode.ack)\n this.socket.send('A0000-ACK'); /* Ack w/o header = normal message */\n }\n\n /**\n * Remote has acknowledged the receipt of a message fragment. When in ack mode, this\n * triggers us to send more messages from the txPending queue. We always ack event when \n * not in ack-mode. See processExtendedMessage() comment for header description.\n *\n * Future: this is where maxInFlight might be tuned, based on how fast we get here\n *\n * @param {string} _msg\n */\n processAcknowledgement(_msg)\n {\n if (this.txNumInFlight > 0)\n this.txNumInFlight--;\n else\n {\n if (this.options.strict || !this.mode.ack)\n throw new DCPError(this.debugLabel, `Synchronization error: received unexpected acknowledgement message from ${this.debugInfo.remoteLabel}`, 'DCPC-1109');\n else\n this.mode.ack = true;\n }\n \n this.drainTxPending_soon();\n }\n\n /**\n * Extended messages have the following format:\n * Octet(s) Description\n * 0 E (69) or A (65)\n * 1..4 Header size (N) in hexadecimal\n * 5 Header Type\n * C (67) => concatenation\n * 6..6 + N: Header\n *\n * Upon receipt, extended messages are acknowledged by retransmitting their header, but\n * with the first character changed from E to A.\n *\n * @param {string} msg\n */\n processExtendedMessage(msg)\n {\n const headerSize = parseInt(msg.slice(1,5), 16);\n const headerType = msg[5];\n const header = msg.slice(6, 6 + headerSize);\n\n switch(headerType)\n {\n case 'M':\n this.processExtendedMessage_mode(header, msg.slice(6 + headerSize));\n break;\n case 'B':\n this.processExtendedMessage_maxBufferSize(msg.slice(6 + headerSize));\n break;\n case 'C':\n this.processExtendedMessage_concatenated(header, msg.slice(6 + headerSize));\n break;\n default:\n if (this.options.strict)\n throw new DCPError(`Unrecognized extended message header type indicator (${msg.charCodeAt(5)})`, 'DCPC-1101');\n }\n\n if (this.mode.ack)\n this.socket.send('A' + msg.slice(1, 6 + headerSize));\n }\n\n /**\n * We have received a fragment of a concatenation extended message. Memoize the fragment, and\n * if this is the last fragment, transform the fragments list into a pending message string and\n * then emits all ready messages (pending message strings).\n *\n * This code is capable of handling messages whose fragments arrive out of order. Complete messages \n * are emitted in the order that their first parts are received. Socketio is supposed to be total\n * order-aware but I wanted a safety belt for intermingled messages without occupying the entire\n * socketio network buffer on the first pass of the event loop. Because the first message fragment\n * is emitted to socketio on the same event loop pass as their Connection::send calls, I believe\n * that are no cases where messages will be emitted out of order at the other end, even when \n * intermingled on the wire, and the Nth message is much longer than any (N+k)th messages.\n *\n * @param {object} header the extended message header\n * @param {string} payload the payload of the extended message, i.e. a message fragment\n */\n processExtendedMessage_concatenated(header, payload)\n {\n try\n {\n header = JSON.parse(header);\n }\n catch(e)\n {\n throw new DCPError(`invalid extended message header '${header}'`, 'DCPC-1103');\n }\n\n if (!this.options.strict && !this.mode.concat)\n this.mode.concat = true;\n \n if (!(header.total <= this.options.maxFragmentCount))\n throw new DCPError('excessive message fragmentation', 'DCPC-1107'); /* make it more difficult for attacker to force OOM us */\n\n if (!(header.seq < header.total))\n throw new DCPError(`corrupt header for part ${header.seq + 1} of ${header.total} of message ${header.msgId}`, 'DCPC-1108');\n \n /* First fragment received, initial data structures and memoize the message's arrival order via pending list */\n if (!this.rxFragments[header.msgId])\n {\n const fragments = [];\n this.rxFragments[header.msgId] = fragments;\n this.rxPending.push(fragments);\n }\n\n this.rxFragments[header.msgId][header.seq] = payload;\n debugging('socketio') && console.debug(this.debugLabel, 'received part', header.seq + 1, 'of', header.total, 'for message', header.msgId);\n\n if (header.total !== this.rxFragments[header.msgId].filter(el => el !== undefined).length)\n return;\n\n debugging('socketio') && console.debug(this.debugLabel, 'received all parts of message', header.msgId);\n \n const idx = this.rxPending.indexOf(this.rxFragments[header.msgId])\n this.rxPending[idx] = this.rxFragments[header.msgId].join('');\n delete this.rxFragments[header.msgId];\n this.emitReadyMessages();\n }\n\n processExtendedMessage_mode(_header, payload)\n {\n var modeList;\n \n try\n {\n modeList = JSON.parse(payload);\n }\n catch(e)\n {\n throw new DCPError(`invalid mode message payload '${payload}'`, 'DCPC-1105');\n }\n\n for (let mode of modeList)\n {\n debugging('socketio') && console.debug(this.debugLabel, ` - remote supports ${mode} mode`); \n switch(mode)\n {\n case 'ack':\n this.mode.ack = true;\n break;\n case 'concat':\n this.mode.concat = true;\n break;\n default:\n debugging('socketio') && console.debug(this.debugLabel, ' - remote requested unrecognized mode:', mode);\n break;\n }\n }\n\n // true if we are connecting a new transport, expecting the initial mode reply, and have received the target's buffer size\n if (this.initTimeout && this.bufferSizeSet)\n {\n clearTimeout(this.initTimeout);\n delete this.initTimeout;\n this.emit('connect');\n }\n this.modeSet = true;\n this.sendModeList();\n }\n \n processExtendedMessage_maxBufferSize(payload)\n {\n var receivedOptions = JSON.parse(payload); \n \n this.options.txMaxFragmentSize = Math.floor(receivedOptions.maxFragmentSize / 11); /* XXX EMERGENCY BUGFIX - Aug 2022 /wg */\n this.options.txMaxFragmentCount = receivedOptions.maxFragmentCount;\n\n debugging() && console.debug(this.debugLabel, 'Set max transmit fragment size to', this.options.txMaxFragmentSize);\n\n if (receivedOptions.txMaxInFlight < this.txMaxInFlight)\n this.txMaxInFlight = receivedOptions.txMaxInFlight;\n if (this.options.maxFragmentSize < 0)\n throw new DCPError(`maxFragmentSize was set to < 1, (${this.options.txMaxFragmentSize}). Server requested bad fragment size, cannot connect.`, 'DCPC-1111');\n if (this.txMaxInFlight < 1)\n throw new DCPError(`txMaxInFlight was set to < 1, (${this.txMaxInFlight}). Cannot send messages with fewer than 1 in flight.`, 'DCPC-1111');\n \n // true if we are connecting a new transport, expecting the initial buffer reply, and have received the target's mode designation\n if (this.initTimeout && this.modeSet)\n {\n clearTimeout(this.initTimeout);\n delete this.initTimeout;\n this.emit('connect');\n }\n this.bufferSizeSet = true;\n\n this.sendMaxMessageInfo(); \n }\n\n /* Internal method to tell peer what we can receive */\n sendModeList()\n {\n var rawMessage;\n var modeList = [];\n\n if (this.sentModeList)\n return;\n\n this.sentModeList = true;\n this.options.enableModeAck == true && modeList.push('ack');\n this.options.enableModeConcat !== false && modeList.push('concat');\n rawMessage = 'E0000M' + JSON.stringify(modeList);\n\n debugging('socketio') && console.debug(`${this.debugLabel} sending mode list to ${this.debugInfo.remoteLabel}`)\n if (this.socket)\n this.socket.send(rawMessage);\n else\n this.txPending.unshift({ message: rawMessage, msgSequence: ++this.msgSequence});\n }\n \n /* Internal method to exchange desired fragment size/max inflight messages/max fragments per message */\n sendMaxMessageInfo()\n {\n if (this.sentMessageInfo)\n return;\n\n /* Calculate the optimal maxFragmentSize based on maxHttpBufferSize.\n * Currently unsure of exactly what happens when socketio serializes our serialized\n * JSON. I suspect we may double-encode UTF-8 > 0x80. The largest UTF-8 character is\n * 5 bytes...so we ensure the maxFragmentSize is (maxHttpBufferSize / 5) - (fragmentOverhead * 5)\n * to allow for worst-case scenario\n */\n const fragmentOverhead = 1000 + 64 * 1024; /* derived from concatenation algorithm */\n\n let desiredMaxFragmentSize;\n if (this.options.maxHttpBufferSize)\n desiredMaxFragmentSize = (this.options.maxHttpBufferSize / 5) - (fragmentOverhead * 5);\n else\n desiredMaxFragmentSize = this.options.maxFragmentSize;\n const options = {\n maxFragmentSize: desiredMaxFragmentSize,\n maxFragments: this.options.maxFragmentCount,\n txMaxInFlight: this.txMaxInFlight,\n };\n\n const message = 'E0000B' + JSON.stringify(options);\n\n debugging('socketio') && console.debug(`${this.debugLabel} sending max http buffer size to ${this.debugInfo.remoteLabel}`);\n if (this.socket)\n this.socket.send(message);\n else\n this.txPending.push({ message: message, msgSequence: ++this.msgSequence });\n \n this.sentMessageInfo = true;\n }\n\n /**\n * Emit message events so that the connection can process them. The pending property is an\n * array which is used to order messages, so that they are emitted in the order they were\n * sent. There are edge cases in the concatenation code where a sender could theoretically\n * interleave multiple messages, and have a second, shorter, message fully received before \n * the first is fully transmitted.\n *\n * The pending property stores only strings or objects; objects are effectively positional\n * memoes which are turned into strings when they are ready to be emitted to the connection\n * for processing.\n */\n emitReadyMessages()\n {\n while (this.rxReady && this.rxPending.length && typeof this.rxPending[0] === 'string' && this.listenerCount('message') > 0)\n this.emit('message', this.rxPending.shift());\n }\n \n /** \n * Close the current instance of Socket.\n *\n * This function effectively finalizes the socket so that it can't be used any more and\n * (hopefully) does not entrain garbage. A 'close' event is emitted at the end of socket\n * finalization and should be hooked by things that might entrain this transport instance to free\n * up memory, like maybe a list of active transports, etc.\n *\n * We add few extra hops on/off the event loop here to try and clean up any messages halfway in/out\n * the door that we might want to process. That happens between the setting of the isClosed and \n * isFinal flags.\n *\n * @param {string} reason\n */\n #close(reason)\n {\n const that = this;\n debugging('socketio') && console.debug(this.debugLabel, 'closing connection' + (reason ? ` (${reason})` : ''));\n\n function finalize(socket)\n {\n if (that.isFinal)\n return;\n\n if (socket)\n socket.removeAllListeners();\n that.isFinal = true;\n\n /* Free up memory that might be huge in case something entrains us */\n that.rxPending.length = 0;\n that.txPending.length = 0;\n for (let msgId in that.rxFragments)\n that.rxFragments[msgId].length = 0;\n }\n\n function closeSocket()\n {\n const socket = that.socket;\n \n try\n {\n if (socket)\n socket.disconnect();\n that.isClosed = true;\n that.emit('close', reason);\n delete that.socket;\n }\n finally\n {\n finalize(socket);\n }\n }\n\n if (!this.isClosed)\n {\n /* If we called close, drain any pending messages and then disconnect */\n if (reason === 'api-call')\n {\n this.drainTxPending();\n setImmediate(()=>closeSocket());\n }\n else\n closeSocket();\n }\n }\n\n /* API - close the transport, free up memory. */\n close()\n {\n assert(arguments.length === 0)\n this.#close('api-call');\n }\n\n /**\n * Handle the socketio connect_error event, which is fired when:\n * - the low-level connection cannot be established\n * - the connection is denied by the server in a middleware function\n *\n * In the first case, socket.io will automatically try to reconnect, after a delay,\n * except that will cancel that reconnection attempt here by killing the transport\n * instance so that the Connection class can try failing over to an alternate transport.\n *\n * @param {Error} error optional instance of error describing the underlying reason for the\n * connect failure. If the underlying reason is expressible by an\n * HTTP status code, this code will be placed as a Number in the\n * description property.\n */\n handleConnectErrorEvent(error)\n {\n debugging('socketio') && console.debug(this.debugLabel, `unable to connect to ${this.url}`, error);\n if (error.type === 'TransportError' && typeof error.description === 'number')\n error.httpStatus = Number(error.description);\n\n this.emit('connect-failed', error);\n this.#close(error.message);\n }\n\n /** \n * Handle the socketio connect event, which is fired by the Socket instance upon initiator\n * connection and reconnection to a target.\n */\n handleConnectEvent()\n {\n if (this.hasConnected === true)\n {\n /* this transport is not supposed to reuse connections, and we call .close() on disconnect \n * to prevent this. However, the API does not guarantee that will be race-free.\n */\n debugging('socketio') && console.debug(this.debugLabel, `*** reconnected to ${this.debugInfo.remoteLabel} (ignoring)`);\n return;\n }\n\n /* initial connection */\n this.hasConnected = true;\n debugging('socketio') && console.debug(this.debugLabel, 'connected to', this.debugInfo.remoteLabel);\n\n this.initTimeout = setTimeout(() => this.#close('no mode message received'), this.options.establishTimeout) /* give target 10 second to reply with capabilities before assuming bad connection and crashing */\n if (dcpEnv.platform === 'nodejs')\n this.initTimeout.unref();\n \n this.drainTxPending();\n }\n\n /**\n * API: Send a message to the transport instance on the other side of this connection.\n *\n * @param {string} message the message to send\n */\n send(message)\n {\n const msgSequence = ++this.msgSequence;\n\n debugging('socketio') && console.debug(this.debugLabel, `sending message ${msgSequence} to`, this.debugInfo.remoteLabel);\n debugging('socketio') && debugging('verbose') && !debugging('all') && console.debug(this.debugLabel, message);\n\n if (!this.socket)\n throw new DCPError('SocketIOTransport.send: Not connected', 'DCPC-1110');\n \n if ( false\n || !(message.length > this.options.txMaxFragmentSize)\n || !this.mode.concat)\n this.txPending.push({ message, msgSequence });\n else\n { /* This is a large message. Use message concatenation to send in fragments. A failure of any\n * fragment is a failure of the entire message and will be handled at the connection layer.\n */\n let total = Math.ceil(message.length / this.options.txMaxFragmentSize);\n \n if (total > this.options.txMaxFragmentCount)\n throw new DCPError('extended message has more fragments than the peer will allow', 'DCP-1112');\n \n for (let seq=0; seq < total; seq++)\n {\n let start = seq * this.options.txMaxFragmentSize;\n let fragment = message.slice(start, start + this.options.txMaxFragmentSize);\n let header = JSON.stringify({ msgId: msgSequence, seq, total });\n let extMessage = 'E' + header.length.toString(16).padStart(4, '0') + 'C' + header + fragment;\n\n this.txPending.push({ message: extMessage, msgSequence, seq, total });\n\n /* This should be impossible, but it also effectively kills the connection. */\n if (header.length > 0xFFFF)\n throw new DCPError('extended message header too long', 'DCPC-1104');\n }\n }\n\n this.drainTxPending_soon();\n }\n\n drainTxPending_soon()\n {\n if (this.drainingSoon)\n return;\n this.drainingSoon = true;\n\n setImmediate(() => this.drainTxPending());\n }\n \n /**\n * Drain messages from the txPending queue out to socketio's network buffer. We throw away traffic\n * when it can't be sent; this is no different from packets already on the wire when the other end\n * has changed IP number, etc.\n *\n * In ack mode, this drain is controlled by acknowledgement messages so that socketio can have the\n * opportunity to insert ping/pong messages in the data stream while sending a message large enough\n * to need to be sent via concatenation. [ NOTE - it appears to send ping/pong OOB in version 4.4, \n * so currently ack mode is off by default / wg Mar 2022 ]\n *\n * The reason we put non-concat messages through here is to avoid special casing a bunch of nasty\n * things for the sake of a little tiny bit of performance. Sending message<-ACK and queueing through\n * the drain code is necessary if we wind up interleaving multiple message sends in some interesting\n * edge cases; notably, fast-teardown, but a future Connection class that ran without nonces (or multiple\n * nonces) would need this, too.\n *\n * In non-ack mode, the drain is mostly uncontrolled, but we throw each message on a separate pass of\n * the event loop, to try and keep huge message blasts one connection from overwhelming a busy server,\n * as well allowing socketio the chance to send its heartbeat messages and make it less likely that it\n * will hit its http buffer size limit, because that causes an immediate disconnect.\n *\n * Non-ack-mode is compatible with previous dcp5 versions and probably a little faster.\n */\n drainTxPending()\n {\n const that = this;\n\n if (this.txPending.length)\n debugging('socketio') && console.debug(this.debugLabel, `drain tx pending queue, ${this.txPending.length} ready`\n + (this.mode.ack ? `; ${this.txMaxInFlight - this.txNumInFlight} slots available` : ''));\n delete this.drainingSoon;\n\n if (this.isClosed)\n return;\n \n for (let i=0;\n !this.isClosed && this.txPending.length && (this.txNumInFlight < this.txMaxInFlight || !this.mode.ack);\n i++)\n {\n const pendingElement = this.txPending.shift();\n\n if (i === 0)\n writeToSocket(pendingElement);\n else\n setImmediateN(() => writeToSocket(pendingElement), i);\n \n if (this.mode.ack)\n this.txNumInFlight++;\n\n if (this.txPending.length === 0) /* just sent the last message */\n this.emit('drain');\n }\n\n if (this.txPending.length)\n this.drainTxPending_soon();\n \n function writeToSocket(pendingElement)\n {\n if (that.isClosed)\n return;\n\n const { message, msgSequence, seq, total } = pendingElement;\n that.socket.send(message);\n\n if (typeof seq !== 'undefined')\n debugging('socketio') && console.debug(that.debugLabel, `sent fragment ${seq + 1}/${total} of message ${msgSequence} (${message.length} bytes)`);\n else\n debugging('socketio') && console.debug(that.debugLabel, `sent message ${msgSequence} (${message.length} bytes)`);\n }\n }\n}\nSocketIOTransport.prototype.buildOptions = buildOptions;\n\n/** \n * Build the socketio options object, based on the passed-in options, dcp debug state, and internal\n * object state. Method shared between SocketIOTransport and SocketIOListener.\n */\nfunction buildOptions(options)\n{\n options = leafMerge(\n /* Baked-in Defaults */\n ({\n autoUnref: dcpEnv.platform === 'nodejs' ? true : undefined, /* socket.io + webpack 5 { node:global } work-around */\n perMessageDeflate: dcpConfig.build !== 'debug',\n maxFragmentCount: 1000, /* used to limit memory on receiver */\n maxFragmentSize: 1e6, /* bytes */\n txMaxFragmentCount:1000, /* peer's limit for fragments */\n txMaxFragmentSize: 1e6, /* peer's limit for fragment size */\n maxHttpBufferSize: false, /* bytes; false = auto */\n establishTimeout: 10 * 1000, /* 10 seconds */\n pingTimeout: 30 * 1e3, /* s */\n pingInterval: 90 * 1e3, /* s */\n transports: ['polling', 'websocket'],\n upgrade: true,\n rememberUpgrade: true,\n autoConnect: true,\n cors: {\n origin: '*',\n methods: ['GET', 'POST']\n },\n }),\n options,\n /* Absolutely mandatory */\n ({\n path: this.url.pathname // eslint-disable-line no-invalid-this\n }));\n\n /* draw out errors quickly in dev */\n if ((process.env.DCP_NETWORK_CONFIG_BUILD || dcpConfig.build) === 'debug')\n {\n /* short timeouts and debuggers don't get along well */\n if (dcpEnv.platform === 'nodejs' && !(requireNative('module')._cache.niim instanceof requireNative('module').Module))\n {\n options.pingTimeout /= 5;\n options.pingInterval /= 5;\n }\n }\n\n return options;\n}\n \n/**\n * @constructor\n * API - Factory which creates a new socketio listener, or throws. Emits 'listening' if the server \n * parameter is not provided and we create it, once it is ready for connections.\n * \n * @bug It is possible for a connection event to sneak in before the 'listening' event is fired, due to \n * the two-step event trigger. The right fix is to proxy the event, but dcp-events::event-emitter.proxy\n * itself currently has the same bug. For now, we just have callbacks in API. /wg apr 2022\n * \n * @param {object} config dcpConfig fragment - has .listen and .location\n * @param {object} target the instance of Target to bind to\n * @param {object} options [optional] A DCP config options variable (like dcpConfig.dcp.listenOptions)\n */\nfunction SocketIOListener(config, target, options)\n{\n var matchRegex;\n \n this.url = config.location;\n this.options = this.buildOptions(options);\n this.connSequence = 0;\n this.seq = ++listenerSeq;\n this.debugLabel = `socketio(l:${this.seq}):`;\n this.isShutdown = false;\n\n if (dcpEnv.platform !== 'nodejs')\n throw new Error('target mode only supported in nodejs');\n\n if (target.httpServer.listening)\n setImmediate(() => this.emit('listening', target.httpServer)); /* setImmediate to allow event listeners to be added before emitting */\n else\n target.httpServer.on('listening', () => this.emit('listening', target.httpServer));\n\n matchRegex = '^' + this.url.pathname;\n if (matchRegex.slice(-1) !== '/')\n matchRegex += '\\\\/';\n else\n matchRegex = matchRegex.slice(0, -1) + \"\\\\/\"\n\n matchRegex += '\\\\?EIO=[0-9]';\n debugging('socketio') && console.debug(this.debugLabel, `expecting requests to match regex ${matchRegex}`);\n\n target.all(new RegExp(matchRegex), (request) => {\n /* Socket.io does this intrinsically because it plays with the server's events directly,\n * but we \"handle\" it explicitly here so that we don't trip the 404 handler. \n */\n debugging('socketio') && console.debug(this.debugLabel, 'Handling', request.url);\n }, true);\n\n /* While we normally want to inherit options from parent to child socket via this.options,\n * we don't want this to happen for maxHttpBufferSize due to the way it is used and calculated\n * in the transport constructor. A buffer that is too small (eg false) will cause us to not\n * recognized socket.io SIDs.\n */\n \n this.options.maxHttpBufferSize = 1e7;\n \n const socketOptions = this.options;\n\n this.socketServer = requireNative('socket.io')(target.httpServer, socketOptions);\n this.socketServer.on('connection', (socket) => !this.isShutdown && this.handleConnectionEvent(config, socket));\n debugging() && console.debug(this.debugLabel, 'Socketio listener initialized for path', this.options.path);\n}\nSocketIOListener.prototype = new EventEmitter('socketio(l)');\nSocketIOListener.prototype.buildOptions = buildOptions;\n\nSocketIOListener.prototype.close = function SocketIOListener$$close()\n{\n if (!this.isShutdown)\n this.shutdown();\n this.socketServer.close(() => this.emit('close'));\n}\n\n/** Stop accepting new connections */\nSocketIOListener.prototype.shutdown = function SocketIOListener$$shutdown()\n{\n this.isShutdown = true;\n}\n\n/** Remove any event loop references used by this transport instance */\nSocketIOListener.prototype.unref = function SocketIOListener$$unref()\n{\n void false;\n}\n\n/**\n * Handle the socketio connection event, which is fired upon a connection from client.\n * Used by Target->SocketIOListener to create transport instances.\n *\n * It is expected that code hooking the 'connection' event will hook the 'message' in\n * the same pass of the event loop as it is invoked; otherwise, messages may be missed.\n *\n * @param {object} socket a new connection emitted by the socket.io server\n */\nSocketIOListener.prototype.handleConnectionEvent = function SocketIOListener$$handleConnectionEvent(config, socket)\n{\n var transport = new SocketIOTransport(config, this.options, socket);\n\n this.url = config.location || config.listen;\n this.connSequence++;\n transport.debugLabel = `socketio(t:${this.seq}.${this.connSequence}):`\n\n /* Ensure we don't emit any message events until the application or Connection class\n * has had the chance to set up message handlers via the connection event.\n */\n transport.rxReady = false;\n this.on('connection', () => setImmediate(() => {\n transport.rxReady = true;\n transport.emitReadyMessages();\n }));\n\n this.emit('connection', transport);\n}\n\n/* Define API */\nexports.TransportClass = SocketIOTransport;\nexports.Listener = SocketIOListener;\n\n\n//# sourceURL=webpack://dcp/./src/protocol-v4/transport/socketio.js?");
|
|
4719
4719
|
|
|
4720
4720
|
/***/ }),
|
|
4721
4721
|
|
|
@@ -5678,7 +5678,7 @@ eval("module.exports = JSON.parse('{\"1.3.132.0.10\":\"secp256k1\",\"1.3.132.0.3
|
|
|
5678
5678
|
/***/ ((module) => {
|
|
5679
5679
|
|
|
5680
5680
|
"use strict";
|
|
5681
|
-
eval("module.exports = JSON.parse('{\"browser\":[\"deny-node\",\"kvin/kvin.js\",\"script-load-wrapper\",\"wrap-event-listeners\",\"event-loop-virtualization\",\"gpu-timers\",\"access-lists\",\"bravojs-init\",\"bravojs/bravo.js\",\"bravojs-env\",\"calculate-capabilities\",\"bootstrap\"],\"node\":[\"kvin/kvin.js\",\"sa-ww-simulation\",\"script-load-wrapper\",\"wrap-event-listeners\",\"event-loop-virtualization\",\"gpu-timers\",\"access-lists\",\"bravojs-init\",\"bravojs/bravo.js\",\"bravojs-env\",\"calculate-capabilities\",\"bootstrap\"],\"native\":[\"deny-node\",\"kvin/kvin.js\",\"sa-ww-simulation\",\"script-load-wrapper\",\"native-event-loop\",\"wrap-event-listeners\",\"event-loop-virtualization\",\"
|
|
5681
|
+
eval("module.exports = JSON.parse('{\"browser\":[\"deny-node\",\"kvin/kvin.js\",\"script-load-wrapper\",\"wrap-event-listeners\",\"event-loop-virtualization\",\"gpu-timers\",\"access-lists\",\"bravojs-init\",\"bravojs/bravo.js\",\"bravojs-env\",\"calculate-capabilities\",\"bootstrap\"],\"node\":[\"kvin/kvin.js\",\"sa-ww-simulation\",\"script-load-wrapper\",\"wrap-event-listeners\",\"event-loop-virtualization\",\"gpu-timers\",\"access-lists\",\"bravojs-init\",\"bravojs/bravo.js\",\"bravojs-env\",\"calculate-capabilities\",\"bootstrap\"],\"native\":[\"deny-node\",\"kvin/kvin.js\",\"sa-ww-simulation\",\"script-load-wrapper\",\"native-event-loop\",\"wrap-event-listeners\",\"event-loop-virtualization\",\"gpu-timers\",\"access-lists\",\"bravojs-init\",\"bravojs/bravo.js\",\"bravojs-env\",\"calculate-capabilities\",\"bootstrap\"],\"webGpuNative\":[\"deny-node\",\"kvin/kvin.js\",\"sa-ww-simulation\",\"script-load-wrapper\",\"native-event-loop\",\"wrap-event-listeners\",\"event-loop-virtualization\",\"webgpu-worker-environment\",\"gpu-timers\",\"access-lists\",\"bravojs-init\",\"bravojs/bravo.js\",\"bravojs-env\",\"calculate-capabilities\",\"bootstrap\"],\"nodeTesting\":[\"kvin/kvin.js\",\"sa-ww-simulation\",\"script-load-wrapper\",\"wrap-event-listeners\",\"event-loop-virtualization\",\"gpu-timers\",\"access-lists\",\"bravojs-init\",\"bravojs/bravo.js\",\"bravojs-env\",\"calculate-capabilities\",\"bootstrap\",\"testing.js\"],\"testing\":[\"deny-node\",\"kvin/kvin.js\",\"sa-ww-simulation\",\"script-load-wrapper\",\"native-event-loop\",\"wrap-event-listeners\",\"event-loop-virtualization\",\"gpu-timers\",\"access-lists\",\"bravojs-init\",\"bravojs/bravo.js\",\"bravojs-env\",\"calculate-capabilities\",\"bootstrap\",\"testing.js\"]}');\n\n//# sourceURL=webpack://dcp/./node_modules/dcp-client/generated/sandbox-definitions.json?");
|
|
5682
5682
|
|
|
5683
5683
|
/***/ }),
|
|
5684
5684
|
|