@scratch/scratch-vm 11.0.0-beta.1

package/src/dispatch/central-dispatch.js

@@ -0,0 +1,143 @@
+const SharedDispatch = require('./shared-dispatch');
+
+const log = require('../util/log');
+
+/**
+ * This class serves as the central broker for message dispatch. It expects to operate on the main thread / Window and
+ * it must be informed of any Worker threads which will participate in the messaging system. From any context in the
+ * messaging system, the dispatcher's "call" method can call any method on any "service" provided in any participating
+ * context. The dispatch system will forward function arguments and return values across worker boundaries as needed.
+ * @see {WorkerDispatch}
+ */
+class CentralDispatch extends SharedDispatch {
+    constructor () {
+        super();
+
+        /**
+         * Map of channel name to worker or local service provider.
+         * If the entry is a Worker, the service is provided by an object on that worker.
+         * Otherwise, the service is provided locally and methods on the service will be called directly.
+         * @see {setService}
+         * @type {object.<Worker|object>}
+         */
+        this.services = {};
+
+        /**
+         * The constructor we will use to recognize workers.
+         * @type {Function}
+         */
+        this.workerClass = (typeof Worker === 'undefined' ? null : Worker);
+
+        /**
+         * List of workers attached to this dispatcher.
+         * @type {Array}
+         */
+        this.workers = [];
+    }
+
+    /**
+     * Synchronously call a particular method on a particular service provided locally.
+     * Calling this function on a remote service will fail.
+     * @param {string} service - the name of the service.
+     * @param {string} method - the name of the method.
+     * @param {*} [args] - the arguments to be copied to the method, if any.
+     * @returns {*} - the return value of the service method.
+     */
+    callSync (service, method, ...args) {
+        const {provider, isRemote} = this._getServiceProvider(service);
+        if (provider) {
+            if (isRemote) {
+                throw new Error(`Cannot use 'callSync' on remote provider for service ${service}.`);
+            }
+
+            // TODO: verify correct `this` after switching from apply to spread
+            // eslint-disable-next-line prefer-spread
+            return provider[method].apply(provider, args);
+        }
+        throw new Error(`Provider not found for service: ${service}`);
+    }
+
+    /**
+     * Synchronously set a local object as the global provider of the specified service.
+     * WARNING: Any method on the provider can be called from any worker within the dispatch system.
+     * @param {string} service - a globally unique string identifying this service. Examples: 'vm', 'gui', 'extension9'.
+     * @param {object} provider - a local object which provides this service.
+     */
+    setServiceSync (service, provider) {
+        if (Object.prototype.hasOwnProperty.call(this.services, service)) {
+            log.warn(`Central dispatch replacing existing service provider for ${service}`);
+        }
+        this.services[service] = provider;
+    }
+
+    /**
+     * Set a local object as the global provider of the specified service.
+     * WARNING: Any method on the provider can be called from any worker within the dispatch system.
+     * @param {string} service - a globally unique string identifying this service. Examples: 'vm', 'gui', 'extension9'.
+     * @param {object} provider - a local object which provides this service.
+     * @returns {Promise} - a promise which will resolve once the service is registered.
+     */
+    setService (service, provider) {
+        /** Return a promise for consistency with {@link WorkerDispatch#setService} */
+        try {
+            this.setServiceSync(service, provider);
+            return Promise.resolve();
+        } catch (e) {
+            return Promise.reject(e);
+        }
+    }
+
+    /**
+     * Add a worker to the message dispatch system. The worker must implement a compatible message dispatch framework.
+     * The dispatcher will immediately attempt to "handshake" with the worker.
+     * @param {Worker} worker - the worker to add into the dispatch system.
+     */
+    addWorker (worker) {
+        if (this.workers.indexOf(worker) === -1) {
+            this.workers.push(worker);
+            worker.onmessage = this._onMessage.bind(this, worker);
+            this._remoteCall(worker, 'dispatch', 'handshake').catch(e => {
+                log.error(`Could not handshake with worker: ${JSON.stringify(e)}`);
+            });
+        } else {
+            log.warn('Central dispatch ignoring attempt to add duplicate worker');
+        }
+    }
+
+    /**
+     * Fetch the service provider object for a particular service name.
+     * @override
+     * @param {string} service - the name of the service to look up
+     * @returns {{provider:(object|Worker), isRemote:boolean}} - the means to contact the service, if found
+     * @protected
+     */
+    _getServiceProvider (service) {
+        const provider = this.services[service];
+        return provider && {
+            provider,
+            isRemote: Boolean(this.workerClass && provider instanceof this.workerClass)
+        };
+    }
+
+    /**
+     * Handle a call message sent to the dispatch service itself
+     * @override
+     * @param {Worker} worker - the worker which sent the message.
+     * @param {DispatchCallMessage} message - the message to be handled.
+     * @returns {Promise|undefined} - a promise for the results of this operation, if appropriate
+     * @protected
+     */
+    _onDispatchMessage (worker, message) {
+        let promise;
+        switch (message.method) {
+        case 'setService':
+            promise = this.setService(message.args[0], worker);
+            break;
+        default:
+            log.error(`Central dispatch received message for unknown method: ${message.method}`);
+        }
+        return promise;
+    }
+}
+
+module.exports = new CentralDispatch();

package/src/dispatch/shared-dispatch.js

@@ -0,0 +1,235 @@
+const log = require('../util/log');
+
+/**
+ * @typedef {object} DispatchCallMessage - a message to the dispatch system representing a service method call
+ * @property {*} responseId - send a response message with this response ID. See {@link DispatchResponseMessage}
+ * @property {string} service - the name of the service to be called
+ * @property {string} method - the name of the method to be called
+ * @property {Array|undefined} args - the arguments to be passed to the method
+ */
+
+/**
+ * @typedef {object} DispatchResponseMessage - a message to the dispatch system representing the results of a call
+ * @property {*} responseId - a copy of the response ID from the call which generated this response
+ * @property {*|undefined} error - if this is truthy, then it contains results from a failed call (such as an exception)
+ * @property {*|undefined} result - if error is not truthy, then this contains the return value of the call (if any)
+ */
+
+/**
+ * @typedef {DispatchCallMessage|DispatchResponseMessage} DispatchMessage
+ * Any message to the dispatch system.
+ */
+
+/**
+ * The SharedDispatch class is responsible for dispatch features shared by
+ * {@link CentralDispatch} and {@link WorkerDispatch}.
+ */
+class SharedDispatch {
+    constructor () {
+        /**
+         * List of callback registrations for promises waiting for a response from a call to a service on another
+         * worker. A callback registration is an array of [resolve,reject] Promise functions.
+         * Calls to local services don't enter this list.
+         * @type {Array.<Function[]>}
+         */
+        this.callbacks = [];
+
+        /**
+         * The next response ID to be used.
+         * @type {int}
+         */
+        this.nextResponseId = 0;
+    }
+
+    /**
+     * Call a particular method on a particular service, regardless of whether that service is provided locally or on
+     * a worker. If the service is provided by a worker, the `args` will be copied using the Structured Clone
+     * algorithm, except for any items which are also in the `transfer` list. Ownership of those items will be
+     * transferred to the worker, and they should not be used after this call.
+     * @example
+     *      dispatcher.call('vm', 'setData', 'cat', 42);
+     *      // this finds the worker for the 'vm' service, then on that worker calls:
+     *      vm.setData('cat', 42);
+     * @param {string} service - the name of the service.
+     * @param {string} method - the name of the method.
+     * @param {*} [args] - the arguments to be copied to the method, if any.
+     * @returns {Promise} - a promise for the return value of the service method.
+     */
+    call (service, method, ...args) {
+        return this.transferCall(service, method, null, ...args);
+    }
+
+    /**
+     * Call a particular method on a particular service, regardless of whether that service is provided locally or on
+     * a worker. If the service is provided by a worker, the `args` will be copied using the Structured Clone
+     * algorithm, except for any items which are also in the `transfer` list. Ownership of those items will be
+     * transferred to the worker, and they should not be used after this call.
+     * @example
+     *      dispatcher.transferCall('vm', 'setData', [myArrayBuffer], 'cat', myArrayBuffer);
+     *      // this finds the worker for the 'vm' service, transfers `myArrayBuffer` to it, then on that worker calls:
+     *      vm.setData('cat', myArrayBuffer);
+     * @param {string} service - the name of the service.
+     * @param {string} method - the name of the method.
+     * @param {Array} [transfer] - objects to be transferred instead of copied. Must be present in `args` to be useful.
+     * @param {*} [args] - the arguments to be copied to the method, if any.
+     * @returns {Promise} - a promise for the return value of the service method.
+     */
+    transferCall (service, method, transfer, ...args) {
+        try {
+            const {provider, isRemote} = this._getServiceProvider(service);
+            if (provider) {
+                if (isRemote) {
+                    return this._remoteTransferCall(provider, service, method, transfer, ...args);
+                }
+
+                // TODO: verify correct `this` after switching from apply to spread
+                // eslint-disable-next-line prefer-spread
+                const result = provider[method].apply(provider, args);
+                return Promise.resolve(result);
+            }
+            return Promise.reject(new Error(`Service not found: ${service}`));
+        } catch (e) {
+            return Promise.reject(e);
+        }
+    }
+
+    /**
+     * Check if a particular service lives on another worker.
+     * @param {string} service - the service to check.
+     * @returns {boolean} - true if the service is remote (calls must cross a Worker boundary), false otherwise.
+     * @private
+     */
+    _isRemoteService (service) {
+        return this._getServiceProvider(service).isRemote;
+    }
+
+    /**
+     * Like {@link call}, but force the call to be posted through a particular communication channel.
+     * @param {object} provider - send the call through this object's `postMessage` function.
+     * @param {string} service - the name of the service.
+     * @param {string} method - the name of the method.
+     * @param {*} [args] - the arguments to be copied to the method, if any.
+     * @returns {Promise} - a promise for the return value of the service method.
+     */
+    _remoteCall (provider, service, method, ...args) {
+        return this._remoteTransferCall(provider, service, method, null, ...args);
+    }
+
+    /**
+     * Like {@link transferCall}, but force the call to be posted through a particular communication channel.
+     * @param {object} provider - send the call through this object's `postMessage` function.
+     * @param {string} service - the name of the service.
+     * @param {string} method - the name of the method.
+     * @param {Array} [transfer] - objects to be transferred instead of copied. Must be present in `args` to be useful.
+     * @param {*} [args] - the arguments to be copied to the method, if any.
+     * @returns {Promise} - a promise for the return value of the service method.
+     */
+    _remoteTransferCall (provider, service, method, transfer, ...args) {
+        return new Promise((resolve, reject) => {
+            const responseId = this._storeCallbacks(resolve, reject);
+
+            /** @TODO: remove this hack! this is just here so we don't try to send `util` to a worker */
+            if ((args.length > 0) && (typeof args[args.length - 1].yield === 'function')) {
+                args.pop();
+            }
+
+            if (transfer) {
+                provider.postMessage({service, method, responseId, args}, transfer);
+            } else {
+                provider.postMessage({service, method, responseId, args});
+            }
+        });
+    }
+
+    /**
+     * Store callback functions pending a response message.
+     * @param {Function} resolve - function to call if the service method returns.
+     * @param {Function} reject - function to call if the service method throws.
+     * @returns {*} - a unique response ID for this set of callbacks. See {@link _deliverResponse}.
+     * @protected
+     */
+    _storeCallbacks (resolve, reject) {
+        const responseId = this.nextResponseId++;
+        this.callbacks[responseId] = [resolve, reject];
+        return responseId;
+    }
+
+    /**
+     * Deliver call response from a worker. This should only be called as the result of a message from a worker.
+     * @param {int} responseId - the response ID of the callback set to call.
+     * @param {DispatchResponseMessage} message - the message containing the response value(s).
+     * @protected
+     */
+    _deliverResponse (responseId, message) {
+        try {
+            const [resolve, reject] = this.callbacks[responseId];
+            delete this.callbacks[responseId];
+            if (message.error) {
+                reject(message.error);
+            } else {
+                resolve(message.result);
+            }
+        } catch (e) {
+            log.error(`Dispatch callback failed: ${JSON.stringify(e)}`);
+        }
+    }
+
+    /**
+     * Handle a message event received from a connected worker.
+     * @param {Worker} worker - the worker which sent the message, or the global object if running in a worker.
+     * @param {MessageEvent} event - the message event to be handled.
+     * @protected
+     */
+    _onMessage (worker, event) {
+        /** @type {DispatchMessage} */
+        const message = event.data;
+        message.args = message.args || [];
+        let promise;
+        if (message.service) {
+            if (message.service === 'dispatch') {
+                promise = this._onDispatchMessage(worker, message);
+            } else {
+                promise = this.call(message.service, message.method, ...message.args);
+            }
+        } else if (typeof message.responseId === 'undefined') {
+            log.error(`Dispatch caught malformed message from a worker: ${JSON.stringify(event)}`);
+        } else {
+            this._deliverResponse(message.responseId, message);
+        }
+        if (promise) {
+            if (typeof message.responseId === 'undefined') {
+                log.error(`Dispatch message missing required response ID: ${JSON.stringify(event)}`);
+            } else {
+                promise.then(
+                    result => worker.postMessage({responseId: message.responseId, result}),
+                    error => worker.postMessage({responseId: message.responseId, error})
+                );
+            }
+        }
+    }
+
+    /**
+     * Fetch the service provider object for a particular service name.
+     * @abstract
+     * @param {string} service - the name of the service to look up
+     * @returns {{provider:(object|Worker), isRemote:boolean}} - the means to contact the service, if found
+     * @protected
+     */
+    _getServiceProvider (service) {
+        throw new Error(`Could not get provider for ${service}: _getServiceProvider not implemented`);
+    }
+
+    /**
+     * Handle a call message sent to the dispatch service itself
+     * @abstract
+     * @param {Worker} worker - the worker which sent the message.
+     * @param {DispatchCallMessage} message - the message to be handled.
+     * @returns {Promise|undefined} - a promise for the results of this operation, if appropriate
+     * @private
+     */
+    _onDispatchMessage (worker, message) {
+        throw new Error(`Unimplemented dispatch message handler cannot handle ${message.method} method`);
+    }
+}
+
+module.exports = SharedDispatch;

package/src/dispatch/worker-dispatch.js

@@ -0,0 +1,110 @@
+const SharedDispatch = require('./shared-dispatch');
+
+const log = require('../util/log');
+
+/**
+ * This class provides a Worker with the means to participate in the message dispatch system managed by CentralDispatch.
+ * From any context in the messaging system, the dispatcher's "call" method can call any method on any "service"
+ * provided in any participating context. The dispatch system will forward function arguments and return values across
+ * worker boundaries as needed.
+ * @see {CentralDispatch}
+ */
+class WorkerDispatch extends SharedDispatch {
+    constructor () {
+        super();
+
+        /**
+         * This promise will be resolved when we have successfully connected to central dispatch.
+         * @type {Promise}
+         * @see {waitForConnection}
+         * @private
+         */
+        this._connectionPromise = new Promise(resolve => {
+            this._onConnect = resolve;
+        });
+
+        /**
+         * Map of service name to local service provider.
+         * If a service is not listed here, it is assumed to be provided by another context (another Worker or the main
+         * thread).
+         * @see {setService}
+         * @type {object}
+         */
+        this.services = {};
+
+        this._onMessage = this._onMessage.bind(this, self);
+        if (typeof self !== 'undefined') {
+            self.onmessage = this._onMessage;
+        }
+    }
+
+    /**
+     * @returns {Promise} a promise which will resolve upon connection to central dispatch. If you need to make a call
+     * immediately on "startup" you can attach a 'then' to this promise.
+     * @example
+     *      dispatch.waitForConnection.then(() => {
+     *          dispatch.call('myService', 'hello');
+     *      })
+     */
+    get waitForConnection () {
+        return this._connectionPromise;
+    }
+
+    /**
+     * Set a local object as the global provider of the specified service.
+     * WARNING: Any method on the provider can be called from any worker within the dispatch system.
+     * @param {string} service - a globally unique string identifying this service. Examples: 'vm', 'gui', 'extension9'.
+     * @param {object} provider - a local object which provides this service.
+     * @returns {Promise} - a promise which will resolve once the service is registered.
+     */
+    setService (service, provider) {
+        if (Object.prototype.hasOwnProperty.call(this.services, service)) {
+            log.warn(`Worker dispatch replacing existing service provider for ${service}`);
+        }
+        this.services[service] = provider;
+        return this.waitForConnection.then(() => this._remoteCall(self, 'dispatch', 'setService', service));
+    }
+
+    /**
+     * Fetch the service provider object for a particular service name.
+     * @override
+     * @param {string} service - the name of the service to look up
+     * @returns {{provider:(object|Worker), isRemote:boolean}} - the means to contact the service, if found
+     * @protected
+     */
+    _getServiceProvider (service) {
+        // if we don't have a local service by this name, contact central dispatch by calling `postMessage` on self
+        const provider = this.services[service];
+        return {
+            provider: provider || self,
+            isRemote: !provider
+        };
+    }
+
+    /**
+     * Handle a call message sent to the dispatch service itself
+     * @override
+     * @param {Worker} worker - the worker which sent the message.
+     * @param {DispatchCallMessage} message - the message to be handled.
+     * @returns {Promise|undefined} - a promise for the results of this operation, if appropriate
+     * @protected
+     */
+    _onDispatchMessage (worker, message) {
+        let promise;
+        switch (message.method) {
+        case 'handshake':
+            promise = this._onConnect();
+            break;
+        case 'terminate':
+            // Don't close until next tick, after sending confirmation back
+            setTimeout(() => self.close(), 0);
+            promise = Promise.resolve();
+            break;
+        default:
+            log.error(`Worker dispatch received message for unknown method: ${message.method}`);
+        }
+        return promise;
+    }
+}
+
+module.exports = new WorkerDispatch();

package/src/engine/adapter.js

@@ -0,0 +1,176 @@
+const mutationAdapter = require('./mutation-adapter');
+const html = require('htmlparser2');
+const uid = require('../util/uid');
+
+/**
+ * Convert and an individual block DOM to the representation tree.
+ * Based on Blockly's `domToBlockHeadless_`.
+ * @param {Element} blockDOM DOM tree for an individual block.
+ * @param {object} blocks Collection of blocks to add to.
+ * @param {boolean} isTopBlock Whether blocks at this level are "top blocks."
+ * @param {?string} parent Parent block ID.
+ * @return {undefined}
+ */
+const domToBlock = function (blockDOM, blocks, isTopBlock, parent) {
+    if (!blockDOM.attribs.id) {
+        blockDOM.attribs.id = uid();
+    }
+
+    // Block skeleton.
+    const block = {
+        id: blockDOM.attribs.id, // Block ID
+        opcode: blockDOM.attribs.type, // For execution, "event_whengreenflag".
+        inputs: {}, // Inputs to this block and the blocks they point to.
+        fields: {}, // Fields on this block and their values.
+        next: null, // Next block in the stack, if one exists.
+        topLevel: isTopBlock, // If this block starts a stack.
+        parent: parent, // Parent block ID, if available.
+        shadow: blockDOM.name === 'shadow', // If this represents a shadow/slot.
+        x: blockDOM.attribs.x, // X position of script, if top-level.
+        y: blockDOM.attribs.y // Y position of script, if top-level.
+    };
+
+    // Add the block to the representation tree.
+    blocks[block.id] = block;
+
+    // Process XML children and find enclosed blocks, fields, etc.
+    for (let i = 0; i < blockDOM.children.length; i++) {
+        const xmlChild = blockDOM.children[i];
+        // Enclosed blocks and shadows
+        let childBlockNode = null;
+        let childShadowNode = null;
+        for (let j = 0; j < xmlChild.children.length; j++) {
+            const grandChildNode = xmlChild.children[j];
+            if (!grandChildNode.name) {
+                // Non-XML tag node.
+                continue;
+            }
+            const grandChildNodeName = grandChildNode.name.toLowerCase();
+            if (grandChildNodeName === 'block') {
+                childBlockNode = grandChildNode;
+            } else if (grandChildNodeName === 'shadow') {
+                childShadowNode = grandChildNode;
+            }
+        }
+
+        // Use shadow block only if there's no real block node.
+        if (!childBlockNode && childShadowNode) {
+            childBlockNode = childShadowNode;
+        }
+
+        // Not all Blockly-type blocks are handled here,
+        // as we won't be using all of them for Scratch.
+        switch (xmlChild.name.toLowerCase()) {
+        case 'field':
+        {
+            // Add the field to this block.
+            const fieldName = xmlChild.attribs.name;
+            // Add id in case it is a variable field
+            const fieldId = xmlChild.attribs.id;
+            let fieldData = '';
+            if (xmlChild.children.length > 0 && xmlChild.children[0].data) {
+                fieldData = xmlChild.children[0].data;
+            } else {
+                // If the child of the field with a data property
+                // doesn't exist, set the data to an empty string.
+                fieldData = '';
+            }
+            block.fields[fieldName] = {
+                name: fieldName,
+                id: fieldId,
+                value: fieldData
+            };
+            const fieldVarType = xmlChild.attribs.variabletype;
+            if (typeof fieldVarType === 'string') {
+                block.fields[fieldName].variableType = fieldVarType;
+            }
+            break;
+        }
+        case 'comment':
+        {
+            block.comment = xmlChild.attribs.id;
+            break;
+        }
+        case 'value':
+        case 'statement':
+        {
+            // Recursively generate block structure for input block.
+            domToBlock(childBlockNode, blocks, false, block.id);
+            if (childShadowNode && childBlockNode !== childShadowNode) {
+                // Also generate the shadow block.
+                domToBlock(childShadowNode, blocks, false, block.id);
+            }
+            // Link this block's input to the child block.
+            const inputName = xmlChild.attribs.name;
+            block.inputs[inputName] = {
+                name: inputName,
+                block: childBlockNode.attribs.id,
+                shadow: childShadowNode ? childShadowNode.attribs.id : null
+            };
+            break;
+        }
+        case 'next':
+        {
+            if (!childBlockNode || !childBlockNode.attribs) {
+                // Invalid child block.
+                continue;
+            }
+            // Recursively generate block structure for next block.
+            domToBlock(childBlockNode, blocks, false, block.id);
+            // Link next block to this block.
+            block.next = childBlockNode.attribs.id;
+            break;
+        }
+        case 'mutation':
+        {
+            block.mutation = mutationAdapter(xmlChild);
+            break;
+        }
+        }
+    }
+};
+
+/**
+ * Convert outer blocks DOM from a Blockly CREATE event
+ * to a usable form for the Scratch runtime.
+ * This structure is based on Blockly xml.js:`domToWorkspace` and `domToBlock`.
+ * @param {Element} blocksDOM DOM tree for this event.
+ * @return {Array.<object>} Usable list of blocks from this CREATE event.
+ */
+const domToBlocks = function (blocksDOM) {
+    // At this level, there could be multiple blocks adjacent in the DOM tree.
+    const blocks = {};
+    for (let i = 0; i < blocksDOM.length; i++) {
+        const block = blocksDOM[i];
+        if (!block.name || !block.attribs) {
+            continue;
+        }
+        const tagName = block.name.toLowerCase();
+        if (tagName === 'block' || tagName === 'shadow') {
+            domToBlock(block, blocks, true, null);
+        }
+    }
+    // Flatten blocks object into a list.
+    const blocksList = [];
+    for (const b in blocks) {
+        if (!Object.prototype.hasOwnProperty.call(blocks, b)) continue;
+        blocksList.push(blocks[b]);
+    }
+    return blocksList;
+};
+
+/**
+ * Adapter between block creation events and block representation which can be
+ * used by the Scratch runtime.
+ * @param {object} e `Blockly.events.create` or `Blockly.events.endDrag`
+ * @return {Array.<object>} List of blocks from this CREATE event.
+ */
+const adapter = function (e) {
+    // Validate input
+    if (typeof e !== 'object') return;
+    if (typeof e.xml !== 'object') return;
+
+    return domToBlocks(html.parseDOM(e.xml.outerHTML, {decodeEntities: true}));
+};
+
+module.exports = adapter;