@remix_labs/machine-starter 1.1890.0-dev → 1.1895.0-dev

package/common.js

@@ -0,0 +1,292 @@
+import { nanoid } from "nanoid";
+import { GROOVEBOX_BUILD } from "./groovebox_build.js";
+
+let terminate = function(f) {};
+if (globalThis.ThisIsNode) {
+    terminate = function(f) {
+        Process.on("exit", () => {
+            console.log("terminating machine worker");
+            f()
+        })
+    }
+};
+
+function CreateVMID() {
+    return nanoid()
+}
+
+let initialMask = 64 + 1024;   // default for web
+globalThis.MixSetMask = (m => {
+    initialMask = m;
+});
+
+function StartWASM(hub, baseURL, org, workspace, vmID, user, token, noOutputViaMQTT) {
+    return StartWASM2(hub, {
+        baseURL: baseURL,
+        org: org,
+        workspace: workspace,
+        token: token,
+        noOutputViaMQTT: noOutputViaMQTT,
+        machType: "WASM"
+    })
+}
+
+function StartWASM2(hub, config) {
+    console.log("GROOVEBOX_BUILD (machine-starter)", GROOVEBOX_BUILD);
+    return hub.newChannel().then(channel => {
+        // injected by either index.js or node.js
+        let worker = globalThis.GetMachineWASMWorker();
+        let localFFIs = {};
+        if (config.localFFIs) {
+            for (const [k, v] of Object.entries(config.localFFIs)) {
+                if (v.isRaw && (v.canFail || v.useJsonDecoder)) {
+                    console.error("A raw, local FFI must neither be failing nor use the JSON decoder: " + k);
+                } else {
+                    localFFIs[k] = {...v};
+                    delete localFFIs[k].run;
+                }
+            }
+        }
+        let config_msg =
+            { "_rmx_type": "msg_vm_configure",
+              "baseURL": config.baseURL,
+              "org": config.org,
+              "workspace": config.workspace,
+              "vmID": config.vmID,
+              "user": config.user,
+              "token": config.token,
+              "hub": channel.port,
+              "outputViaMQTT": !(config.noOutputViaMQTT),
+              "machType": config.machType,
+              "debugMask": initialMask,
+              "localFFIs": localFFIs,
+              "grooveboxUrlPrefix": globalThis.GROOVEBOX_URL_PREFIX,
+              "allowInsecureHttp": globalThis.GROOVEBOX_ALLOW_INSECURE_HTTP,
+              "interceptFFI": config.interceptFFI,
+              "interceptDynloadFFI": config.interceptDynloadFFI,
+            };
+        worker.postMessage(config_msg, [ channel.port ]);
+        terminate(() => worker.terminate());
+        globalThis.MixSetMask = (m => {
+            worker.postMessage({ "_rmx_type": "msg_vm_logMask",
+                                 "mask": m
+                               })
+        });
+        if (config.localFFIs)
+            setupLocalFFIs(hub, config);  // don't await!
+        return worker;
+    })
+}
+
+class Token {
+    constructor(data, enabled, extras) {
+        // data: must be an Uint8Array
+        // enabled: a number with the bitset of enabled operations
+        // extras: a map
+        if (!data instanceof Uint8Array)
+            throw new Error("Token: data must be an Uint8Array");
+        if (typeof(enabled) != "number")
+            throw new Error("Token: enabled_ops must be a number");
+        this.data = data;
+        this.enabled = enabled;
+        this.extras = extras;
+    }
+    encode() {
+        return new Map([
+            [ "_rmx_type", "token" ],
+            [ "data", this.data ],
+            [ "enabled", this.enabled ],
+            [ "extras", this.extras ]
+        ]);
+    }
+}
+
+function decode_Token(m) {
+    return new Token(m.get("data"), m.get("enabled"), m.get("extras"))
+}
+
+class Case {
+    constructor(name, arg) {
+        if (typeof(name) != "string")
+            throw new Error("Case: name must be a string");
+        this.name = name;
+        this.arg = arg;
+    }
+    encode() {
+        return new Map([
+            [ "_rmx_type", "case" ],
+            [ "name", this.name ],
+            [ "arg", this.arg ],
+        ]);
+    }
+}
+
+function decode_Case(m) {
+    return new Case(m.get("name"), m.get("arg"))
+}
+
+class Opaque {
+    constructor(buf) {
+        if (!(buf instanceof Uint32Array))
+            throw new Error("Opaque: must be a Uint32Array");
+        this.data = buf;
+    }
+    encode() {
+        return new Map([
+            [ "_rmx_type", "opaque" ],
+            [ "data", this.data ],
+        ]);
+    }
+}
+
+function decode_Opaque(m) {
+    return new Opaque(m.get("data"))
+}
+
+function encode(val) {
+    switch (typeof(val)) {
+    case "object":
+        if (val === null) return val;
+        if (Array.isArray(val)) {
+            // array: map the array elements
+            val = val.map(encode)
+        } else if (Object.getPrototypeOf(val) === Object.prototype) {
+            // simple object: map the object elements
+            val = Object.fromEntries(Object.entries(val).map(([ key, elem ]) => [ key, encode(elem) ]))
+        } else if (val.encode instanceof Function) {
+            // map after encode(): map the map elements
+            val = new Map(Array.from(val.encode().entries()).map(([ key, elem ]) => [ key, encode(elem) ]))
+        };
+        return val;
+    default:
+        return val;
+    }
+}
+
+function decode(val) {
+    switch (typeof(val)) {
+    case "object":
+        if (val === null) return val;
+        if (Array.isArray(val)) {
+            // array: map the array elements
+            val = val.map(decode)
+        } else if (Object.getPrototypeOf(val) === Object.prototype) {
+            // simple object: map the object elements
+            val = Object.fromEntries(Object.entries(val).map(([ key, elem ]) => [ key, decode(elem) ]))
+        } else if (val instanceof Map) {
+            val = new Map(Array.from(val.entries()).map(([ key, elem ]) => [ key, decode(elem) ]))
+            let ty = val.get("_rmx_type");
+            switch (ty) {
+            case "token":
+                return decode_Token(val);
+            case "case":
+                return decode_Case(val);
+            case "opaque":
+                return decode_Opaque(val);
+            default:
+                throw new Error("decode: bad _rmx_type: " + ty);
+            }
+        };
+        return val;
+    default:
+        return val;
+    }
+}
+
+async function setupLocalFFIs(hub, config) {
+    let channel = await hub.newChannel();
+    let comm = new FFIComm(hub, channel);
+    let localFFIs = config.localFFIs;
+    await channel.setLocalSubTopic("/local/ffi/call");
+    await channel.setLocalPubTopic("/local/ffi/return");
+    let sub = await channel.subscribe("/local/ffi/call");
+    while (true) {
+        let msg = await sub.next();
+        let name = msg.payload.name;
+        let call_id = msg.payload.call_id;
+        let args = decode(msg.payload.args);
+        let fun = localFFIs[name];
+        try {
+            if (!fun)
+                throw new Error("no such local FFI: " + name);
+            let connector =
+                { call_id: call_id,
+                  hub: hub,
+                  channel: channel,
+                  state: config.state,
+                };
+            let r = fun instanceof Function ? fun(connector, args) : fun.run(connector, args);
+            if (r instanceof Promise) {
+                await comm.later(call_id);
+                r.then(
+                    (value) => comm.returnOrFail(fun, call_id, value),
+                    (reason) => comm.error(call_id, reason.message, reason.stack));
+            } else {
+                await comm.returnOrFail(fun, call_id, r);
+            }
+        } catch (reason) {
+            await comm.error(call_id, reason.message, reason.stack);
+        }
+    }
+}
+
+class FFIComm {
+    constructor(hub, channel) {
+        this.hub = hub;
+        this.channel = channel;
+    }
+    later(call_id) {
+        let r_payload =
+            { call_id: call_id,
+            };
+        let response = this.hub.newLocalMessage("msg_ffi_later", "starter", r_payload);
+        return this.channel.publish("/local/ffi/return", response, false);
+    }
+    return_(fun, call_id, result) {
+        let r_payload =
+            { call_id: call_id,
+              value: fun.useJsonDecoder ? result : encode(result)
+            };
+        let response = this.hub.newLocalMessage("msg_ffi_return", "starter", r_payload);
+        return this.channel.publish("/local/ffi/return", response, false);
+    }
+    error(call_id, message, stack) {
+        let r_payload =
+            { call_id: call_id,
+              message: message,
+              stack: stack === undefined ? [] : stack,
+            };
+        let response = this.hub.newLocalMessage("msg_ffi_error", "starter", r_payload);
+        this.channel.publish("/local/ffi/return", response, false);
+    }
+    returnOrFail(fun, call_id, result) {
+        if (fun.canFail) {
+            var name, arg;
+            if (result instanceof Case) {
+                name = result.name;
+                arg = result.arg;
+            } else if (fun.useJsonDecoder && result._rmx_type && result._rmx_type.startsWith("{tag}case:")) {
+                name = result._rmx_type.substring(10);
+                arg = result._rmx_value;
+            } else {
+                return this.error(call_id, "expected result from local FFI that can fail");
+            }
+            switch (name) {
+            case "ok":
+                return this.return_(fun, call_id, arg);
+            case "error":
+                return this.error(call_id, arg);
+            default:
+                return this.error(call_id, "expected result from local FFI that can fail, found case value " + name);
+            }
+        } else {
+            return this.return_(fun, call_id, result);
+        }
+    }
+}
+
+// use worker.terminate() to shut a worker down!
+
+export { CreateVMID, StartWASM, StartWASM2,
+         Token, Case, Opaque
+       }

package/groovebox_build.js

@@ -1,2 +1,2 @@
-var GROOVEBOX_BUILD = "1890";
+var GROOVEBOX_BUILD = "1895";
 export { GROOVEBOX_BUILD }

package/index.js

@@ -1,306 +1,15 @@
-import { nanoid } from "nanoid";
-import { GROOVEBOX_BUILD } from "./groovebox_build.js";
-
-let terminate = function(f) {};
-if (globalThis.ThisIsNode) {
-    terminate = function(f) {
-        Process.on("exit", () => {
-            console.log("terminating machine worker");
-            f()
-        })
-    }
-};
-
-function CreateVMID() {
-    return nanoid()
+function workerify(code) {
+    let blob = new Blob([code], {type: 'application/javascript'});
+    return new Worker(URL.createObjectURL(blob));
 }
 
-let initialMask = 64 + 1024;   // default for web
-globalThis.MixSetMask = (m => {
-    initialMask = m;
-});
+import MachineWASMWorker from './machine-wasm.worker.js';
 
-function StartWASM(hub, baseURL, org, workspace, vmID, user, token, noOutputViaMQTT) {
-    return StartWASM2(hub, {
-        baseURL: baseURL,
-        org: org,
-        workspace: workspace,
-        vmID: vmID,
-        user: user,
-        token: token,
-        noOutputViaMQTT: noOutputViaMQTT,
-        machType: "WASM"
-    })
+function GetMachineWASMWorker() {
+    return workerify(MachineWASMWorker);
 }
 
-function StartWASM2(hub, config) {
-    console.log("GROOVEBOX_BUILD (machine-starter)", GROOVEBOX_BUILD);
-    return hub.newChannel().then(channel => {
-        let worker;
-        if (globalThis.ThisIsNode) {
-            worker = new Worker(new URL("./machine-wasm.mjs", import.meta.url));
-        } else {
-            // FIXME: Stopgap solution until we figure out a way to bundle groovebox correctly
-            let baseURL = globalThis.GROOVEBOX_URL_PREFIX || "/g";
-
-            // already fire off the download of the wasm code:
-            let code_url = new URL(`${baseURL}/machine-wasm-code.wasm`, window.location.href);
-            fetch(code_url, {cache:"default"}).then(resp => { return resp.arrayBuffer() });
-
-            let bundle = new URL(`${baseURL}/machine-wasm-core.js`, window.location.href);
-            worker = new Worker(bundle);
-        }
-        let localFFIs = {};
-        if (config.localFFIs) {
-            for (const [k, v] of Object.entries(config.localFFIs)) {
-                if (v.isRaw && (v.canFail || v.useJsonDecoder)) {
-                    console.error("A raw, local FFI must neither be failing nor use the JSON decoder: " + k);
-                } else {
-                    localFFIs[k] = {...v};
-                    delete localFFIs[k].run;
-                }
-            }
-        }
-        let config_msg =
-            { "_rmx_type": "msg_vm_configure",
-              "baseURL": config.baseURL,
-              "org": config.org,
-              "workspace": config.workspace,
-              "vmID": config.vmID,
-              "user": config.user,
-              "token": config.token,
-              "hub": channel.port,
-              "outputViaMQTT": !(config.noOutputViaMQTT),
-              "machType": config.machType,
-              "debugMask": initialMask,
-              "localFFIs": localFFIs,
-              "grooveboxUrlPrefix": globalThis.GROOVEBOX_URL_PREFIX,
-              "allowInsecureHttp": globalThis.GROOVEBOX_ALLOW_INSECURE_HTTP,
-              "interceptFFI": config.interceptFFI,
-              "interceptDynloadFFI": config.interceptDynloadFFI,
-            };
-        worker.postMessage(config_msg, [ channel.port ]);
-        terminate(() => worker.terminate());
-        globalThis.MixSetMask = (m => {
-            worker.postMessage({ "_rmx_type": "msg_vm_logMask",
-                                 "mask": m
-                               })
-        });
-        if (config.localFFIs)
-            setupLocalFFIs(hub, config);  // don't await!
-        return worker;
-    })
-}
-
-class Token {
-    constructor(data, enabled, extras) {
-        // data: must be an Uint8Array
-        // enabled: a number with the bitset of enabled operations
-        // extras: a map
-        if (!data instanceof Uint8Array)
-            throw new Error("Token: data must be an Uint8Array");
-        if (typeof(enabled) != "number")
-            throw new Error("Token: enabled_ops must be a number");
-        this.data = data;
-        this.enabled = enabled;
-        this.extras = extras;
-    }
-    encode() {
-        return new Map([
-            [ "_rmx_type", "token" ],
-            [ "data", this.data ],
-            [ "enabled", this.enabled ],
-            [ "extras", this.extras ]
-        ]);
-    }
-}
-
-function decode_Token(m) {
-    return new Token(m.get("data"), m.get("enabled"), m.get("extras"))
-}
-
-class Case {
-    constructor(name, arg) {
-        if (typeof(name) != "string")
-            throw new Error("Case: name must be a string");
-        this.name = name;
-        this.arg = arg;
-    }
-    encode() {
-        return new Map([
-            [ "_rmx_type", "case" ],
-            [ "name", this.name ],
-            [ "arg", this.arg ],
-        ]);
-    }
-}
-
-function decode_Case(m) {
-    return new Case(m.get("name"), m.get("arg"))
-}
-
-class Opaque {
-    constructor(buf) {
-        if (!(buf instanceof Uint32Array))
-            throw new Error("Opaque: must be a Uint32Array");
-        this.data = buf;
-    }
-    encode() {
-        return new Map([
-            [ "_rmx_type", "opaque" ],
-            [ "data", this.data ],
-        ]);
-    }
-}
-
-function decode_Opaque(m) {
-    return new Opaque(m.get("data"))
-}
-
-function encode(val) {
-    switch (typeof(val)) {
-    case "object":
-        if (val === null) return val;
-        if (Array.isArray(val)) {
-            // array: map the array elements
-            val = val.map(encode)
-        } else if (Object.getPrototypeOf(val) === Object.prototype) {
-            // simple object: map the object elements
-            val = Object.fromEntries(Object.entries(val).map(([ key, elem ]) => [ key, encode(elem) ]))
-        } else if (val.encode instanceof Function) {
-            // map after encode(): map the map elements
-            val = new Map(Array.from(val.encode().entries()).map(([ key, elem ]) => [ key, encode(elem) ]))
-        };
-        return val;
-    default:
-        return val;
-    }
-}
-
-function decode(val) {
-    switch (typeof(val)) {
-    case "object":
-        if (val === null) return val;
-        if (Array.isArray(val)) {
-            // array: map the array elements
-            val = val.map(decode)
-        } else if (Object.getPrototypeOf(val) === Object.prototype) {
-            // simple object: map the object elements
-            val = Object.fromEntries(Object.entries(val).map(([ key, elem ]) => [ key, decode(elem) ]))
-        } else if (val instanceof Map) {
-            val = new Map(Array.from(val.entries()).map(([ key, elem ]) => [ key, decode(elem) ]))
-            let ty = val.get("_rmx_type");
-            switch (ty) {
-            case "token":
-                return decode_Token(val);
-            case "case":
-                return decode_Case(val);
-            case "opaque":
-                return decode_Opaque(val);
-            default:
-                throw new Error("decode: bad _rmx_type: " + ty);
-            }
-        };
-        return val;
-    default:
-        return val;
-    }
-}
-
-async function setupLocalFFIs(hub, config) {
-    let channel = await hub.newChannel();
-    let comm = new FFIComm(hub, channel);
-    let localFFIs = config.localFFIs;
-    await channel.setLocalSubTopic("/local/ffi/call");
-    await channel.setLocalPubTopic("/local/ffi/return");
-    let sub = await channel.subscribe("/local/ffi/call");
-    while (true) {
-        let msg = await sub.next();
-        let name = msg.payload.name;
-        let call_id = msg.payload.call_id;
-        let args = decode(msg.payload.args);
-        let fun = localFFIs[name];
-        try {
-            if (!fun)
-                throw new Error("no such local FFI: " + name);
-            let connector =
-                { call_id: call_id,
-                  hub: hub,
-                  channel: channel,
-                  state: config.state,
-                };
-            let r = fun instanceof Function ? fun(connector, args) : fun.run(connector, args);
-            if (r instanceof Promise) {
-                await comm.later(call_id);
-                r.then(
-                    (value) => comm.returnOrFail(fun, call_id, value),
-                    (reason) => comm.error(call_id, reason.message, reason.stack));
-            } else {
-                await comm.returnOrFail(fun, call_id, r);
-            }
-        } catch (reason) {
-            await comm.error(call_id, reason.message, reason.stack);
-        }
-    }
-}
-
-class FFIComm {
-    constructor(hub, channel) {
-        this.hub = hub;
-        this.channel = channel;
-    }
-    later(call_id) {
-        let r_payload =
-            { call_id: call_id,
-            };
-        let response = this.hub.newLocalMessage("msg_ffi_later", "starter", r_payload);
-        return this.channel.publish("/local/ffi/return", response, false);
-    }
-    return_(fun, call_id, result) {
-        let r_payload =
-            { call_id: call_id,
-              value: fun.useJsonDecoder ? result : encode(result)
-            };
-        let response = this.hub.newLocalMessage("msg_ffi_return", "starter", r_payload);
-        return this.channel.publish("/local/ffi/return", response, false);
-    }
-    error(call_id, message, stack) {
-        let r_payload =
-            { call_id: call_id,
-              message: message,
-              stack: stack === undefined ? [] : stack,
-            };
-        let response = this.hub.newLocalMessage("msg_ffi_error", "starter", r_payload);
-        this.channel.publish("/local/ffi/return", response, false);
-    }
-    returnOrFail(fun, call_id, result) {
-        if (fun.canFail) {
-            var name, arg;
-            if (result instanceof Case) {
-                name = result.name;
-                arg = result.arg;
-            } else if (fun.useJsonDecoder && result._rmx_type && result._rmx_type.startsWith("{tag}case:")) {
-                name = result._rmx_type.substring(10);
-                arg = result._rmx_value;
-            } else {
-                return this.error(call_id, "expected result from local FFI that can fail");
-            }
-            switch (name) {
-            case "ok":
-                return this.return_(fun, call_id, arg);
-            case "error":
-                return this.error(call_id, arg);
-            default:
-                return this.error(call_id, "expected result from local FFI that can fail, found case value " + name);
-            }
-        } else {
-            return this.return_(fun, call_id, result);
-        }
-    }
-}
+globalThis.GetMachineWASMWorker = GetMachineWASMWorker;
 
-// use worker.terminate() to shut a worker down!
+export * from './common.js';
 
-export { CreateVMID, StartWASM, StartWASM2,
-         Token, Case, Opaque
-       }

package/node.js

@@ -14,8 +14,12 @@ global.Process = Process;
 // or the browser Worker to give the same interface). Webpack says ESM
 // is necessary but I'm not sure if this is true for either the main
 // script or the workers.
-import { MessageChannel, Worker } from "worker_threads";
-global.Worker = Worker;
+import { Worker } from "worker_threads";
+
+function GetMachineWASMWorker() {
+    return new Worker(new URL("./machine-wasm.mjs", import.meta.url));
+}
+global.GetMachineWASMWorker = GetMachineWASMWorker;
 
 import * as Hub from "@remix_labs/hub-client";
 
@@ -24,7 +28,7 @@ global.__filename = fileURLToPath(import.meta.url);
 
 import { CreateVMID, StartWASM, StartWASM2,
          Token, Case, Opaque
-       } from "./index.js";
+       } from "./common.js";
 
 export { CreateVMID, StartWASM, StartWASM2,
          Token, Case, Opaque, Hub

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remix_labs/machine-starter",
-  "version": "1.1890.0-dev",
+  "version": "1.1895.0-dev",
   "description": "start the groove",
   "main": "node.js",
   "browser": "index.js",
@@ -11,7 +11,7 @@
   "author": "Remixlabs staff",
   "license": "ISC",
   "dependencies": {
-    "@remix_labs/hub-client": "1.1890.0-dev",
+    "@remix_labs/hub-client": "1.1895.0-dev",
     "nanoid": "^3.1.12",
     "web-worker": "^1.2.0"
   },

package/webpack.config.js

@@ -6,12 +6,37 @@ const __dirname = path.dirname(__filename);
 
 export default [
     {
+        name: 'web',
+        entry: './index.js',
+        mode: "production",
+        output: {
+            filename: 'machine-starter.js',
+            path: path.resolve(__dirname, '../dist')
+        },
+        optimization: {
+            minimize: false
+        },
+        module: {
+            rules: [
+                {
+                    test: /\.worker.js$/,
+                    type: 'asset/source'
+                },
+                {
+                    // Needed for import.meta.url unless we bundle the web workers
+                    // themselves as separate entries in this bundle.
+                    test: /\.js$/,
+                    loader: '@open-wc/webpack-import-meta-loader'
+                }
+            ]
+        }
+    },
+    {
+        name: 'node',
         target: 'node',
         mode: "production",
         entry: {
             node: './start.mjs',
-            //'hub-worker': './hub-worker.js',
-            //'machine-wasm': './machine-wasm.js',
         },
         optimization: {
             minimize: false,
@@ -19,6 +44,8 @@ export default [
         module: {
             rules: [
                 {
+                    // Needed for import.meta.url unless we bundle the web workers
+                    // themselves as separate entries in this bundle.
                     test: /\.js$/,
                     loader: '@open-wc/webpack-import-meta-loader'
                 }
@@ -29,7 +56,7 @@ export default [
         },
         output: {
             chunkFormat: 'module',
-            filename: 'machine-starter.mjs',
+            filename: 'machine-starter.node.mjs',
             library : {
                 type: 'module',
             },