@remix_labs/machine-starter 1.1924.0-dev → 1.1929.0-dev

package/groovebox_build.js

@@ -1,2 +1,2 @@
-var GROOVEBOX_BUILD = "1924";
+var GROOVEBOX_BUILD = "1929";
 export { GROOVEBOX_BUILD }

package/index.js

@@ -1,15 +1,308 @@
-function workerify(code) {
-    let blob = new Blob([code], {type: 'application/javascript'});
-    return new Worker(URL.createObjectURL(blob));
+import { nanoid } from "nanoid";
+import { GROOVEBOX_BUILD } from "./groovebox_build.js";
+
+let terminate = function(f) {};
+if (globalThis.ThisIsNode) {
+    terminate = function(f) {
+        Process.on("exit", () => {
+            console.log("terminating machine worker");
+            f()
+        })
+    }
+};
+
+function CreateVMID() {
+    return nanoid()
+}
+
+let initialMask = 64 + 1024;   // default for web
+globalThis.MixSetMask = (m => {
+    initialMask = m;
+});
+
+function StartWASM(hub, baseURL, org, workspace, vmID, user, token, noOutputViaMQTT) {
+    return StartWASM2(hub, {
+        baseURL: baseURL,
+        org: org,
+        workspace: workspace,
+        vmID: vmID,
+        user: user,
+        token: token,
+        noOutputViaMQTT: noOutputViaMQTT,
+        machType: "WASM"
+    })
+}
+
+function StartWASM2(hub, config) {
+    console.log("GROOVEBOX_BUILD (machine-starter)", GROOVEBOX_BUILD);
+    return hub.newChannel().then(channel => {
+        let bundle;
+        if (globalThis.ThisIsNode) {
+            bundle = new URL("../../machine-wasm/dist/node-bundle.js", import.meta.url);
+        } else if (globalThis.GROOVEBOX_URL_PREFIX) {
+            // FIXME: Stopgap solution until we figure out a way to bundle groovebox correctly
+            // already fire off the download of the wasm code:
+            let code_url = new URL(`${globalThis.GROOVEBOX_URL_PREFIX}/machine-wasm-code.wasm`, window.location.href);
+            fetch(code_url, {cache:"default"}).then(resp => { return resp.arrayBuffer() });
+            bundle = new URL(`${globalThis.GROOVEBOX_URL_PREFIX}/machine-wasm-core.js`, window.location.href);
+        } else {
+            // already fire off the download of the wasm code:
+            let code_url = new URL("/g/machine-wasm-code.wasm", window.location.href);
+            fetch(code_url, {cache:"default"}).then(resp => { return resp.arrayBuffer() });
+            bundle = new URL("/g/machine-wasm-core.js", window.location.href);
+        }
+        let localFFIs = {};
+        if (config.localFFIs) {
+            for (const [k, v] of Object.entries(config.localFFIs)) {
+                if (v.isRaw && (v.canFail || v.useJsonDecoder)) {
+                    console.error("A raw, local FFI must neither be failing nor use the JSON decoder: " + k);
+                } else {
+                    localFFIs[k] = {...v};
+                    delete localFFIs[k].run;
+                }
+            }
+        }
+        let worker = new Worker(bundle);
+        let config_msg =
+            { "_rmx_type": "msg_vm_configure",
+              "baseURL": config.baseURL,
+              "org": config.org,
+              "workspace": config.workspace,
+              "vmID": config.vmID,
+              "user": config.user,
+              "token": config.token,
+              "hub": channel.port,
+              "outputViaMQTT": !(config.noOutputViaMQTT),
+              "machType": config.machType,
+              "debugMask": initialMask,
+              "localFFIs": localFFIs,
+              "grooveboxUrlPrefix": globalThis.GROOVEBOX_URL_PREFIX,
+              "allowInsecureHttp": globalThis.GROOVEBOX_ALLOW_INSECURE_HTTP,
+              "interceptFFI": config.interceptFFI,
+              "interceptDynloadFFI": config.interceptDynloadFFI,
+            };
+        worker.postMessage(config_msg, [ channel.port ]);
+        terminate(() => worker.terminate());
+        globalThis.MixSetMask = (m => {
+            worker.postMessage({ "_rmx_type": "msg_vm_logMask",
+                                 "mask": m
+                               })
+        });
+        if (config.localFFIs)
+            setupLocalFFIs(hub, config);  // don't await!
+        return worker;
+    })
+}
+
+class Token {
+    constructor(data, enabled, extras) {
+        // data: must be an Uint8Array
+        // enabled: a number with the bitset of enabled operations
+        // extras: a map
+        if (!data instanceof Uint8Array)
+            throw new Error("Token: data must be an Uint8Array");
+        if (typeof(enabled) != "number")
+            throw new Error("Token: enabled_ops must be a number");
+        this.data = data;
+        this.enabled = enabled;
+        this.extras = extras;
+    }
+    encode() {
+        return new Map([
+            [ "_rmx_type", "token" ],
+            [ "data", this.data ],
+            [ "enabled", this.enabled ],
+            [ "extras", this.extras ]
+        ]);
+    }
 }
 
-import MachineWASMWorker from './machine-wasm.worker.js';
+function decode_Token(m) {
+    return new Token(m.get("data"), m.get("enabled"), m.get("extras"))
+}
+
+class Case {
+    constructor(name, arg) {
+        if (typeof(name) != "string")
+            throw new Error("Case: name must be a string");
+        this.name = name;
+        this.arg = arg;
+    }
+    encode() {
+        return new Map([
+            [ "_rmx_type", "case" ],
+            [ "name", this.name ],
+            [ "arg", this.arg ],
+        ]);
+    }
+}
 
-function GetMachineWASMWorker() {
-    return workerify(MachineWASMWorker);
+function decode_Case(m) {
+    return new Case(m.get("name"), m.get("arg"))
 }
 
-globalThis.GetMachineWASMWorker = GetMachineWASMWorker;
+class Opaque {
+    constructor(buf) {
+        if (!(buf instanceof Uint32Array))
+            throw new Error("Opaque: must be a Uint32Array");
+        this.data = buf;
+    }
+    encode() {
+        return new Map([
+            [ "_rmx_type", "opaque" ],
+            [ "data", this.data ],
+        ]);
+    }
+}
+
+function decode_Opaque(m) {
+    return new Opaque(m.get("data"))
+}
+
+function encode(val) {
+    switch (typeof(val)) {
+    case "object":
+        if (val === null) return val;
+        if (Array.isArray(val)) {
+            // array: map the array elements
+            val = val.map(encode)
+        } else if (Object.getPrototypeOf(val) === Object.prototype) {
+            // simple object: map the object elements
+            val = Object.fromEntries(Object.entries(val).map(([ key, elem ]) => [ key, encode(elem) ]))
+        } else if (val.encode instanceof Function) {
+            // map after encode(): map the map elements
+            val = new Map(Array.from(val.encode().entries()).map(([ key, elem ]) => [ key, encode(elem) ]))
+        };
+        return val;
+    default:
+        return val;
+    }
+}
+
+function decode(val) {
+    switch (typeof(val)) {
+    case "object":
+        if (val === null) return val;
+        if (Array.isArray(val)) {
+            // array: map the array elements
+            val = val.map(decode)
+        } else if (Object.getPrototypeOf(val) === Object.prototype) {
+            // simple object: map the object elements
+            val = Object.fromEntries(Object.entries(val).map(([ key, elem ]) => [ key, decode(elem) ]))
+        } else if (val instanceof Map) {
+            val = new Map(Array.from(val.entries()).map(([ key, elem ]) => [ key, decode(elem) ]))
+            let ty = val.get("_rmx_type");
+            switch (ty) {
+            case "token":
+                return decode_Token(val);
+            case "case":
+                return decode_Case(val);
+            case "opaque":
+                return decode_Opaque(val);
+            default:
+                throw new Error("decode: bad _rmx_type: " + ty);
+            }
+        };
+        return val;
+    default:
+        return val;
+    }
+}
+
+async function setupLocalFFIs(hub, config) {
+    let channel = await hub.newChannel();
+    let comm = new FFIComm(hub, channel);
+    let localFFIs = config.localFFIs;
+    await channel.setLocalSubTopic("/local/ffi/call");
+    await channel.setLocalPubTopic("/local/ffi/return");
+    let sub = await channel.subscribe("/local/ffi/call");
+    while (true) {
+        let msg = await sub.next();
+        let name = msg.payload.name;
+        let call_id = msg.payload.call_id;
+        let args = decode(msg.payload.args);
+        let fun = localFFIs[name];
+        try {
+            if (!fun)
+                throw new Error("no such local FFI: " + name);
+            let connector =
+                { call_id: call_id,
+                  hub: hub,
+                  channel: channel,
+                  state: config.state,
+                };
+            let r = fun instanceof Function ? fun(connector, args) : fun.run(connector, args);
+            if (r instanceof Promise) {
+                await comm.later(call_id);
+                r.then(
+                    (value) => comm.returnOrFail(fun, call_id, value),
+                    (reason) => comm.error(call_id, reason.message, reason.stack));
+            } else {
+                await comm.returnOrFail(fun, call_id, r);
+            }
+        } catch (reason) {
+            await comm.error(call_id, reason.message, reason.stack);
+        }
+    }
+}
+
+class FFIComm {
+    constructor(hub, channel) {
+        this.hub = hub;
+        this.channel = channel;
+    }
+    later(call_id) {
+        let r_payload =
+            { call_id: call_id,
+            };
+        let response = this.hub.newLocalMessage("msg_ffi_later", "starter", r_payload);
+        return this.channel.publish("/local/ffi/return", response, false);
+    }
+    return_(fun, call_id, result) {
+        let r_payload =
+            { call_id: call_id,
+              value: fun.useJsonDecoder ? result : encode(result)
+            };
+        let response = this.hub.newLocalMessage("msg_ffi_return", "starter", r_payload);
+        return this.channel.publish("/local/ffi/return", response, false);
+    }
+    error(call_id, message, stack) {
+        let r_payload =
+            { call_id: call_id,
+              message: message,
+              stack: stack === undefined ? [] : stack,
+            };
+        let response = this.hub.newLocalMessage("msg_ffi_error", "starter", r_payload);
+        this.channel.publish("/local/ffi/return", response, false);
+    }
+    returnOrFail(fun, call_id, result) {
+        if (fun.canFail) {
+            var name, arg;
+            if (result instanceof Case) {
+                name = result.name;
+                arg = result.arg;
+            } else if (fun.useJsonDecoder && result._rmx_type && result._rmx_type.startsWith("{tag}case:")) {
+                name = result._rmx_type.substring(10);
+                arg = result._rmx_value;
+            } else {
+                return this.error(call_id, "expected result from local FFI that can fail");
+            }
+            switch (name) {
+            case "ok":
+                return this.return_(fun, call_id, arg);
+            case "error":
+                return this.error(call_id, arg);
+            default:
+                return this.error(call_id, "expected result from local FFI that can fail, found case value " + name);
+            }
+        } else {
+            return this.return_(fun, call_id, result);
+        }
+    }
+}
 
-export * from './common.js';
+// use worker.terminate() to shut a worker down!
 
+export { CreateVMID, StartWASM, StartWASM2,
+         Token, Case, Opaque
+       }

package/node.js

@@ -1,31 +1,7 @@
+import Worker from "web-worker";
+global.Worker = Worker;
+import WT from "worker_threads";
+global.MessageChannel = WT.MessageChannel;
 import Process from "process";
 global.Process = Process;
-
-// NB: The only way I've made this work in node+webpack is if:
-//
-// * we use Worker from worker_threads directly
-// * we build each worker's module with webpack as an ESM .mjs bundle
-// * we put all the bundles in one directory
-// * we load the worker with a URL('./foo.mjs')
-//
-// Other ways to package/load might work, but definitely seems like
-// using worker_threads.Worker directly is needed (rather than through,
-// say, the web-worker packages that wraps either worker_threads.Worker
-// or the browser Worker to give the same interface). Webpack says ESM
-// is necessary but I'm not sure if this is true for either the main
-// script or the workers.
-import { Worker } from "worker_threads";
-
-function GetMachineWASMWorker() {
-    return new Worker(new URL("./machine-wasm.mjs", import.meta.url));
-}
-global.GetMachineWASMWorker = GetMachineWASMWorker;
-
-// Can't just `export *` because we need hub-client below in common.js
-import * as Hub from "@remix_labs/hub-client";
-export { Hub };
-
-import { fileURLToPath } from 'url';
-global.__filename = fileURLToPath(import.meta.url);
-
-export * from "./common.js";
+export * from "./index.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remix_labs/machine-starter",
-  "version": "1.1924.0-dev",
+  "version": "1.1929.0-dev",
   "description": "start the groove",
   "main": "node.js",
   "browser": "index.js",
@@ -11,7 +11,7 @@
   "author": "Remixlabs staff",
   "license": "ISC",
   "dependencies": {
-    "@remix_labs/hub-client": "1.1924.0-dev",
+    "@remix_labs/hub-client": "1.1929.0-dev",
     "nanoid": "^3.1.12",
     "web-worker": "^1.2.0"
   },

package/start.js

@@ -1,6 +1,6 @@
 import Process from "process";
-import { StartWASM2, Hub } from "./node.js";
-
+import { StartWASM2 } from "./node.js";
+import * as Hub from "@remix_labs/hub-client";
 if (Process.stdout._handle)
     Process.stdout._handle.setBlocking(true);
 if (Process.stderr._handle)

package/common.js

@@ -1,292 +0,0 @@
-import { nanoid } from "nanoid";
-import { GROOVEBOX_BUILD } from "./groovebox_build.js";
-
-let terminate = function(f) {};
-if (globalThis.ThisIsNode) {
-    terminate = function(f) {
-        Process.on("exit", () => {
-            console.log("terminating machine worker");
-            f()
-        })
-    }
-};
-
-function CreateVMID() {
-    return nanoid()
-}
-
-let initialMask = 64 + 1024;   // default for web
-globalThis.MixSetMask = (m => {
-    initialMask = m;
-});
-
-function StartWASM(hub, baseURL, org, workspace, vmID, user, token, noOutputViaMQTT) {
-    return StartWASM2(hub, {
-        baseURL: baseURL,
-        org: org,
-        workspace: workspace,
-        token: token,
-        noOutputViaMQTT: noOutputViaMQTT,
-        machType: "WASM"
-    })
-}
-
-function StartWASM2(hub, config) {
-    console.log("GROOVEBOX_BUILD (machine-starter)", GROOVEBOX_BUILD);
-    return hub.newChannel().then(channel => {
-        // injected by either index.js or node.js
-        let worker = globalThis.GetMachineWASMWorker();
-        let localFFIs = {};
-        if (config.localFFIs) {
-            for (const [k, v] of Object.entries(config.localFFIs)) {
-                if (v.isRaw && (v.canFail || v.useJsonDecoder)) {
-                    console.error("A raw, local FFI must neither be failing nor use the JSON decoder: " + k);
-                } else {
-                    localFFIs[k] = {...v};
-                    delete localFFIs[k].run;
-                }
-            }
-        }
-        let config_msg =
-            { "_rmx_type": "msg_vm_configure",
-              "baseURL": config.baseURL,
-              "org": config.org,
-              "workspace": config.workspace,
-              "vmID": config.vmID,
-              "user": config.user,
-              "token": config.token,
-              "hub": channel.port,
-              "outputViaMQTT": !(config.noOutputViaMQTT),
-              "machType": config.machType,
-              "debugMask": initialMask,
-              "localFFIs": localFFIs,
-              "grooveboxUrlPrefix": globalThis.GROOVEBOX_URL_PREFIX,
-              "allowInsecureHttp": globalThis.GROOVEBOX_ALLOW_INSECURE_HTTP,
-              "interceptFFI": config.interceptFFI,
-              "interceptDynloadFFI": config.interceptDynloadFFI,
-            };
-        worker.postMessage(config_msg, [ channel.port ]);
-        terminate(() => worker.terminate());
-        globalThis.MixSetMask = (m => {
-            worker.postMessage({ "_rmx_type": "msg_vm_logMask",
-                                 "mask": m
-                               })
-        });
-        if (config.localFFIs)
-            setupLocalFFIs(hub, config);  // don't await!
-        return worker;
-    })
-}
-
-class Token {
-    constructor(data, enabled, extras) {
-        // data: must be an Uint8Array
-        // enabled: a number with the bitset of enabled operations
-        // extras: a map
-        if (!data instanceof Uint8Array)
-            throw new Error("Token: data must be an Uint8Array");
-        if (typeof(enabled) != "number")
-            throw new Error("Token: enabled_ops must be a number");
-        this.data = data;
-        this.enabled = enabled;
-        this.extras = extras;
-    }
-    encode() {
-        return new Map([
-            [ "_rmx_type", "token" ],
-            [ "data", this.data ],
-            [ "enabled", this.enabled ],
-            [ "extras", this.extras ]
-        ]);
-    }
-}
-
-function decode_Token(m) {
-    return new Token(m.get("data"), m.get("enabled"), m.get("extras"))
-}
-
-class Case {
-    constructor(name, arg) {
-        if (typeof(name) != "string")
-            throw new Error("Case: name must be a string");
-        this.name = name;
-        this.arg = arg;
-    }
-    encode() {
-        return new Map([
-            [ "_rmx_type", "case" ],
-            [ "name", this.name ],
-            [ "arg", this.arg ],
-        ]);
-    }
-}
-
-function decode_Case(m) {
-    return new Case(m.get("name"), m.get("arg"))
-}
-
-class Opaque {
-    constructor(buf) {
-        if (!(buf instanceof Uint32Array))
-            throw new Error("Opaque: must be a Uint32Array");
-        this.data = buf;
-    }
-    encode() {
-        return new Map([
-            [ "_rmx_type", "opaque" ],
-            [ "data", this.data ],
-        ]);
-    }
-}
-
-function decode_Opaque(m) {
-    return new Opaque(m.get("data"))
-}
-
-function encode(val) {
-    switch (typeof(val)) {
-    case "object":
-        if (val === null) return val;
-        if (Array.isArray(val)) {
-            // array: map the array elements
-            val = val.map(encode)
-        } else if (Object.getPrototypeOf(val) === Object.prototype) {
-            // simple object: map the object elements
-            val = Object.fromEntries(Object.entries(val).map(([ key, elem ]) => [ key, encode(elem) ]))
-        } else if (val.encode instanceof Function) {
-            // map after encode(): map the map elements
-            val = new Map(Array.from(val.encode().entries()).map(([ key, elem ]) => [ key, encode(elem) ]))
-        };
-        return val;
-    default:
-        return val;
-    }
-}
-
-function decode(val) {
-    switch (typeof(val)) {
-    case "object":
-        if (val === null) return val;
-        if (Array.isArray(val)) {
-            // array: map the array elements
-            val = val.map(decode)
-        } else if (Object.getPrototypeOf(val) === Object.prototype) {
-            // simple object: map the object elements
-            val = Object.fromEntries(Object.entries(val).map(([ key, elem ]) => [ key, decode(elem) ]))
-        } else if (val instanceof Map) {
-            val = new Map(Array.from(val.entries()).map(([ key, elem ]) => [ key, decode(elem) ]))
-            let ty = val.get("_rmx_type");
-            switch (ty) {
-            case "token":
-                return decode_Token(val);
-            case "case":
-                return decode_Case(val);
-            case "opaque":
-                return decode_Opaque(val);
-            default:
-                throw new Error("decode: bad _rmx_type: " + ty);
-            }
-        };
-        return val;
-    default:
-        return val;
-    }
-}
-
-async function setupLocalFFIs(hub, config) {
-    let channel = await hub.newChannel();
-    let comm = new FFIComm(hub, channel);
-    let localFFIs = config.localFFIs;
-    await channel.setLocalSubTopic("/local/ffi/call");
-    await channel.setLocalPubTopic("/local/ffi/return");
-    let sub = await channel.subscribe("/local/ffi/call");
-    while (true) {
-        let msg = await sub.next();
-        let name = msg.payload.name;
-        let call_id = msg.payload.call_id;
-        let args = decode(msg.payload.args);
-        let fun = localFFIs[name];
-        try {
-            if (!fun)
-                throw new Error("no such local FFI: " + name);
-            let connector =
-                { call_id: call_id,
-                  hub: hub,
-                  channel: channel,
-                  state: config.state,
-                };
-            let r = fun instanceof Function ? fun(connector, args) : fun.run(connector, args);
-            if (r instanceof Promise) {
-                await comm.later(call_id);
-                r.then(
-                    (value) => comm.returnOrFail(fun, call_id, value),
-                    (reason) => comm.error(call_id, reason.message, reason.stack));
-            } else {
-                await comm.returnOrFail(fun, call_id, r);
-            }
-        } catch (reason) {
-            await comm.error(call_id, reason.message, reason.stack);
-        }
-    }
-}
-
-class FFIComm {
-    constructor(hub, channel) {
-        this.hub = hub;
-        this.channel = channel;
-    }
-    later(call_id) {
-        let r_payload =
-            { call_id: call_id,
-            };
-        let response = this.hub.newLocalMessage("msg_ffi_later", "starter", r_payload);
-        return this.channel.publish("/local/ffi/return", response, false);
-    }
-    return_(fun, call_id, result) {
-        let r_payload =
-            { call_id: call_id,
-              value: fun.useJsonDecoder ? result : encode(result)
-            };
-        let response = this.hub.newLocalMessage("msg_ffi_return", "starter", r_payload);
-        return this.channel.publish("/local/ffi/return", response, false);
-    }
-    error(call_id, message, stack) {
-        let r_payload =
-            { call_id: call_id,
-              message: message,
-              stack: stack === undefined ? [] : stack,
-            };
-        let response = this.hub.newLocalMessage("msg_ffi_error", "starter", r_payload);
-        this.channel.publish("/local/ffi/return", response, false);
-    }
-    returnOrFail(fun, call_id, result) {
-        if (fun.canFail) {
-            var name, arg;
-            if (result instanceof Case) {
-                name = result.name;
-                arg = result.arg;
-            } else if (fun.useJsonDecoder && result._rmx_type && result._rmx_type.startsWith("{tag}case:")) {
-                name = result._rmx_type.substring(10);
-                arg = result._rmx_value;
-            } else {
-                return this.error(call_id, "expected result from local FFI that can fail");
-            }
-            switch (name) {
-            case "ok":
-                return this.return_(fun, call_id, arg);
-            case "error":
-                return this.error(call_id, arg);
-            default:
-                return this.error(call_id, "expected result from local FFI that can fail, found case value " + name);
-            }
-        } else {
-            return this.return_(fun, call_id, result);
-        }
-    }
-}
-
-// use worker.terminate() to shut a worker down!
-
-export { CreateVMID, StartWASM, StartWASM2,
-         Token, Case, Opaque
-       }