@remix_labs/machine-starter 1.1872.0-dev → 1.1878.0-dev

package/groovebox_build.js

@@ -1,2 +1,2 @@
-var GROOVEBOX_BUILD = "1872";
+var GROOVEBOX_BUILD = "1878";
 export { GROOVEBOX_BUILD }

package/index.js

@@ -1,34 +1,14 @@
 import { nanoid } from "nanoid";
 import { GROOVEBOX_BUILD } from "./groovebox_build.js";
 
-let workers = {};
-let parents = {};
-
-function terminateAll() {
-    for (let [vmID, worker] of Object.entries(workers)) {
-        worker.terminate();
-    };
-    workers = {};
-    parents = {};
-}
-
-function terminateVM(vmID) {
-    let worker = workers[vmID];
-    if (worker) {
-        delete workers[vmID];
-        delete parents[vmID];
-        worker.terminate();
-        for (let [subID, parentID] of Object.entries(parents)) {
-            terminateVM(subID);
-        };
-    }
-}
-
+let terminate = function(f) {};
 if (globalThis.ThisIsNode) {
-    Process.on("exit", () => {
-        console.log("terminating machine workers");
-        terminateAll()
-    })
+    terminate = function(f) {
+        Process.on("exit", () => {
+            console.log("terminating machine worker");
+            f()
+        })
+    }
 };
 
 function CreateVMID() {
@@ -36,14 +16,8 @@ function CreateVMID() {
 }
 
 let initialMask = 64 + 1024;   // default for web
-
 globalThis.MixSetMask = (m => {
     initialMask = m;
-    for (let [vmID, worker] of Object.entries(workers)) {
-        worker.postMessage({ "_rmx_type": "msg_vm_logMask",
-                             "mask": m
-                           })
-    }
 });
 
 function StartWASM(hub, baseURL, org, workspace, vmID, user, token, noOutputViaMQTT) {
@@ -62,44 +36,41 @@ function StartWASM(hub, baseURL, org, workspace, vmID, user, token, noOutputViaM
 function StartWASM2(hub, config) {
     console.log("GROOVEBOX_BUILD (machine-starter)", GROOVEBOX_BUILD);
     return hub.newChannel().then(channel => {
-        let bundle;
+        let worker;
         if (globalThis.ThisIsNode) {
-            bundle = new URL("../../machine-wasm/dist/node-bundle.js", import.meta.url);
-        } else if (globalThis.GROOVEBOX_URL_PREFIX) {
-            // FIXME: Stopgap solution until we figure out a way to bundle groovebox correctly
-            // already fire off the download of the wasm code:
-            let code_url = new URL(`${globalThis.GROOVEBOX_URL_PREFIX}/machine-wasm-code.wasm`, window.location.href);
-            fetch(code_url, {cache:"default"}).then(resp => { return resp.arrayBuffer() });
-            bundle = new URL(`${globalThis.GROOVEBOX_URL_PREFIX}/machine-wasm-core.js`, window.location.href);
+            console.log("loading node bundle")
+            //bundle = new URL("file:///Users/verm/remix/harmony/components/groovebox/machine-wasm/dist/node-bundle.js");
+            worker = globalThis.getMachineWasmWorker();
         } else {
-            // already fire off the download of the wasm code:
-            let code_url = new URL("/g/machine-wasm-code.wasm", window.location.href);
-            fetch(code_url, {cache:"default"}).then(resp => { return resp.arrayBuffer() });
-            bundle = new URL("/g/machine-wasm-core.js", window.location.href);
+            console.log("non-node bundle");
+            let bundle;
+            if (globalThis.GROOVEBOX_URL_PREFIX) {
+                // FIXME: Stopgap solution until we figure out a way to bundle groovebox correctly
+                // already fire off the download of the wasm code:
+                let code_url = new URL(`${globalThis.GROOVEBOX_URL_PREFIX}/machine-wasm-code.wasm`, window.location.href);
+                fetch(code_url, {cache:"default"}).then(resp => { return resp.arrayBuffer() });
+                bundle = new URL(`${globalThis.GROOVEBOX_URL_PREFIX}/machine-wasm.js`, window.location.href);
+            } else {
+                // already fire off the download of the wasm code:
+                let code_url = new URL("/g/machine-wasm-code.wasm", window.location.href);
+                fetch(code_url, {cache:"default"}).then(resp => { return resp.arrayBuffer() });
+                bundle = new URL("/g/machine-wasm.js", window.location.href);
+            }
+            worker = new Worker(bundle);
         }
-        let localFFIkind = {};
-        let localFFIfuns = {};
+        let localFFIs = {};
         if (config.localFFIs) {
             for (const [k, v] of Object.entries(config.localFFIs)) {
-                localFFIfuns[k] = {...v};
                 if (v.isRaw && (v.canFail || v.useJsonDecoder)) {
                     console.error("A raw, local FFI must neither be failing nor use the JSON decoder: " + k);
                 } else {
-                    localFFIkind[k] = {...v};
-                    delete localFFIkind[k].run;
+                    localFFIs[k] = {...v};
+                    delete localFFIs[k].run;
                 }
-            };
-            let vm_kind = { isRaw:false, canFail:false, useJsonDecoder:false};
-            localFFIkind["$vm_start"] = vm_kind;
-            localFFIfuns["$vm_start"] = ffi_vm_start;
-            localFFIkind["$vm_stop"] = vm_kind;
-            localFFIfuns["$vm_stop"] = ffi_vm_stop;
+            }
         }
-        let worker = new Worker(bundle);
-        workers[config.vmID] = worker;
         let config_msg =
             { "_rmx_type": "msg_vm_configure",
-              "workerURL": bundle.toString(),
               "baseURL": config.baseURL,
               "org": config.org,
               "workspace": config.workspace,
@@ -110,44 +81,25 @@ function StartWASM2(hub, config) {
               "outputViaMQTT": !(config.noOutputViaMQTT),
               "machType": config.machType,
               "debugMask": initialMask,
-              "localFFIs": localFFIkind,
+              "localFFIs": localFFIs,
               "grooveboxUrlPrefix": globalThis.GROOVEBOX_URL_PREFIX,
               "allowInsecureHttp": globalThis.GROOVEBOX_ALLOW_INSECURE_HTTP,
               "interceptFFI": config.interceptFFI,
               "interceptDynloadFFI": config.interceptDynloadFFI,
             };
         worker.postMessage(config_msg, [ channel.port ]);
+        terminate(() => worker.terminate());
+        globalThis.MixSetMask = (m => {
+            worker.postMessage({ "_rmx_type": "msg_vm_logMask",
+                                 "mask": m
+                               })
+        });
         if (config.localFFIs)
-            setupLocalFFIs(hub, config, localFFIfuns);  // don't await!
+            setupLocalFFIs(hub, config);  // don't await!
         return worker;
     })
 }
 
-async function ffi_vm_start(conn, args) {
-    let vm_id = "embedded." + nanoid();
-    let config = {};
-    for (const [k, v] of Object.entries(conn.config)) {
-        config[k] = v;
-    }
-    config["vmID"] = vm_id;
-    parents[vm_id] = conn.config.vmID;
-    let arg_config = args[0];
-    if (arg_config.interceptFFI !== undefined)
-        config["interceptFFI"] = arg_config.interceptFFI;
-    if (arg_config.interceptDynloadFFI !== undefined)
-        config["interceptDynloadFFI"] = arg_config.interceptDynloadFFI;
-    let worker = await StartWASM2(conn.hub, config);
-    return vm_id;
-}
-
-async function ffi_vm_stop(conn, args) {
-    let vm_id = args[0];
-    let worker = workers[vm_id];
-    if (!worker) throw new Error("no such VM: " + vm_id);
-    terminateVM(vm_id);
-    return null;
-}
-
 class Token {
     constructor(data, enabled, extras) {
         // data: must be an Uint8Array
@@ -263,13 +215,13 @@ function decode(val) {
     }
 }
 
-async function setupLocalFFIs(hub, config, localFFIs) {
-    let vmID = config.vmID;
+async function setupLocalFFIs(hub, config) {
     let channel = await hub.newChannel();
-    let comm = new FFIComm(vmID, hub, channel);
-    await channel.setLocalSubTopic("/local/" + vmID);
-    await channel.setLocalPubTopic("/local/" + vmID);
-    let sub = await channel.subscribe("/local/" + vmID + "/ffi/call");
+    let comm = new FFIComm(hub, channel);
+    let localFFIs = config.localFFIs;
+    await channel.setLocalSubTopic("/local/ffi/call");
+    await channel.setLocalPubTopic("/local/ffi/return");
+    let sub = await channel.subscribe("/local/ffi/call");
     while (true) {
         let msg = await sub.next();
         let name = msg.payload.name;
@@ -283,15 +235,14 @@ async function setupLocalFFIs(hub, config, localFFIs) {
                 { call_id: call_id,
                   hub: hub,
                   channel: channel,
-                  config: config,
                   state: config.state,
                 };
             let r = fun instanceof Function ? fun(connector, args) : fun.run(connector, args);
             if (r instanceof Promise) {
-                r.catch(_ => null);  // prevent "unhandled rejection"
                 await comm.later(call_id);
-                let value = await r;
-                await comm.returnOrFail(fun, call_id, value);
+                r.then(
+                    (value) => comm.returnOrFail(fun, call_id, value),
+                    (reason) => comm.error(call_id, reason.message, reason.stack));
             } else {
                 await comm.returnOrFail(fun, call_id, r);
             }
@@ -302,8 +253,7 @@ async function setupLocalFFIs(hub, config, localFFIs) {
 }
 
 class FFIComm {
-    constructor(vmID, hub, channel) {
-        this.vmID = vmID;
+    constructor(hub, channel) {
         this.hub = hub;
         this.channel = channel;
     }
@@ -312,7 +262,7 @@ class FFIComm {
             { call_id: call_id,
             };
         let response = this.hub.newLocalMessage("msg_ffi_later", "starter", r_payload);
-        return this.channel.publish("/local/" + this.vmID + "/ffi/return", response, false);
+        return this.channel.publish("/local/ffi/return", response, false);
     }
     return_(fun, call_id, result) {
         let r_payload =
@@ -320,7 +270,7 @@ class FFIComm {
               value: fun.useJsonDecoder ? result : encode(result)
             };
         let response = this.hub.newLocalMessage("msg_ffi_return", "starter", r_payload);
-        return this.channel.publish("/local/" + this.vmID + "/ffi/return", response, false);
+        return this.channel.publish("/local/ffi/return", response, false);
     }
     error(call_id, message, stack) {
         let r_payload =
@@ -329,7 +279,7 @@ class FFIComm {
               stack: stack === undefined ? [] : stack,
             };
         let response = this.hub.newLocalMessage("msg_ffi_error", "starter", r_payload);
-        this.channel.publish("/local/" + this.vmID + "/ffi/return", response, false);
+        this.channel.publish("/local/ffi/return", response, false);
     }
     returnOrFail(fun, call_id, result) {
         if (fun.canFail) {
@@ -360,6 +310,5 @@ class FFIComm {
 // use worker.terminate() to shut a worker down!
 
 export { CreateVMID, StartWASM, StartWASM2,
-         terminateAll, terminateVM,
          Token, Case, Opaque
        }

package/node.js

@@ -1,7 +1,49 @@
-import Worker from "web-worker";
-global.Worker = Worker;
-import WT from "worker_threads";
-global.MessageChannel = WT.MessageChannel;
+//import Worker from "web-worker";
+//global.Worker = Worker;
+import { MessageChannel, Worker } from "worker_threads";
+global.MessageChannel = MessageChannel;
 import Process from "process";
 global.Process = Process;
-export * from "./index.js";
+
+// NB: The only way I've made this work in node+webpack is if:
+//
+// * we use Worker from worker_threads directly
+// * we build each worker's module with webpack as an ESM .mjs bundle
+// * we put all the bundles in one directory
+// * we load the worker as below with a URL('./foo.mjs')
+//
+// Other ways to package/load might work, but definitely seems like
+// using worker_threads.Worker directly is needed. Webpack says ESM
+// is necessary but I'm not sure if this is true for either the main
+// script or the workers.
+global.getHubWorker = () => {
+    console.log(`getting hub-worker worker`);
+    return new Worker(new URL("./hub-worker.mjs", import.meta.url))
+}
+
+global.getMachineWasmWorker = () => {
+    console.log("getting machine-wasm worker");
+    return new Worker(new URL("./machine-wasm.mjs", import.meta.url))
+}
+
+import * as Hub from "@remix_labs/hub-client";
+
+import { fileURLToPath } from 'url';
+global.__filename = fileURLToPath(import.meta.url);
+
+import { CreateVMID, StartWASM, StartWASM2,
+         Token, Case, Opaque
+       } from "./index.js";
+
+export { CreateVMID, StartWASM, StartWASM2,
+         Token, Case, Opaque, Hub
+       }
+
+global.CreateVMID = CreateVMID;
+global.StartWASM2 = StartWASM2;
+global.StartWASM = StartWASM;
+global.Token = Token;
+global.Case = Case;
+global.Opaque = Opaque;
+global.Hub = Hub;
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remix_labs/machine-starter",
-  "version": "1.1872.0-dev",
+  "version": "1.1878.0-dev",
   "description": "start the groove",
   "main": "node.js",
   "browser": "index.js",
@@ -11,7 +11,7 @@
   "author": "Remixlabs staff",
   "license": "ISC",
   "dependencies": {
-    "@remix_labs/hub-client": "1.1872.0-dev",
+    "@remix_labs/hub-client": "1.1878.0-dev",
     "nanoid": "^3.1.12",
     "web-worker": "^1.2.0"
   },

package/{start.js → start.mjs}

@@ -1,6 +1,14 @@
+//import { fileURLToPath } from 'url';
+//const __filename = fileURLToPath(import.meta.url);
+
 import Process from "process";
-import { StartWASM2 } from "./node.js";
-import * as Hub from "@remix_labs/hub-client";
+import * as Starter from "./node.mjs";
+//import { StartWASM2 } from "../dist/node.js";
+//import * as Hub from "@remix_labs/hub-client";
+
+StartWASM2 = Starter.StartWASM2;
+Hub = Starter.Hub;
+
 if (Process.stdout._handle)
     Process.stdout._handle.setBlocking(true);
 if (Process.stderr._handle)
@@ -45,8 +53,7 @@ w.configure({wsURL: mqttURL, user:mqttUser, token:mqttToken}).then(resp => {
             user: mqttUser,
             token: mqttToken,
             noOutputViaMQTT: noOutputViaMQTT,
-            machType: variant,
-            localFFIs: {},
+            machType: variant
         });
         break
     default:

package/webpack.config.js

@@ -0,0 +1,39 @@
+import path from 'path';
+
+import {fileURLToPath} from 'url'
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+export default [
+    {
+        target: 'node',
+        mode: "production",
+        entry: {
+            node: './node.js',
+            //'hub-worker': './hub-worker.js',
+            //'machine-wasm': './machine-wasm.js',
+        },
+        optimization: {
+            minimize: false,
+        },
+        module: {
+            rules: [
+                {
+                    test: /\.js$/,
+                    loader: '@open-wc/webpack-import-meta-loader'
+                }
+            ]
+        },
+        experiments: {
+            outputModule: true
+        },
+        output: {
+            chunkFormat: 'module',
+            filename: '[name].mjs',
+            library : {
+                type: 'module',
+            },
+            path: path.resolve(__dirname, '../dist'),
+        }
+    }
+]