@remix_labs/machine-starter 2.3239.0 → 2.3241.0-dev

package/common.js

@@ -2,61 +2,34 @@ import { GROOVEBOX_BUILD } from "./groovebox_build.js";
 import { nanoid } from "nanoid";
 import { MixcoreTauri, filterFFINames } from "./mixcore.lib.mjs";
 
-const DEBUG_STATE = 4;
-
-let last_hub = null;
-
 let workers = {};
 let children = {};
 let ffiLoops = {};
-let blobs = {};
-let blobsLoops = {};
 
 function terminateAll() {
-    console.log("[machine-starter] terminate all VMs");
+    console.log("[groovebox-starter] terminate all VMs");
     for (let [vmID, worker] of Object.entries(workers)) {
         worker.terminate();
     };
-    for (let [vmID, blist] of Object.entries(blobs)) {
-        for (let blob_url of blist) {
-            console.log("[machine-starter] revoke ", blob_url);
-            URL.revokeObjectURL(blob_url);
-        }
-    };
     for (let ffiLoop of Object.values(ffiLoops)) {
         ffiLoop.terminate();
     }
-    for (let blobsLoop of Object.values(blobsLoops)) {
-        blobsLoop.terminate();
-    }
     workers = {};
     children = {};
     ffiLoops = {};
-    blobs = {};
-    blobsLoops = {};
 }
 
 function terminateVM(vmID) {
-    console.log("[machine-starter] terminate VM", vmID);
+    console.log("[groovebox-starter] terminate VM", vmID);
     let worker = workers[vmID];
     if (worker) {
         let vmChildren = children[vmID];
-        let blobList = blobs[vmID] || [];
         let ffiLoop = ffiLoops[vmID];
-        let blobsLoop = blobsLoops[vmID];
         delete workers[vmID];
         delete children[vmID];
         delete ffiLoops[vmID];
-        delete blobsLoops[vmID];
         worker.terminate();
         ffiLoop.terminate();
-        if (blobsLoop) {
-            blobsLoop.terminate();
-        }
-        for (let blob_url of blobList) {
-            console.log("[machine-starter] revoke ", blob_url);
-            URL.revokeObjectURL(blob_url);
-        };
         if (vmChildren) {
             for (let subID of vmChildren) {
                 terminateVM(subID)
@@ -67,37 +40,19 @@ function terminateVM(vmID) {
 
 if (globalThis.Process) {
     Process.on("exit", () => {
+        console.log("terminating machine workers");
         terminateAll()
     })
 };
 
-let initialMask = 0;
-const logMaskTopic = "/logMask";
-
-globalThis.MixExplain = (_ => {
-    console.debug("Debug levels (call MixSetMask to change):");
-    console.debug("DEBUG_MEMORY        1");
-    console.debug("DEBUG_SCHEDULER     2");
-    console.debug("DEBUG_STATE         4");
-    console.debug("DEBUG_STACK         8");
-    console.debug("DEBUG_LOOP         16");
-    console.debug("DEBUG_MEMORY_OG    32");
-    console.debug("DEBUG_GC           64");
-    console.debug("DEBUG_INTERPRETER 128");
-    console.debug("DEBUG_MIXERROR    256");
-    console.debug("DEBUG_MESSAGES   1024");
-    console.debug("DEBUG_QUEUES     2048");
-    console.debug("DEBUG_OUTPUT     4096");
-    console.debug("DEBUG_COMPILER   8192");
-});
+let initialMask = 1024;   // default for web
 
 globalThis.MixSetMask = (m => {
     initialMask = m;
-    if (last_hub) {
-        last_hub.newChannel().then(channel => {
-            let msg = channel.newJSONMessage("_rmx_setLogMask", "MixSetMask", { mask: m });
-            channel.set(logMaskTopic, msg)
-        });
+    for (let [vmID, worker] of Object.entries(workers)) {
+        worker.postMessage({ "_rmx_type": "msg_vm_logMask",
+                             "mask": m
+                           })
     }
 });
 
@@ -113,10 +68,7 @@ function StartWASM(hub, baseURL, org, workspace, vmID, user, token, noOutputViaM
 }
 
 async function StartWASM2(hub, config) {
-    if (!last_hub) {
-        console.log("[machine-starter] Call MixExplain() for explaining debug levels, and MixSetMask(m) for changing these")
-    };
-    last_hub = hub;
+    console.log("[groovebox-starter] start VM", GROOVEBOX_BUILD, config.vmID, config.mixcore);
     let channel = await hub.newChannel();
     // injected by either index.js or node.js
     let worker = globalThis.GetMachineWASMWorker();
@@ -126,7 +78,7 @@ async function StartWASM2(hub, config) {
     let localFFIs = config.localFFIs || {};
     let mixcoreFFIs = await getMixcoreFFIs(config.mixcore);
     if (mixcoreFFIs) {
-        console.log("[machine-starter] mixcore FFIs", Object.keys(mixcoreFFIs));
+        console.log("[groovebox-starter] mixcore FFIs", Object.keys(mixcoreFFIs));
         // merge the ffis and store them back into the config so sub vms also get them
         config.localFFIs = Object.assign(localFFIs, mixcoreFFIs);
         // delete the mixcore config to avoid being handled further by the worker
@@ -148,23 +100,7 @@ async function StartWASM2(hub, config) {
     localFFIfuns["$vm_start"] = ffi_vm_start;
     localFFIkind["$vm_stop"] = vm_kind;
     localFFIfuns["$vm_stop"] = ffi_vm_stop;
-    let mixrtCode = config.mixrtCode;
-    if (typeof(mixrtCode) == "string" && mixrtCode.startsWith("remix://")) {
-        // for the desktop only: precompile the mixrt module to get faster
-        // startup times
-        if (globalThis.mixrtCodeModule && globalThis.mixrtCodeURL == mixrtCode) {
-            console.log("[machine-starter] reusing mixrt");
-            mixrtCode = globalThis.mixrtCodeModule;
-        } else {
-            let url = mixrtCode;
-            console.log("[machine-starter] pre-compiling mixrt");
-            mixrtCode = await WebAssembly.compileStreaming(fetch(url));
-            globalThis.mixrtCodeURL = url;
-            globalThis.mixrtCodeModule = mixrtCode;
-        }
-    };
     ffiLoops[config.vmID] = setupLocalFFIs(hub, config, localFFIfuns);
-    blobsLoops[config.vmID] = setupBlobs(hub, config);
     let config_msg =
         { "_rmx_type": "msg_vm_configure",
           "baseURL": config.baseURL,
@@ -182,7 +118,7 @@ async function StartWASM2(hub, config) {
           "interceptFFI": config.interceptFFI,
           "interceptDynloadFFI": config.interceptDynloadFFI,
           "mixcore": config.mixcore,
-          "mixrtCode": mixrtCode
+          "mixrtCode": config.mixrtCode
           // can be: base64-encoded string, Uint8Array, or a URL as string
           // (http, https, data)
         };
@@ -192,7 +128,6 @@ async function StartWASM2(hub, config) {
 
 async function ffi_vm_start(conn, args) {
     let vm_id = "embedded." + nanoid();
-    console.log("[machine-starter] starting VM: ", vm_id);
     let config = {};
     for (const [k, v] of Object.entries(conn.config)) {
         config[k] = v;
@@ -355,15 +290,14 @@ async function localFFIsMessageLoop(hub, config, localFFIs, terminate) {
             terminate.then(() => ({terminated: true})),
         ]);
         if (terminated) {
-            // console.log("[machine-starter] terminate FFI loop", vmID);
+            console.log("[groovebox-starter] terminate FFI loop", vmID);
             return;
         }
         let name = msg.payload.name;
         let call_id = msg.payload.call_id;
         let args = decode(msg.payload.args);
         let fun = localFFIs[name];
-        if (initialMask & DEBUG_STATE)
-            console.debug("[groovebox-starter] local FFI call", vmID, call_id, name);
+        console.log("[groovebox-starter] local FFI call", vmID, call_id, name);
         try {
             if (!fun) throw new Error("No such local FFI in groovebox starter: " + name);
             let connector =
@@ -376,57 +310,18 @@ async function localFFIsMessageLoop(hub, config, localFFIs, terminate) {
             let r = fun instanceof Function ? fun(connector, args) : fun.run(connector, args);
             if (r instanceof Promise) {
                 // r.catch(_ => null);  // prevent "unhandled rejection"
-                if (initialMask & DEBUG_STATE)
-                    console.debug("[groovebox-starter] running FFI call asynchronously", vmID, call_id, name);
                 await comm.later(call_id);
                 let value = await r;
-                if (initialMask & DEBUG_STATE)
-                    console.debug("[groovebox-starter] async FFI call returned", vmID, call_id, name);
                 await comm.returnOrFail(fun, call_id, value);
             } else {
-                if (initialMask & DEBUG_STATE)
-                    console.debug("[groovebox-starter] sync FFI call returned", vmID, call_id, name);
                 await comm.returnOrFail(fun, call_id, r);
             }
         } catch (reason) {
-            if (initialMask & DEBUG_STATE) {
-                console.debug("[groovebox-starter] FFI call exception", vmID, call_id, name);
-                console.debug("[groovebox-starter] exception", reason);
-            };
             await comm.error(call_id, reason.message, reason.stack);
         }
     }
 }
 
-function setupBlobs(hub, config) {
-    let vmID = config.vmID;
-    let terminate = Promise.withResolvers();
-    blobsMessageLoop(hub, config, terminate.promise); // don't wait
-    return {terminate: terminate.resolve};
-}
-async function blobsMessageLoop(hub, config, terminate) {
-    let vmID = config.vmID;
-    let channel = await hub.newChannel();
-    let comm = new FFIComm(vmID, hub, channel);
-    let sub = await channel.subscribe("/local/" + vmID + "/blobs");
-    while (true) {
-        let {msg, terminated} = await Promise.race([
-            sub.next().then((msg) => ({msg})),
-            terminate.then(() => ({terminated: true})),
-        ]);
-        if (terminated) {
-            // console.log("[machine-starter] terminate blobs loop", vmID);
-            return;
-        }
-        let name = msg.payload.name;
-        let url = msg.payload.url;
-        let blobList = blobs[vmID];
-        if (!blobList) blobList = [];
-        blobList.push(url);
-        blobs[vmID] = blobList;
-    }
-}
-
 class FFIComm {
     constructor(vmID, hub, channel) {
         this.vmID = vmID;
@@ -493,7 +388,6 @@ async function getMixcoreFFIs(config) {
         mixcore = await MixcoreTauri.create(config.workspace, config.app);
         break;
     default:
-        console.warn("[machine-starter] not going to configure mixcore - kind: ", config?.kind);
         return;
     }
 

package/groovebox_build.js

@@ -1,2 +1,2 @@
-var GROOVEBOX_BUILD = "3239";
+var GROOVEBOX_BUILD = "3610";
 export { GROOVEBOX_BUILD }

package/index.js

@@ -1,10 +1,12 @@
-import MachineWASMWorker from './machine-wasm.worker.mjs'
+function workerify(code) {
+    let blob = new Blob([code], {type: 'application/javascript'});
+    return new Worker(URL.createObjectURL(blob));
+}
 
-const blob = new Blob([MachineWASMWorker], {type: 'application/javascript'});
-const url = URL.createObjectURL(blob);
+import MachineWASMWorker from './machine-wasm.worker.mjs'
 
 function GetMachineWASMWorker() {
-    return new Worker(url);
+    return workerify(MachineWASMWorker);
 }
 
 globalThis.GetMachineWASMWorker = GetMachineWASMWorker;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remix_labs/machine-starter",
-  "version": "2.3239.0",
+  "version": "2.3241.0-dev",
   "description": "start the groove",
   "main": "node.js",
   "browser": "index.js",
@@ -11,11 +11,9 @@
   "author": "Remixlabs staff",
   "license": "ISC",
   "dependencies": {
-    "@remix_labs/hub-client": "2.3239.0",
+    "@remix_labs/hub-client": "2.3241.0-dev",
     "nanoid": "^5.0.2",
     "web-worker": "^1.2.0"
   },
-  "repository": {
-    "url": "git+https://github.com/remixlabs/groovebox.git"
-  }
+  "repository": "https://github.com/remixlabs/groovebox.git"
 }

package/package_pub.json

@@ -15,7 +15,5 @@
     "nanoid": "^5.0.2",
     "web-worker": "^1.2.0"
   },
-  "repository": {
-    "url": "git+https://github.com/remixlabs/groovebox.git"
-  }
+  "repository": "https://github.com/remixlabs/groovebox.git"
 }

package/start.js

@@ -1,16 +1,6 @@
 import Process from "process";
 import { StartWASM2, Hub } from "./node.js";
-import { MixcoreWasm, MixcoreWasmApi } from "./mixcore.lib.mjs";
-import { mixcoreWasmCode } from './mixcore-wasm.js';
-import { WASI, OpenFile, File, Directory, PreopenDirectory, ConsoleStdout } from "@bjorn3/browser_wasi_shim";
-function importCode(base64) {
-    let binstr = atob(base64);
-    let code = new Uint8Array(binstr.length);
-    for (let i = 0; i < binstr.length; i++)
-        code[i] = binstr.charCodeAt(i);
-    return code;
-}
-let mixcoreWasm = importCode(mixcoreWasmCode);
+import { mixcoreWasmBase64, mixcoreJsBase64 } from './mixcore-wasm.js';
 
 if (Process.stdout._handle)
     Process.stdout._handle.setBlocking(true);
@@ -42,46 +32,47 @@ let noOutputViaMQTT = Process.env["MIX_MQTT_NO_OUTPUT"] !== undefined;
 console.log("Machine params: variant=" + variant + " baseURL=" + baseURL, " mqttURL=" + mqttURL, " mqttUser=" + mqttUser, " mqttToken=" + mqttToken, " id=" + id, " noOutputViaMQTT=" + noOutputViaMQTT);
 let w = new Hub.Worker();
 globalThis.GROOVEBOX_ALLOW_INSECURE_HTTP = true;
-
-let DB_DIR = "db";
-let wasi = new WASI([], [], [
-    new OpenFile(new File([])),
-    ConsoleStdout.lineBuffered(msg => console.log(`[WASI stdout] ${msg}`)),
-    ConsoleStdout.lineBuffered(msg => console.warn(`[WASI stderr] ${msg}`)),
-    new PreopenDirectory(".", new Map([[DB_DIR, new Directory(new Map())]]))
-]);
-MixcoreWasmApi.create(mixcoreWasm, wasi).then(mixcoreWasmApi => {
-    let mixcore = MixcoreWasm.create(DB_DIR, "compiler-testing", "test@user", mixcoreWasmApi);
-    let mixcoreConfig = { kind: "mixcore", mixcore };
-    w.configure({wsURL: mqttURL, user:mqttUser, token:mqttToken}).then(resp => {
-        if (variant == "BYTE") variant = "QCODE";
-        switch(variant) {
-        case "WASM":
-        case "WAT":
-        case "QCODE":
-            let startVM = (_ => {
-                MixSetMask(64 + 256 + 512 + 1024);
-                StartWASM2(w, {
-                    baseURL: baseURL,
-                    org: "local",
-                    workspace: "local",
-                    vmID: id,
-                    user: mqttUser,
-                    token: mqttToken,
-                    noOutputViaMQTT: noOutputViaMQTT,
-                    machType: variant,
-                    localFFIs: {},
-                    mixcore: mixcoreConfig,
-                });
+let mixcoreWasmBlob = new Blob([mixcoreWasm], { type: "application/wasm" });
+let mixcoreWasmUrl = Url.createObjectURL(mixcoreWasmBlob); // leaks, but hey we are on node
+let mixcoreJsBlob = new Blob([MixcoreJs], { type: "text/javascript" });
+let mixcoreJsUrl = Url.createObjectURL(mixcoreJsBlob);
+let mixcore = {
+    kind: "wasm-opfs",
+    mixcoreWasmUrl,
+    mixcoreJsUrl,
+    dbDir: "/databases",
+    dbUser: "test@user",
+    dbName: "compiler-testing",
+    createDbs: ["compiler-testing"],
+};
+w.configure({wsURL: mqttURL, user:mqttUser, token:mqttToken}).then(resp => {
+    if (variant == "BYTE") variant = "QCODE";
+    switch(variant) {
+    case "WASM":
+    case "WAT":
+    case "QCODE":
+        let startVM = (_ => {
+            MixSetMask(64 + 256 + 512 + 1024);
+            StartWASM2(w, {
+                baseURL: baseURL,
+                org: "local",
+                workspace: "local",
+                vmID: id,
+                user: mqttUser,
+                token: mqttToken,
+                noOutputViaMQTT: noOutputViaMQTT,
+                machType: variant,
+                localFFIs: {},
+                mixcore,
             });
-            if (globalThis.EnableAndStartMixCompiler) {
-                EnableAndStartMixCompiler(w).then(startVM);
-            } else {
-                startVM()
-            };
-            break
-        default:
-            throw(new Error("unknown MIX_MACHINE_VARIANT: " + variant))
-        }
-    })
+        });
+        if (globalThis.EnableAndStartMixCompiler) {
+            EnableAndStartMixCompiler(w).then(startVM);
+        } else {
+            startVM()
+        };
+        break
+    default:
+        throw(new Error("unknown MIX_MACHINE_VARIANT: " + variant))
+    }
 });