@remix_labs/machine-starter 2.12816.0 → 2.12820.0-dev

package/common.js

@@ -8,10 +8,32 @@ let last_hub = null;
 
 let workers = {};
 let worker_topics = {};
+// hub channel handed to each VM worker; the hub's end must be released when the worker dies
+let worker_chans = {};
 let children = {};
 let ffiLoops = {};
 let blobs = {};
 let blobsLoops = {};
+// vmID -> { cancelled } for a StartWASM2 that has not finished setting itself
+// up. terminateVM marks it cancelled so the start can unwind instead of
+// registering loops for a VM that was already stopped.
+let starting = {};
+
+// Diagnostics for slowdowns and "the ID is used" errors. Call
+// globalThis.__rmxMachineDiag() in the console: every map below should be empty
+// when no VM is running. A non-empty ffiLoops/blobsLoops/worker_topics entry
+// without a worker is a leak.
+const diag = { started: 0, terminated: 0, cancelledStarts: 0, terminateWhileStarting: 0, loopsClosed: 0, killPublishFailed: 0 };
+globalThis.__rmxMachineDiag = () => ({
+    ...diag,
+    workers: Object.keys(workers),
+    worker_topics: Object.keys(worker_topics),
+    ffiLoops: Object.keys(ffiLoops),
+    blobsLoops: Object.keys(blobsLoops),
+    children: Object.keys(children),
+    blobs: Object.keys(blobs),
+    starting: Object.keys(starting),
+});
 
 function terminateAll() {
     console.log("[machine-starter] terminate all VMs");
@@ -40,6 +62,19 @@ function terminateAll() {
 function terminateVM(vmID) {
     console.log("[machine-starter] terminate VM", vmID);
     let worker = workers[vmID];
+    if (starting[vmID]) {
+        // Stopped while StartWASM2 was still awaiting (fetching/compiling mixrt,
+        // creating the starter mixcore). Tell it to unwind; it removes what it
+        // registered. Without this it registered ffiLoops[vmID] after this
+        // call and every later start of the same vmID failed with
+        // "Cannot start VM, the ID is used".
+        starting[vmID].cancelled = true;
+        // free the id now: a start that follows this stop must not see the cancelled one
+        // (the cancelled start still holds its token and unwinds itself)
+        delete starting[vmID];
+        diag.terminateWhileStarting++;
+        console.warn("[machine-starter] terminate VM while it is still starting", vmID);
+    }
     if (worker) {
         let vmChildren = children[vmID];
         let blobList = blobs[vmID] || [];
@@ -48,8 +83,12 @@ function terminateVM(vmID) {
         delete children[vmID];
         delete ffiLoops[vmID];
         delete blobsLoops[vmID];
+        delete blobs[vmID];
         killVM(vmID);
-        ffiLoop.terminate();
+        diag.terminated++;
+        if (ffiLoop) {
+            ffiLoop.terminate();
+        }
         if (blobsLoop) {
             blobsLoop.terminate();
         }
@@ -72,15 +111,39 @@ function killVM(vmID) {
     delete worker_topics[vmID];
     if (topic && last_hub) {
         let payload = { _rmx_type:"msg_vm_register", id: vmID, alive: false };
+        // The worker must be terminated even when the unannounce cannot be published:
+        // a chain of .then() skipped worker.terminate() on any failure, leaking a Web Worker
+        // (with its wasm instance) for every VM stopped that way.
         last_hub.newChannel().then(channel => {
             let msg = channel.newJSONMessage("msg_vm_register", "vm", payload);
-            channel.publish(topic + "/announce", msg, true)
-        }).then(_ => worker.terminate())
+            return channel.publish(topic + "/announce", msg, true).finally(() => closeChannel(channel));
+        }).catch(e => {
+            diag.killPublishFailed++;
+            console.warn("[machine-starter] could not announce VM stop", vmID, e);
+        }).finally(() => { worker.terminate(); releaseWorkerChannel(last_hub, vmID); });
     } else {
-        worker.terminate()
+        worker.terminate();
+        releaseWorkerChannel(last_hub, vmID);
     }
 }
 
+// A hub channel has no explicit close: drop its MessagePort so the hub worker can release it.
+function closeChannel(channel) {
+    try {
+        if (typeof channel.close === "function") channel.close();
+        else channel.port && channel.port.close && channel.port.close();
+        // a message sent just before close() may not arrive: ask via the hub itself as well
+        if (last_hub && last_hub.closeChannelById) last_hub.closeChannelById(channel.chanId);
+    } catch (_) {}
+}
+
+// The channel given to a VM worker was transferred, so only the hub can release its end.
+function releaseWorkerChannel(hub, vmID) {
+    let chanId = worker_chans[vmID];
+    delete worker_chans[vmID];
+    try { if (hub && hub.closeChannelById) hub.closeChannelById(chanId); } catch (_) {}
+}
+
 if (globalThis.Process) {
     Process.on("exit", () => {
         terminateAll()
@@ -133,12 +196,60 @@ async function StartWASM2(hub, config, getWorker) {
         console.log("[machine-starter] Call MixExplain() for explaining debug levels, and MixSetMask(m) for changing these")
     };
     last_hub = hub;
-    let channel = await hub.newChannel();
-    // injected by either index.js or node.js
-    let worker = await getWorker();
-    if (workers[config.vmID] || ffiLoops[config.vmID])
+    if (starting[config.vmID]) {
+        // another start of the same id has not finished setting itself up
         throw new Error("Cannot start VM, the ID is used: " + config.vmID);
+    }
+    // Registered before the first await: a terminateVM that arrives while the channel and worker
+    // are still being created finds the token and cancels this start. Without it that stop was a
+    // no-op, the VM registered and ran with no owner, and every later start failed with
+    // "Cannot start VM, the ID is used".
+    const startToken = { cancelled: false };
+    starting[config.vmID] = startToken;
+    let channel;
+    let worker;
+    try {
+        channel = await hub.newChannel();
+        // injected by either index.js or node.js
+        worker = await getWorker();
+    } catch (e) {
+        if (starting[config.vmID] === startToken) delete starting[config.vmID];
+        if (channel) closeChannel(channel);
+        throw e;
+    }
+    if (startToken.cancelled) {
+        diag.cancelledStarts++;
+        console.warn("[machine-starter] start cancelled before the VM was registered", config.vmID);
+        if (starting[config.vmID] === startToken) delete starting[config.vmID];
+        closeChannel(channel);
+        try { worker.terminate(); } catch (_) {}
+        return worker;
+    }
+    if (workers[config.vmID] || ffiLoops[config.vmID]) {
+        if (starting[config.vmID] === startToken) delete starting[config.vmID];
+        closeChannel(channel);
+        throw new Error("Cannot start VM, the ID is used: " + config.vmID);
+    }
     workers[config.vmID] = worker;
+    worker_chans[config.vmID] = channel.chanId;
+    diag.started++;
+    // Called after every await below: if terminateVM ran meanwhile, undo and stop.
+    const abortIfCancelled = () => {
+        if (!startToken.cancelled) return false;
+        diag.cancelledStarts++;
+        console.warn("[machine-starter] start cancelled, VM was terminated while starting", config.vmID);
+        if (starting[config.vmID] === startToken) delete starting[config.vmID];
+        // only undo what this start registered: a newer start of the same id may own the maps now
+        if (workers[config.vmID] === worker) {
+            delete ffiLoops[config.vmID];
+            delete blobsLoops[config.vmID];
+            delete worker_topics[config.vmID];
+            delete workers[config.vmID];
+            releaseWorkerChannel(hub, config.vmID);
+        }
+        try { worker.terminate(); } catch (_) {}
+        return true;
+    };
     if (config.topicPrefix != undefined && config.topicPrefix != "") {
         worker_topics[config.vmID] = config.topicPrefix + "/vm/" + config.vmID;
     } else {
@@ -148,9 +259,11 @@ async function StartWASM2(hub, config, getWorker) {
     // from now on, the "local" FFIs are called "starter" FFIs, because this
     // distinguishes these from other locally defined FFIs
     let mixcore = await createStarterMixcore(config.mixcore);
+    if (abortIfCancelled()) return worker;
     if (mixcore) {
         // merge the ffis and store them back into the config so sub vms also get them
         config.localFFIs = Object.assign(starterFFIs, await mixcoreStarterFFIs(mixcore));
+        if (abortIfCancelled()) return worker;
         // delete the mixcore config to avoid being handled further by the worker
         delete config.mixcore;
     }
@@ -184,7 +297,9 @@ async function StartWASM2(hub, config, getWorker) {
             globalThis.mixrtCodeURL = url;
             globalThis.mixrtCodeModule = mixrtCode;
         }
+        if (abortIfCancelled()) return worker;
     };
+    if (abortIfCancelled()) return worker;
     ffiLoops[config.vmID] = setupStarterFFIs(hub, config, starterFFIfuns);
     blobsLoops[config.vmID] = setupBlobs(hub, config);
     let config_msg =
@@ -210,6 +325,7 @@ async function StartWASM2(hub, config, getWorker) {
           // (http, https, data)
         };
     worker.postMessage(config_msg, [ channel.port ]);
+    if (starting[config.vmID] === startToken) delete starting[config.vmID];
     return worker;
 }
 
@@ -234,6 +350,7 @@ async function ffi_vm_start(conn, args, getWorker) {
     let vmChildren = children[conn.config.vmID];
     if (!vmChildren) vmChildren = [];
     vmChildren.push(vm_id);
+    children[conn.config.vmID] = vmChildren;   // was never stored: embedded VMs were not stopped with their parent
     let arg_config = args[0];
     if (arg_config.interceptFFI !== undefined)
         config["interceptFFI"] = arg_config.interceptFFI;
@@ -247,6 +364,10 @@ async function ffi_vm_stop(conn, args) {
     let vm_id = args[0];
     let worker = workers[vm_id];
     if (!worker) throw new Error("no such VM: " + vm_id);
+    for (let parent of Object.keys(children)) {
+        let i = children[parent].indexOf(vm_id);
+        if (i >= 0) children[parent].splice(i, 1);
+    }
     terminateVM(vm_id);
     return null;
 }
@@ -389,6 +510,11 @@ async function starterFFIsMessageLoop(hub, config, starterFFIs, terminate) {
         ]);
         if (terminated) {
             // console.log("[machine-starter] terminate FFI loop", vmID);
+            // Release the hub subscription and channel; without this each VM (one per build in the
+            // builder) left both behind for the lifetime of the page.
+            await sub.unsubscribe().catch(() => {});
+            closeChannel(channel);
+            diag.loopsClosed++;
             return;
         }
         let name = msg.payload.name;
@@ -449,6 +575,9 @@ async function blobsMessageLoop(hub, config, terminate) {
         ]);
         if (terminated) {
             // console.log("[machine-starter] terminate blobs loop", vmID);
+            await sub.unsubscribe().catch(() => {});
+            closeChannel(channel);
+            diag.loopsClosed++;
             return;
         }
         let name = msg.payload.name;
@@ -516,7 +645,20 @@ class FFIComm {
     }
 }
 
+// Stops every VM that `parentVmID` started (through $vm_start) and has not stopped itself, including
+// ones that are still starting. Returns how many were stopped. A parent that outlives its children
+// (the L2 session agent starts one embedded VM per session) can leave one running when its owner
+// goes away before the session finished starting; the caller decides when that has happened.
+function terminateChildren(parentVmID) {
+    let list = children[parentVmID];
+    if (!list || list.length === 0) return 0;
+    let ids = list.slice();
+    children[parentVmID] = [];
+    for (let id of ids) terminateVM(id);
+    return ids.length;
+}
+
 // use terminateVM to shut a worker down!
-export { StartWASM, StartWASM2, terminateAll, terminateVM,
+export { StartWASM, StartWASM2, terminateAll, terminateVM, terminateChildren,
          Token, Case, Opaque
        }

package/groovebox_build.js

@@ -1,2 +1,2 @@
-var GROOVEBOX_BUILD = "12816";
+var GROOVEBOX_BUILD = "12820";
 export { GROOVEBOX_BUILD }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remix_labs/machine-starter",
-  "version": "2.12816.0",
+  "version": "2.12820.0-dev",
   "description": "start the groove",
   "main": "node.js",
   "browser": "index.js",
@@ -11,7 +11,7 @@
   "author": "Remixlabs staff",
   "license": "ISC",
   "dependencies": {
-    "@remix_labs/hub-client": "2.12816.0",
+    "@remix_labs/hub-client": "2.12820.0-dev",
     "nanoid": "^5.0.2",
     "web-worker": "^1.2.0"
   },