@multiplayer-app/sandbox 7.0.1 → 7.0.2

package/dist/cli.js

@@ -28,7 +28,7 @@ var import_config5 = require("dotenv/config");
 var import_commander = require("commander");
 
 // package.json
-var version = "7.0.1";
+var version = "7.0.2";
 
 // src/config.ts
 var import_os2 = __toESM(require("os"));
@@ -51,6 +51,14 @@ var WorkerEvent = {
   // worker → sandbox, fire-and-forget like HEARTBEAT — sent once when a 'job' VM's
   // entrypoint exits on its own (the worker has already torn the resource down by then).
   VM_RUN_EXITED: "vm:run-exited",
+  // worker → sandbox after an asynchronous VM_CREATE finishes. Socket.IO acknowledgements
+  // cannot be relied on across the detached Redis-adapter emitter used by the API.
+  VM_CREATE_RESULT: "vm:create-result",
+  // worker → sandbox, fire-and-forget — sent when periodic reconciliation finds the
+  // underlying resource gone without this worker ever issuing a destroy (e.g. AWS's own
+  // idle-suspend/max-duration policies terminating a Lambda MicroVM). Only engines whose
+  // VMs can disappear outside an explicit command from this process emit it.
+  VM_EXTERNALLY_TERMINATED: "vm:externally-terminated",
   // sandbox → worker, generated by the control plane and reused by this worker process
   // on Socket.IO reconnects. This keeps duplicate configured WORKER_ID values isolated.
   ASSIGNED_ID: "worker:assigned-id",
@@ -78,6 +86,8 @@ var WorkerEvent = {
   // sandbox → worker (browser input forwarded downstream)
   VNC_MOUSE: "vnc:mouse",
   VNC_KEYBOARD: "vnc:keyboard",
+  TERMINAL_ATTACH: "terminal:attach",
+  TERMINAL_DETACH: "terminal:detach",
   TERMINAL_INPUT: "terminal:input"
 };
 var VM_ACTION_EVENTS = {
@@ -122,7 +132,20 @@ function s3Key(prefix, arch, stack, file) {
 async function makeS3Client(endpoint) {
   const { S3Client } = await import("@aws-sdk/client-s3");
   return new S3Client({
-    ...endpoint ? { endpoint, forcePathStyle: true } : {},
+    ...endpoint ? {
+      endpoint,
+      forcePathStyle: true,
+      // A custom endpoint means a local/self-hosted store (MinIO, etc.), not real AWS.
+      // Use its own static credentials explicitly rather than the ambient default
+      // provider chain — otherwise, whenever AWS_PROFILE is also set (e.g. for the
+      // lambda-microvm engine's real AWS calls), the SDK unconditionally skips static env
+      // credentials process-wide and sends the real AWS SSO session to this local store
+      // instead, which rejects it since it only recognizes its own configured user.
+      credentials: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? "minioadmin",
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? "minioadmin"
+      }
+    } : {},
     ...process.env.AWS_REGION ? {} : { region: "us-east-1" },
     // aws-chunked checksum streaming breaks older S3-compatible stores (MinIO)
     requestChecksumCalculation: "WHEN_REQUIRED"
@@ -143,7 +166,10 @@ async function downloadFromHttp(opts, stack, file, dest) {
   if (!response.ok || !response.body) {
     throw new Error(`GET ${url} failed: ${response.status} ${response.statusText}`);
   }
-  await (0, import_promises.pipeline)(import_stream.Readable.fromWeb(response.body), import_fs.default.createWriteStream(dest));
+  await (0, import_promises.pipeline)(
+    import_stream.Readable.fromWeb(response.body),
+    import_fs.default.createWriteStream(dest)
+  );
 }
 async function pullImages(opts, destDir) {
   const isS3 = opts.location.startsWith("s3://");
@@ -178,11 +204,13 @@ async function pushImages(opts, srcDir) {
         throw new Error(`${src} not found \u2014 run scripts/setup-vm-images.sh first`);
       }
       const key = s3Key(prefix, opts.arch, stack, file);
-      await client.send(new PutObjectCommand({
-        Bucket: bucket,
-        Key: key,
-        Body: import_fs.default.readFileSync(src)
-      }));
+      await client.send(
+        new PutObjectCommand({
+          Bucket: bucket,
+          Key: key,
+          Body: import_fs.default.readFileSync(src)
+        })
+      );
       uploaded.push(`s3://${bucket}/${key}`);
     }
   }
@@ -221,12 +249,14 @@ function resolveSandboxEndpoint(rawUrl) {
 }
 function loadWorkerConfigFromEnv(env = process.env) {
   const workerId = env.WORKER_ID?.trim() || import_os2.default.hostname();
+  const workerIdFile = env.WORKER_ID_FILE?.trim() || import_path2.default.join(import_os2.default.tmpdir(), `multiplayer-sandbox-worker-${workerId}.id`);
   const token = env.WORKER_TOKEN?.trim() || void 0;
   const sandboxEndpoint = resolveSandboxEndpoint(
     env.SANDBOX_URL ?? "https://api.sandbox.filingramp.com/v0/gapi"
   );
   return {
     workerId,
+    workerIdFile,
     token,
     sandboxUrl: sandboxEndpoint.sandboxUrl,
     workerSocketPath: sandboxEndpoint.workerSocketPath,
@@ -261,7 +291,10 @@ function loadWorkerConfigFromEnv(env = process.env) {
       // 0 disables auto-suspend; the sandbox drives pause/resume explicitly by default
       maxIdleSeconds: Number(env.MICROVM_MAX_IDLE_SECONDS ?? 900),
       suspendedDurationSeconds: Number(env.MICROVM_SUSPENDED_DURATION_SECONDS ?? 28800),
-      maximumDurationInSeconds: Number(env.MICROVM_MAX_DURATION_SECONDS ?? 28800)
+      maximumDurationInSeconds: Number(env.MICROVM_MAX_DURATION_SECONDS ?? 28800),
+      // A real cold MicroVM boot can comfortably exceed the old 25s hardcoded default —
+      // must stay well under the api's own VM_CREATE ack timeout (see vm.service.ts).
+      createTimeoutMs: Number(env.MICROVM_CREATE_TIMEOUT_MS ?? 6e4)
     },
     fargate: {
       region: env.FARGATE_AWS_REGION ?? env.AWS_REGION,
@@ -2466,8 +2499,8 @@ var RemoteExecutor = class {
       body: result.body ? Buffer.from(result.body, "base64") : Buffer.alloc(0)
     };
   }
-  request(method, path7, body) {
-    const url = new URL(this.base + path7);
+  request(method, path8, body) {
+    const url = new URL(this.base + path8);
     const lib = url.protocol === "https:" ? import_https.default : import_http5.default;
     const payload = body !== void 0 ? JSON.stringify(body) : void 0;
     return new Promise((resolve, reject) => {
@@ -2488,7 +2521,7 @@ var RemoteExecutor = class {
           res.on("end", () => {
             const text = Buffer.concat(chunks).toString();
             if (res.statusCode && res.statusCode >= 400) {
-              reject(new Error(`vm-agent ${method} ${path7} \u2192 ${res.statusCode}: ${text}`));
+              reject(new Error(`vm-agent ${method} ${path8} \u2192 ${res.statusCode}: ${text}`));
               return;
             }
             if (res.statusCode === 204 || !text) {
@@ -2515,6 +2548,23 @@ var import_https2 = __toESM(require("https"));
 var import_client_lambda_microvms = require("@aws-sdk/client-lambda-microvms");
 var AUTH_TOKEN_TTL_MINUTES = 60;
 var AUTH_TOKEN_REFRESH_MARGIN_MS = 2 * 6e4;
+var SHELL_PORT = 8022;
+function managedConnectorArn(imageIdentifier, name) {
+  const [arn, partition, service, region] = imageIdentifier.split(":");
+  if (arn !== "arn" || !partition || service !== "lambda" || !region) {
+    throw new Error(`cannot derive Lambda MicroVM connector ARN from image "${imageIdentifier}"`);
+  }
+  return `arn:${partition}:lambda:${region}:aws:network-connector:aws-network-connector:${name}`;
+}
+function runtimeEnv(spec) {
+  const env = {
+    ...spec.env,
+    // A release URL is unique and short-lived, so it belongs in per-VM run data rather than
+    // the image-level environment shared by every MicroVM launched from that image.
+    ...spec.appUrl ? { APP_URL: spec.appUrl } : {}
+  };
+  return Object.keys(env).length ? env : void 0;
+}
 var LambdaMicrovmExecutor = class {
   constructor(config) {
     this.config = config;
@@ -2524,10 +2574,15 @@ var LambdaMicrovmExecutor = class {
   async create(spec) {
     const imageIdentifier = this.config.stackImageArns?.[spec.stack] ?? this.config.imageArn;
     const maxIdle = this.config.maxIdleSeconds ?? 900;
+    const shellConnectorArn = managedConnectorArn(imageIdentifier, "SHELL_INGRESS");
+    const ingressNetworkConnectors = [
+      .../* @__PURE__ */ new Set([...this.config.ingressConnectorArns ?? [], shellConnectorArn])
+    ];
     const run = await this.client.send(
       new import_client_lambda_microvms.RunMicrovmCommand({
         imageIdentifier,
-        ingressNetworkConnectors: this.config.ingressConnectorArns,
+        // Required by CreateMicrovmShellAuthToken and the /shell WebSocket used by vm-console.
+        ingressNetworkConnectors,
         egressNetworkConnectors: this.config.egressConnectorArns,
         executionRoleArn: this.config.executionRoleArn,
         maximumDurationInSeconds: this.config.maximumDurationInSeconds ?? 28800,
@@ -2544,9 +2599,11 @@ var LambdaMicrovmExecutor = class {
           vmId: spec.vmId,
           sessionId: spec.sessionId,
           stack: spec.stack,
-          appUrl: spec.appUrl,
-          env: spec.env
-        })
+          env: runtimeEnv(spec)
+        }),
+        // VM_CREATE may be delivered more than once by the Redis-backed control plane.
+        // AWS returns the original request result instead of provisioning another MicroVM.
+        clientToken: spec.vmId
       })
     );
     if (!run.microvmId || !run.endpoint) {
@@ -2609,6 +2666,31 @@ var LambdaMicrovmExecutor = class {
   list() {
     return [...this.vms.values()].map((v) => v.handle);
   }
+  getSerialSource(vmId) {
+    const vm = this.vms.get(vmId);
+    if (!vm) return void 0;
+    return {
+      type: "websocket",
+      url: `wss://${vm.endpoint}/shell`,
+      protocols: async () => {
+        const response = await this.client.send(
+          new import_client_lambda_microvms.CreateMicrovmShellAuthTokenCommand({
+            microvmIdentifier: vm.microvmId,
+            expirationInMinutes: AUTH_TOKEN_TTL_MINUTES
+          })
+        );
+        const token = Object.entries(response.authToken ?? {}).find(
+          ([key]) => key.toLowerCase() === "x-aws-proxy-auth"
+        )?.[1];
+        if (!token) throw new Error("create-microvm-shell-auth-token returned no auth token");
+        return [
+          "lambda-microvms",
+          `lambda-microvms.authentication.${token}`,
+          `lambda-microvms.port.${SHELL_PORT}`
+        ];
+      }
+    };
+  }
   async proxyHttp(vmId, request) {
     const vm = this.mustGet(vmId);
     const auth = await this.getAuthHeaders(vm);
@@ -2676,6 +2758,47 @@ var LambdaMicrovmExecutor = class {
     );
     return handle;
   }
+  /**
+   * AWS can suspend or terminate a MicroVM on its own (idle-suspend policy, the suspended-
+   * duration cap, the hard maximumDurationInSeconds lifetime) without this process ever
+   * issuing a command — nothing else here polls for that. Called periodically by the
+   * worker: re-describes every tracked VM, quietly reconciles running/suspended drift, and
+   * reports (then forgets) any that AWS has actually terminated so the control plane's
+   * record doesn't stay stuck saying "running" forever.
+   */
+  async reconcile() {
+    const externallyTerminated = [];
+    for (const [vmId, vm] of [...this.vms.entries()]) {
+      try {
+        const described = await this.client.send(
+          new import_client_lambda_microvms.GetMicrovmCommand({ microvmIdentifier: vm.microvmId })
+        );
+        if (described.state === "TERMINATED" || described.state === "TERMINATING") {
+          this.vms.delete(vmId);
+          externallyTerminated.push(vmId);
+          src_default.warn(
+            { vmId, microvmId: vm.microvmId, state: described.state },
+            "[LambdaMicrovm] vm terminated outside our control"
+          );
+        } else if (typeof described.state === "string") {
+          const running = described.state === "RUNNING";
+          if (running !== vm.handle.running) vm.handle = { ...vm.handle, running };
+        }
+      } catch (error) {
+        if (error instanceof Error && error.name === "ResourceNotFoundException") {
+          this.vms.delete(vmId);
+          externallyTerminated.push(vmId);
+          src_default.warn(
+            { vmId, microvmId: vm.microvmId },
+            "[LambdaMicrovm] vm no longer exists in AWS"
+          );
+        } else {
+          src_default.error({ vmId, err: error }, "[LambdaMicrovm] reconcile check failed");
+        }
+      }
+    }
+    return externallyTerminated;
+  }
   mustGet(vmId) {
     const vm = this.vms.get(vmId);
     if (!vm) throw new Error(`vm ${vmId} not found`);
@@ -3248,6 +3371,8 @@ function detectEngines(config) {
 }
 
 // src/executor/index.ts
+var VmResourceGoneError = class extends Error {
+};
 function findNotAttachedVms(executor, records) {
   return records.filter((record) => !executor.get(record.vmId));
 }
@@ -3277,11 +3402,25 @@ var MultiEngineExecutor = class {
   constructor(executors, defaultEngine) {
     this.executors = executors;
     this.defaultEngine = defaultEngine;
+    this.creations = /* @__PURE__ */ new Map();
   }
   get engines() {
     return [...this.executors.keys()];
   }
   async create(spec) {
+    const existing = this.get(spec.vmId);
+    if (existing) return existing;
+    const pending = this.creations.get(spec.vmId);
+    if (pending) return pending;
+    const creation = this.createOnce(spec);
+    this.creations.set(spec.vmId, creation);
+    try {
+      return await creation;
+    } finally {
+      this.creations.delete(spec.vmId);
+    }
+  }
+  async createOnce(spec) {
     const engine = spec.engine ?? this.defaultEngine;
     const executor = this.executors.get(engine);
     if (!executor) {
@@ -3292,22 +3431,27 @@ var MultiEngineExecutor = class {
     const handle = await executor.create(spec);
     return { ...handle, engine };
   }
-  pause(vmId) {
+  // async here isn't a no-op: route() throws synchronously when the vmId isn't tracked in any
+  // executor's in-memory map (e.g. worker restarted and hasn't reattached this VM yet). Without
+  // `async`, that throw escapes as a real exception instead of a rejected promise — which,
+  // called from socket-client.ts's wrap(), skips its .catch() entirely and leaves the command
+  // un-acked until the caller's timeout, looking like the action "does nothing".
+  async pause(vmId) {
     return this.route(vmId).pause(vmId);
   }
-  resume(vmId) {
+  async resume(vmId) {
     return this.route(vmId).resume(vmId);
   }
-  stop(vmId) {
+  async stop(vmId) {
     return this.route(vmId).stop(vmId);
   }
-  restore(vmId) {
+  async restore(vmId) {
     return this.route(vmId).restore(vmId);
   }
-  destroy(vmId) {
+  async destroy(vmId) {
     return this.route(vmId).destroy(vmId);
   }
-  proxyHttp(vmId, request) {
+  async proxyHttp(vmId, request) {
     return this.route(vmId).proxyHttp(vmId, request);
   }
   get(vmId) {
@@ -3326,15 +3470,14 @@ var MultiEngineExecutor = class {
     }
     return void 0;
   }
-  exec(vmId, request) {
+  async exec(vmId, request) {
     const executor = this.route(vmId);
-    if (!executor.exec) return Promise.reject(new Error("executor does not support guest exec"));
+    if (!executor.exec) throw new Error("executor does not support guest exec");
     return executor.exec(vmId, request);
   }
-  writeFiles(vmId, files) {
+  async writeFiles(vmId, files) {
     const executor = this.route(vmId);
-    if (!executor.writeFiles)
-      return Promise.reject(new Error("executor does not support guest file writes"));
+    if (!executor.writeFiles) throw new Error("executor does not support guest file writes");
     return executor.writeFiles(vmId, files);
   }
   onExit(vmId, handler) {
@@ -3342,6 +3485,15 @@ var MultiEngineExecutor = class {
     if (!executor.onExit) throw new Error("executor does not support job run mode");
     executor.onExit(vmId, handler);
   }
+  /** Fans out to every sub-executor that supports it (currently just lambda-microvm). */
+  async reconcile() {
+    const externallyTerminated = [];
+    for (const executor of this.executors.values()) {
+      if (!executor.reconcile) continue;
+      externallyTerminated.push(...await executor.reconcile());
+    }
+    return externallyTerminated;
+  }
   /**
    * Recover VMs sandbox says this workerId owns but that aren't currently tracked (a worker
    * restart, most likely). Runs in parallel — probing many VMs serially would needlessly
@@ -3350,9 +3502,16 @@ var MultiEngineExecutor = class {
    * never thrown — a VM that can't be reattached just stays unavailable until something
    * touches it again (create/proxy/lifecycle), which already errors gracefully today.
    */
+  /**
+   * Returns vmIds found genuinely gone (executor.reattach resolved undefined — the
+   * resource itself no longer exists, e.g. AWS already terminated it while this worker
+   * was down) as distinct from any other reattach failure (engine unavailable, no reattach
+   * support, a transient describe error, ...), which leave the VM merely unavailable
+   * rather than confirmed gone.
+   */
   async reattachAll(records) {
     const pending = findNotAttachedVms(this, records);
-    if (!pending.length) return;
+    if (!pending.length) return [];
     const results = await Promise.allSettled(
       pending.map(async (record) => {
         const executor = record.engine ? this.executors.get(record.engine) : void 0;
@@ -3361,18 +3520,25 @@ var MultiEngineExecutor = class {
         if (!executor.reattach)
           throw new Error(`engine "${record.engine}" does not support reattach`);
         const handle = await executor.reattach(record);
-        if (!handle) throw new Error("resource no longer exists");
+        if (!handle) throw new VmResourceGoneError();
       })
     );
+    const externallyTerminated = [];
     results.forEach((result, i) => {
       const vmId = pending[i].vmId;
-      if (result.status === "fulfilled") src_default.info({ vmId }, "vm reattached");
-      else
+      if (result.status === "fulfilled") {
+        src_default.info({ vmId }, "vm reattached");
+      } else if (result.reason instanceof VmResourceGoneError) {
+        externallyTerminated.push(vmId);
+        src_default.warn({ vmId }, "vm reattach found resource already gone");
+      } else {
         src_default.warn(
           { vmId, err: result.reason },
           "vm reattach failed \u2014 staying unavailable until next touch"
         );
+      }
     });
+    return externallyTerminated;
   }
   route(vmId) {
     for (const executor of this.executors.values()) {
@@ -3394,8 +3560,36 @@ function createExecutor(config) {
 // src/socket-client.ts
 var import_os9 = __toESM(require("os"));
 var import_socket = require("socket.io-client");
+
+// src/worker-identity.ts
+var import_fs3 = __toESM(require("fs"));
+var import_path7 = __toESM(require("path"));
+function loadPersistedWorkerId(filePath) {
+  try {
+    const id = import_fs3.default.readFileSync(filePath, "utf8").trim();
+    return id || void 0;
+  } catch {
+    return void 0;
+  }
+}
+function savePersistedWorkerId(filePath, workerId) {
+  try {
+    import_fs3.default.mkdirSync(import_path7.default.dirname(filePath), { recursive: true });
+    import_fs3.default.writeFileSync(filePath, workerId, "utf8");
+  } catch (error) {
+    src_default.warn(
+      { err: error, filePath },
+      "failed to persist assigned worker id \u2014 reattach after a restart may not find its VMs"
+    );
+  }
+}
+
+// src/socket-client.ts
 var HEARTBEAT_INTERVAL_MS = 1e4;
+var RECONCILE_INTERVAL_MS = 6e4;
 var REATTACH_FALLBACK_MS = 5e3;
+var RECONNECTION_DELAY_MS = 1e3;
+var RECONNECTION_DELAY_MAX_MS = 1e4;
 function primaryIp() {
   for (const ifaces of Object.values(import_os9.default.networkInterfaces())) {
     for (const iface of ifaces ?? []) {
@@ -3424,7 +3618,11 @@ function wrap(handler) {
   return (payload, callback) => {
     handler(payload).then((result) => callback({ ok: true, result })).catch((error) => {
       src_default.error({ err: error }, "command failed");
-      callback({ ok: false, error: error.message });
+      try {
+        callback({ ok: false, error: error.message });
+      } catch (ackError) {
+        src_default.error({ err: ackError }, "failed to send command failure ack");
+      }
     });
   };
 }
@@ -3432,20 +3630,32 @@ function registerCommandHandlers(socket, executor) {
   socket.on(
     WorkerEvent.VM_CREATE,
     wrap(async (spec) => {
-      const handle = await executor.create(spec);
-      if (spec.runMode === "job") {
-        try {
+      try {
+        const handle = await executor.create(spec);
+        if (spec.runMode === "job") {
           executor.onExit(
             spec.vmId,
-            (result) => socket.emit(WorkerEvent.VM_RUN_EXITED, { vmId: spec.vmId, ...result })
+            (result2) => socket.emit(WorkerEvent.VM_RUN_EXITED, { vmId: spec.vmId, ...result2 })
           );
-        } catch (error) {
-          await executor.destroy(spec.vmId).catch(() => {
-          });
-          throw error;
         }
+        const result = {
+          vmId: spec.vmId,
+          ok: true,
+          metadata: handle.metadata
+        };
+        socket.emit(WorkerEvent.VM_CREATE_RESULT, result);
+        return handle;
+      } catch (error) {
+        if (spec.runMode === "job") await executor.destroy(spec.vmId).catch(() => {
+        });
+        const result = {
+          vmId: spec.vmId,
+          ok: false,
+          error: error instanceof Error ? error.message : String(error)
+        };
+        socket.emit(WorkerEvent.VM_CREATE_RESULT, result);
+        throw error;
       }
-      return handle;
     })
   );
   socket.on(
@@ -3497,7 +3707,7 @@ function registerCommandHandlers(socket, executor) {
   );
 }
 function connectToSandbox(config, executor) {
-  let assignedWorkerId;
+  let assignedWorkerId = loadPersistedWorkerId(config.workerIdFile);
   const socket = (0, import_socket.io)(config.sandboxUrl, {
     path: config.workerSocketPath,
     auth: (callback) => callback({
@@ -3506,12 +3716,17 @@ function connectToSandbox(config, executor) {
       ...config.token ? { token: config.token } : {}
     }),
     transports: ["websocket"],
-    reconnection: true
+    reconnection: true,
+    reconnectionAttempts: Infinity,
+    reconnectionDelay: RECONNECTION_DELAY_MS,
+    reconnectionDelayMax: RECONNECTION_DELAY_MAX_MS
   });
   let heartbeatTimer;
+  let reconcileTimer;
   let reattachFallbackTimer;
   socket.on(WorkerEvent.ASSIGNED_ID, ({ workerId }) => {
     assignedWorkerId = workerId;
+    savePersistedWorkerId(config.workerIdFile, workerId);
     src_default.info({ workerId: config.workerId, assignedWorkerId }, "worker runtime ID assigned");
   });
   const sendHeartbeat = () => socket.emit(WorkerEvent.HEARTBEAT, buildHeartbeat(config, executor));
@@ -3520,6 +3735,17 @@ function connectToSandbox(config, executor) {
     sendHeartbeat();
     heartbeatTimer = setInterval(sendHeartbeat, HEARTBEAT_INTERVAL_MS);
   }
+  const runReconcile = () => {
+    executor.reconcile().then((externallyTerminated) => {
+      for (const vmId of externallyTerminated) {
+        socket.emit(WorkerEvent.VM_EXTERNALLY_TERMINATED, { vmId });
+      }
+    }).catch((error) => src_default.error({ err: error }, "vm reconcile pass failed"));
+  };
+  function startReconcile() {
+    if (reconcileTimer) return;
+    reconcileTimer = setInterval(runReconcile, RECONCILE_INTERVAL_MS);
+  }
   socket.on("connect", () => {
     src_default.info(
       { workerId: config.workerId, assignedWorkerId, url: config.sandboxUrl },
@@ -3532,10 +3758,26 @@ function connectToSandbox(config, executor) {
         "no reattach handshake from sandbox \u2014 starting heartbeat anyway"
       );
       startHeartbeat();
+      startReconcile();
     }, REATTACH_FALLBACK_MS);
     socket.once(WorkerEvent.REATTACH, (records) => {
       clearTimeout(reattachFallbackTimer);
-      executor.reattachAll(records).catch((error) => src_default.error({ err: error }, "reattach batch failed")).finally(() => {
+      executor.reattachAll(records).then((externallyTerminated) => {
+        for (const vmId of externallyTerminated) {
+          socket.emit(WorkerEvent.VM_EXTERNALLY_TERMINATED, { vmId });
+        }
+      }).catch((error) => src_default.error({ err: error }, "reattach batch failed")).finally(() => {
+        for (const record of records) {
+          const handle = executor.get(record.vmId);
+          if (record.state === "creating" && handle) {
+            const result = {
+              vmId: record.vmId,
+              ok: true,
+              metadata: handle.metadata
+            };
+            socket.emit(WorkerEvent.VM_CREATE_RESULT, result);
+          }
+        }
         for (const record of records) {
           if (record.runMode !== "job" || !executor.get(record.vmId)) continue;
           try {
@@ -3551,21 +3793,36 @@ function connectToSandbox(config, executor) {
           }
         }
         startHeartbeat();
+        startReconcile();
       });
     });
   });
   socket.on("disconnect", (reason) => {
     src_default.warn({ workerId: config.workerId, reason }, "disconnected from sandbox");
     clearInterval(heartbeatTimer);
+    clearInterval(reconcileTimer);
     clearTimeout(reattachFallbackTimer);
     heartbeatTimer = void 0;
+    reconcileTimer = void 0;
     socket.off(WorkerEvent.REATTACH);
   });
   socket.on("connect_error", (error) => {
-    src_default.error({
-      err: error,
-      sandboxUrl: config.sandboxUrl
-    }, "sandbox connection error");
+    src_default.error(
+      {
+        err: error,
+        sandboxUrl: config.sandboxUrl
+      },
+      "sandbox connection error"
+    );
+  });
+  socket.io.on("reconnect_attempt", (attempt) => {
+    src_default.warn({ workerId: config.workerId, attempt }, "reconnecting to sandbox");
+  });
+  socket.io.on("reconnect_error", (error) => {
+    src_default.warn({ workerId: config.workerId, err: error }, "sandbox reconnect attempt failed");
+  });
+  socket.io.on("reconnect", (attempt) => {
+    src_default.info({ workerId: config.workerId, attempt }, "reconnected to sandbox");
   });
   registerCommandHandlers(socket, executor);
   return socket;
@@ -3675,19 +3932,42 @@ var VncRelay = class {
 
 // src/services/serial-relay.ts
 var import_net3 = __toESM(require("net"));
+var import_ws = __toESM(require("ws"));
 var SerialRelay = class {
   constructor(vmId, source, socket) {
     this.vmId = vmId;
     this.source = source;
     this.socket = socket;
     this.conn = null;
+    this.webSocket = null;
+    this.reconnectTimer = null;
+    this.connecting = false;
+    this.browserCount = 0;
+    this.pendingInput = [];
+    this.destroyed = false;
     this.inputHandler = null;
+    this.attachHandler = null;
+    this.detachHandler = null;
   }
   connect() {
+    this.destroyed = false;
     if (this.source.type === "tcp") {
       this.connectTcp();
-    } else {
+    } else if (this.source.type === "pipe") {
       this.connectPipe();
+    } else {
+      this.attachHandler = ({ vmId }) => {
+        if (vmId !== this.vmId) return;
+        this.browserCount += 1;
+        if (this.browserCount === 1) void this.connectWebSocket();
+      };
+      this.detachHandler = ({ vmId }) => {
+        if (vmId !== this.vmId) return;
+        this.browserCount = Math.max(0, this.browserCount - 1);
+        if (this.browserCount === 0) this.disconnectWebSocket();
+      };
+      this.socket.on(WorkerEvent.TERMINAL_ATTACH, this.attachHandler);
+      this.socket.on(WorkerEvent.TERMINAL_DETACH, this.detachHandler);
     }
     this.inputHandler = (e) => {
       if (e.vmId !== this.vmId) return;
@@ -3696,6 +3976,15 @@ var SerialRelay = class {
       } else if (this.source.type === "pipe") {
         const proc = this.source.process;
         proc.stdin?.write(e.data);
+      } else if (this.source.type === "websocket") {
+        if (this.webSocket?.readyState === import_ws.default.OPEN) {
+          this.webSocket.send(e.data);
+        } else {
+          this.browserCount = Math.max(1, this.browserCount);
+          this.pendingInput.push(e.data);
+          if (this.pendingInput.length > 100) this.pendingInput.shift();
+          void this.connectWebSocket();
+        }
       }
     };
     this.socket.on(WorkerEvent.TERMINAL_INPUT, this.inputHandler);
@@ -3714,7 +4003,9 @@ var SerialRelay = class {
         sock.on("end", () => src_default.info({ vmId: this.vmId }, "[Serial] TCP ended"));
         sock.on("error", (err) => src_default.error({ vmId: this.vmId, err }, "[Serial] TCP error"));
       });
-      sock.once("error", () => setTimeout(attempt, 1e3));
+      sock.once("error", () => {
+        if (!this.destroyed) setTimeout(attempt, 1e3);
+      });
     };
     attempt();
   }
@@ -3729,13 +4020,65 @@ var SerialRelay = class {
       (d) => this.socket.emit(WorkerEvent.TERMINAL_DATA, { vmId: this.vmId, data: d })
     );
   }
+  async connectWebSocket() {
+    if (this.destroyed || this.browserCount === 0 || this.source.type !== "websocket" || this.webSocket || this.connecting) {
+      return;
+    }
+    try {
+      this.connecting = true;
+      const protocols = await this.source.protocols();
+      if (this.destroyed || this.browserCount === 0) return;
+      const webSocket = new import_ws.default(this.source.url, protocols);
+      this.webSocket = webSocket;
+      webSocket.on("open", () => {
+        for (const data of this.pendingInput.splice(0)) webSocket.send(data);
+        src_default.info({ vmId: this.vmId }, "[Serial] Lambda shell connected");
+      });
+      webSocket.on("message", (data) => {
+        const chunk = Array.isArray(data) ? Buffer.concat(data) : Buffer.isBuffer(data) ? data : Buffer.from(data);
+        this.socket.emit(WorkerEvent.TERMINAL_DATA, { vmId: this.vmId, data: chunk });
+      });
+      webSocket.on("error", (error) => {
+        src_default.error({ vmId: this.vmId, err: error }, "[Serial] Lambda shell error");
+      });
+      webSocket.on("close", () => {
+        if (this.webSocket === webSocket) this.webSocket = null;
+        src_default.info({ vmId: this.vmId }, "[Serial] Lambda shell disconnected");
+        this.scheduleWebSocketReconnect();
+      });
+    } catch (error) {
+      src_default.error({ vmId: this.vmId, err: error }, "[Serial] Lambda shell connect failed");
+      this.scheduleWebSocketReconnect();
+    } finally {
+      this.connecting = false;
+    }
+  }
+  scheduleWebSocketReconnect() {
+    if (this.destroyed || this.browserCount === 0 || this.reconnectTimer) return;
+    this.reconnectTimer = setTimeout(() => {
+      this.reconnectTimer = null;
+      void this.connectWebSocket();
+    }, 1e3);
+  }
+  disconnectWebSocket() {
+    if (this.reconnectTimer) clearTimeout(this.reconnectTimer);
+    this.reconnectTimer = null;
+    this.pendingInput = [];
+    const webSocket = this.webSocket;
+    this.webSocket = null;
+    webSocket?.close();
+  }
   destroy() {
+    this.destroyed = true;
     if (this.inputHandler) this.socket.off(WorkerEvent.TERMINAL_INPUT, this.inputHandler);
+    if (this.attachHandler) this.socket.off(WorkerEvent.TERMINAL_ATTACH, this.attachHandler);
+    if (this.detachHandler) this.socket.off(WorkerEvent.TERMINAL_DETACH, this.detachHandler);
     try {
       this.conn?.destroy();
     } catch {
     }
     this.conn = null;
+    this.disconnectWebSocket();
     src_default.info({ vmId: this.vmId }, "[Serial] relay destroyed");
   }
 };
@@ -3743,9 +4086,8 @@ var SerialRelay = class {
 // src/console-relays.ts
 function attachConsoleRelays(executor, sandboxSocket) {
   const relays = /* @__PURE__ */ new Map();
-  const originalCreate = executor.create.bind(executor);
-  executor.create = async (spec) => {
-    const handle = await originalCreate(spec);
+  const attach = (handle) => {
+    if (relays.has(handle.vmId)) return;
     const entry = {};
     if (handle.vncPort) {
       const vnc = new VncRelay(handle.vmId, "127.0.0.1", handle.vncPort, sandboxSocket);
@@ -3759,8 +4101,22 @@ function attachConsoleRelays(executor, sandboxSocket) {
       entry.serial = serial;
     }
     if (entry.vnc || entry.serial) relays.set(handle.vmId, entry);
+  };
+  const originalCreate = executor.create.bind(executor);
+  executor.create = async (spec) => {
+    const handle = await originalCreate(spec);
+    attach(handle);
     return handle;
   };
+  const originalReattachAll = executor.reattachAll.bind(executor);
+  executor.reattachAll = async (records) => {
+    const externallyTerminated = await originalReattachAll(records);
+    for (const record of records) {
+      const handle = executor.get(record.vmId);
+      if (handle) attach(handle);
+    }
+    return externallyTerminated;
+  };
   const originalDestroy = executor.destroy.bind(executor);
   executor.destroy = async (vmId) => {
     const entry = relays.get(vmId);
@@ -3855,68 +4211,111 @@ async function startSandboxHealthServer(options) {
 }
 
 // src/cli.ts
+process.on("unhandledRejection", (reason) => {
+  src_default.error({ err: reason }, "unhandled rejection \u2014 sandbox worker continues running");
+});
+process.on("uncaughtException", (error) => {
+  src_default.error({ err: error }, "uncaught exception \u2014 sandbox worker continues running");
+});
 var program = new import_commander.Command();
 program.name("sandbox").description("Run and manage a Multiplayer sandbox worker").version(version);
-program.command("start").description("start a worker and connect it to the sandbox control plane").option("--token <token>", "worker authentication token (env: WORKER_TOKEN)").option("--worker-id <id>", "worker pool/authentication ID (env: WORKER_ID; default: hostname)").option("--sandbox-url <url>", "control-plane base URL (env: SANDBOX_URL; default: https://api.sandbox.filingramp.com/v0)").option("--max-vms <count>", "maximum concurrent VMs (env: MAX_VMS)").action(async (opts) => {
-  const config = loadWorkerConfigFromEnv({
-    ...process.env,
-    ...opts.token ? { WORKER_TOKEN: opts.token } : {},
-    ...opts.workerId ? { WORKER_ID: opts.workerId } : {},
-    ...opts.sandboxUrl ? { SANDBOX_URL: opts.sandboxUrl } : {},
-    ...opts.maxVms ? { MAX_VMS: opts.maxVms } : {}
-  });
-  const socketRef = {};
-  const healthServer = await startSandboxHealthServer({
-    host: config.healthHost,
-    port: config.healthPort,
-    pathPrefix: config.healthPathPrefix,
-    isReady: () => socketRef.current?.connected === true
-  });
-  await prepareWorkerImages(config);
-  const executor = createExecutor(config);
-  const socket = connectToSandbox(config, executor);
-  socketRef.current = socket;
-  attachConsoleRelays(executor, socket);
-  let stopping = false;
-  const stop = async (signal) => {
-    if (stopping) return;
-    stopping = true;
-    console.log(`
+program.command("start").description("start a worker and connect it to the sandbox control plane").option("--token <token>", "worker authentication token (env: WORKER_TOKEN)").option("--worker-id <id>", "worker pool/authentication ID (env: WORKER_ID; default: hostname)").option(
+  "--sandbox-url <url>",
+  "control-plane base URL (env: SANDBOX_URL; default: https://api.sandbox.filingramp.com/v0)"
+).option("--max-vms <count>", "maximum concurrent VMs (env: MAX_VMS)").action(
+  async (opts) => {
+    const config = loadWorkerConfigFromEnv({
+      ...process.env,
+      ...opts.token ? { WORKER_TOKEN: opts.token } : {},
+      ...opts.workerId ? { WORKER_ID: opts.workerId } : {},
+      ...opts.sandboxUrl ? { SANDBOX_URL: opts.sandboxUrl } : {},
+      ...opts.maxVms ? { MAX_VMS: opts.maxVms } : {}
+    });
+    const socketRef = {};
+    const healthServer = await startSandboxHealthServer({
+      host: config.healthHost,
+      port: config.healthPort,
+      pathPrefix: config.healthPathPrefix,
+      isReady: () => socketRef.current?.connected === true
+    });
+    await prepareWorkerImages(config);
+    const executor = createExecutor(config);
+    const socket = connectToSandbox(config, executor);
+    socketRef.current = socket;
+    attachConsoleRelays(executor, socket);
+    let stopping = false;
+    const stop = async (signal) => {
+      if (stopping) return;
+      stopping = true;
+      console.log(`
 ${signal} received; shutting down sandbox worker...`);
-    socket.disconnect();
-    healthServer.close();
-    const results = await Promise.allSettled(executor.list().map((vm) => executor.destroy(vm.vmId)));
-    const failed = results.filter((result) => result.status === "rejected");
-    if (failed.length) console.error(`Failed to destroy ${failed.length} VM(s) during shutdown`);
-    process.exit(failed.length ? 1 : 0);
-  };
-  process.once("SIGINT", () => {
-    void stop("SIGINT");
-  });
-  process.once("SIGTERM", () => {
-    void stop("SIGTERM");
-  });
-  src_default.info(`Starting sandbox worker ${config.workerId} \u2192 ${config.sandboxUrl}`);
-});
+      socket.disconnect();
+      healthServer.close();
+      const results = await Promise.allSettled(
+        executor.list().map((vm) => executor.destroy(vm.vmId))
+      );
+      const failed = results.filter((result) => result.status === "rejected");
+      if (failed.length) console.error(`Failed to destroy ${failed.length} VM(s) during shutdown`);
+      process.exit(failed.length ? 1 : 0);
+    };
+    process.once("SIGINT", () => {
+      void stop("SIGINT");
+    });
+    process.once("SIGTERM", () => {
+      void stop("SIGTERM");
+    });
+    src_default.info(`Starting sandbox worker ${config.workerId} \u2192 ${config.sandboxUrl}`);
+  }
+);
 program.command("detect").description("probe this host and report which micro-VM engines it can run").action(() => {
-  const probes = probeEngines(loadWorkerConfigFromEnv({ ...process.env, WORKER_ID: "-", WORKER_TOKEN: "-" }));
+  const probes = probeEngines(
+    loadWorkerConfigFromEnv({ ...process.env, WORKER_ID: "-", WORKER_TOKEN: "-" })
+  );
   for (const probe of probes) {
     const status = probe.available ? "available" : `unavailable \u2014 ${probe.reason}`;
     console.log(`${probe.engine.padEnd(15)} ${status}`);
   }
 });
 var images = program.command("images").description("manage engine base images (kernel + initramfs for QEMU/Firecracker)");
-images.command("pull").description("download per-stack base images from S3 or an HTTPS mirror into the standard location").requiredOption("--from <location>", "s3://bucket[/prefix] or https:// base URL (default derived from S3_BUCKET_ARTIFACTS)", artifactImageLocation()).option("--arch <arch>", "image architecture", hostArch()).option("--stack <stack...>", "stacks to pull (default: all)").option("--dest <dir>", "destination directory (default: ~/.multiplayer/vm-images/<arch>)").option("--endpoint <url>", "custom S3 endpoint, e.g. a MinIO URL (env: S3_ARTIFACTS_ENDPOINT)", process.env.S3_ARTIFACTS_ENDPOINT).action(async (opts) => {
-  const dest = opts.dest ?? defaultImagesDir(opts.arch);
-  const files = await pullImages({ location: opts.from, arch: opts.arch, stacks: opts.stack, endpoint: opts.endpoint }, dest);
-  for (const file of files) console.log(`pulled ${file}`);
-  console.log('\nbase images ready \u2014 "sandbox detect" shows which engines they enable');
-});
-images.command("push").description("upload locally built per-stack base images (scripts/setup-vm-images.sh) to S3").requiredOption("--to <location>", "s3://bucket[/prefix] (default derived from S3_BUCKET_ARTIFACTS)", artifactImageLocation()).option("--arch <arch>", "image architecture", hostArch()).option("--stack <stack...>", "stacks to push (default: all found locally)").option("--src <dir>", "source directory (default: ~/.multiplayer/vm-images/<arch>)").option("--endpoint <url>", "custom S3 endpoint, e.g. a MinIO URL (env: S3_ARTIFACTS_ENDPOINT)", process.env.S3_ARTIFACTS_ENDPOINT).action(async (opts) => {
-  const src = opts.src ?? defaultImagesDir(opts.arch);
-  const uploaded = await pushImages({ location: opts.to, arch: opts.arch, stacks: opts.stack, endpoint: opts.endpoint }, src);
-  for (const uri of uploaded) console.log(`pushed ${uri}`);
-});
+images.command("pull").description(
+  "download per-stack base images from S3 or an HTTPS mirror into the standard location"
+).requiredOption(
+  "--from <location>",
+  "s3://bucket[/prefix] or https:// base URL (default derived from S3_BUCKET_ARTIFACTS)",
+  artifactImageLocation()
+).option("--arch <arch>", "image architecture", hostArch()).option("--stack <stack...>", "stacks to pull (default: all)").option("--dest <dir>", "destination directory (default: ~/.multiplayer/vm-images/<arch>)").option(
+  "--endpoint <url>",
+  "custom S3 endpoint, e.g. a MinIO URL (env: S3_ARTIFACTS_ENDPOINT)",
+  process.env.S3_ARTIFACTS_ENDPOINT
+).action(
+  async (opts) => {
+    const dest = opts.dest ?? defaultImagesDir(opts.arch);
+    const files = await pullImages(
+      { location: opts.from, arch: opts.arch, stacks: opts.stack, endpoint: opts.endpoint },
+      dest
+    );
+    for (const file of files) console.log(`pulled ${file}`);
+    console.log('\nbase images ready \u2014 "sandbox detect" shows which engines they enable');
+  }
+);
+images.command("push").description("upload locally built per-stack base images (scripts/setup-vm-images.sh) to S3").requiredOption(
+  "--to <location>",
+  "s3://bucket[/prefix] (default derived from S3_BUCKET_ARTIFACTS)",
+  artifactImageLocation()
+).option("--arch <arch>", "image architecture", hostArch()).option("--stack <stack...>", "stacks to push (default: all found locally)").option("--src <dir>", "source directory (default: ~/.multiplayer/vm-images/<arch>)").option(
+  "--endpoint <url>",
+  "custom S3 endpoint, e.g. a MinIO URL (env: S3_ARTIFACTS_ENDPOINT)",
+  process.env.S3_ARTIFACTS_ENDPOINT
+).action(
+  async (opts) => {
+    const src = opts.src ?? defaultImagesDir(opts.arch);
+    const uploaded = await pushImages(
+      { location: opts.to, arch: opts.arch, stacks: opts.stack, endpoint: opts.endpoint },
+      src
+    );
+    for (const uri of uploaded) console.log(`pushed ${uri}`);
+  }
+);
 program.parseAsync().catch((error) => {
   console.error(`Error: ${error.message}`);
   process.exit(1);