@rainbow-robotics/muscat-plugin-kit 0.1.2-dev.9 → 0.10.1-dev.1

package/cli/src/mcpProxy.mjs

@@ -0,0 +1,110 @@
+// `muscat-pk mcp` — AI clients keep one stdio MCP server per session, so a `muscat-pk update`
+// never reached a running one. This relays stdio to the real server (`mcp --serve`) and, when the
+// installed kit changes, swaps that child for the new one: it replays the client's initialize and
+// sends tools/list_changed, so the connection stays up and the next calls use the new tools.
+import { spawn } from "node:child_process";
+import { statSync } from "node:fs";
+import { resolve } from "node:path";
+import { createInterface } from "node:readline";
+import { kitRoot, resolveLocalCliBin } from "./util.mjs";
+
+const isBinaryMode = typeof Bun !== "undefined";
+const REINIT_ID = "__muscat_pk_reinit__";
+const POLL_MS = 3000;
+
+// Re-resolved on every (re)start: an update may install, bump or remove the project's local kit.
+function serverCommand() {
+  const local = resolveLocalCliBin(process.cwd());
+  if (local) {
+    return { cmd: isBinaryMode ? "node" : process.execPath, args: [local, "mcp", "--serve"], watch: resolve(local, "../../../package.json") };
+  }
+  if (isBinaryMode) return { cmd: process.execPath, args: ["mcp", "--serve"], watch: process.execPath };
+  return { cmd: process.execPath, args: [process.argv[1], "mcp", "--serve"], watch: resolve(kitRoot, "package.json") };
+}
+
+const stamp = (file) => {
+  try {
+    return `${file}:${statSync(file).mtimeMs}`;
+  } catch {
+    return file;
+  }
+};
+
+export function runMcpProxy() {
+  let child = null;
+  let initRequest = null;
+  const pending = new Set(); // client request ids the current child hasn't answered
+  const toClient = (msg) => process.stdout.write(JSON.stringify(msg) + "\n");
+
+  const failPending = () => {
+    for (const id of pending) {
+      toClient({ jsonrpc: "2.0", id, error: { code: -32603, message: "muscat-pk was updated during this call — retry it." } });
+    }
+    pending.clear();
+  };
+
+  const start = () => {
+    const { cmd, args } = serverCommand();
+    const c = spawn(cmd, args, { stdio: ["pipe", "pipe", "inherit"] });
+    createInterface({ input: c.stdout }).on("line", (line) => {
+      let msg = null;
+      try {
+        msg = JSON.parse(line);
+      } catch {
+        /* not JSON — pass through */
+      }
+      if (msg?.id === REINIT_ID) {
+        // Our replayed initialize: the client already has its answer from the first child.
+        c.stdin.write(JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }) + "\n");
+        toClient({ jsonrpc: "2.0", method: "notifications/tools/list_changed" });
+        return;
+      }
+      if (msg && msg.id !== undefined && !msg.method) pending.delete(msg.id);
+      process.stdout.write(line + "\n");
+    });
+    c.on("exit", () => {
+      if (c !== child) return; // replaced on purpose
+      child = null;
+      failPending();
+    });
+    child = c;
+    if (initRequest) c.stdin.write(JSON.stringify({ ...initRequest, id: REINIT_ID }) + "\n");
+  };
+
+  const restart = () => {
+    const old = child;
+    child = null;
+    old?.kill();
+    failPending();
+    start();
+  };
+
+  start();
+  let last = stamp(serverCommand().watch);
+  setInterval(() => {
+    const now = stamp(serverCommand().watch);
+    if (now !== last) {
+      last = now;
+      process.stderr.write("[muscat-pk] kit updated — restarting the MCP server\n");
+      restart();
+    }
+  }, POLL_MS).unref();
+
+  createInterface({ input: process.stdin })
+    .on("line", (line) => {
+      let msg = null;
+      try {
+        msg = JSON.parse(line);
+      } catch {
+        /* forward as-is */
+      }
+      if (msg?.method === "initialize") initRequest = msg;
+      if (!child) start(); // the server crashed — bring it back (it replays initialize)
+      if (msg && msg.id !== undefined && msg.method) pending.add(msg.id);
+      child.stdin.write(line + "\n");
+    })
+    .on("close", () => {
+      child?.kill();
+      process.exit(0);
+    });
+}

package/cli/src/muscatLocal.mjs

@@ -0,0 +1,292 @@
+// Local Muscat runtime for `dev` / `dev:ui`, with or without a robot. It is the plugin gateway
+// (:10000, plugin-socketio — same Socket.IO protocol as on the robot), MediaMTX, and `common`:
+// preview-host hits http://localhost:3000/common/* (baked into the nexus bundle), so program
+// preview works with no front-end change. The backend container joins its netns. We reuse any
+// Muscat already publishing :10000 (e.g. a `muscat` / `muscat-dev` container) and only otherwise
+// start our own: the S3 runtime image with `common` (plus the services `--sim` needs).
+import { spawnSync } from "node:child_process";
+import { createServer } from "node:net";
+import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, resolve } from "node:path";
+import { mediaPortFlags, MEDIA_PORTS, publishPortFlags } from "./ports.mjs";
+import { canSkipImageBuild, hasBin, log, buildCacheFlags, pruneMuskatImages } from "./util.mjs";
+import { LOCAL_HOSTS } from "./zenohDev.mjs";
+
+export const MUSCAT_CONTAINER = "muscat-pk-muscat";
+// Ports published for the backend (it shares this container's netns); a different set recreates it.
+const PORTS_LABEL = "com.rainbow-robotics.muscat-pk.ports";
+const IMAGE = "muscat-pk-runtime";
+const S3 = process.env.MUSCAT_PK_MUSCAT_S3 ?? "https://rainbow-deploy.s3.ap-northeast-2.amazonaws.com/muscat/main";
+const CACHE = resolve(homedir(), ".muscat-pk", "muscat");
+const API_PORT = 3000;
+export const GATEWAY_PORT = 10000;
+// The bin runs inside Docker, so follow the Docker engine's arch, not this Node's
+// (Rosetta/x64 Node on an Apple Silicon host would otherwise pick amd64).
+const dockerArch = () => {
+  const r = spawnSync("docker", ["version", "--format", "{{.Server.Arch}}"], { encoding: "utf8" });
+  const a = (r.stdout || "").trim();
+  return a === "arm64" || a === "amd64" ? a : process.arch === "arm64" ? "arm64" : "amd64";
+};
+
+// Node's fetch reports every network error as "fetch failed"; the real reason (ECONNRESET, ...) is in `cause`.
+const fetchErrorReason = (e) => e.cause?.code ?? e.code ?? e.cause?.message ?? e.message;
+
+// The S3 zips are ~200MB, so a dropped connection mid-download is retried. 4xx is not.
+export async function fetchBuffer(url, attempts = 3) {
+  for (let i = 1; ; i++) {
+    let err;
+    try {
+      const res = await fetch(url);
+      if (res.ok) return Buffer.from(await res.arrayBuffer());
+      err = new Error(`HTTP ${res.status}: ${url}`);
+      if (res.status < 500 && res.status !== 429) throw err;
+    } catch (e) {
+      if (e === err) throw e;
+      err = new Error(`fetch failed (${fetchErrorReason(e)}): ${url}`, { cause: e });
+    }
+    if (i >= attempts) throw err;
+    log.warn(`${err.message} — retrying (${i + 1}/${attempts})…`);
+    await new Promise((r) => setTimeout(r, 1000 * i));
+  }
+}
+
+// macOS/Windows `tar` reads zip; GNU tar does not, Linux has `unzip`.
+function extractZip(zip, dir, members = []) {
+  mkdirSync(dir, { recursive: true });
+  const tries = [
+    ["unzip", ["-o", "-q", zip, ...members, "-d", dir]],
+    ["tar", ["-xf", zip, "-C", dir, ...members]],
+  ];
+  for (const [bin, args] of tries) {
+    if (hasBin(bin) && spawnSync(bin, args, { stdio: "ignore" }).status === 0) return;
+  }
+  throw new Error(`could not extract ${zip} (needs unzip or tar)`);
+}
+
+async function resolveVersions() {
+  const file = resolve(CACHE, "version.json");
+  try {
+    const data = await fetchBuffer(`${S3}/version.json`);
+    mkdirSync(CACHE, { recursive: true });
+    writeFileSync(file, data);
+    return JSON.parse(data.toString());
+  } catch (e) {
+    if (!existsSync(file)) throw e;
+    log.warn(`version.json unreachable (${e.message}) — using cached Muscat version`);
+    return JSON.parse(readFileSync(file, "utf8"));
+  }
+}
+
+// A new Muscat version replaces the previous runtime image (~2.8GB) and its scripts. One still in
+// use by the running local Muscat stays until a later run (rmi refuses it).
+export function removeOldRuntimes(total) {
+  const tags = spawnSync("docker", ["images", IMAGE, "--format", "{{.Tag}}"], { encoding: "utf8" }).stdout ?? "";
+  for (const tag of tags.split("\n").map((t) => t.trim()).filter((t) => t && t !== total)) {
+    spawnSync("docker", ["rmi", `${IMAGE}:${tag}`], { stdio: "ignore" });
+  }
+  const dir = resolve(CACHE, "runtime");
+  for (const old of existsSync(dir) ? readdirSync(dir) : []) {
+    if (old !== total) rmSync(resolve(dir, old), { recursive: true, force: true });
+  }
+}
+
+async function ensureRuntimeImage(total) {
+  const image = `${IMAGE}:${total}`;
+  removeOldRuntimes(total);
+  if (canSkipImageBuild(image)) return image;
+  const dir = resolve(CACHE, "runtime", total);
+  if (!existsSync(resolve(dir, "Dockerfile.runtime"))) {
+    log.step(`downloading Muscat runtime scripts (muscat.zip ${total})…`);
+    const zip = resolve(CACHE, "muscat.zip");
+    writeFileSync(zip, await fetchBuffer(`${S3}/muscat.zip`));
+    extractZip(zip, dir);
+    rmSync(zip, { force: true });
+  }
+  log.step(`docker build -t ${image} (first run downloads MongoDB/InfluxDB/ROS base — slow, then cached)…`);
+  const r = spawnSync(
+    "docker",
+    ["build", ...buildCacheFlags(), "-t", image, "-f", resolve(dir, "Dockerfile.runtime"), "--label", "com.rainbow-robotics.muscat-pk=true", dir],
+    { stdio: "inherit" },
+  );
+  if (r.status !== 0) throw new Error("docker build failed (Muscat runtime image).");
+  pruneMuskatImages();
+  return image;
+}
+
+async function ensureServiceBinary(svc, version) {
+  const ARCH = dockerArch();
+  const dir = resolve(CACHE, "services", svc);
+  const stamp = resolve(dir, ".version");
+  if (existsSync(stamp) && readFileSync(stamp, "utf8").trim() === version && existsSync(resolve(dir, `${svc}.${ARCH}.bin`))) return dir;
+  log.step(`downloading Muscat ${svc} service ${version} (${ARCH})…`);
+  const zip = resolve(CACHE, `${svc}-${version}.zip`);
+  writeFileSync(zip, await fetchBuffer(`${S3}/${svc}/${version}.zip`));
+  extractZip(zip, dir, ["config.env", `${svc}.${ARCH}.bin`]);
+  rmSync(zip, { force: true });
+  chmodSync(resolve(dir, `${svc}.${ARCH}.bin`), 0o755);
+  writeFileSync(stamp, `${version}\n`);
+  return dir;
+}
+
+// Sim containers live on this network and connect to common's Zenoh by container name.
+export const MUSCAT_NETWORK = "muscat-pk";
+
+function ensureNetwork() {
+  if (spawnSync("docker", ["network", "inspect", MUSCAT_NETWORK], { stdio: "ignore" }).status === 0) return;
+  spawnSync("docker", ["network", "create", "--label", "com.rainbow-robotics.muscat-pk=true", MUSCAT_NETWORK], { stdio: "ignore" });
+}
+
+// common (127.0.0.1:7447) and the services (127.0.0.1:0) listen on loopback by default; rb-ctl's
+// listen-endpoints file opens both to the network. Sims reach common as muscat-pk-muscat:7447, and
+// gossip has them dial each service back — on loopback only the sim that happened to connect first
+// answers manipulate's receive_one.
+// Mounted as the only file of /app/data; the rest of the runtime's data stays in the container.
+function zenohListenMount(open) {
+  if (!open) return [];
+  const file = resolve(CACHE, "zenoh_listen_endpoints.json5");
+  mkdirSync(dirname(file), { recursive: true });
+  writeFileSync(file, `{\n  common: ["tcp/0.0.0.0:7447"],\n  service: ["tcp/0.0.0.0:0"],\n}\n`);
+  return ["-v", `${file}:/app/data/zenoh_listen_endpoints.json5:ro`];
+}
+
+async function waitForApi(timeoutMs = 180_000) {
+  const start = Date.now();
+  for (;;) {
+    try {
+      const res = await fetch(`http://127.0.0.1:${API_PORT}/common/docs`);
+      if (res.ok) return;
+    } catch { /* not listening yet */ }
+    if (Date.now() - start > timeoutMs) throw new Error(`Muscat did not answer on :${API_PORT} in time — see \`docker logs ${MUSCAT_CONTAINER}\``);
+    await new Promise((r) => setTimeout(r, 1000));
+  }
+}
+
+// Does the robot at `ip` run Muscat (common answers /is-muscat)? Then the dev UI and backend use its
+// gateway and no local Muscat is started. Local addresses and unreachable hosts → false.
+async function isMuscat(ip) {
+  try {
+    const res = await fetch(`http://${ip}:${API_PORT}/common/is-muscat`, { signal: AbortSignal.timeout(3000) });
+    return res.ok && (await res.json())?.result === true;
+  } catch {
+    return false;
+  }
+}
+
+export async function robotHasMuscat(ip) {
+  if (!ip || LOCAL_HOSTS.has(ip)) return false;
+  return isMuscat(ip);
+}
+
+// rby2's full Muscat dev stack on this machine — unlike muscat-pk's own minimal runtime it has the
+// whole UI backend, so `dev` without a robot IP shows its real Muscat UI, like a robot's.
+const LOCAL_FULL_MUSCAT = "muscat-dev";
+export async function localFullMuscat(ip) {
+  if (!LOCAL_HOSTS.has(ip ?? "") || !hasBin("docker")) return false;
+  const r = spawnSync("docker", ["ps", "--filter", `name=^${LOCAL_FULL_MUSCAT}$`, "--format", "{{.Names}}"], { encoding: "utf8" });
+  return !!(r.stdout || "").trim() && isMuscat("127.0.0.1");
+}
+
+// Is a host TCP port already bound?
+export function isPortInUse(port) {
+  return new Promise((res) => {
+    const srv = createServer();
+    srv.once("error", (err) => res(err.code === "EADDRINUSE"));
+    srv.once("listening", () => srv.close(() => res(false)));
+    srv.listen(port, "0.0.0.0");
+  });
+}
+
+/** A running Muscat container (ours or e.g. muscat-dev) that owns host port 10000, or null. */
+export function localMuscatContainer() {
+  const r = spawnSync("docker", ["ps", "--filter", `publish=${GATEWAY_PORT}`, "--format", "{{.Names}}"], { encoding: "utf8" });
+  return (r.stdout || "").split("\n").map((n) => n.trim()).find(Boolean) ?? null;
+}
+
+const containerLabel = (name, label) =>
+  spawnSync("docker", ["inspect", "-f", `{{index .Config.Labels "${label}"}}`, name], { encoding: "utf8" }).stdout?.trim() ?? "";
+
+/**
+ * Make sure a Muscat answers on localhost:3000 / :10000. Returns the container name,
+ * or null when one can't be started (no Docker, foreign process on the ports).
+ * `dev` removes it on exit; `dev:ui` alone leaves it running (`docker rm -f muscat-pk-muscat`).
+ */
+export async function ensureLocalMuscat({ pluginPorts = [], zenohPort = null, services = [], env = {} } = {}) {
+  if (process.env.MUSCAT_PK_NO_LOCAL_MUSCAT === "1") return null;
+  // Extra services (log, manipulate, rby1, ...) and env come with --sim; a different set recreates it too.
+  const svcs = ["common", ...services.filter((s) => s !== "common")];
+  const envFlags = Object.entries(env).flatMap(([k, v]) => ["-e", `${k}=${v}`]);
+  const portsKey = [
+    ...pluginPorts,
+    ...(zenohPort ? [`zenoh:${zenohPort}`] : []),
+    "ipc:host",
+    ...(services.length ? [`svc:${svcs.join("+")}`] : []),
+    ...Object.entries(env).map(([k, v]) => `${k}=${v}`),
+  ].join(",");
+  const existing = hasBin("docker") ? localMuscatContainer() : null;
+  if (existing === MUSCAT_CONTAINER && containerLabel(existing, PORTS_LABEL) !== portsKey) {
+    log.step(`recreating local Muscat for ${portsKey || "no extra ports / services"}`);
+    spawnSync("docker", ["rm", "-f", MUSCAT_CONTAINER], { stdio: "ignore" });
+  } else if (existing) {
+    if (existing !== MUSCAT_CONTAINER && portsKey)
+      log.warn(`'${existing}' is not managed by muscat-pk — port(s) ${portsKey} are not published from it.`);
+    return existing;
+  }
+  if (!hasBin("docker")) {
+    log.warn("Docker not found — no local Muscat; program preview will have no back end.");
+    return null;
+  }
+  // Ports taken but no container publishing them: usually a Muscat on host networking (robot PC,
+  // CI runner) — the caller uses it as is.
+  for (const port of [API_PORT, GATEWAY_PORT]) {
+    if (await isPortInUse(port)) return null;
+  }
+
+  const versions = await resolveVersions();
+  const total = versions.total;
+  if (!total) throw new Error("version.json has no total");
+  const missing = svcs.filter((s) => !versions.services?.[s]?.version);
+  if (missing.length) throw new Error(`version.json has no services.${missing.join(" / ")}.version`);
+
+  const image = await ensureRuntimeImage(total);
+  const svcMounts = [];
+  for (const s of svcs) svcMounts.push("-v", `${await ensureServiceBinary(s, versions.services[s].version)}:/app/services/${s}`);
+  ensureNetwork();
+  // Earlier kits kept DB, logs and programs here across runs.
+  rmSync(resolve(CACHE, "data"), { recursive: true, force: true });
+
+  let media = true;
+  for (const port of MEDIA_PORTS) if (await isPortInUse(port)) media = false;
+  // A busy plugin port would make `docker run` fail with a raw daemon error — skip it and say so.
+  const busy = [];
+  for (const port of pluginPorts) if (await isPortInUse(port)) busy.push(port);
+  if (busy.length) log.warn(`plugin port(s) already in use on the host: ${busy.join(", ")} — not published (the backend still listens inside).`);
+  const published = pluginPorts.filter((port) => !busy.includes(port));
+
+  spawnSync("docker", ["rm", "-f", MUSCAT_CONTAINER], { stdio: "ignore" });
+  log.step(`starting local Muscat (${svcs.map((s) => `${s} ${versions.services[s].version}`).join(", ")}) → http://localhost:${API_PORT}/common/docs`);
+  const r = spawnSync("docker", [
+    "run", "-d", "--name", MUSCAT_CONTAINER, "--label", "com.rainbow-robotics.muscat-pk=true", "--label", `${PORTS_LABEL}=${portsKey}`,
+    "--network", MUSCAT_NETWORK,
+    // Like the robot, where plugins run with --ipc host: zenoh SHM and plugin shm_open reach the services.
+    "--ipc", "host",
+    "-e", `RB_ENABLED_SERVICES=${svcs.join(",")}`,
+    "-e", "RB_RUNTIME_ENABLE_ZENOH_BRIDGE=0",
+    ...envFlags,
+    ...svcMounts,
+    ...zenohListenMount(services.length > 0),
+    "-p", `127.0.0.1:${API_PORT}:${API_PORT}`,
+    "-p", `127.0.0.1:${GATEWAY_PORT}:${GATEWAY_PORT}`,
+    ...(media ? mediaPortFlags() : []),
+    ...publishPortFlags(published),
+    // The robot reverse-dials the backend's Zenoh session from the LAN, so not loopback-only.
+    ...(zenohPort ? ["-p", `${zenohPort}:${zenohPort}`] : []),
+    image,
+  ], { stdio: ["ignore", "ignore", "inherit"] });
+  if (r.status !== 0) throw new Error("failed to start the local Muscat container.");
+  await waitForApi();
+  log.step("local Muscat ready");
+  if (published.length) log.step(`plugin backend ports published (TCP/UDP): ${published.join(", ")}`);
+  if (zenohPort) log.step(`Zenoh dev peer port published on :${zenohPort}`);
+  return MUSCAT_CONTAINER;
+}