@mmerterden/multi-agent-toolkit-mcp 3.14.0 → 3.15.0

package/tools/policy/index.js

@@ -0,0 +1,283 @@
+// Opt-in egress and read policy.
+//
+// The web family opens whatever URL it is handed, and the context family
+// indexes whatever file it is handed. On a developer's own machine that is the
+// point. On a host that feeds the server instructions it did not write (a
+// crawled page, a ticket, a PR description), the same reach turns into a way to
+// read the cloud metadata endpoint, a service on localhost or the office
+// network, or ~/.ssh.
+//
+// MCP_TOOLKIT_URL_POLICY=strict narrows it. Unset, every function here answers
+// "allowed" without looking, so the default behaviour is exactly what it was.
+//
+//   URL policy    anything that is not http(s) is refused, and so is any host
+//                 that is, or resolves to, a loopback, unspecified, link-local,
+//                 private (RFC1918, CGNAT 100.64/10, IPv6 ULA fc00::/7, the
+//                 deprecated IPv6 site-local fec0::/10),
+//                 0.0.0.0/8, IETF protocol (192.0.0/24), benchmarking
+//                 (198.18/15), multicast or reserved (224/4, 240/4, ff00::/8)
+//                 address, or a cloud metadata endpoint. An IPv6 address that
+//                 carries an IPv4 one (mapped, compatible, translated, NAT64,
+//                 6to4, Teredo) is judged by the IPv4 address it carries.
+//                 MCP_TOOLKIT_URL_ALLOW (comma-separated hosts, IPs and CIDRs)
+//                 names exceptions: a listed host name, a literal IP inside a
+//                 listed range, or a name whose every refused answer falls
+//                 inside a listed range is let through. It never widens the
+//                 scheme check, and it has no effect without strict.
+//   Index policy  context_index, context_get and context_search refuse paths
+//                 under the deny list. Strict brings a default list of
+//                 credential stores. MCP_TOOLKIT_INDEX_DENY (comma-separated)
+//                 adds to that list and applies with or without strict; a value
+//                 that starts with "!" replaces the defaults instead, and one
+//                 that leaves nothing denied is reported on stderr.
+//
+// The server applies the URL check to every request the browser context makes
+// (navigations, subresources, popups, WebSockets), and again to the URL a
+// navigation landed on. A DNS answer can still change between the check and
+// the browser's own lookup.
+
+import { lookup as dnsLookup } from "node:dns/promises";
+import { isIP } from "node:net";
+import { realpathSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname, join, resolve, sep } from "node:path";
+
+export const DEFAULT_INDEX_DENY = [
+  "~/.ssh", "~/.aws", "~/.gnupg", "~/Library/Keychains",
+  "~/.netrc", "~/.git-credentials", "~/.config/gh", "~/.docker", "~/.kube", "~/.npmrc",
+  "~/.config/gcloud", "~/.azure", "~/.pypirc", "~/.cargo/credentials", "~/.cargo/credentials.toml",
+];
+
+// Hostnames that answer with instance credentials on the major clouds. The
+// addresses behind them (169.254.169.254, fd00:ec2::254, 100.100.100.200) are
+// refused by address as well, for a caller that uses the IP directly.
+const METADATA_HOSTS = new Set([
+  "metadata",
+  "metadata.google.internal",
+  "metadata.goog",
+  "instance-data",
+  "instance-data.ec2.internal",
+]);
+const METADATA_ADDRESSES = new Set(["100.100.100.200"]);
+
+// Anything other than "strict" or "off" is a typo in a security setting, so it
+// is reported and treated as strict rather than silently ignored.
+export function urlPolicy(env = process.env, warn = (m) => process.stderr.write(m)) {
+  const raw = String(env.MCP_TOOLKIT_URL_POLICY ?? "").trim().toLowerCase();
+  if (!raw || raw === "off") return "off";
+  if (raw !== "strict") warn(`multi-agent-toolkit: unknown MCP_TOOLKIT_URL_POLICY value "${raw}", applying strict\n`);
+  return "strict";
+}
+
+function ipv6Groups(address) {
+  let a = address.toLowerCase().split("%")[0];
+  const dotted = a.match(/(\d+)\.(\d+)\.(\d+)\.(\d+)$/);
+  if (dotted) {
+    const [p0, p1, p2, p3] = dotted.slice(1).map(Number);
+    a = `${a.slice(0, -dotted[0].length)}${((p0 << 8) | p1).toString(16)}:${((p2 << 8) | p3).toString(16)}`;
+  }
+  const [head, tail] = a.split("::");
+  const h = head ? head.split(":") : [];
+  if (tail === undefined) return h.map((g) => parseInt(g, 16));
+  const t = tail ? tail.split(":") : [];
+  return [...h, ...Array(Math.max(0, 8 - h.length - t.length)).fill("0"), ...t].map((g) => parseInt(g || "0", 16));
+}
+
+// IPv4-mapped IPv6 (::ffff:a.b.c.d) is judged as the IPv4 address it carries.
+function normalizeIp(address) {
+  const ip = String(address).replace(/^\[|\]$/g, "");
+  const kind = isIP(ip);
+  if (kind === 4) return { kind: 4, parts: ip.split(".").map(Number) };
+  if (kind === 6) {
+    const g = ipv6Groups(ip);
+    if (g.slice(0, 5).every((x) => x === 0) && g[5] === 0xffff) {
+      return { kind: 4, parts: [g[6] >> 8, g[6] & 255, g[7] >> 8, g[7] & 255] };
+    }
+    return { kind: 6, parts: g };
+  }
+  return null;
+}
+
+function blockedV4([a, b, c]) {
+  return a === 0 || a === 10 || a === 127
+    || (a === 169 && b === 254)
+    || (a === 172 && b >= 16 && b <= 31)
+    || (a === 192 && b === 168)
+    || (a === 192 && b === 0 && c === 0)
+    || (a === 198 && (b === 18 || b === 19))
+    || (a === 100 && b >= 64 && b <= 127)
+    || a >= 224;
+}
+
+const v4From = (hi, lo) => [hi >> 8, hi & 255, lo >> 8, lo & 255];
+
+// IPv6 forms that carry an IPv4 address a relay or translator will deliver to:
+// NAT64 64:ff9b::/96, IPv4-compatible ::a.b.c.d, IPv4-translated
+// ::ffff:0:a.b.c.d, 6to4 2002::/16 and Teredo 2001::/32 (server in bits 32-63,
+// client inverted in the last 32).
+function embeddedV4(g) {
+  const out = [];
+  if (g[0] === 0x64 && g[1] === 0xff9b && g.slice(2, 6).every((x) => x === 0)) out.push(v4From(g[6], g[7]));
+  if (g.slice(0, 6).every((x) => x === 0)) out.push(v4From(g[6], g[7]));
+  if (g.slice(0, 4).every((x) => x === 0) && g[4] === 0xffff && g[5] === 0) out.push(v4From(g[6], g[7]));
+  if (g[0] === 0x2002) out.push(v4From(g[1], g[2]));
+  if (g[0] === 0x2001 && g[1] === 0) out.push(v4From(g[2], g[3]), v4From(g[6] ^ 0xffff, g[7] ^ 0xffff));
+  return out;
+}
+
+export function isBlockedAddress(address) {
+  const n = normalizeIp(address);
+  if (!n) return false;
+  if (n.kind === 4) return blockedV4(n.parts) || METADATA_ADDRESSES.has(n.parts.join("."));
+  const g = n.parts;
+  if (g.slice(0, 7).every((x) => x === 0) && (g[7] === 0 || g[7] === 1)) return true;
+  if ((g[0] & 0xffc0) === 0xfe80) return true;
+  if ((g[0] & 0xffc0) === 0xfec0) return true;
+  if ((g[0] & 0xfe00) === 0xfc00) return true;
+  if ((g[0] & 0xff00) === 0xff00) return true;
+  if (g[0] === 0x64 && g[1] === 0xff9b && g[2] === 1) return true;
+  return embeddedV4(g).some((v4) => blockedV4(v4) || METADATA_ADDRESSES.has(v4.join(".")));
+}
+
+function bitsOf(n) {
+  const width = n.kind === 4 ? 8 : 16;
+  return n.parts.map((p) => p.toString(2).padStart(width, "0")).join("");
+}
+
+function inCidr(address, cidr) {
+  const n = normalizeIp(address);
+  if (!n || n.kind !== cidr.kind) return false;
+  return bitsOf(n).slice(0, cidr.prefix) === cidr.bits;
+}
+
+// Entries are host names (matched exactly, case-insensitively), IP literals,
+// or CIDRs. A malformed entry is reported and dropped: a typo in an allow list
+// must narrow it, never turn into "allow everything".
+export function urlAllowList(env = process.env, warn = (m) => process.stderr.write(m)) {
+  const out = [];
+  for (const raw of String(env.MCP_TOOLKIT_URL_ALLOW ?? "").split(",")) {
+    const entry = raw.trim().toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, "");
+    if (!entry) continue;
+    const slash = entry.indexOf("/");
+    const ipPart = slash >= 0 ? entry.slice(0, slash).replace(/^\[|\]$/g, "") : entry;
+    const n = normalizeIp(ipPart);
+    if (slash < 0 && !n) {
+      if (/^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$/.test(entry)) out.push({ host: entry });
+      else warn(`multi-agent-toolkit: ignoring MCP_TOOLKIT_URL_ALLOW entry "${raw.trim()}" (not a host, IP or CIDR)\n`);
+      continue;
+    }
+    const max = isIP(ipPart) === 4 ? 32 : 128;
+    const prefixText = slash >= 0 ? entry.slice(slash + 1) : String(max);
+    const prefix = /^\d{1,3}$/.test(prefixText) ? Number(prefixText) : NaN;
+    // A mapped IPv6 literal normalises to IPv4, so its prefix is re-based.
+    const effective = n && n.kind === 4 && max === 128 ? prefix - 96 : prefix;
+    if (!n || !(prefix >= 0 && prefix <= max) || effective < 0) {
+      warn(`multi-agent-toolkit: ignoring MCP_TOOLKIT_URL_ALLOW entry "${raw.trim()}" (not a valid CIDR)\n`);
+      continue;
+    }
+    out.push({ kind: n.kind, prefix: effective, bits: bitsOf(n).slice(0, effective) });
+  }
+  return out;
+}
+
+const hostAllowed = (host, allow) => allow.some((e) => e.host === host);
+const addressAllowed = (address, allow) => allow.some((e) => e.kind && inCidr(address, e));
+
+// null when the URL may be opened, otherwise the reason it may not.
+export async function urlPolicyError(rawUrl, { policy = "off", lookup = dnsLookup, allow = [] } = {}) {
+  if (policy !== "strict") return null;
+  const refuse = (why) => `URL policy strict refuses ${rawUrl}: ${why}`;
+  let url;
+  try {
+    url = new URL(String(rawUrl));
+  } catch {
+    return refuse("not a valid URL");
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") return refuse(`${url.protocol} is not http or https`);
+  const host = url.hostname.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
+  if (hostAllowed(host, allow)) return null;
+  if (isIP(host)) {
+    if (addressAllowed(host, allow)) return null;
+    return isBlockedAddress(host) ? refuse("loopback, private, link-local or metadata address") : null;
+  }
+  if (host === "localhost" || host.endsWith(".localhost")) return refuse("loopback host");
+  if (METADATA_HOSTS.has(host)) return refuse("cloud metadata host");
+  let answers;
+  try {
+    answers = await lookup(host, { all: true });
+  } catch (e) {
+    return refuse(`could not resolve ${host} (${e.code || e.message}), so a local address cannot be ruled out`);
+  }
+  const bad = answers.find((a) => isBlockedAddress(a.address) && !addressAllowed(a.address, allow));
+  return bad ? refuse(`${host} resolves to ${bad.address}`) : null;
+}
+
+function expandHome(p, home) {
+  if (p === "~") return home;
+  if (p.startsWith("~/")) return join(home, p.slice(2));
+  return p;
+}
+
+// Symlinks are resolved on both sides, so a link that points into a denied
+// directory is judged by where it points. A path that does not exist yet is
+// resolved through its nearest existing ancestor, so /var and /private/var (or
+// a symlinked parent) cannot make the same place look like two.
+function real(p) {
+  const abs = resolve(p);
+  try {
+    return realpathSync.native(abs);
+  } catch {
+    const parent = dirname(abs);
+    return parent === abs ? abs : join(real(parent), basename(abs));
+  }
+}
+
+// APFS and HFS+ are case-insensitive by default, so on darwin ~/.SSH/id_rsa is
+// ~/.ssh/id_rsa. realpath does not always return the on-disk spelling, so the
+// comparison itself ignores case there.
+const foldCase = (p, platform) => (platform === "darwin" ? p.toLowerCase() : p);
+
+export function indexDenyList({ env = process.env, policy = "off", home = homedir(), warn = (m) => process.stderr.write(m) } = {}) {
+  const raw = String(env.MCP_TOOLKIT_INDEX_DENY ?? "").trim();
+  const replace = raw.startsWith("!");
+  const own = (replace ? raw.slice(1) : raw).split(",");
+  const entries = (replace ? own : raw ? [...DEFAULT_INDEX_DENY, ...own] : policy === "strict" ? DEFAULT_INDEX_DENY : [])
+    .map((e) => e.trim())
+    .filter(Boolean);
+  if (replace && !entries.length) {
+    const dropped = policy === "strict" ? ", including the credential stores the strict policy denies by default" : "";
+    warn(`multi-agent-toolkit: MCP_TOOLKIT_INDEX_DENY="${raw}" empties the index deny list; nothing is refused${dropped}\n`);
+  }
+  return entries.map((e) => real(resolve(expandHome(e, home))));
+}
+
+export function indexDenied(path, denyList, platform = process.platform) {
+  if (!denyList.length) return null;
+  const target = foldCase(real(resolve(String(path))), platform);
+  const root = denyList.find((raw) => {
+    const d = foldCase(raw, platform);
+    return target === d || target.startsWith(d.endsWith(sep) ? d : d + sep);
+  });
+  return root ? `path denied by the index policy (${root}): ${path}` : null;
+}
+
+// Browser launch options under the strict policy. Chromium sends loopback
+// traffic around a proxy unless told otherwise, and loopback is exactly what
+// strict refuses. WebRTC opens UDP sockets that no proxy or route sees, so
+// Chromium is held to proxied transports and Firefox has peer connections off.
+export function strictLaunchOptions(engineName, proxyUrl) {
+  if (engineName === "chromium") {
+    return {
+      headless: true,
+      proxy: { server: proxyUrl, bypass: "<-loopback>" },
+      args: [
+        "--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
+        "--webrtc-ip-handling-policy=disable_non_proxied_udp",
+      ],
+    };
+  }
+  if (engineName === "firefox") {
+    return { headless: true, proxy: { server: proxyUrl }, firefoxUserPrefs: { "media.peerconnection.enabled": false } };
+  }
+  return { headless: true, proxy: { server: proxyUrl } };
+}

package/tools/spawn-collect/index.js

@@ -0,0 +1,141 @@
+// spawn-based collector for the long-runners that also want incremental output
+// (progress heartbeats read the last line). Interleaves stdout+stderr the way a
+// terminal would, honors an AbortSignal, and never rejects.
+//
+// Output is capped at the 64MB the execSync it replaced enforced via maxBuffer:
+// unbounded `output += chunk` on a verbose xcodebuild can exceed V8's max
+// string length, and that throw fires inside a stream 'data' handler - outside
+// the CallTool try/catch - killing the whole stdio server. The oldest chunks
+// are dropped; errors and test verdicts land at the tail of a build log.
+//
+// A timeout or an abort sends SIGTERM, and SIGKILL after KILL_GRACE_MS if the
+// child is still there. A child that traps or ignores SIGTERM would otherwise
+// hold the call open forever, and the timeout would bound nothing.
+//
+// The command runs in its own process group and the signal goes to the group.
+// `/bin/sh -c` forks: a pipeline or a background job is a grandchild that a
+// signal to the shell never reaches, and it keeps the output pipe open, so the
+// call would wait for it however long it runs. Once the shell has exited, for
+// whatever reason, the result is returned after EXIT_GRACE_MS even if the pipe
+// has not closed: a background job the command started keeps it open, and
+// waiting for it would turn a clean exit into a timeout. The timeout stops
+// counting at exit, so timedOut means the timer fired while the command was
+// still running. Whatever is left in the group once the result is returned is
+// sent SIGTERM, then SIGKILL after the grace period, so no background job
+// outlives the call that started it.
+//
+// Both pipes decode as UTF-8 with their own decoder, so a multibyte character
+// split across two chunks arrives whole.
+
+import { spawn } from "node:child_process";
+
+export const SPAWN_OUTPUT_CAP = 64 * 1024 * 1024;
+export const KILL_GRACE_MS = 5000;
+export const EXIT_GRACE_MS = 250;
+
+// Process groups still running, so a server shutting down can take them with
+// it: a detached group does not receive the terminal's SIGINT.
+const liveGroups = new Set();
+
+function groupAlive(child) {
+  if (!child.pid) return false;
+  try {
+    process.kill(-child.pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function signalGroup(child, sig) {
+  if (!child.pid) return;
+  try {
+    process.kill(-child.pid, sig);
+  } catch {
+    try { child.kill(sig); } catch { /* already gone */ }
+  }
+}
+
+export function killAllSpawned(sig = "SIGKILL") {
+  for (const child of liveGroups) signalGroup(child, sig);
+}
+
+export function spawnCollect(cmd, { timeout = 600000, signal, env, cwd, onLine, killGraceMs = KILL_GRACE_MS } = {}) {
+  return new Promise((resolve) => {
+    const child = spawn("/bin/sh", ["-c", cmd], { env, cwd, detached: true });
+    liveGroups.add(child);
+    const chunks = [];
+    let size = 0;
+    let truncated = false;
+    let timedOut = false;
+    let aborted = false;
+    let settled = false;
+    let killTimer = null;
+    let exitTimer = null;
+    let exited = false;
+    const stop = () => {
+      signalGroup(child, "SIGTERM");
+      // Not cleared when the result is returned: a grandchild that ignores
+      // SIGTERM can outlive the shell, and still gets SIGKILL on schedule.
+      killTimer ??= setTimeout(() => { signalGroup(child, "SIGKILL"); liveGroups.delete(child); }, killGraceMs);
+      killTimer.unref?.();
+    };
+    const timer = setTimeout(() => { timedOut = true; stop(); }, timeout);
+    const onAbort = () => { aborted = true; stop(); };
+    if (signal) {
+      if (signal.aborted) onAbort();
+      else signal.addEventListener("abort", onAbort, { once: true });
+    }
+    const finish = (result) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      clearTimeout(exitTimer);
+      if (exited && !killTimer && groupAlive(child)) stop();
+      if (!killTimer) liveGroups.delete(child);
+      if (signal) signal.removeEventListener("abort", onAbort);
+      resolve(result);
+    };
+    const collect = (chunk) => {
+      const s = chunk;
+      chunks.push(s);
+      size += s.length;
+      while (size > SPAWN_OUTPUT_CAP && chunks.length > 1) {
+        size -= chunks[0].length;
+        chunks.shift();
+        truncated = true;
+      }
+      if (size > SPAWN_OUTPUT_CAP) {
+        chunks[0] = chunks[0].slice(size - SPAWN_OUTPUT_CAP);
+        size = SPAWN_OUTPUT_CAP;
+        truncated = true;
+      }
+      if (onLine) {
+        const lines = s.split("\n");
+        for (let i = lines.length - 1; i >= 0; i--) {
+          const line = lines[i].trim();
+          if (line) { onLine(line); break; }
+        }
+      }
+    };
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", collect);
+    child.stderr.on("data", collect);
+    child.on("error", (e) => {
+      finish({ code: 127, output: `${chunks.join("")}\n${e.message}`, truncated, timedOut, aborted });
+    });
+    child.on("exit", (code) => {
+      exited = true;
+      clearTimeout(timer);
+      exitTimer = setTimeout(() => {
+        child.stdout.destroy();
+        child.stderr.destroy();
+        finish({ code: code ?? 1, output: chunks.join(""), truncated, timedOut, aborted });
+      }, EXIT_GRACE_MS);
+    });
+    child.on("close", (code) => {
+      finish({ code: code ?? 1, output: chunks.join(""), truncated, timedOut, aborted });
+    });
+  });
+}