@mmerterden/multi-agent-pipeline 12.7.0 → 12.9.0

package/pipeline/scripts/lint-mcp-refs.mjs

@@ -1,218 +0,0 @@
-#!/usr/bin/env node
-// lint-mcp-refs.mjs  -  every mcp__<server>__<tool> reference must be callable.
-//
-// A host composes an MCP tool name as mcp__<registered-server-name>__<tool>.
-// The pipeline had 100 references to `mcp__dev_toolkit__*` while the server is
-// registered as `dev-toolkit`, so the real names are `mcp__dev-toolkit__*`.
-// Nothing caught it: prose is not executed, and a wrong name inside an
-// `allowed-tools:` allowlist silently denies the tool instead of erroring.
-//
-// Checks (errors - fail the gate):
-//   1. the server segment must be a known registered server name
-//   2. the known-bad snake_case spellings are called out with their fix
-//
-// Optional deep check (advisory - never fails):
-//   3. when the companion dev-toolkit repo resolves locally (via
-//      prefs.global.devToolkit.localPath, or the mcpServers registration in
-//      ~/.claude.json / ~/.claude/settings.json), handshake it over stdio and
-//      confirm every referenced dev-toolkit tool actually exists in tools/list.
-//      CI has no local checkout, so an unresolvable repo is a skip, not a fail.
-//
-// Zero-dependency. Runs inside `npm test` and ci-lite.
-//
-// Usage: node pipeline/scripts/lint-mcp-refs.mjs [--strict-tools]
-// Exit: 0 clean, 1 unknown server segment (or a missing tool with --strict-tools).
-
-import { readFileSync, existsSync, readdirSync, statSync } from "node:fs";
-import { homedir } from "node:os";
-import { dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-import { spawnSync } from "node:child_process";
-
-const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
-const strictTools = process.argv.includes("--strict-tools");
-
-// Servers the pipeline is allowed to reference. The key is the registered
-// server name as it appears in an mcpServers config, which is what the host
-// interpolates into the tool name.
-const KNOWN_SERVERS = {
-  "dev-toolkit": "companion device/browser toolkit (@mmerterden/dev-toolkit-mcp)",
-  claude_ai_Figma: "Figma MCP (analysis phase only)",
-  XcodeBuildMCP: "optional third-party Xcode build server",
-};
-// Spellings that look right but cannot resolve to a registered server.
-const KNOWN_BAD = { dev_toolkit: "dev-toolkit", "claude-ai-Figma": "claude_ai_Figma" };
-
-const SCAN_DIRS = ["pipeline/commands", "pipeline/skills", "pipeline/multi-agent-refs"];
-const SCAN_EXT = /\.(md|sh|mjs|json)$/;
-const PRUNE = new Set([
-  "node_modules",
-  ".git",
-  ".worktrees",
-  "Pods",
-  ".build",
-  "DerivedData",
-  ".next",
-]);
-
-function walk(dir, out = []) {
-  let entries;
-  try {
-    entries = readdirSync(dir, { withFileTypes: true });
-  } catch {
-    return out;
-  }
-  for (const e of entries) {
-    if (PRUNE.has(e.name)) continue;
-    const p = join(dir, e.name);
-    if (e.isDirectory()) walk(p, out);
-    else if (SCAN_EXT.test(e.name)) out.push(p);
-  }
-  return out;
-}
-
-const files = SCAN_DIRS.flatMap((d) => walk(join(repoRoot, d)));
-const REF = /mcp__([A-Za-z0-9_-]+?)__([a-z0-9_]+)/g;
-
-const errors = [];
-const refs = new Map(); // server -> Map(tool -> Set(file:line))
-
-for (const file of files) {
-  const rel = file.slice(repoRoot.length + 1);
-  const lines = readFileSync(file, "utf8").split("\n");
-  lines.forEach((line, i) => {
-    for (const m of line.matchAll(REF)) {
-      const [, server, tool] = m;
-      // Prose writes families as mcp__dev-toolkit__ios_* - a wildcard, not a tool.
-      const next = line[m.index + m[0].length];
-      if (next === "*" || tool.endsWith("_")) continue;
-      const where = `${rel}:${i + 1}`;
-      if (!refs.has(server)) refs.set(server, new Map());
-      const tools = refs.get(server);
-      if (!tools.has(tool)) tools.set(tool, new Set());
-      tools.get(tool).add(where);
-    }
-  });
-}
-
-for (const [server, tools] of refs) {
-  if (server in KNOWN_SERVERS) continue;
-  const fix = KNOWN_BAD[server];
-  const sample = [...tools.values()][0].values().next().value;
-  errors.push(
-    fix
-      ? `unknown MCP server segment "${server}" (${tools.size} tool refs, e.g. ${sample}) - the registered server is "${fix}", so the prefix must be mcp__${fix}__`
-      : `unknown MCP server segment "${server}" (${tools.size} tool refs, e.g. ${sample}) - add it to KNOWN_SERVERS in lint-mcp-refs.mjs if this server is expected, or fix the prefix to match the registered server name`,
-  );
-}
-
-// ---- optional deep check: do the referenced dev-toolkit tools exist? --------
-
-function resolveToolkitRepo() {
-  const prefsPath = join(homedir(), ".claude", "multi-agent-preferences.json");
-  if (existsSync(prefsPath)) {
-    try {
-      const dt = JSON.parse(readFileSync(prefsPath, "utf8"))?.global?.devToolkit;
-      if (dt && dt.enabled === false) return null;
-      if (dt?.localPath) {
-        const p = dt.localPath.replace(/^~|\$HOME/, homedir());
-        if (existsSync(join(p, "package.json"))) return p;
-      }
-    } catch {
-      /* fall through to registration lookup */
-    }
-  }
-  const candidates = [];
-  const collect = (obj) => {
-    for (const [name, cfg] of Object.entries(obj || {})) {
-      const arg = Array.isArray(cfg?.args) ? cfg.args.find((a) => /\.(mjs|js)$/.test(a)) : null;
-      if (name.includes("dev") && arg) candidates.push(dirname(resolve(arg)));
-    }
-  };
-  for (const cfgPath of [
-    join(homedir(), ".claude.json"),
-    join(homedir(), ".claude", "settings.json"),
-  ]) {
-    if (!existsSync(cfgPath)) continue;
-    try {
-      const json = JSON.parse(readFileSync(cfgPath, "utf8"));
-      collect(json.mcpServers);
-      for (const proj of Object.values(json.projects || {})) collect(proj.mcpServers);
-    } catch {
-      /* ignore unreadable config */
-    }
-  }
-  for (const c of candidates) {
-    try {
-      const pkg = JSON.parse(readFileSync(join(c, "package.json"), "utf8"));
-      if (pkg.dependencies?.["@modelcontextprotocol/sdk"] && statSync(join(c, ".git"))) return c;
-    } catch {
-      /* not the toolkit */
-    }
-  }
-  return null;
-}
-
-function listTools(repo) {
-  const frames =
-    [
-      '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"lint-mcp-refs","version":"1"}}}',
-      '{"jsonrpc":"2.0","method":"notifications/initialized"}',
-      '{"jsonrpc":"2.0","id":2,"method":"tools/list"}',
-    ].join("\n") + "\n";
-  const r = spawnSync(process.execPath, ["index.js"], {
-    cwd: repo,
-    input: frames,
-    encoding: "utf8",
-    timeout: 20000,
-    maxBuffer: 32 * 1024 * 1024,
-  });
-  if (r.error || !r.stdout) return null;
-  for (const line of r.stdout.split("\n")) {
-    if (!line.trim()) continue;
-    try {
-      const msg = JSON.parse(line);
-      if (msg.id === 2 && msg.result?.tools) return msg.result.tools.map((t) => t.name);
-    } catch {
-      /* not a frame */
-    }
-  }
-  return null;
-}
-
-const referencedToolkitTools = refs.get("dev-toolkit");
-let deep = "skipped (no local dev-toolkit checkout)";
-if (referencedToolkitTools) {
-  const repo = resolveToolkitRepo();
-  const served = repo ? listTools(repo) : null;
-  if (!served) {
-    deep = repo ? `skipped (${repo} did not answer tools/list)` : deep;
-  } else {
-    const missing = [...referencedToolkitTools.keys()].filter((t) => !served.includes(t));
-    if (missing.length === 0) {
-      deep = `all ${referencedToolkitTools.size} referenced tools exist in the live server (${served.length} served)`;
-    } else {
-      const msg = `dev-toolkit tools referenced but not served (${served.length} served): ${missing
-        .map((t) => `${t} [${[...referencedToolkitTools.get(t)][0]}]`)
-        .join(", ")}`;
-      deep = `${missing.length} referenced tool(s) not served`;
-      if (strictTools) errors.push(msg);
-      else console.warn(`  WARNING - ${msg}`);
-    }
-  }
-}
-
-const totalRefs = [...refs.values()].reduce((n, tools) => n + tools.size, 0);
-console.log(
-  `lint-mcp-refs: ${totalRefs} distinct tool references across ${refs.size} server(s) in ${files.length} files`,
-);
-for (const [server, tools] of refs) {
-  console.log(`  ${server in KNOWN_SERVERS ? "✓" : "✗"} mcp__${server}__* : ${tools.size} tools`);
-}
-console.log(`  tool-existence check: ${deep}`);
-
-if (errors.length > 0) {
-  console.error(`lint-mcp-refs: ${errors.length} error(s)`);
-  for (const e of errors) console.error("  " + e);
-  process.exit(1);
-}

package/pipeline/scripts/lint-skills.mjs

@@ -1,154 +0,0 @@
-#!/usr/bin/env node
-// Frontmatter linter for every SKILL.md shipped by the pipeline.
-//
-// Until v10.0.0 the shipped SKILL.md files were counted by smokes but never
-// content-checked, so a skill could ship without the frontmatter that
-// Claude Code / Copilot CLI need to register it. This gate enforces the
-// minimal contract; it is zero-dependency (line-based YAML subset) so it
-// runs inside `npm test` and ci-lite without installs.
-//
-// Contract per SKILL.md (errors - fail the gate):
-//   - frontmatter block present (--- ... ---) starting at line 1
-//   - name: required, lowercase kebab-case (underscores tolerated for
-//     legacy skills already published under a snake_case name)
-//   - description: required, non-empty, single line, <= 1024 chars
-//   - user-invocable: when present, must be true or false
-//   - platform: when present, must be ios | android | common | all
-//   - description: must carry a routing/trigger clause ("use when ..."), the
-//     one thing it is for - blocking since all 193 skills have one
-//
-// Spec-conformance signals (warnings - never fail the gate):
-//   - name should match the agent-skills spec pattern
-//     (lowercase letters/digits/hyphens, <= 64 chars)
-//   - a body over ~500 lines with no companion-file reference should be
-//     split into references/ files loaded on demand
-
-import { readFileSync } from "node:fs";
-import { execSync } from "node:child_process";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
-// Scope to pipeline/skills only. Slash commands under pipeline/commands also use
-// the <name>/SKILL.md layout now, but they are command definitions
-// (description + allowed-tools, name derived from the dir), NOT skills, so the
-// skill frontmatter contract (required `name:`) does not apply to them.
-const files = execSync("find pipeline/skills -name SKILL.md", { cwd: repoRoot, encoding: "utf-8" })
-  .trim()
-  .split("\n")
-  .filter(Boolean)
-  .sort();
-
-const errors = [];
-const fail = (file, msg) => errors.push(`${file}: ${msg}`);
-// Non-fatal quality signals (a terse description routes poorly). Warnings never
-// fail the gate; they surface skills whose frontmatter could trigger better.
-const warnings = [];
-const warn = (file, msg) => warnings.push(`${file}: ${msg}`);
-
-for (const file of files) {
-  const raw = readFileSync(join(repoRoot, file), "utf-8");
-
-  if (!raw.startsWith("---\n")) {
-    fail(file, "missing frontmatter block at line 1");
-    continue;
-  }
-  const end = raw.indexOf("\n---", 4);
-  if (end === -1) {
-    fail(file, "frontmatter block never closed");
-    continue;
-  }
-
-  const fields = {};
-  for (const line of raw.slice(4, end).split("\n")) {
-    const m = line.match(/^([a-zA-Z][a-zA-Z0-9-]*):\s*(.*)$/);
-    if (m) fields[m[1]] = m[2].replace(/^["']|["']$/g, "").trim();
-  }
-
-  if (!fields.name) {
-    fail(file, "missing required field: name");
-  } else if (!/^[a-z0-9]+([-_][a-z0-9]+)*$/.test(fields.name)) {
-    fail(file, `name is not lowercase kebab/snake-case: "${fields.name}"`);
-  } else {
-    // Spec conformance (warn-only): the agent-skills spec allows lowercase
-    // letters, digits, and hyphens, max 64 chars. Snake_case passes the
-    // legacy error check above but is out of spec.
-    if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(fields.name)) {
-      warn(file, `name is outside the agent-skills spec pattern (hyphens only): "${fields.name}"`);
-    }
-    if (fields.name.length > 64) {
-      warn(file, `name exceeds the spec's 64-char limit (${fields.name.length})`);
-    }
-  }
-
-  if (!("description" in fields)) {
-    fail(file, "missing required field: description");
-  } else if (fields.description.length === 0) {
-    fail(file, "description is empty");
-  } else if (fields.description.length > 1024) {
-    fail(file, `description exceeds 1024 chars (${fields.description.length})`);
-  } else {
-    // Routing quality: a description of a few words rarely says what the skill
-    // does AND when to use it, so the model cannot decide to trigger it.
-    // Skip YAML block scalars (`description: |` / `>`): this line-based parser
-    // only sees the indicator, not the real multi-line text, so a word count
-    // would be a false positive.
-    const desc = fields.description;
-    const isBlockScalar = /^[|>][+-]?\d*$/.test(desc);
-    const words = desc.split(/\s+/).filter(Boolean).length;
-    if (!isBlockScalar && words < 6) {
-      warn(file, `description is terse (${words} words) - say what it does AND when to use it`);
-    }
-    // Routing quality (BLOCKING since the backlog reached zero). A description
-    // without a trigger clause tells the model what the skill is and nothing
-    // about when to reach for it, which is the one job the description has -
-    // it is the routing rule, not a summary.
-    //
-    // This was warn-only while 93 of 193 skills were missing one, because a
-    // gate that fires on half the tree gets ignored. All 93 were written, so
-    // the baseline is zero and it can hold the line. Block scalars are skipped
-    // for the same parser reason as above.
-    const ROUTING_RE =
-      /\b(use (this|it|when|for|whenever)|when (the|a|an|you|asked|working|building|implementing|writing|reviewing)|triggers?|invoked?( when| for)?|applies when|activated?)\b/i;
-    if (!isBlockScalar && !ROUTING_RE.test(desc)) {
-      fail(file, 'description has no routing/trigger clause (e.g. "use when ...")');
-    }
-  }
-
-  // Body size (warn-only): the agent-skills guidance is to keep SKILL.md
-  // lean and push detail into companion files loaded on demand. A ~500+
-  // line body that references no companion file loads whole every time.
-  const body = raw.slice(end + 4);
-  const bodyLines = body.split("\n").length;
-  if (bodyLines > 500) {
-    const referencesCompanion =
-      /\]\((?!https?:|mailto:|#)[^)]+\)/.test(body) ||
-      /`[^`\n]*\/[^`\n]*\.(md|sh|mjs|json|py|txt)`/.test(body);
-    if (!referencesCompanion) {
-      warn(
-        file,
-        `body is ${bodyLines} lines with no companion-file reference - consider splitting into references/`,
-      );
-    }
-  }
-
-  if ("user-invocable" in fields && !["true", "false"].includes(fields["user-invocable"])) {
-    fail(file, `user-invocable must be true|false, got "${fields["user-invocable"]}"`);
-  }
-
-  if ("platform" in fields && !["ios", "android", "common", "all"].includes(fields.platform)) {
-    fail(file, `platform must be ios|android|common|all, got "${fields.platform}"`);
-  }
-}
-
-if (warnings.length > 0) {
-  console.warn(`lint-skills: ${warnings.length} warning(s) (non-fatal):`);
-  for (const w of warnings) console.warn("  " + w);
-}
-
-if (errors.length > 0) {
-  console.error(`lint-skills: ${errors.length} error(s) across ${files.length} SKILL.md files`);
-  for (const e of errors) console.error("  " + e);
-  process.exit(1);
-}
-console.log(`lint-skills: ${files.length} SKILL.md files OK`);

package/pipeline/scripts/run-smokes.mjs

@@ -1,130 +0,0 @@
-#!/usr/bin/env node
-// Structured runner for pipeline/scripts/smoke-*.sh.
-//
-// Replaces the bare `for f in smoke-*.sh; do bash "$f" || exit 1; done`
-// loop with per-suite timeouts, timing, and a summary, so a hung smoke
-// (network wait, interactive prompt) can never block `npm test` or CI
-// indefinitely.
-//
-// Env:
-//   SMOKE_TIMEOUT_SECONDS  per-suite timeout (default 180)
-//   SMOKE_NO_BAIL=1        run every suite and summarize instead of
-//                          stopping at the first failure
-// Args:
-//   optional substring filters: `node run-smokes.mjs adapters keychain`
-//   runs only suites whose filename contains one of the terms.
-
-import { spawnSync } from "node:child_process";
-import { readdirSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const scriptsDir = dirname(fileURLToPath(import.meta.url));
-const timeoutSec = Number(process.env.SMOKE_TIMEOUT_SECONDS || 180);
-const bail = process.env.SMOKE_NO_BAIL !== "1";
-const argv = process.argv.slice(2);
-const reportCounts = argv.includes("--counts");
-const filters = argv.filter((a) => !a.startsWith("--"));
-
-// How many assertions did a suite actually make?
-//
-// Exit code alone cannot answer that, and the difference matters: three smokes
-// were exiting 0 having checked nothing. smoke-workflow-audit skipped both its
-// linters because neither was installed anywhere, so the GitHub Actions
-// security audit had never run while showing green - it was hiding a
-// high-severity finding. smoke-plugin-validate had no manifest to validate in
-// this repo at all. A gate that cannot fail is worse than no gate, because the
-// green tick is read as coverage.
-//
-// The four output shapes actually in use here, surveyed across all suites
-// rather than assumed. Anything a suite prints that matches none of them
-// counts as zero, which is the intended answer: an unrecognisable result is
-// not evidence that something was checked.
-//
-//   1. "══ name: 22 passed, 0 failed ══"   most suites
-//   2. "check-md-links: 212 files OK"      counted sweeps
-//   3. per-line "✓" / "ok " / "PASS:"      suites with no summary line
-//   4. "smoke-language-axis: ok"           terse single-verdict suites
-function countAssertions(stdout) {
-  const passed = [...stdout.matchAll(/(\d+) passed/g)];
-  if (passed.length > 0) return Number(passed[passed.length - 1][1]);
-
-  const swept = [...stdout.matchAll(/(\d+) (?:files|patterns|checks|entries) (?:OK|clean)/gi)];
-  if (swept.length > 0) return Number(swept[swept.length - 1][1]);
-
-  const marks = stdout.match(/^[ \t]*(✓|ok |PASS[: ])/gm);
-  if (marks) return marks.length;
-
-  if (/^\S+: ok\s*$/m.test(stdout)) return 1;
-
-  return 0;
-}
-
-const suites = readdirSync(scriptsDir)
-  .filter((f) => f.startsWith("smoke-") && f.endsWith(".sh"))
-  .filter((f) => filters.length === 0 || filters.some((t) => f.includes(t)))
-  .sort();
-
-if (suites.length === 0) {
-  console.error("run-smokes: no smoke suites matched");
-  process.exit(1);
-}
-
-const failures = [];
-let ran = 0;
-const startedAt = Date.now();
-
-for (const suite of suites) {
-  ran++;
-  const t0 = Date.now();
-  const res = spawnSync("bash", [join(scriptsDir, suite)], {
-    encoding: "utf-8",
-    timeout: timeoutSec * 1000,
-    maxBuffer: 16 * 1024 * 1024,
-  });
-  const secs = ((Date.now() - t0) / 1000).toFixed(1);
-
-  const timedOut = res.error && res.error.code === "ETIMEDOUT";
-  const asserted = countAssertions(res.stdout || "");
-  const ok = !timedOut && res.status === 0 && (reportCounts || asserted > 0);
-
-  if (reportCounts) {
-    console.log(`${String(asserted).padStart(4)}  ${suite}`);
-    continue;
-  }
-
-  if (ok) {
-    console.log(`ok   ${suite} (${secs}s, ${asserted} checks)`);
-    continue;
-  }
-
-  if (!timedOut && res.status === 0 && asserted === 0) {
-    console.error(
-      `FAIL ${suite} (exited 0 having asserted nothing, ${secs}s)\n` +
-        "     A suite that checks nothing reports coverage it does not have.\n" +
-        "     Give it something to assert, or delete it.",
-    );
-    if (res.stdout) console.error(res.stdout.split("\n").slice(-15).join("\n"));
-    failures.push(suite);
-    if (bail) break;
-    continue;
-  }
-
-  const reason = timedOut ? `timeout after ${timeoutSec}s` : `exit ${res.status}`;
-  console.error(`FAIL ${suite} (${reason}, ${secs}s)`);
-  if (res.stdout) console.error(res.stdout.split("\n").slice(-30).join("\n"));
-  if (res.stderr) console.error(res.stderr.split("\n").slice(-30).join("\n"));
-  failures.push(suite);
-  if (bail) break;
-}
-
-const totalSecs = ((Date.now() - startedAt) / 1000).toFixed(1);
-const skipped = suites.length - ran;
-console.log(
-  `run-smokes: ${ran - failures.length}/${suites.length} passed in ${totalSecs}s` +
-    (skipped > 0 ? ` (${skipped} not run after bail)` : ""),
-);
-if (failures.length > 0) {
-  console.error(`run-smokes: failed -> ${failures.join(", ")}`);
-  process.exit(1);
-}