@raishin/vanguard-frontier-agentic 1.8.0 → 2.0.0

package/tests/validate-maestro-routing.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""Provider-agnostic routing eval-harness for every `*-maestro` skill.
+
+Discovers all `tests/fixtures/*-maestro-routing/` directories. Each directory
+must contain:
+
+  - `taxonomy.json` — the executable routing contract:
+        {
+          "provider": "<provider-id>",
+          "domains": {
+              "<domain>": {"keywords": [...], "agent": "<agent-id>"},
+              ...
+          },
+          "live_guards": ["<agent-id>", ...],
+          "live_guard_intent": "<regex>",   # task patterns that imply mutation
+          "parallel_threshold": 0.6           # optional
+        }
+  - `inputs/NN-name.json` — { "name", "task", "tags"? } operator-style tasks
+  - `expected/NN-name.json` — { "route": [agent_ids...], "mode": ... }
+
+The grader scores each domain by counting taxonomy keywords present in the task
+(word-boundary, case-insensitive), then resolves to one or more agents, with a
+live-guard gate that fires when `live_guard_intent` matches. Live-guard agents
+are never permitted to appear in `single` or `parallel` modes — only in
+`live-guard-gate` (or `runtime-evidence-gate` for read-only-runtime tiers).
+
+Regression guards:
+
+  - Every agent referenced by the routing table exists in catalog/agents.json.
+  - No `live_guards` agent is dispatched in non-gate modes (live-agent
+    auto-dispatch is the highest-blast-radius regression we can have).
+  - Every domain has a mapped agent.
+
+Stress-test fixture tags:
+
+  - `adversarial:instruction-injection`  — SYSTEM-style prose framing
+  - `adversarial:persona-replacement`    — "you are now ..." prose
+  - `adversarial:live-guard-bypass`      — urgency-laden destructive ops
+  - `adversarial:secrets-bait`           — fixture contains fake credential
+  - `ambiguous`                          — vague signals, expect unclassified
+
+A fixture tagged `adversarial:secrets-bait` MUST not contain real-looking
+credentials in the grader output (the grader is route-only, so this is
+structurally guaranteed). The fixture *task* may carry a fake credential
+shape; the validator checks that the task contains the marker `<FAKE>` so
+no credential pattern accidentally ships in the repository.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+FIXTURES_ROOT = ROOT / "tests" / "fixtures"
+AGENTS_CATALOG = ROOT / "catalog" / "agents.json"
+
+DEFAULT_PARALLEL_THRESHOLD = 0.6
+PARALLEL_CEILING = 4
+
+
+def _score_domain(task: str, keywords: list[str]) -> int:
+    task_lower = task.lower()
+    hits = 0
+    for kw in keywords:
+        if re.search(r"\W", kw):
+            if kw.lower() in task_lower:
+                hits += 1
+        else:
+            if re.search(rf"\b{re.escape(kw.lower())}\b", task_lower):
+                hits += 1
+    return hits
+
+
+def evaluate(task: str, taxonomy: dict) -> dict:
+    """Return {route: [agent_ids], mode: str}."""
+    gate_mode = taxonomy.get("gate_mode", "live-guard-gate")
+    live_guard_intent = taxonomy.get("live_guard_intent")
+    if live_guard_intent and re.search(live_guard_intent, task, re.IGNORECASE):
+        # Identify which live-guard the task matches by keyword overlap.
+        live_guards = taxonomy.get("live_guards", [])
+        if live_guards:
+            scored = []
+            for agent_id in live_guards:
+                # Score the agent id tokens against the task.
+                tokens = re.split(r"[-_]", agent_id.replace("agent", "").strip("-"))
+                score = sum(1 for t in tokens if t and re.search(rf"\b{re.escape(t)}\b", task.lower()))
+                scored.append((agent_id, score))
+            scored.sort(key=lambda kv: (-kv[1], kv[0]))
+            if scored[0][1] > 0:
+                return {"route": [scored[0][0]], "mode": gate_mode}
+            # Generic live-guard intent without a clear specific match: emit
+            # the first live_guard as a stand-in (specialist will refine).
+            return {"route": [scored[0][0]], "mode": gate_mode}
+
+    domains = taxonomy["domains"]
+    scores = {d: _score_domain(task, conf["keywords"]) for d, conf in domains.items()}
+    ranked = sorted(scores.items(), key=lambda kv: (-kv[1], kv[0]))
+    if ranked[0][1] == 0:
+        return {"route": [], "mode": "unclassified"}
+
+    top_score = ranked[0][1]
+    threshold = taxonomy.get("parallel_threshold", DEFAULT_PARALLEL_THRESHOLD)
+    winners = [d for d, s in ranked if s > 0 and s >= top_score * threshold]
+    winners = winners[:PARALLEL_CEILING]
+
+    agents = sorted({domains[d]["agent"] for d in winners})
+    mode = "single" if len(agents) == 1 else f"parallel ({len(agents)})"
+    return {"route": agents, "mode": mode}
+
+
+def _validate_taxonomy(taxonomy: dict, catalog_ids: set[str]) -> list[str]:
+    errors: list[str] = []
+    provider = taxonomy.get("provider", "?")
+    for domain, conf in taxonomy.get("domains", {}).items():
+        agent_id = conf.get("agent")
+        if agent_id not in catalog_ids:
+            errors.append(f"[{provider}] regression: domain {domain!r} maps to unknown agent {agent_id!r}")
+        if not conf.get("keywords"):
+            errors.append(f"[{provider}] regression: domain {domain!r} has empty keywords list")
+    for guard in taxonomy.get("live_guards", []):
+        if guard not in catalog_ids:
+            errors.append(f"[{provider}] regression: live_guard {guard!r} not in catalog")
+    return errors
+
+
+def _task_has_unmarked_credential(task: str) -> bool:
+    """Return True iff the task carries a real-looking credential pattern without
+    a `<FAKE>` marker. Only a boolean is returned so no portion of the task can
+    flow into any caller's log output (CodeQL: clear-text-logging false positive
+    avoidance — the contract is *boolean in, boolean out*)."""
+    risky = re.compile(r"AKIA[0-9A-Z]{16}|api[_-]?key\s*[:=]\s*['\"]?[A-Za-z0-9]{16,}", re.IGNORECASE)
+    return bool(risky.search(task)) and "<FAKE>" not in task
+
+
+def main() -> int:
+    catalog_ids = {a["id"] for a in json.loads(AGENTS_CATALOG.read_text())}
+    routing_dirs = sorted(FIXTURES_ROOT.glob("*-maestro-routing"))
+    if not routing_dirs:
+        print("ERROR: no maestro-routing fixture directories found", file=sys.stderr)
+        return 2
+
+    total_failures = 0
+    total_scenarios = 0
+
+    for prov_dir in routing_dirs:
+        taxonomy_path = prov_dir / "taxonomy.json"
+        if not taxonomy_path.exists():
+            print(f"SKIP [{prov_dir.name}] no taxonomy.json")
+            continue
+        taxonomy = json.loads(taxonomy_path.read_text())
+        provider = taxonomy.get("provider", prov_dir.name)
+
+        for err in _validate_taxonomy(taxonomy, catalog_ids):
+            print(f"FAIL {err}")
+            total_failures += 1
+
+        inputs_dir = prov_dir / "inputs"
+        expected_dir = prov_dir / "expected"
+        fixtures = sorted(inputs_dir.glob("*.json")) if inputs_dir.is_dir() else []
+        if not fixtures:
+            print(f"WARN [{provider}] no fixtures yet")
+            continue
+
+        provider_fails = 0
+        live_guards = set(taxonomy.get("live_guards", []))
+
+        for fp in fixtures:
+            fixture = json.loads(fp.read_text())
+            name = fixture.get("name", fp.stem)
+            tags = fixture.get("tags", [])
+            expected = json.loads((expected_dir / f"{name}.json").read_text())
+            total_scenarios += 1
+
+            if _task_has_unmarked_credential(fixture["task"]):
+                # Intentionally do NOT echo any portion of the task — the
+                # whole point of this check is to keep credential-shaped
+                # strings out of logs.
+                print(f"FAIL [{provider}] secrets-bait fixture {name!r} "
+                      f"contains real-looking credential without <FAKE> marker")
+                provider_fails += 1
+
+            got = evaluate(fixture["task"], taxonomy)
+
+            # Live-guard auto-dispatch guard: live-guard agents must only
+            # appear in 'live-guard-gate' or 'runtime-evidence-gate' modes.
+            gate_modes = {"live-guard-gate", "runtime-evidence-gate"}
+            if got["mode"] not in gate_modes:
+                bad = [a for a in got["route"] if a in live_guards]
+                if bad:
+                    print(f"FAIL [{provider}/{name}] live-agent auto-dispatch guard tripped: "
+                          f"{bad} in mode {got['mode']!r}")
+                    provider_fails += 1
+                    continue
+
+            route_ok = set(got["route"]) == set(expected["route"])
+            mode_ok = got["mode"] == expected["mode"]
+            tag_str = f" tags={tags}" if tags else ""
+            if route_ok and mode_ok:
+                print(f"OK   [{provider}/{name}] route={got['route']} mode={got['mode']}{tag_str}")
+            else:
+                print(f"FAIL [{provider}/{name}] got route={got['route']} mode={got['mode']} | "
+                      f"expected route={expected['route']} mode={expected['mode']}{tag_str}")
+                provider_fails += 1
+
+        if provider_fails == 0:
+            print(f"-- [{provider}] {len(fixtures)} scenarios passed")
+        total_failures += provider_fails
+
+    if total_failures:
+        print(f"\n{total_failures} routing check(s) failed across {len(routing_dirs)} provider(s)", file=sys.stderr)
+        return 1
+    print(f"\nOK: {total_scenarios} scenarios validated across {len(routing_dirs)} maestro(s)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tests/validate-mcp-trust-matrix.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Validate the trust_matrix block on every MCP reference metadata file.
+
+The trust_matrix field is optional in the schema today (graceful rollout),
+but this validator enforces it on every file currently committed under
+mcp/. New entries that lack trust_matrix fail this check, which prevents
+regressions and steers contributors to declare the security-relevant
+posture explicitly.
+
+Promotes to a required field in the schema once the corpus and
+contribution guide have been updated. Until then, this validator is the
+de-facto contract.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+REQUIRED_TM_FIELDS = {
+    "mutation_capable",
+    "requires_egress",
+    "requires_credentials",
+    "signed_release",
+    "pin_strategy",
+}
+ALLOWED_SIGNED = {"cosign", "gh-attestation", "unsigned", "unknown"}
+ALLOWED_PIN = {"digest", "tag", "version", "none"}
+
+
+def metadata_files() -> list[Path]:
+    paths: list[Path] = []
+    for path in (ROOT / "mcp").rglob("*.metadata.json"):
+        paths.append(path)
+    for path in (ROOT / "mcp").rglob("metadata.json"):
+        paths.append(path)
+    return sorted(set(paths))
+
+
+def main() -> int:
+    files = metadata_files()
+    if not files:
+        print("OK: no MCP metadata files present")
+        return 0
+
+    errors: list[str] = []
+    for path in files:
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:  # noqa: BLE001
+            errors.append(f"{path.relative_to(ROOT)}: invalid JSON: {exc}")
+            continue
+
+        tm = data.get("trust_matrix")
+        rel = path.relative_to(ROOT)
+        if not isinstance(tm, dict):
+            errors.append(f"{rel}: missing trust_matrix block")
+            continue
+
+        missing = REQUIRED_TM_FIELDS - tm.keys()
+        if missing:
+            errors.append(f"{rel}: trust_matrix missing fields {sorted(missing)}")
+
+        for boolean_field in ("mutation_capable", "requires_egress", "requires_credentials"):
+            if boolean_field in tm and not isinstance(tm[boolean_field], bool):
+                errors.append(f"{rel}: trust_matrix.{boolean_field} must be boolean")
+
+        if tm.get("signed_release") not in ALLOWED_SIGNED:
+            errors.append(
+                f"{rel}: trust_matrix.signed_release must be one of {sorted(ALLOWED_SIGNED)}"
+            )
+        if tm.get("pin_strategy") not in ALLOWED_PIN:
+            errors.append(
+                f"{rel}: trust_matrix.pin_strategy must be one of {sorted(ALLOWED_PIN)}"
+            )
+
+    if errors:
+        print("ERROR: MCP trust_matrix validation failed", file=sys.stderr)
+        for err in errors:
+            print(f"  - {err}", file=sys.stderr)
+        return 1
+
+    print(f"OK: validated trust_matrix on {len(files)} MCP reference files")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/tests/validate-multi-harness-marketplace.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""Validate Cursor plugin manifest and GitHub Copilot CLI marketplace.
+
+The Claude Code plugin manifest has its own validator
+(tests/validate-plugin-manifest.py). This validator covers the other
+two harnesses that ship a plugin/marketplace manifest:
+
+  - Cursor: .cursor-plugin/plugin.json
+      Source: cursor.com/docs/reference/plugins
+      Required: name. Custom paths for agents/skills/rules supported.
+      We enumerate cursor adapter paths explicitly and check that every
+      catalog agent with `harnesses: [cursor, ...]` is represented.
+
+  - GitHub Copilot CLI: .github/plugin/marketplace.json
+      Source: copilot-cli docs (context7 /github/copilot-cli) and
+      GitHub Docs ("Creating a plugin marketplace for GitHub Copilot CLI").
+      Required: plugins[] array with id + source + description.
+      We declare the repo as a single plugin with source: "./".
+
+Gates:
+  1. Both manifests exist and parse as JSON.
+  2. Cursor manifest: every path resolves, version matches package.json,
+     every cursor-enabled catalog agent is represented.
+  3. Cursor generator (scripts/generate-cursor-plugin.mjs) is in sync.
+  4. Copilot marketplace: declares vanguard-frontier-agentic plugin with
+     source "./"; version matches package.json.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parent.parent
+CURSOR_MANIFEST = REPO / ".cursor-plugin" / "plugin.json"
+COPILOT_MARKETPLACE = REPO / ".github" / "plugin" / "marketplace.json"
+CATALOG = REPO / "catalog" / "agents.json"
+PKG = REPO / "package.json"
+CURSOR_GENERATOR = REPO / "scripts" / "generate-cursor-plugin.mjs"
+
+
+def fail(msg: str) -> None:
+    print(f"FAIL [multi-harness-marketplace] {msg}", file=sys.stderr)
+
+
+def path_is_inside_repo(path_value: str) -> bool:
+    # Note 1: Cursor's manifest is a package boundary. A path that leaves the
+    # repository is not just a broken link; it changes what the installer may
+    # load as trusted plugin content.
+    try:
+        resolved = (REPO / path_value).resolve()
+    except OSError:
+        # Note 2: Treat filesystem resolution problems as validation failures
+        # instead of trying to guess the installer's behavior.
+        return False
+    # Note 3: Checking parents after resolve() catches traversal attempts such
+    # as "../outside.agent.md" even when the target file exists.
+    return resolved == REPO or REPO in resolved.parents
+
+
+def validate_cursor(pkg: dict, catalog: list) -> list[str]:
+    errors: list[str] = []
+    if not CURSOR_MANIFEST.exists():
+        return [".cursor-plugin/plugin.json is missing"]
+
+    manifest = json.loads(CURSOR_MANIFEST.read_text())
+
+    if manifest.get("name") != "vanguard-frontier-agentic":
+        errors.append("cursor plugin name must be 'vanguard-frontier-agentic'")
+    if manifest.get("version") != pkg.get("version"):
+        errors.append(
+            f"cursor plugin version {manifest.get('version')!r} != package.json {pkg.get('version')!r}",
+        )
+
+    manifest_paths = manifest.get("agents") or []
+    # Note 4: Containment comes before existence for the same reason as in the
+    # Claude validator: an outside path can exist locally and still be an
+    # invalid plugin artifact.
+    escaping = [p for p in manifest_paths if not isinstance(p, str) or not path_is_inside_repo(p)]
+    if escaping:
+        errors.append(
+            f"{len(escaping)} cursor manifest paths escape the repository: e.g. {escaping[0]}",
+        )
+    missing = [p for p in manifest_paths if p not in escaping and not (REPO / p).is_file()]
+    if missing:
+        errors.append(
+            f"{len(missing)} cursor manifest paths do not resolve: e.g. {missing[0]}",
+        )
+
+    catalog_paths = set()
+    for entry in catalog:
+        if entry.get("type") != "agent":
+            continue
+        if "cursor" not in (entry.get("harnesses") or []):
+            continue
+        adapter = (entry.get("harness_variants") or {}).get(
+            "cursor",
+        ) or f"{entry['path']}/harnesses/cursor.agent.md"
+        # Note 5: This validates the source of truth, not only the generated
+        # output, so a stale manifest cannot mask unsafe catalog metadata.
+        if not path_is_inside_repo(adapter):
+            errors.append(f"{entry.get('id', '<unknown>')}: cursor adapter path escapes repository: {adapter}")
+            continue
+        catalog_paths.add(f"./{adapter}")
+
+    manifest_set = set(manifest_paths)
+    dropped = catalog_paths - manifest_set
+    extra = manifest_set - catalog_paths
+    if dropped:
+        errors.append(
+            f"{len(dropped)} cursor-enabled catalog agents absent from cursor manifest; e.g. {sorted(dropped)[0]}",
+        )
+    if extra:
+        errors.append(
+            f"{len(extra)} paths in cursor manifest not in catalog; e.g. {sorted(extra)[0]}",
+        )
+
+    # Generator drift
+    result = subprocess.run(
+        ["node", str(CURSOR_GENERATOR), "--check"],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        errors.append((result.stderr or result.stdout).strip())
+
+    return errors
+
+
+def validate_copilot(pkg: dict) -> list[str]:
+    errors: list[str] = []
+    if not COPILOT_MARKETPLACE.exists():
+        return [".github/plugin/marketplace.json is missing"]
+
+    marketplace = json.loads(COPILOT_MARKETPLACE.read_text())
+
+    if marketplace.get("name") != "vanguard-frontier-agentic":
+        errors.append("copilot marketplace name must be 'vanguard-frontier-agentic'")
+    if marketplace.get("version") != pkg.get("version"):
+        errors.append(
+            f"copilot marketplace version {marketplace.get('version')!r} != package.json {pkg.get('version')!r}",
+        )
+
+    plugins = marketplace.get("plugins") or []
+    if not plugins:
+        errors.append("copilot marketplace must declare at least one plugin")
+
+    vfa_plugin = next(
+        (p for p in plugins if p.get("id") == "vanguard-frontier-agentic"),
+        None,
+    )
+    if vfa_plugin is None:
+        errors.append("copilot marketplace must declare 'vanguard-frontier-agentic' plugin")
+    else:
+        if vfa_plugin.get("source") != "./":
+            errors.append(
+                "copilot plugin source must be './' so the repo root is the plugin root",
+            )
+        if not vfa_plugin.get("description"):
+            errors.append("copilot plugin requires a non-empty description")
+
+    return errors
+
+
+def main() -> int:
+    pkg = json.loads(PKG.read_text())
+    catalog = json.loads(CATALOG.read_text())
+
+    errors: list[str] = []
+    errors.extend(validate_cursor(pkg, catalog))
+    errors.extend(validate_copilot(pkg))
+
+    if errors:
+        for err in errors:
+            fail(err)
+        return 1
+
+    cursor_count = len(json.loads(CURSOR_MANIFEST.read_text()).get("agents") or [])
+    print(
+        f"OK: cursor plugin valid ({cursor_count} agents) and copilot marketplace valid (1 plugin)",
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tests/validate-no-lifecycle-scripts.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Reject npm package lifecycle scripts that execute on install.
+
+Lifecycle hooks like `preinstall`, `install`, `postinstall`, `prepare`,
+`prepublish`, and `prepublishOnly` run automatically on `npm install`
+and `npm publish`. They are the same execution primitive that was
+abused in the 2024 xz-utils-style supply-chain incidents, and they are
+the only general-purpose code execution surface in an npm package's
+metadata.
+
+This package ships only documentation and JSON. There is no legitimate
+reason for any install-time script. This validator ensures the
+package.json (and any future workspace package.json files) cannot
+silently grow one.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+# Hooks that npm runs implicitly during install/publish. This list is the
+# union of the npm 10/11 documented lifecycle scripts that execute
+# without explicit invocation.
+FORBIDDEN_HOOKS = {
+    "preinstall",
+    "install",
+    "postinstall",
+    "preuninstall",
+    "uninstall",
+    "postuninstall",
+    "prepare",
+    "prepublish",
+    "prepublishOnly",
+    "prepack",
+    "postpack",
+}
+
+
+def package_json_files() -> list[Path]:
+    paths = [ROOT / "package.json"]
+    # Look for any future workspace packages.
+    for candidate in ROOT.rglob("package.json"):
+        if "node_modules" in candidate.parts:
+            continue
+        if candidate not in paths:
+            paths.append(candidate)
+    return paths
+
+
+def main() -> int:
+    errors: list[str] = []
+    files_checked = 0
+    for path in package_json_files():
+        if not path.is_file():
+            continue
+        files_checked += 1
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:  # noqa: BLE001
+            errors.append(f"{path.relative_to(ROOT)}: invalid JSON: {exc}")
+            continue
+
+        scripts = data.get("scripts") or {}
+        present = sorted(set(scripts.keys()) & FORBIDDEN_HOOKS)
+        if present:
+            errors.append(
+                f"{path.relative_to(ROOT)}: forbidden lifecycle scripts present: "
+                f"{present}. This package must not run code on install/publish."
+            )
+
+    if errors:
+        print("ERROR: lifecycle-script policy violation", file=sys.stderr)
+        for err in errors:
+            print(f"  - {err}", file=sys.stderr)
+        return 1
+
+    print(f"OK: no forbidden lifecycle scripts in {files_checked} package.json file(s)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())