@raishin/vanguard-frontier-agentic 1.9.0 → 2.0.1

package/tests/validate-agent-frontmatter-schema.py

@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+"""Validate AGENT.md frontmatter against schemas/agent.frontmatter.schema.json.
+
+Mirrors tests/validate-skill-frontmatter-schema.py. Walks every
+agents/<provider>/<name>-agent/AGENT.md, parses the YAML frontmatter with a
+stdlib-only restricted parser, and validates against the schema using
+jsonschema if available, else a hand-rolled fallback.
+
+Required fields (empirical from current 141-file corpus):
+  - metadata.author (non-empty string)
+  - metadata.version (semver)
+
+Optional but typed when present: name, description, model, allowed-tools,
+tools, color. additionalProperties is permitted so harness-specific fields
+do not break validation.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+AGENTS_DIR = ROOT / "agents"
+SCHEMA_PATH = ROOT / "schemas" / "agent.frontmatter.schema.json"
+
+# ──────────────────────────────────────────────────────────────────────────────
+# YAML frontmatter parser (stdlib only, mirrors validate-skill-frontmatter-schema.py)
+# ──────────────────────────────────────────────────────────────────────────────
+
+def parse_frontmatter_raw(text: str) -> dict | None:
+    if not text.startswith("---\n"):
+        return None
+    end = text.find("\n---", 4)
+    if end == -1:
+        return None
+    block = text[4:end]
+    return _minimal_yaml_parse(block)
+
+
+def _minimal_yaml_parse(block: str) -> dict:
+    """Parse a restricted YAML subset into a Python dict."""
+    result: dict = {}
+    lines = block.splitlines()
+    i = 0
+
+    def parse_scalar(s: str):
+        s = s.strip()
+        if (s.startswith('"') and s.endswith('"')) or \
+           (s.startswith("'") and s.endswith("'")):
+            return s[1:-1]
+        if s.lower() == "true":
+            return True
+        if s.lower() == "false":
+            return False
+        return s
+
+    while i < len(lines):
+        line = lines[i]
+        if line and not line.startswith(" ") and ":" in line:
+            key, _, rest = line.partition(":")
+            key = key.strip()
+            rest = rest.strip()
+
+            if rest == "":
+                children_list: list = []
+                children_dict: dict = {}
+                i += 1
+                while i < len(lines) and (lines[i].startswith("  ") or lines[i] == ""):
+                    child = lines[i]
+                    if child.strip() == "":
+                        i += 1
+                        continue
+                    if child.startswith("  - "):
+                        children_list.append(child[4:].strip())
+                    elif child.startswith("  ") and ":" in child:
+                        ckey, _, crest = child.strip().partition(":")
+                        children_dict[ckey.strip()] = parse_scalar(crest)
+                    i += 1
+                if children_list:
+                    result[key] = children_list
+                elif children_dict:
+                    result[key] = children_dict
+                else:
+                    result[key] = None
+                continue
+            elif rest.startswith("[") and rest.endswith("]"):
+                inner = rest[1:-1]
+                result[key] = [t.strip().strip("'\"") for t in inner.split(",") if t.strip()]
+            else:
+                result[key] = parse_scalar(rest)
+        i += 1
+
+    return result
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Schema-based validation
+# ──────────────────────────────────────────────────────────────────────────────
+
+NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
+VERSION_RE = re.compile(r"^\d+\.\d+\.\d+(-[\w.-]+)?$")
+
+
+def _try_jsonschema_validate(instance: dict, schema: dict) -> list[str]:
+    try:
+        import jsonschema  # type: ignore
+        errors = []
+        validator_cls = jsonschema.validators.validator_for(schema)
+        validator = validator_cls(schema)
+        for err in sorted(validator.iter_errors(instance), key=lambda e: list(e.path)):
+            path = ".".join(str(p) for p in err.absolute_path) or "<root>"
+            errors.append(f"  field '{path}': {err.message}")
+        return errors
+    except ImportError:
+        return []
+
+
+def _hand_rolled_validate(fm: dict) -> list[str]:
+    errors: list[str] = []
+
+    meta = fm.get("metadata")
+    if meta is None:
+        errors.append("  field 'metadata': required field is missing")
+    elif not isinstance(meta, dict):
+        errors.append(
+            f"  field 'metadata': must be a mapping/object, got {type(meta).__name__}"
+        )
+    else:
+        author = meta.get("author")
+        if author is None:
+            errors.append("  field 'metadata.author': required field is missing")
+        elif not isinstance(author, str) or len(author) < 1:
+            errors.append("  field 'metadata.author': must be a non-empty string")
+
+        version = meta.get("version")
+        if version is None:
+            errors.append("  field 'metadata.version': required field is missing")
+        elif not isinstance(version, str):
+            errors.append(
+                f"  field 'metadata.version': must be a string, got {type(version).__name__}"
+            )
+        elif not VERSION_RE.match(version):
+            errors.append(
+                f"  field 'metadata.version': '{version}' does not match semver pattern"
+            )
+
+    # Optional name pattern
+    name = fm.get("name")
+    if name is not None:
+        if not isinstance(name, str):
+            errors.append(f"  field 'name': must be a string, got {type(name).__name__}")
+        elif not NAME_RE.match(name):
+            errors.append(
+                f"  field 'name': '{name}' does not match pattern ^[a-z0-9][a-z0-9-]*$"
+            )
+
+    # Optional description bounds
+    desc = fm.get("description")
+    if desc is not None:
+        if not isinstance(desc, str):
+            errors.append(
+                f"  field 'description': must be a string, got {type(desc).__name__}"
+            )
+        else:
+            if len(desc) < 20:
+                errors.append(
+                    f"  field 'description': length {len(desc)} is below minimum 20"
+                )
+            if len(desc) > 4000:
+                errors.append(
+                    f"  field 'description': length {len(desc)} exceeds maximum 4000"
+                )
+
+    # allowed-tools / tools shapes
+    for key in ("allowed-tools", "tools"):
+        v = fm.get(key)
+        if v is None:
+            continue
+        if isinstance(v, str):
+            if len(v.strip()) == 0:
+                errors.append(f"  field '{key}': string value must not be empty")
+        elif isinstance(v, list):
+            if len(v) == 0:
+                errors.append(f"  field '{key}': list must contain at least one item")
+            for idx, item in enumerate(v):
+                if not isinstance(item, str) or len(item.strip()) == 0:
+                    errors.append(
+                        f"  field '{key}[{idx}]': each item must be a non-empty string"
+                    )
+        else:
+            errors.append(
+                f"  field '{key}': must be a string or array, got {type(v).__name__}"
+            )
+
+    return errors
+
+
+def validate_agent(agent_md: Path, schema: dict) -> list[str]:
+    text = agent_md.read_text(encoding="utf-8")
+    fm = parse_frontmatter_raw(text)
+    if fm is None or not isinstance(fm, dict):
+        return ["  no valid YAML frontmatter block found"]
+
+    errors = _try_jsonschema_validate(fm, schema)
+    if not errors:
+        # double-check with hand-rolled to catch anything jsonschema might miss
+        errors = _hand_rolled_validate(fm)
+    return errors
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Entry point
+# ──────────────────────────────────────────────────────────────────────────────
+
+def main() -> int:
+    if not SCHEMA_PATH.exists():
+        print(f"ERROR: schema not found at {SCHEMA_PATH}", file=sys.stderr)
+        return 2
+
+    schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))
+
+    agent_files = sorted(AGENTS_DIR.glob("*/*/AGENT.md"))
+    if not agent_files:
+        print("ERROR: no AGENT.md files found", file=sys.stderr)
+        return 2
+
+    failed: list[tuple[Path, list[str]]] = []
+
+    for agent_md in agent_files:
+        errors = validate_agent(agent_md, schema)
+        if errors:
+            failed.append((agent_md, errors))
+
+    if failed:
+        print(
+            f"FAIL: {len(failed)} agent(s) failed AGENT.md frontmatter schema validation "
+            f"(out of {len(agent_files)} checked):",
+            file=sys.stderr,
+        )
+        for path, errs in failed:
+            print(f"\n  {path}", file=sys.stderr)
+            for e in errs:
+                print(f"    {e}", file=sys.stderr)
+        return 1
+
+    print(
+        f"OK: all {len(agent_files)} agents passed AGENT.md frontmatter schema validation"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tests/validate-asset-integrity.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""Generate or validate the cross-asset integrity manifest.
+
+The skill-manifest covers SKILL.md trees only. This manifest covers every
+asset surface a downstream consumer trusts at runtime: agents/, rules/,
+mcp/, catalog/, schemas/, plus root-level governance files
+(README, SECURITY, LICENSE, CONTRIBUTING, CODE_OF_CONDUCT, CLAUDE,
+AGENTS, GEMINI). Each file is hashed individually (sha256) and rolled
+into a per-tree aggregate hash, plus a single top-level aggregate.
+
+The manifest itself is the artifact attested at release time
+(see .github/workflows/release.yml). A consumer who trusts the
+attestation can then verify any asset by recomputing its sha256 and
+comparing against the manifest entry, without having to trust the
+git history or the npm tarball file list separately.
+
+Use --write when asset content changes intentionally. Default mode is
+verification, suitable for CI gates.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+MANIFEST = ROOT / "catalog" / "asset-integrity.json"
+
+# Trees walked recursively. Order is stable (preserved in manifest).
+TREES = [
+    "agents",
+    "rules",
+    "mcp",
+    "schemas",
+    "catalog",
+    # The shipped CLI and generator scripts (npm bin + manifest generators)
+    # are runtime-critical: a malicious change here would not change any
+    # other tracked path but would compromise downstream installs.
+    "scripts",
+    # Marketplace surface — each plugin manifest is consumed verbatim by
+    # the corresponding harness at install time. Cover them all.
+    "powers",
+    "plugins",
+    ".claude-plugin",
+    ".cursor-plugin",
+    ".github/plugin",
+    ".agents/plugins",
+    # Validator scripts execute with full CI credentials during release
+    # (scripts/release-prepare.mjs calls validate-asset-integrity.py --write).
+    # A backdoor in tests/ would not trigger manifest drift unless tests/
+    # is also covered, giving an attacker a blind spot for supply-chain
+    # compromise. Include tests/ so any change here shows up in CI.
+    "tests",
+]
+
+# Top-level governance files. These are part of the trust surface for
+# any downstream consumer evaluating the project (procurement reviews,
+# compliance attestation, supply-chain due diligence).
+ROOT_FILES = [
+    "README.md",
+    "SECURITY.md",
+    "LICENSE",
+    "CONTRIBUTING.md",
+    "CODE_OF_CONDUCT.md",
+    "CLAUDE.md",
+    "AGENTS.md",
+    "GEMINI.md",
+    "package.json",
+    ".releaserc.js",
+]
+
+# Files inside the trees that are not part of the trust surface and
+# would create false drift on every CI run. Kept narrow on purpose.
+EXCLUDED_NAMES = {".DS_Store"}
+
+# Directories whose contents are generated build artifacts, never trust
+# surface. Pruned during the walk to keep the manifest stable.
+EXCLUDED_DIR_NAMES = {"__pycache__", ".pytest_cache", "node_modules"}
+
+# Bootstrap exception: the manifest cannot hash itself.
+EXCLUDED_RELATIVE_PATHS = {"catalog/asset-integrity.json"}
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def file_record(path: Path) -> dict:
+    data = path.read_bytes()
+    return {
+        "path": path.relative_to(ROOT).as_posix(),
+        "sha256": sha256_bytes(data),
+        "bytes": len(data),
+    }
+
+
+def aggregate_hash(records: list[dict]) -> str:
+    h = hashlib.sha256()
+    for item in records:
+        h.update(item["path"].encode("utf-8"))
+        h.update(b"\0")
+        h.update(item["sha256"].encode("ascii"))
+        h.update(b"\0")
+        h.update(str(item["bytes"]).encode("ascii"))
+        h.update(b"\n")
+    return h.hexdigest()
+
+
+def walk_tree(tree: str) -> list[dict]:
+    base = ROOT / tree
+    if not base.is_dir():
+        raise AssertionError(f"missing tree: {tree}")
+    files: list[dict] = []
+    for path in sorted(p for p in base.rglob("*") if p.is_file()):
+        if path.is_symlink():
+            # Symlinks are followed by is_file() but their target may lie
+            # outside the repo, producing a misleading integrity guarantee.
+            # Reject any symlink in the trust surface.
+            raise AssertionError(f"symlink in trust surface: {path.relative_to(ROOT)}")
+        if path.name in EXCLUDED_NAMES:
+            continue
+        if any(part in EXCLUDED_DIR_NAMES for part in path.parts):
+            continue
+        if path.relative_to(ROOT).as_posix() in EXCLUDED_RELATIVE_PATHS:
+            continue
+        files.append(file_record(path))
+    return files
+
+
+def build_manifest() -> dict:
+    trees: list[dict] = []
+    for tree in TREES:
+        records = walk_tree(tree)
+        trees.append({
+            "tree": tree,
+            "aggregate_sha256": aggregate_hash(records),
+            "files": records,
+        })
+
+    root_records: list[dict] = []
+    for name in ROOT_FILES:
+        path = ROOT / name
+        if not path.is_file():
+            raise AssertionError(f"missing root file: {name}")
+        root_records.append(file_record(path))
+
+    manifest = {
+        "manifest_version": 1,
+        "algorithm": "sha256",
+        "scope": {
+            "trees": TREES,
+            "root_files": ROOT_FILES,
+        },
+        "trees": trees,
+        "root_files": root_records,
+    }
+
+    # Top-level aggregate covers every per-tree aggregate plus every
+    # root-file record. Tampering with any single byte changes this hash.
+    h = hashlib.sha256()
+    for tree_entry in trees:
+        h.update(tree_entry["tree"].encode("utf-8"))
+        h.update(b"\0")
+        h.update(tree_entry["aggregate_sha256"].encode("ascii"))
+        h.update(b"\n")
+    for record in root_records:
+        h.update(record["path"].encode("utf-8"))
+        h.update(b"\0")
+        h.update(record["sha256"].encode("ascii"))
+        h.update(b"\n")
+    manifest["aggregate_sha256"] = h.hexdigest()
+
+    return manifest
+
+
+def render(manifest: dict) -> str:
+    return json.dumps(manifest, indent=2, sort_keys=False) + "\n"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--write", action="store_true",
+        help="Rewrite catalog/asset-integrity.json from disk state.",
+    )
+    args = parser.parse_args()
+
+    try:
+        current = build_manifest()
+    except AssertionError as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 1
+
+    rendered = render(current)
+
+    if args.write:
+        MANIFEST.write_text(rendered, encoding="utf-8")
+        n_files = sum(len(t["files"]) for t in current["trees"]) + len(current["root_files"])
+        print(
+            f"OK: wrote {MANIFEST.relative_to(ROOT)} "
+            f"({n_files} files, top-level sha256 {current['aggregate_sha256'][:12]}...)"
+        )
+        return 0
+
+    if not MANIFEST.exists():
+        print(
+            f"ERROR: missing {MANIFEST.relative_to(ROOT)}; "
+            "run tests/validate-asset-integrity.py --write",
+            file=sys.stderr,
+        )
+        return 1
+
+    expected = MANIFEST.read_text(encoding="utf-8")
+    if expected != rendered:
+        print(
+            f"ERROR: {MANIFEST.relative_to(ROOT)} is stale; asset content has drifted. "
+            "Run tests/validate-asset-integrity.py --write if the change is intentional, "
+            "then re-commit.",
+            file=sys.stderr,
+        )
+        return 1
+
+    print(
+        f"OK: asset integrity manifest matches "
+        f"(top-level sha256 {current['aggregate_sha256'][:12]}...)"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/tests/validate-aws-progressive-disclosure.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Validate AWS skills use progressive disclosure via references/."""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+AWS_DIR = ROOT / "skills" / "aws"
+CATALOG = ROOT / "catalog" / "skills.json"
+REQUIRED_REFERENCES = {
+    "references/workflow-and-output.md",
+    "references/official-sources.md",
+    "references/safety-checklist.md",
+}
+AGENTCORE_EXTRA_TERMS = [
+    "runtime",
+    "Memory",
+    "Gateway",
+    "Identity",
+    "Observability",
+    "Browser",
+    "Code Interpreter",
+    "MCP",
+]
+
+
+def err(msg: str, errors: list[str]) -> None:
+    errors.append(msg)
+
+
+def main() -> int:
+    errors: list[str] = []
+    catalog = {item["id"]: item for item in json.loads(CATALOG.read_text()) if item.get("provider") == "aws"}
+    for skill_dir in sorted(p for p in AWS_DIR.iterdir() if p.is_dir()):
+        sid = skill_dir.name
+        skill = skill_dir / "SKILL.md"
+        text = skill.read_text(encoding="utf-8")
+        if sid not in catalog:
+            err(f"{sid}: missing from catalog/skills.json", errors)
+        for rel in REQUIRED_REFERENCES:
+            ref = skill_dir / rel
+            if not ref.exists():
+                err(f"{sid}: missing {rel}", errors)
+            elif len(ref.read_text(encoding="utf-8").strip()) < 200:
+                err(f"{sid}: {rel} is too thin for useful lazy loading", errors)
+            if f"]({rel})" not in text:
+                err(f"{sid}: SKILL.md does not link {rel}", errors)
+        if len(text.splitlines()) > 90:
+            err(f"{sid}: SKILL.md has {len(text.splitlines())} lines; too heavy for trigger-time loading", errors)
+        if not re.search(r"Load these only when needed", text, re.IGNORECASE):
+            err(f"{sid}: missing lazy-load instruction", errors)
+        if sid == "aws-agentcore":
+            joined = "\n".join((skill_dir / rel).read_text(encoding="utf-8") for rel in REQUIRED_REFERENCES if (skill_dir / rel).exists()) + text
+            for term in AGENTCORE_EXTRA_TERMS:
+                if term.lower() not in joined.lower():
+                    err(f"{sid}: AgentCore references missing {term}", errors)
+            if "Do not hardcode credentials" not in joined and "Do not ask for" not in joined:
+                err(f"{sid}: AgentCore references missing credential safety language", errors)
+    if errors:
+        for item in errors:
+            print(f"ERROR: {item}", file=sys.stderr)
+        return 1
+    print(f"OK: validated progressive references for {len([p for p in AWS_DIR.iterdir() if p.is_dir()])} AWS skills")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())