@raishin/vanguard-frontier-agentic 1.9.0 → 2.0.0

package/tests/validate-finops-price-fixtures.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""Grader for finops-cloud-price-advisor v0.2.0 integration test fixtures.
+
+Validates structure and provenance labels of the expected output fixtures
+against their paired inputs. No network calls are made — all checks are
+deterministic against static JSON files.
+
+Exit codes:
+  0 — all fixtures pass
+  1 — one or more fixtures fail
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+FIXTURE_DIR = ROOT / "tests" / "fixtures" / "finops-cloud-price-advisor"
+INPUTS_DIR = FIXTURE_DIR / "inputs"
+EXPECTED_DIR = FIXTURE_DIR / "expected"
+TAXONOMY_PATH = FIXTURE_DIR / "taxonomy.json"
+
+VALID_PROVIDERS = {"aws", "azure", "oci", "scaleway", "gandi", "alibaba", "tencent", "comparative"}
+VALID_CURRENCIES = {"USD", "EUR", "CNY"}
+VALID_PROVENANCE_LABELS = {"live-price", "documentation-based", "assumed", "excluded"}
+REQUIRED_TAXONOMY_PROVIDERS = {"aws", "azure", "oci", "scaleway", "gandi", "alibaba", "tencent"}
+
+# Credential patterns — must be wrapped in <FAKE> when present in fixtures.
+# Each tuple is (name, compiled pattern).
+_CREDENTIAL_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
+    ("AWS access key", re.compile(r"AKIA[0-9A-Z]{16}")),
+    ("Alibaba access key", re.compile(r"LTAI[0-9A-Za-z]{12,20}")),
+    ("Tencent secret ID", re.compile(r"AKID[0-9A-Za-z]{13,28}")),
+]
+
+
+def _load_json(path: Path) -> dict:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _input_has_unmarked_credential(inp: dict) -> list[str]:
+    """Return a list of violation descriptions for any real-looking credential
+    patterns that are not wrapped in <FAKE> tags.
+
+    The raw string value is never echoed to output — only the field name and
+    pattern name are returned to avoid leaking credential-shaped strings into
+    CI logs.
+    """
+    violations: list[str] = []
+    raw = json.dumps(inp)
+    for name, pattern in _CREDENTIAL_PATTERNS:
+        for match in pattern.finditer(raw):
+            start = max(0, match.start() - 7)
+            # Check a small window around the match for the <FAKE> marker.
+            window = raw[start : match.end() + 7]
+            if "<FAKE>" not in window:
+                violations.append(f"unmarked {name} pattern found in input JSON")
+    return violations
+
+
+def _validate_expected(inp: dict, exp: dict) -> list[str]:
+    """Return a list of failure reasons for a single fixture pair."""
+    failures: list[str] = []
+    fixture_id = inp.get("id", "?")
+
+    # --- provider ---
+    provider = exp.get("provider")
+    if provider is None:
+        failures.append("expected: missing 'provider' field")
+    elif provider not in VALID_PROVIDERS:
+        failures.append(f"expected: provider {provider!r} not in valid set {sorted(VALID_PROVIDERS)}")
+
+    # --- currency ---
+    currency = exp.get("currency")
+    if currency is None:
+        failures.append("expected: missing 'currency' field")
+    elif currency not in VALID_CURRENCIES:
+        failures.append(f"expected: currency {currency!r} not in {sorted(VALID_CURRENCIES)}")
+
+    # --- provenance_label ---
+    provenance_label = exp.get("provenance_label")
+    if provenance_label is None:
+        failures.append("expected: missing 'provenance_label' field")
+    elif provenance_label not in VALID_PROVENANCE_LABELS:
+        failures.append(
+            f"expected: provenance_label {provenance_label!r} not in {sorted(VALID_PROVENANCE_LABELS)}"
+        )
+
+    # --- key_stored must be false ---
+    key_stored = exp.get("key_stored")
+    if key_stored is not False:
+        failures.append(f"expected: key_stored must be false, got {key_stored!r}")
+
+    # --- CNY fixtures must declare requires_usd_conversion: true ---
+    if currency == "CNY":
+        if exp.get("requires_usd_conversion") is not True:
+            failures.append("expected: CNY fixture must have requires_usd_conversion: true")
+
+    # --- Gandi with-key fixture: disclaimer_required and key_stored checks ---
+    auth_mode = inp.get("auth_mode") or exp.get("auth_mode")
+    if inp.get("provider") == "gandi" and auth_mode == "user-provided":
+        if exp.get("disclaimer_required") is not True:
+            failures.append("expected: Gandi user-provided-key fixture must have disclaimer_required: true")
+        if exp.get("key_stored") is not False:
+            failures.append("expected: Gandi user-provided-key fixture must have key_stored: false")
+
+    # --- Input credential sweep (skip for adversarial fixtures — intentional payloads) ---
+    if not inp.get("intent"):
+        cred_violations = _input_has_unmarked_credential(inp)
+        for v in cred_violations:
+            failures.append(f"input: {v}")
+
+    # --- Fixture 004: user_key_marker must carry <FAKE> prefix ---
+    if fixture_id == "004":
+        marker = inp.get("user_key_marker", "")
+        if not marker.startswith("<FAKE>"):
+            failures.append("input 004: user_key_marker must start with <FAKE>")
+
+    return failures
+
+
+def _validate_taxonomy() -> list[str]:
+    """Check that taxonomy.json covers all required providers and counts."""
+    failures: list[str] = []
+    if not TAXONOMY_PATH.exists():
+        return ["taxonomy.json not found"]
+    taxonomy = _load_json(TAXONOMY_PATH)
+    coverage = set(taxonomy.get("provider_coverage", []))
+    missing = REQUIRED_TAXONOMY_PROVIDERS - coverage
+    if missing:
+        failures.append(f"taxonomy.json provider_coverage missing: {sorted(missing)}")
+    extra = coverage - REQUIRED_TAXONOMY_PROVIDERS
+    if extra:
+        failures.append(f"taxonomy.json provider_coverage has unexpected entries: {sorted(extra)}")
+    # Verify adversarial_count is declared when adversarial fixtures exist
+    if taxonomy.get("adversarial_count", 0) == 0:
+        adv_inputs = list(INPUTS_DIR.glob("adv-*.json")) if INPUTS_DIR.is_dir() else []
+        if adv_inputs:
+            failures.append("taxonomy.json missing adversarial_count but adversarial fixtures exist")
+    return failures
+
+
+def main() -> int:
+    passes = 0
+    fails = 0
+
+    # Validate taxonomy first.
+    taxonomy_errors = _validate_taxonomy()
+    for err in taxonomy_errors:
+        print(f"FAIL: taxonomy — {err}")
+        fails += 1
+    if not taxonomy_errors:
+        print("PASS: taxonomy — provider_coverage validated")
+
+    # Discover input fixtures in sorted order.
+    if not INPUTS_DIR.is_dir():
+        print(f"ERROR: inputs directory not found at {INPUTS_DIR}", file=sys.stderr)
+        return 1
+
+    input_files = sorted(INPUTS_DIR.glob("*.json"))
+    if not input_files:
+        print("ERROR: no input fixtures found", file=sys.stderr)
+        return 1
+
+    for inp_path in input_files:
+        stem = inp_path.stem
+        exp_path = EXPECTED_DIR / f"{stem}.json"
+
+        if not exp_path.exists():
+            print(f"FAIL: {stem} — missing expected file {exp_path.name}")
+            fails += 1
+            continue
+
+        try:
+            inp = _load_json(inp_path)
+            exp = _load_json(exp_path)
+        except json.JSONDecodeError as exc:
+            print(f"FAIL: {stem} — JSON parse error: {exc}")
+            fails += 1
+            continue
+
+        errors = _validate_expected(inp, exp)
+        if errors:
+            for err in errors:
+                print(f"FAIL: {stem} — {err}")
+            fails += 1
+        else:
+            print(f"PASS: {stem}")
+            passes += 1
+
+    total = passes + fails
+    print(f"\nResults: {passes}/{total} fixtures PASS")
+    return 0 if fails == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tests/validate-kiro-powers.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+"""Validate Kiro Powers under powers/ for spec conformance.
+
+Kiro Powers (kirodotdev/powers) frontmatter is strictly limited to FIVE
+fields: name, displayName, description, keywords, author. Any extra
+field — version, repository, license, tags — is forbidden and will be
+rejected by the Kiro Powers UI.
+
+Gates:
+  1. Every powers/<name>/ subdirectory has a POWER.md.
+  2. POWER.md frontmatter contains exactly the five allowed fields.
+  3. name is lowercase kebab-case.
+  4. description has at most three sentences (Kiro constraint).
+  5. keywords is a non-empty list of specific (non-broad) terms.
+  6. Generator (scripts/generate-kiro-powers.mjs) is in sync.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parent.parent
+POWERS = REPO / "powers"
+GENERATOR = REPO / "scripts" / "generate-kiro-powers.mjs"
+
+ALLOWED_FIELDS = {"name", "displayName", "description", "keywords", "author"}
+NAME_RE = re.compile(r"^[a-z][a-z0-9-]*$")
+# Broad terms that Kiro docs warn against — these are too generic to scope
+# activation reliably and produce false positives across unrelated tasks.
+BROAD_KEYWORDS = {
+    "cloud",
+    "code",
+    "devops",
+    "infrastructure",
+    "infra",
+    "agent",
+    "agents",
+    "ai",
+    "ml",
+    "ops",
+    "automation",
+    "tool",
+    "tools",
+    "general",
+}
+
+
+def fail(msg: str) -> None:
+    print(f"FAIL [kiro-powers] {msg}", file=sys.stderr)
+
+
+def _unquote(value: str) -> str:
+    value = value.strip()
+    if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
+        return value[1:-1]
+    return value
+
+
+def _parse_flow_list(value: str) -> list[str] | None:
+    value = value.strip()
+    if not (value.startswith("[") and value.endswith("]")):
+        return None
+    inner = value[1:-1].strip()
+    if not inner:
+        return []
+    return [_unquote(item.strip()) for item in inner.split(",") if item.strip()]
+
+
+def parse_frontmatter(text: str, path: Path) -> dict | None:
+    """Parse the strict-5 Kiro Powers frontmatter without a YAML library.
+
+    Accepted shapes:
+      key: value
+      key: "quoted value"
+      key: [item, item]   # flow list
+      key:                # block list
+        - item
+        - item
+    Anything outside this shape is rejected with a clear error.
+    """
+    if not text.startswith("---\n"):
+        fail(f"{path.relative_to(REPO)}: POWER.md must start with YAML frontmatter delimiter")
+        return None
+    end = text.find("\n---\n", 4)
+    if end == -1:
+        fail(f"{path.relative_to(REPO)}: POWER.md frontmatter is not terminated")
+        return None
+    block = text[4:end]
+
+    data: dict = {}
+    lines = block.split("\n")
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+        if not line.strip() or line.lstrip().startswith("#"):
+            i += 1
+            continue
+        if ":" not in line:
+            fail(f"{path.relative_to(REPO)}: malformed frontmatter line: {line!r}")
+            return None
+        key, _, rest = line.partition(":")
+        key = key.strip()
+        rest = rest.strip()
+        if rest == "":
+            items: list[str] = []
+            i += 1
+            while i < len(lines) and lines[i].lstrip().startswith("- "):
+                items.append(_unquote(lines[i].lstrip()[2:].strip()))
+                i += 1
+            data[key] = items
+            continue
+        flow = _parse_flow_list(rest)
+        if flow is not None:
+            data[key] = flow
+        else:
+            data[key] = _unquote(rest)
+        i += 1
+
+    if not data:
+        fail(f"{path.relative_to(REPO)}: frontmatter must be a mapping")
+        return None
+    return data
+
+
+_DECIMAL_RE = re.compile(r"\d\.\d")
+
+
+def count_sentences(text: str) -> int:
+    # Count terminal punctuation only when followed by whitespace or EOL,
+    # excluding periods inside decimal numbers (e.g. "MLPS 2.0", "v1.2.3")
+    # and abbreviation dots (e.g. "i.e.", "e.g.") that are not at a word
+    # boundary. Matches the same algorithm used in test-marketplace-validators.py
+    # to keep test expectations and live validation in sync.
+    text = text.strip()
+    if not text:
+        return 0
+    masked = _DECIMAL_RE.sub(lambda m: m.group(0).replace(".", "_"), text)
+    return len(re.findall(r"[.!?](?:\s|$)", masked))
+
+
+def validate_power(power_dir: Path) -> list[str]:
+    errs: list[str] = []
+    md = power_dir / "POWER.md"
+    if not md.is_file():
+        return [f"{power_dir.relative_to(REPO)}: POWER.md is missing"]
+
+    data = parse_frontmatter(md.read_text(), md)
+    if data is None:
+        return [f"{md.relative_to(REPO)}: frontmatter could not be parsed"]
+
+    keys = set(data.keys())
+    extra = keys - ALLOWED_FIELDS
+    missing = ALLOWED_FIELDS - keys
+    if extra:
+        errs.append(
+            f"{md.relative_to(REPO)}: forbidden frontmatter fields (Kiro strict-5 rule): {sorted(extra)}",
+        )
+    if missing:
+        errs.append(
+            f"{md.relative_to(REPO)}: missing required frontmatter fields: {sorted(missing)}",
+        )
+    if extra or missing:
+        return errs
+
+    name = data["name"]
+    if not isinstance(name, str) or not NAME_RE.match(name):
+        errs.append(f"{md.relative_to(REPO)}: name must be lowercase kebab-case ({name!r})")
+    if name != power_dir.name:
+        errs.append(
+            f"{md.relative_to(REPO)}: name {name!r} must match directory name {power_dir.name!r}",
+        )
+
+    description = data["description"]
+    if not isinstance(description, str) or not description.strip():
+        errs.append(f"{md.relative_to(REPO)}: description must be a non-empty string")
+    else:
+        n = count_sentences(description)
+        if n > 3:
+            errs.append(
+                f"{md.relative_to(REPO)}: description has {n} sentences; Kiro limit is 3",
+            )
+
+    keywords = data["keywords"]
+    if not isinstance(keywords, list) or not any(
+        isinstance(k, str) and k.strip() for k in keywords
+    ):
+        errs.append(f"{md.relative_to(REPO)}: keywords must be a non-empty list")
+    else:
+        for k in keywords:
+            if not isinstance(k, str):
+                errs.append(f"{md.relative_to(REPO)}: keyword {k!r} is not a string")
+            elif k.lower() in BROAD_KEYWORDS:
+                errs.append(
+                    f"{md.relative_to(REPO)}: keyword {k!r} is too broad; "
+                    "Kiro docs warn that broad keywords cause false activations",
+                )
+
+    if not isinstance(data.get("author"), str) or not data["author"].strip():
+        errs.append(f"{md.relative_to(REPO)}: author must be a non-empty string")
+    if not isinstance(data.get("displayName"), str) or not data["displayName"].strip():
+        errs.append(f"{md.relative_to(REPO)}: displayName must be a non-empty string")
+
+    return errs
+
+
+def main() -> int:
+    if not POWERS.is_dir():
+        fail("powers/ directory is missing")
+        return 1
+
+    errors: list[str] = []
+    power_dirs = sorted(p for p in POWERS.iterdir() if p.is_dir())
+    if not power_dirs:
+        fail("no powers found under powers/")
+        return 1
+
+    for d in power_dirs:
+        errors.extend(validate_power(d))
+
+    # Generator drift
+    result = subprocess.run(
+        ["node", str(GENERATOR), "--check"],
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+    if result.returncode != 0:
+        errors.append((result.stderr or result.stdout).strip())
+
+    if errors:
+        for err in errors:
+            fail(err)
+        return 1
+
+    print(f"OK: {len(power_dirs)} Kiro Powers valid (strict-5 frontmatter, kebab-case names, ≤3-sentence descriptions, generator in sync)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tests/validate-links.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Validate README local links and catalog official links."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+LOCAL_LINK_RE = re.compile(r"\[([^\]]+)\]\((?!https?://|mailto:|#)([^)]+)\)")
+URL_RE = re.compile(r"https?://[^\s>)]+")
+
+
+def load_json(path: Path):
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def catalog_urls() -> list[str]:
+    urls: list[str] = []
+    for path in (ROOT / "catalog").glob("*.json"):
+        data = load_json(path)
+        if isinstance(data, list):
+            for item in data:
+                urls.extend(item.get("official_docs", []))
+                if item.get("official_project_url"):
+                    urls.append(item["official_project_url"])
+    return sorted(set(urls))
+
+
+def markdown_urls() -> list[str]:
+    urls: list[str] = []
+    for path in [ROOT / "README.md", *ROOT.glob("mcp/**/*.md"), *ROOT.glob("docs/**/*.md")]:
+        if path.exists():
+            urls.extend(URL_RE.findall(path.read_text(encoding="utf-8", errors="ignore")))
+    return sorted(set(urls))
+
+
+def validate_local_readme_links(errors: list[str]) -> None:
+    readme = ROOT / "README.md"
+    text = readme.read_text(encoding="utf-8")
+    for _, raw in LOCAL_LINK_RE.findall(text):
+        target = raw.split("#", 1)[0]
+        if not target:
+            continue
+        if not (ROOT / target).exists():
+            errors.append(f"README local link does not exist: {raw}")
+
+
+def check_url(url: str, errors: list[str]) -> None:
+    req = urllib.request.Request(url, method="HEAD", headers={"User-Agent": "vanguard-frontier-agentic-link-check/1.0"})
+    try:
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            if resp.status >= 400:
+                errors.append(f"{url}: HTTP {resp.status}")
+    except Exception:
+        # Some sites reject HEAD. Retry GET with a small range.
+        req = urllib.request.Request(url, method="GET", headers={"User-Agent": "vanguard-frontier-agentic-link-check/1.0", "Range": "bytes=0-1024"})
+        try:
+            with urllib.request.urlopen(req, timeout=15) as resp:
+                if resp.status >= 400:
+                    errors.append(f"{url}: HTTP {resp.status}")
+        except Exception as exc:
+            errors.append(f"{url}: {exc}")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--offline", action="store_true", help="Only validate local links and URL syntax.")
+    args = parser.parse_args()
+
+    errors: list[str] = []
+    validate_local_readme_links(errors)
+    urls = sorted(set(catalog_urls() + markdown_urls()))
+    for url in urls:
+        if not URL_RE.match(url):
+            errors.append(f"invalid URL syntax: {url}")
+        elif not args.offline:
+            check_url(url, errors)
+
+    if errors:
+        for error in errors:
+            print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    mode = "offline" if args.offline else "online"
+    print(f"OK: validated README links and {len(urls)} URLs ({mode})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())