lgtm-specs 0.0.4

package/scripts/validate_specs.py

@@ -0,0 +1,730 @@
+"""Repository integrity validation for lgtm-specs.
+
+This script validates the Knowledge Graph structure at a coarse level.
+
+Design goals:
+- Zero third-party dependencies.
+- Deterministic output suitable for CI.
+
+Policy:
+- Validation is pass/fail.
+- WARN findings fail validation unless baselined.
+
+Notes:
+- Errors are reserved for broken invariants (e.g., spec version mismatch).
+- Most content shape checks are emitted as warnings; they still fail validation
+  unless ignored via `--baseline`.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+import argparse
+import urllib.error
+import urllib.parse
+import urllib.request
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+
+@dataclass(frozen=True)
+class Finding:
+    """A single validation finding.
+
+    Attributes:
+        level: One of "ERROR" or "WARN".
+        message: Human-readable message.
+    """
+
+    level: str
+    message: str
+
+
+def repo_rel(p: Path) -> str:
+    """Return a repo-relative path for printing."""
+
+    # Normalize to forward slashes for stable output across platforms.
+    return str(p.relative_to(REPO_ROOT)).replace(os.sep, "/")
+
+
+def walk_files(root: Path) -> list[Path]:
+    """Recursively enumerate all files under root.
+
+    We use os.walk to keep behavior consistent across platforms.
+    """
+
+    out: list[Path] = []
+    for dirpath, _dirnames, filenames in os.walk(root):
+        for fn in filenames:
+            out.append(Path(dirpath) / fn)
+    return out
+
+
+def read_text(p: Path) -> str:
+    """Read UTF-8 text from a file."""
+
+    return p.read_text(encoding="utf-8")
+
+
+def is_markdown(p: Path) -> bool:
+    """True if file is a Markdown file."""
+
+    return p.suffix == ".md"
+
+
+def is_readme(p: Path) -> bool:
+    """True if file is a README.md."""
+
+    return p.name == "README.md"
+
+
+def is_rule_file(p: Path) -> bool:
+    """True if this is an atomic rule file (constitution or law).
+
+    We intentionally exclude:
+    - Directory READMEs (indexes)
+    - `rules/laws/default.md` (fallback rules, not an atomic law)
+    """
+
+    if not is_markdown(p) or is_readme(p):
+        return False
+
+    rel = repo_rel(p).replace(os.sep, "/")
+
+    if rel.startswith("rules/constitution/"):
+        return True
+
+    if rel == "rules/laws/default.md":
+        return False
+
+    return rel.startswith("rules/laws/")
+
+
+def is_policy_file(p: Path) -> bool:
+    """True if this is a policy spec file under rules/policies/."""
+
+    if not is_markdown(p) or is_readme(p):
+        return False
+    rel = repo_rel(p).replace(os.sep, "/")
+    return rel.startswith("rules/policies/")
+
+
+def is_rules_template_file(p: Path) -> bool:
+    """True if this is the rule template file."""
+
+    if not is_markdown(p) or is_readme(p):
+        return False
+    return repo_rel(p).replace(os.sep, "/") == "rules/RULE-00000-EXAMPLE.md"
+
+
+def is_default_law_file(p: Path) -> bool:
+    """True if this is the non-atomic fallback law file."""
+
+    if not is_markdown(p) or is_readme(p):
+        return False
+    return repo_rel(p).replace(os.sep, "/") == "rules/laws/default.md"
+
+
+RULE_FILENAME_RE = re.compile(r"^[A-Z]+-\d{5}-[a-z0-9][a-z0-9-]*\.md$")
+
+# Every atomic rule should include at least one high-level "dimension" tag.
+DIMENSION_TAGS = {"#structural", "#behavioral", "#runtime", "#operational"}
+
+
+def find_meta_line(text: str, key: str) -> str | None:
+    """Return the first metadata line matching **Key**: ... if present."""
+
+    for line in text.splitlines():
+        if line.startswith(f"**{key}**:"):
+            return line
+        if line.startswith(f"**{key}** :"):
+            return line
+    return None
+
+
+# External URL checks are best-effort and may need temporary suppressions.
+# Keep this empty by default (fork-friendly); use `--url-ignore` for project-specific ignores.
+DEFAULT_IGNORED_URLS: set[str] = set()
+
+# Common badge endpoints are often flaky/permission-dependent for HEAD checks.
+IGNORED_URL_PATTERNS = [
+    re.compile(r"^https?://github\.com/.+/actions/workflows/.+/badge\.svg(\?.*)?$"),
+]
+
+
+def extract_first_link(line: str) -> str | None:
+    """Extract the first Markdown link target from a line."""
+
+    links = extract_markdown_links(line)
+    return links[0] if links else None
+
+
+def extract_markdown_links(text: str) -> list[str]:
+    """Extract all Markdown link targets from a text blob."""
+
+    # Minimal parser for `[label](target)` that tolerates nested `(...)` in the target.
+    out: list[str] = []
+    i = 0
+    while True:
+        lb = text.find("[", i)
+        if lb == -1:
+            break
+
+        # Ignore image syntax: ![alt](url)
+        if lb > 0 and text[lb - 1] == "!":
+            i = lb + 1
+            continue
+
+        rb = text.find("]", lb + 1)
+        if rb == -1:
+            break
+
+        if rb + 1 >= len(text) or text[rb + 1] != "(":
+            i = rb + 1
+            continue
+
+        j = rb + 2
+        depth = 1
+        start = j
+        while j < len(text) and depth > 0:
+            ch = text[j]
+            if ch == "(":
+                depth += 1
+            elif ch == ")":
+                depth -= 1
+                if depth == 0:
+                    out.append(text[start:j])
+                    j += 1
+                    break
+            j += 1
+
+        i = j
+
+    return out
+
+
+def normalize_link_target(target: str) -> str:
+    """Normalize Markdown link target.
+
+    Handles common forms such as:
+    - [x](https://example.com)
+    - [x](https://example.com "title")
+    - [x](<https://example.com>)
+    """
+
+    t = target.strip()
+    if t.startswith("<") and t.endswith(">"):
+        t = t[1:-1].strip()
+
+    # Strip optional title suffix from markdown link target.
+    # Example: https://example.com "title"
+    if t.startswith("http://") or t.startswith("https://"):
+        t = t.split()[0]
+
+    return t
+
+
+def is_external_http_url(target: str) -> bool:
+    """True if target is an external http(s) URL."""
+
+    parsed = urllib.parse.urlparse(target)
+    return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
+
+
+def probe_url_status(url: str, timeout: float) -> tuple[int | None, str | None]:
+    """Probe URL with HEAD first, then fallback GET for unsupported HEAD.
+
+    Returns:
+        (status_code, error_text)
+    """
+
+    headers = {"User-Agent": "lgtm-specs-validator/1.0"}
+    transient_statuses = {429, 500, 502, 503, 504}
+
+    def do_request(method: str) -> tuple[int | None, str | None]:
+        req_headers = dict(headers)
+        if method == "GET":
+            # Minimize payload when falling back from HEAD.
+            req_headers["Range"] = "bytes=0-0"
+
+        req = urllib.request.Request(url, method=method, headers=req_headers)
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                return getattr(resp, "status", 200), None
+        except urllib.error.HTTPError as e:
+            return e.code, None
+        except urllib.error.URLError as e:
+            return None, str(e.reason)
+
+    status, err = do_request("HEAD")
+    for attempt in range(2):
+        if status is None or status in transient_statuses:
+            time.sleep(0.5 * (attempt + 1))
+            status, err = do_request("HEAD")
+
+    # Some endpoints don't support HEAD.
+    if status in {405, 501}:
+        status, err = do_request("GET")
+        for attempt in range(2):
+            if status is None or status in transient_statuses:
+                time.sleep(0.5 * (attempt + 1))
+                status, err = do_request("GET")
+
+    return status, err
+
+
+def validate_external_urls(findings: list[Finding], timeout: float, ignored_urls: set[str]) -> None:
+    """Validate external URLs referenced in Markdown files.
+
+    Notes:
+    - Internal repo links are intentionally skipped.
+    - External URL checks are network-dependent, so this should be opt-in.
+    """
+
+    url_sources: dict[str, list[str]] = {}
+
+    for p in walk_files(REPO_ROOT):
+        if not is_markdown(p):
+            continue
+
+        rel = repo_rel(p)
+        if rel.startswith(".git/"):
+            continue
+        if rel.startswith("node_modules/"):
+            continue
+        if rel.startswith("__pycache__/"):
+            continue
+        for raw_target in extract_markdown_links(read_text(p)):
+            target = normalize_link_target(raw_target)
+            if not is_external_http_url(target):
+                continue
+
+            url_sources.setdefault(target, []).append(rel)
+
+    for url, sources in sorted(url_sources.items()):
+        if url in ignored_urls:
+            continue
+
+        if any(p.search(url) for p in IGNORED_URL_PATTERNS):
+            continue
+
+        status, err = probe_url_status(url, timeout)
+
+        # 2xx/3xx are good.
+        if status is not None and 200 <= status < 400:
+            continue
+
+        # Many doc sites return auth/forbidden for bots but are still valid links.
+        if status in {401, 403}:
+            continue
+
+        where = ", ".join(sorted(set(sources)))
+        if status in {404, 410}:
+            findings.append(
+                Finding(
+                    "WARN",
+                    f"Broken external URL ({status}): {url} (found in {where})",
+                )
+            )
+        elif status is not None:
+            findings.append(
+                Finding(
+                    "WARN",
+                    f"External URL returned status {status}: {url} (found in {where})",
+                )
+            )
+        else:
+            findings.append(
+                Finding(
+                    "WARN",
+                    f"External URL probe failed ({err}): {url} (found in {where})",
+                )
+            )
+
+
+def validate_spec_version(findings: list[Finding]) -> None:
+    """Ensure VERSION matches integrate/README.md Spec Version.
+
+    This intentionally enforces strict equality.
+    Keep `VERSION` and `integrate/README.md` `**Spec Version**:` in sync.
+    """
+
+    version_file = REPO_ROOT / "VERSION"
+    integrate_readme = REPO_ROOT / "integrate" / "README.md"
+
+    if not version_file.exists():
+        findings.append(Finding("ERROR", "Missing VERSION file"))
+        return
+
+    version = read_text(version_file).strip()
+    integrate = read_text(integrate_readme)
+
+    m = re.search(r"\*\*Spec Version\*\*:\s*([^\r\n]+)", integrate)
+    if not m:
+        findings.append(Finding("ERROR", "Missing **Spec Version** in integrate/README.md"))
+        return
+
+    spec_version = m.group(1).strip()
+    if spec_version != version:
+        findings.append(
+            Finding(
+                "ERROR",
+                f"Spec version mismatch: VERSION={version} integrate/README.md={spec_version}",
+            )
+        )
+
+
+def validate_rule_shape(p: Path, findings: list[Finding]) -> None:
+    """Validate rule-like file shape.
+
+    We emit WARN-level findings for legacy gaps.
+    """
+
+    validate_rule_shape_with_options(p, findings, enforce_filename=True)
+
+
+def validate_rule_shape_with_options(
+    p: Path,
+    findings: list[Finding],
+    *,
+    enforce_filename: bool,
+) -> None:
+    """Validate rule-like file shape.
+
+    Args:
+        enforce_filename: If true, enforce atomic rule filename format.
+    """
+
+    base = p.name
+    if enforce_filename and not RULE_FILENAME_RE.match(base):
+        findings.append(
+            Finding("WARN", f"Bad filename (expected PREFIX-00000-slug.md): {repo_rel(p)}")
+        )
+
+    text = read_text(p)
+    if not text.startswith("# "):
+        findings.append(Finding("WARN", f"Missing H1 title: {repo_rel(p)}"))
+        return
+
+    for key in ("Source", "Tags", "Related"):
+        if find_meta_line(text, key) is None:
+            findings.append(Finding("WARN", f"Missing **{key}** metadata: {repo_rel(p)}"))
+
+    # Validate that metadata links resolve to real files (internal links only).
+    for key in ("Source", "Related"):
+        meta = find_meta_line(text, key)
+        if not meta:
+            continue
+
+        for raw_target in extract_markdown_links(meta):
+            target = normalize_link_target(raw_target)
+
+            # Skip external http(s) URLs.
+            if is_external_http_url(target):
+                continue
+
+            # Skip mailto links.
+            if target.startswith("mailto:"):
+                continue
+
+            # Strip anchor for path existence checks.
+            path_part = target.split("#", 1)[0]
+            if not path_part:
+                # Anchor-only link (same file).
+                continue
+
+            if path_part.startswith("/"):
+                resolved = (REPO_ROOT / path_part.lstrip("/")).resolve(strict=False)
+            else:
+                resolved = (p.parent / path_part).resolve(strict=False)
+
+            # Allow directory links by treating them as README.md.
+            if resolved.is_dir():
+                resolved = resolved / "README.md"
+
+            # Guard against repo-escape paths.
+            try:
+                resolved.relative_to(REPO_ROOT)
+            except ValueError:
+                findings.append(
+                    Finding(
+                        "WARN",
+                        f"Metadata link escapes repo root (**{key}**): {target} (in {repo_rel(p)})",
+                    )
+                )
+                continue
+
+            if not resolved.exists():
+                findings.append(
+                    Finding(
+                        "WARN",
+                        f"Broken **{key}** link: {target} (in {repo_rel(p)})",
+                    )
+                )
+
+    tags_line = find_meta_line(text, "Tags")
+    if tags_line is not None:
+        tags = {t.lower() for t in re.findall(r"#[A-Za-z0-9_-]+", tags_line)}
+        if not (tags & DIMENSION_TAGS):
+            findings.append(
+                Finding(
+                    "WARN",
+                    f"Missing dimension tag in **Tags** metadata (need one of: {', '.join(sorted(DIMENSION_TAGS))}): {repo_rel(p)}",
+                )
+            )
+
+    for hdr in ("## Definition", "## Requirements", "## Anti-Patterns", "## Examples"):
+        if hdr not in text:
+            findings.append(Finding("WARN", f"Missing section '{hdr}': {repo_rel(p)}"))
+
+    if "## Examples" in text:
+        if "**Bad:**" not in text:
+            findings.append(Finding("WARN", f"Missing **Bad:** example: {repo_rel(p)}"))
+        if "**Good:**" not in text:
+            findings.append(Finding("WARN", f"Missing **Good:** example: {repo_rel(p)}"))
+
+    src = find_meta_line(text, "Source")
+    if src:
+        link = extract_first_link(src)
+        # Internal meta docs should prefer anchors.
+        if link and "docs/meta/" in link and "#" not in link:
+            findings.append(
+                Finding("WARN", f"Internal Source link missing #anchor: {repo_rel(p)}")
+            )
+
+
+def validate_rule_indexing(p: Path, findings: list[Finding]) -> None:
+    """Validate that atomic rules are indexed by their parent README."""
+
+    parent = p.parent
+    readme = parent / "README.md"
+    if not readme.exists():
+        findings.append(
+            Finding("WARN", f"Missing directory index README.md: {repo_rel(parent)}")
+        )
+        return
+
+    readme_text = read_text(readme)
+    if p.name not in readme_text:
+        findings.append(
+            Finding(
+                "WARN",
+                f"File not indexed in {repo_rel(readme)}: {p.name}",
+            )
+        )
+
+
+def parse_registry_capabilities() -> set[str]:
+    """Parse capability keys from models/registry.yaml.
+
+    This is a minimal YAML parser by convention:
+    - Capabilities live under the top-level key: `capabilities:`
+    - Each capability is a 2-space indented key.
+    """
+
+    p = REPO_ROOT / "models" / "registry.yaml"
+    caps: set[str] = set()
+    in_caps = False
+
+    for line in read_text(p).splitlines():
+        if line.strip() == "capabilities:":
+            in_caps = True
+            continue
+        if not in_caps:
+            continue
+
+        m = re.match(r"^\s{2}([a-z][a-z0-9_-]*):\s*$", line)
+        if m:
+            caps.add(m.group(1))
+
+    return caps
+
+
+def parse_agent_capabilities() -> tuple[list[tuple[str, str]], list[str]]:
+    """Extract (file, capability) pairs from agents/roles specs.
+
+    Role specs declare `Capability:` inside a `<Meta>` block.
+    """
+
+    roles_dir = REPO_ROOT / "agents" / "roles"
+    out: list[tuple[str, str]] = []
+    missing: list[str] = []
+
+    meta_re = re.compile(r"(?s)<Meta>\s*(.*?)\s*</Meta>")
+    cap_re = re.compile(r"(?m)^\s*Capability:\s*([a-z][a-z0-9_-]*)\s*$")
+
+    for p in walk_files(roles_dir):
+        if not is_markdown(p) or is_readme(p):
+            continue
+
+        text = read_text(p)
+        meta = meta_re.search(text)
+        if not meta:
+            # Not a role spec (e.g., shared Markdown fragments).
+            continue
+
+        rel = repo_rel(p)
+        m = cap_re.search(meta.group(1))
+        if not m:
+            missing.append(rel)
+            continue
+
+        out.append((rel, m.group(1)))
+
+    return out, missing
+
+
+def validate_agent_capabilities(findings: list[Finding]) -> None:
+    """Ensure every agent capability exists in the registry."""
+
+    caps = parse_registry_capabilities()
+
+    pairs, missing = parse_agent_capabilities()
+    if not pairs and not missing:
+        findings.append(
+            Finding(
+                "ERROR",
+                "No agent capabilities found under agents/roles (Capability:<...> in <Meta>); parsing may be broken",
+            )
+        )
+
+    for file_path in missing:
+        findings.append(Finding("ERROR", f"Agent spec missing Capability in <Meta>: {file_path}"))
+
+    for file_path, cap in pairs:
+        if cap not in caps:
+            findings.append(
+                Finding("ERROR", f"Agent capability not in registry: {file_path} -> {cap}")
+            )
+
+
+def main() -> int:
+    """Run validation and return a shell exit code.
+
+    WARN findings fail validation unless baselined.
+    """
+
+    parser = argparse.ArgumentParser(add_help=True)
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help="DEPRECATED (no-op): validation is always strict; retained for compatibility.",
+    )
+    parser.add_argument(
+        "--baseline",
+        type=str,
+        default=None,
+        help="Path to a newline-delimited list of known warnings to ignore.",
+    )
+    parser.add_argument(
+        "--write-baseline",
+        type=str,
+        default=None,
+        help="Write current warning messages to the given file and exit 0.",
+    )
+    parser.add_argument(
+        "--check-urls",
+        action="store_true",
+        help="Opt-in: validate external http(s) URLs found in Markdown files.",
+    )
+    parser.add_argument(
+        "--url-timeout",
+        type=float,
+        default=8.0,
+        help="Timeout in seconds for external URL checks (default: 8.0).",
+    )
+    parser.add_argument(
+        "--url-ignore",
+        action="append",
+        default=[],
+        help="External URL to ignore during --check-urls (repeatable).",
+    )
+    args = parser.parse_args()
+
+    findings: list[Finding] = []
+
+    # Spec versioning is a hard invariant.
+    validate_spec_version(findings)
+
+    # Validate rules (atomic rules + policies + templates).
+    for p in walk_files(REPO_ROOT / "rules"):
+        if is_rule_file(p):
+            validate_rule_shape_with_options(p, findings, enforce_filename=True)
+            validate_rule_indexing(p, findings)
+            continue
+
+        if is_policy_file(p) or is_rules_template_file(p) or is_default_law_file(p):
+            validate_rule_shape_with_options(p, findings, enforce_filename=False)
+            validate_rule_indexing(p, findings)
+
+    # Ensure agents only use declared capabilities.
+    validate_agent_capabilities(findings)
+
+    if args.check_urls:
+        ignored_urls = set(DEFAULT_IGNORED_URLS)
+        ignored_urls.update(args.url_ignore)
+        validate_external_urls(findings, timeout=args.url_timeout, ignored_urls=ignored_urls)
+
+    warns = [f for f in findings if f.level == "WARN"]
+    if args.write_baseline:
+        out_path = REPO_ROOT / args.write_baseline
+        out_lines = sorted({f.message for f in warns})
+        out_path.write_text("\n".join(out_lines) + "\n", encoding="utf-8")
+        print(f"OK: wrote baseline to {repo_rel(out_path)}")
+        return 0
+
+    baseline: set[str] = set()
+    if args.baseline:
+        baseline_path = (
+            (REPO_ROOT / args.baseline).resolve()
+            if not os.path.isabs(args.baseline)
+            else Path(args.baseline)
+        )
+        if not baseline_path.exists():
+            findings.append(
+                Finding(
+                    "ERROR",
+                    f"Baseline file does not exist: {repo_rel(baseline_path)}",
+                )
+            )
+        else:
+            baseline = {
+                line.strip()
+                for line in read_text(baseline_path).splitlines()
+                if line.strip() and not line.strip().startswith("#")
+            }
+
+    errors = [f for f in findings if f.level == "ERROR"]
+    warns = [f for f in findings if f.level == "WARN"]
+
+    if baseline:
+        warns = [w for w in warns if w.message not in baseline]
+
+    if errors:
+        print(f"FAILED: {len(errors)} error(s)", file=sys.stderr)
+        for f in errors:
+            print(f"- {f.message}", file=sys.stderr)
+
+    if warns:
+        print(f"WARN: {len(warns)} warning(s)", file=sys.stderr)
+        for f in warns:
+            print(f"- {f.message}", file=sys.stderr)
+
+    if errors:
+        return 1
+
+    if warns:
+        return 1
+
+    print("OK: validation passed")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())