@event4u/agent-config 5.10.0 → 6.0.0

package/src/scripts/check_condensation.py

@@ -0,0 +1,375 @@
+#!/usr/bin/env python3
+"""
+Condensation quality checker for agent-config packages.
+
+Compares .agent-src.uncondensed/ source files with their dist/agent-src/ condensed versions.
+Checks that condensation preserved structural integrity:
+- All headings from source present in condensed
+- All code blocks preserved exactly
+- YAML frontmatter identical
+- Word count reduction within healthy range (10-60%)
+- Iron Law sections (## Iron Law / ### Iron Law / ## The Iron Law / Iron Laws / numbered)
+  preserved per `preservation-guard`: heading verbatim at original level, ≤ 15% reduction
+
+Exit codes: 0 = clean, 1 = issues found, 3 = internal error
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from dataclasses import dataclass, asdict
+from pathlib import Path
+from typing import List, Literal
+
+# Import the rewriter so frontmatter comparison can normalise the source
+# side through the same path transformations the condenseor applies. Without
+# this, every `load_context:` logical name (e.g. `contexts/foo.md`) and every
+# `../../docs/...` body link looks like a frontmatter / body mismatch even
+# though the rewriter is doing exactly what road-to-path-fixes.md P2/P3
+# specified.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from condense import _rewrite_paths  # noqa: E402
+
+Severity = Literal["error", "warning", "info"]
+
+SOURCE_DIR = Path(".agent-src.uncondensed")
+TARGET_DIR = Path("dist/agent-src")
+
+
+@dataclass
+class Issue:
+    file: str
+    check: str
+    severity: Severity
+    message: str
+
+
+def extract_headings(text: str) -> list[str]:
+    """Extract all markdown headings (outside code blocks)."""
+    headings = []
+    in_code = False
+    for line in text.splitlines():
+        if line.strip().startswith("```"):
+            in_code = not in_code
+            continue
+        if not in_code and re.match(r"^#{1,6}\s+.+$", line):
+            headings.append(line)
+    return headings
+
+
+def extract_code_blocks(text: str) -> list[str]:
+    """Extract content of fenced code blocks."""
+    return re.findall(r"```[^\n]*\n(.*?)```", text, re.DOTALL)
+
+
+def extract_frontmatter(text: str) -> str:
+    """Extract YAML frontmatter."""
+    m = re.match(r"^---\n(.*?\n)---", text, re.DOTALL)
+    return m.group(1).strip() if m else ""
+
+
+# Matches `## Iron Law`, `## The Iron Law`, `## Iron Laws`, `### Iron Law — …`,
+# `## Iron Law 1 — …`, etc. Any heading level 2-6.
+IRON_LAW_HEADING = re.compile(r"^(#{2,6})\s+(The\s+)?Iron Laws?\b")
+
+LIST_ITEM_RE = re.compile(r"^(?:[-*+]|\d+\.)\s")
+INNER_HEADING_RE = re.compile(r"^#{1,6}\s")
+
+
+def count_iron_law_structure(body: str) -> dict:
+    """Count structural units in an Iron Law body.
+
+    Returns counts of paragraphs (blank-line-separated prose blocks),
+    list items (bullet + numbered), and fenced code blocks. Telegraph
+    condensation may shorten word count freely; what must NOT change is
+    the count of these structural units. Each represents a passage of
+    the law that the source decided to keep.
+
+    Multi-line list items (bullet text wrapped to indented continuation
+    lines, no blank line between) count as ONE list item, not as a
+    list item plus a paragraph.
+    """
+    paragraphs = 0
+    list_items = 0
+    code_blocks = 0
+    in_code = False
+    state = "blank"  # "blank" | "paragraph" | "list"
+    for line in body.splitlines():
+        stripped = line.strip()
+        if stripped.startswith("```"):
+            if not in_code:
+                code_blocks += 1
+            in_code = not in_code
+            state = "blank"
+            continue
+        if in_code:
+            continue
+        if not stripped:
+            state = "blank"
+            continue
+        if LIST_ITEM_RE.match(stripped):
+            list_items += 1
+            state = "list"
+            continue
+        if INNER_HEADING_RE.match(stripped):
+            state = "blank"
+            continue
+        # Indented non-empty line right after a list item is a wrap
+        # continuation of that item, not a new paragraph.
+        if state == "list" and line.startswith((" ", "\t")):
+            continue
+        if state != "paragraph":
+            paragraphs += 1
+            state = "paragraph"
+    return {"paragraphs": paragraphs, "list_items": list_items, "code_blocks": code_blocks}
+
+
+def extract_iron_law_sections(text: str) -> list[tuple[str, int, str]]:
+    """Return [(heading, level, body)] for each Iron Law section.
+
+    Body is everything after the heading until the next heading at the same
+    or higher (numerically lower) level — fenced code blocks included verbatim.
+    """
+    lines = text.splitlines()
+    sections: list[tuple[str, int, str]] = []
+    i = 0
+    in_code = False
+    while i < len(lines):
+        line = lines[i]
+        if line.strip().startswith("```"):
+            in_code = not in_code
+            i += 1
+            continue
+        if not in_code:
+            m = IRON_LAW_HEADING.match(line)
+            if m:
+                heading = line.rstrip()
+                level = len(m.group(1))
+                body_lines: list[str] = []
+                j = i + 1
+                inner_code = False
+                while j < len(lines):
+                    jline = lines[j]
+                    if jline.strip().startswith("```"):
+                        inner_code = not inner_code
+                    if not inner_code:
+                        hm = re.match(r"^(#{1,6})\s", jline)
+                        if hm and len(hm.group(1)) <= level:
+                            break
+                    body_lines.append(jline)
+                    j += 1
+                sections.append((heading, level, "\n".join(body_lines)))
+                i = j
+                continue
+        i += 1
+    return sections
+
+
+def check_pair(rel_path: str, source: str, condensed: str) -> List[Issue]:
+    """Compare source and condensed versions of a file."""
+    issues: List[Issue] = []
+
+    # Frontmatter check
+    src_fm = extract_frontmatter(source)
+    cmp_fm = extract_frontmatter(condensed)
+    if src_fm and src_fm != cmp_fm:
+        issues.append(Issue(rel_path, "frontmatter_mismatch", "error",
+                            "YAML frontmatter differs between source and condensed"))
+
+    # Heading preservation — check H1 and H2 headings (H3+ may be merged during condensation)
+    src_headings = extract_headings(source)
+    cmp_headings = extract_headings(condensed)
+    for h in src_headings:
+        # Only check H1 and H2 (## level) — H3+ subheadings may be merged
+        if h.startswith("# ") or (h.startswith("## ") and not h.startswith("### ")):
+            if h not in cmp_headings:
+                issues.append(Issue(rel_path, "missing_heading", "warning",
+                                    f"H1/H2 heading lost during condensation: {h}"))
+    # Also flag if comment lines inside code blocks are being treated as headings
+    # Filter out false positives from code blocks
+
+    # Code block preservation
+    src_blocks = extract_code_blocks(source)
+    cmp_blocks = extract_code_blocks(condensed)
+    if len(src_blocks) > len(cmp_blocks):
+        issues.append(Issue(rel_path, "lost_code_blocks", "error",
+                            f"Code blocks lost: source has {len(src_blocks)}, condensed has {len(cmp_blocks)}"))
+    for i, block in enumerate(src_blocks):
+        if i < len(cmp_blocks) and block.strip() != cmp_blocks[i].strip():
+            # Only flag if content actually changed (not just whitespace)
+            if block.replace(" ", "").replace("\n", "") != cmp_blocks[i].replace(" ", "").replace("\n", ""):
+                issues.append(Issue(rel_path, "modified_code_block", "error",
+                                    f"Code block {i+1} content changed during condensation"))
+
+    # Iron Law preservation — non-negotiable behavioral rules, see preservation-guard
+    src_laws = extract_iron_law_sections(source)
+    cmp_laws = extract_iron_law_sections(condensed)
+    cmp_law_map = {h: (lvl, body) for h, lvl, body in cmp_laws}
+    # Build a level-agnostic lookup so we can detect heading-level downgrades
+    # (`## Iron Law` → `### Iron Law`).
+    cmp_law_by_text = {h.lstrip("# ").strip(): (lvl, h, body)
+                       for h, lvl, body in cmp_laws}
+    for src_heading, src_level, src_body in src_laws:
+        src_text = src_heading.lstrip("# ").strip()
+        if src_heading not in cmp_law_map:
+            # Heading text may exist at a different level → downgrade
+            if src_text in cmp_law_by_text:
+                cmp_level, cmp_heading, _ = cmp_law_by_text[src_text]
+                if cmp_level != src_level:
+                    issues.append(Issue(rel_path, "iron_law_heading_downgrade", "error",
+                                        f"Iron Law heading level changed: "
+                                        f"{'#' * src_level} → {'#' * cmp_level} "
+                                        f"({src_heading.strip()})"))
+                    continue
+            issues.append(Issue(rel_path, "iron_law_missing", "error",
+                                f"Iron Law section removed during condensation: "
+                                f"{src_heading.strip()}"))
+            continue
+        # Section exists at correct level — check structural-unit survival.
+        # Telegraph condensation is fine (drop articles, terse phrasing); what
+        # must NOT change is the count of paragraphs, list items, and code
+        # blocks. Each is a passage the source kept on purpose.
+        _, cmp_body = cmp_law_map[src_heading]
+        src_struct = count_iron_law_structure(src_body)
+        cmp_struct = count_iron_law_structure(cmp_body)
+        for kind, src_n in src_struct.items():
+            cmp_n = cmp_struct[kind]
+            if cmp_n < src_n:
+                issues.append(Issue(rel_path, "iron_law_passage_dropped", "error",
+                                    f"Iron Law section dropped "
+                                    f"{src_n - cmp_n} {kind} "
+                                    f"({src_n} → {cmp_n}): "
+                                    f"{src_heading.strip()}"))
+
+    # Word count ratio
+    src_words = len(source.split())
+    cmp_words = len(condensed.split())
+    if src_words > 0:
+        reduction = (1 - cmp_words / src_words) * 100
+        if reduction > 60:
+            issues.append(Issue(rel_path, "excessive_reduction", "warning",
+                                f"Condensation reduced {reduction:.0f}% — possible content loss "
+                                f"({src_words} → {cmp_words} words)"))
+        elif reduction < 5 and src_words > 100:
+            issues.append(Issue(rel_path, "minimal_reduction", "info",
+                                f"Condensation only reduced {reduction:.0f}% "
+                                f"({src_words} → {cmp_words} words)"))
+
+    return issues
+
+
+def scan_all(root: Path) -> List[Issue]:
+    """Scan all .md file pairs for condensation quality issues."""
+    issues: List[Issue] = []
+    source_dir = root / SOURCE_DIR
+    target_dir = root / TARGET_DIR
+
+    if not source_dir.exists() or not target_dir.exists():
+        return issues
+
+    for source_file in sorted(source_dir.rglob("*.md")):
+        rel = source_file.relative_to(source_dir)
+        target_file = target_dir / rel
+
+        if not target_file.exists():
+            continue  # sync-check handles missing files
+
+        # Skip commands — they are copied verbatim, not condensed
+        rel_str = str(rel)
+        if rel_str.startswith("commands/"):
+            continue
+
+        source_text = source_file.read_text(encoding="utf-8")
+        target_text = target_file.read_text(encoding="utf-8")
+        # Normalise source through the path rewriter (idempotent) so logical
+        # `load_context:` names and `../../docs/...` body links match the
+        # depth-aware form the condenseor produced. Condensation word-count
+        # checks downstream are unaffected because rewriting only edits
+        # frontmatter list values and link targets, not prose tokens.
+        source_text = _rewrite_paths(source_text, rel_str)
+        issues.extend(check_pair(rel_str, source_text, target_text))
+
+    return issues
+
+
+def format_text(issues: List[Issue]) -> str:
+    if not issues:
+        return "✅  Condensation quality check passed."
+    icons = {"error": "🔴", "warning": "🟡", "info": "ℹ️"}
+    lines = [f"Found {len(issues)} condensation quality issue(s):\n"]
+    for i in issues:
+        lines.append(f"  {icons[i.severity]} [{i.check}] {i.file}: {i.message}")
+    errors = sum(1 for i in issues if i.severity == "error")
+    if errors:
+        lines.append(f"\n❌  {errors} error(s) must be fixed.")
+    return "\n".join(lines)
+
+
+def scan_summary(root: Path) -> str:
+    """Generate per-category condensation summary stats."""
+    source_dir = root / SOURCE_DIR
+    target_dir = root / TARGET_DIR
+    if not source_dir.exists() or not target_dir.exists():
+        return "No source/target directories found."
+
+    categories: dict[str, list[tuple[int, int]]] = {}
+    for source_file in sorted(source_dir.rglob("*.md")):
+        rel = source_file.relative_to(source_dir)
+        target_file = target_dir / rel
+        if not target_file.exists() or str(rel).startswith("commands/"):
+            continue
+        src_words = len(source_file.read_text(encoding="utf-8").split())
+        cmp_words = len(target_file.read_text(encoding="utf-8").split())
+        parts = str(rel).split("/")
+        cat = parts[0] if len(parts) > 1 else "root"
+        categories.setdefault(cat, []).append((src_words, cmp_words))
+
+    lines = ["Category         | Files | Avg Source | Avg Condensed | Avg Reduction",
+             "---              | ---   | ---        | ---            | ---"]
+    total_src = total_cmp = total_files = 0
+    for cat in sorted(categories):
+        pairs = categories[cat]
+        n = len(pairs)
+        avg_src = sum(s for s, _ in pairs) // n
+        avg_cmp = sum(c for _, c in pairs) // n
+        reduction = (1 - avg_cmp / avg_src) * 100 if avg_src > 0 else 0
+        lines.append(f"{cat:<17}| {n:>5} | {avg_src:>10} | {avg_cmp:>14} | {reduction:>5.0f}%")
+        total_src += sum(s for s, _ in pairs)
+        total_cmp += sum(c for _, c in pairs)
+        total_files += n
+    overall = (1 - total_cmp / total_src) * 100 if total_src > 0 else 0
+    lines.append(f"{'TOTAL':<17}| {total_files:>5} | {total_src // max(total_files, 1):>10} | {total_cmp // max(total_files, 1):>14} | {overall:>5.0f}%")
+    return "\n".join(lines)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Check condensation quality")
+    parser.add_argument("--format", choices=["text", "json"], default="text")
+    parser.add_argument("--summary", action="store_true", help="Show per-category condensation stats")
+    parser.add_argument("--root", type=Path, default=Path("."))
+    args = parser.parse_args()
+
+    if args.summary:
+        print(scan_summary(args.root))
+        return 0
+
+    try:
+        issues = scan_all(args.root)
+    except Exception as e:
+        print(f"Internal error: {e}", file=sys.stderr)
+        return 3
+
+    if args.format == "json":
+        print(json.dumps([asdict(i) for i in issues], indent=2))
+    else:
+        print(format_text(issues))
+
+    errors = [i for i in issues if i.severity == "error"]
+    return 1 if errors else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/src/scripts/check_condensed_paths.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""Validate condensed-output paths in `dist/agent-src/rules/*.md`.
+
+Runs after `scripts/condense.py` projects sources to `dist/agent-src/`. The
+rewriter in `condense.py` is the load-bearing primitive (road-to-path-fixes
+P1.2); this script is the post-condition gate (P5.1) — every `load_context:`
+entry in `dist/agent-src/rules/*.md` must resolve relative to the rule file's
+directory to an existing file, and forbidden substrings must not survive
+the rewrite (unless declared in `validator_ignore`).
+
+Forbidden substrings (load_context + body):
+  - `.agent-src.uncondensed/`            unless declared in validator_ignore
+  - `../../docs/`                         body-link two-up form (rewriter
+                                           collapses to single-up)
+  - `../../agents/`                       same shape, different root
+
+Body-link checks (Council Decision 2, 2026-05-06):
+  - `load_context:` entries MUST resolve to an existing file under
+    `dist/agent-src/`.
+  - Body markdown links to `../contexts/...md` MUST resolve.
+  - Body markdown links to `../docs/guidelines/...md` are NOT checked
+    (P3.1 was cancelled; resolution is intentionally out of scope, the
+    Copilot suppression floor in P6 is the silencer).
+
+`validator_ignore:` frontmatter primitive:
+  - Per-rule allowlist for rules that *describe* a forbidden substring as
+    their subject matter (e.g. `augment-edit-discipline` documents the
+    `.agent-src.uncondensed/` boundary). Each entry: `{type, pattern,
+    reason}`. The validator emits an audit line per matched ignore so
+    drift cannot hide.
+
+Exit codes: 0 = clean, 1 = violations found, 3 = internal error.
+"""
+from __future__ import annotations
+
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+QUIET = "--quiet" in sys.argv
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+RULES_DIR = ROOT / "dist/agent-src" / "rules"
+
+FORBIDDEN_SUBSTRINGS = (
+    ".agent-src.uncondensed/",
+    "../../docs/",
+    "../../agents/",
+)
+
+# Markdown links: `[text](path)` — capture path. Skip URLs and anchors.
+_LINK_RE = re.compile(r'\[[^\]]*\]\(([^)#\s]+)(?:#[^)]*)?\)')
+
+
+# Body-link prefixes whose resolution is intentionally out of scope.
+# Council Decision 2 (2026-05-06): P3.1 was cancelled, so guideline links
+# under `dist/agent-src/rules/` cannot resolve in the projected tree. Copilot
+# suppression (P6) is the silencer for the noise. `docs/contracts/` shares
+# the same shape as `docs/guidelines/` — both live at repo root and the
+# rewriter collapses `../../docs/{contracts,guidelines}/...` to a
+# `../docs/...` form that cannot resolve under `dist/agent-src/`.
+UNCHECKED_LINK_PREFIXES = (
+    "../docs/guidelines/",
+    "../../docs/guidelines/",
+    "../docs/contracts/",
+    "../../docs/contracts/",
+    # Consumer-layout root-escaping links (ADR-058): since the condensed
+    # output moved to `dist/agent-src/` the projected tree sits one level
+    # deeper than the consumer install (`.augment/rules/` — 2 deep), so the
+    # two-up forms below are authored for the consumer layout and cannot
+    # resolve in the package repo by construction. Same rationale as the
+    # docs/guidelines carve-out above; `validator_ignore` still audits the
+    # substring hits per rule.
+    "../../docs/",
+    "../../agents/",
+    "../../src/",
+)
+
+
+@dataclass
+class Violation:
+    file: str
+    line: int
+    kind: str
+    detail: str
+
+
+@dataclass
+class IgnoreEntry:
+    """Frontmatter `validator_ignore:` entry."""
+    kind: str       # "substring" | "link"
+    pattern: str    # exact substring or link prefix to ignore
+    reason: str     # human-readable rationale (audited)
+
+
+def _split_frontmatter(text: str):
+    if not text.startswith("---\n"):
+        return None, text
+    end = text.find("\n---\n", 4)
+    if end == -1:
+        return None, text
+    fm_text = text[4:end]
+    body = text[end + len("\n---\n"):]
+    try:
+        fm = yaml.safe_load(fm_text)
+    except yaml.YAMLError:
+        return None, text
+    return fm if isinstance(fm, dict) else {}, body
+
+
+def _parse_ignores(fm: dict) -> list[IgnoreEntry]:
+    entries = fm.get("validator_ignore") or []
+    if not isinstance(entries, list):
+        return []
+    out: list[IgnoreEntry] = []
+    for raw in entries:
+        if not isinstance(raw, dict):
+            continue
+        kind = str(raw.get("type") or "").strip()
+        pattern = str(raw.get("pattern") or "").strip()
+        reason = str(raw.get("reason") or "").strip()
+        if kind in ("substring", "link") and pattern and reason:
+            out.append(IgnoreEntry(kind=kind, pattern=pattern, reason=reason))
+    return out
+
+
+def _ignored(needle: str, ignores: list[IgnoreEntry], kind: str) -> IgnoreEntry | None:
+    for ig in ignores:
+        if ig.kind == kind and ig.pattern == needle:
+            return ig
+    return None
+
+
+def _check_load_context(rule_file: Path, fm: dict, viols: list[Violation],
+                        ignores: list[IgnoreEntry], audited: list[tuple[str, IgnoreEntry]]) -> None:
+    rule_dir = rule_file.parent
+    for key in ("load_context", "load_context_eager"):
+        entries = fm.get(key) or []
+        if not isinstance(entries, list):
+            continue
+        for entry in entries:
+            if not isinstance(entry, str):
+                continue
+            blocked = False
+            for needle in FORBIDDEN_SUBSTRINGS:
+                if needle in entry:
+                    ig = _ignored(needle, ignores, "substring")
+                    if ig:
+                        audited.append((str(rule_file.relative_to(ROOT)), ig))
+                        continue
+                    viols.append(Violation(
+                        str(rule_file.relative_to(ROOT)), 0, f"{key}-forbidden",
+                        f"forbidden substring {needle!r} in entry {entry!r}",
+                    ))
+                    blocked = True
+                    break
+            if blocked:
+                continue
+            target = (rule_dir / entry).resolve()
+            if not target.is_file():
+                viols.append(Violation(
+                    str(rule_file.relative_to(ROOT)), 0, f"{key}-missing",
+                    f"{entry!r} does not resolve to an existing file",
+                ))
+
+
+def _check_body(rule_file: Path, body: str, viols: list[Violation],
+                ignores: list[IgnoreEntry], audited: list[tuple[str, IgnoreEntry]]) -> None:
+    rule_dir = rule_file.parent
+    for line_num, line in enumerate(body.splitlines(), start=1):
+        for needle in FORBIDDEN_SUBSTRINGS:
+            if needle in line:
+                ig = _ignored(needle, ignores, "substring")
+                if ig:
+                    audited.append((f"{rule_file.relative_to(ROOT)}:{line_num}", ig))
+                    continue
+                viols.append(Violation(
+                    str(rule_file.relative_to(ROOT)), line_num, "body-forbidden",
+                    f"forbidden substring {needle!r}",
+                ))
+        for m in _LINK_RE.finditer(line):
+            link = m.group(1)
+            if link.startswith(("http://", "https://", "mailto:", "#")):
+                continue
+            if not link.endswith(".md"):
+                continue
+            if any(link.startswith(p) for p in UNCHECKED_LINK_PREFIXES):
+                continue
+            target = (rule_dir / link).resolve()
+            if not target.is_file():
+                viols.append(Violation(
+                    str(rule_file.relative_to(ROOT)), line_num, "body-link-missing",
+                    f"link target {link!r} does not resolve",
+                ))
+
+
+def main() -> int:
+    if not RULES_DIR.is_dir():
+        print(f"❌  {RULES_DIR} not found — run condensation first", file=sys.stderr)
+        return 3
+    viols: list[Violation] = []
+    audited: list[tuple[str, IgnoreEntry]] = []
+    for rule_file in sorted(RULES_DIR.glob("*.md")):
+        text = rule_file.read_text(encoding="utf-8")
+        fm, body = _split_frontmatter(text)
+        ignores: list[IgnoreEntry] = _parse_ignores(fm) if fm is not None else []
+        if fm is not None:
+            _check_load_context(rule_file, fm, viols, ignores, audited)
+        _check_body(rule_file, body, viols, ignores, audited)
+    if audited:
+        print("ℹ️   validator_ignore audit:")
+        for loc, ig in audited:
+            print(f"    {loc} — [{ig.kind}] {ig.pattern!r} → {ig.reason}")
+        print()
+    if viols:
+        for v in viols:
+            loc = f"{v.file}:{v.line}" if v.line else v.file
+            print(f"❌  [{v.kind}] {loc} — {v.detail}")
+        print(f"\n{len(viols)} violation(s) in dist/agent-src/rules/")
+        return 1
+    rule_count = len(list(RULES_DIR.glob('*.md')))
+    if not QUIET:
+        print(f"✅  condensed-path check clean ({rule_count} rules, {len(audited)} ignore(s) audited)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())