@event4u/agent-config 5.10.1 → 6.1.0

package/src/scripts/lint_bench_corpus.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+"""Lint benchmark corpora under tests/eval/corpus-*.yaml.
+
+Enforces the contract from docs/contracts/benchmark-corpus-spec.md:
+  - Required top-level keys (version, corpus_id, prompts) present.
+  - version == 1.
+  - selection_accuracy_target in [0.0, 1.0].
+  - Per-prompt schema (id format, category enum, language enum,
+    expected_skills non-empty + referencing real skills, destructive
+    prompts carry expected_carve_outs, prompt text non-empty).
+  - No duplicate ids within a corpus.
+
+Hooked into `task ci` via `task lint-bench`. Step-4 Phase 1 Step 3.
+
+Exit codes:
+  0  contract holds across every corpus
+  1  one or more violations
+  2  invocation error (missing PyYAML, no corpora found)
+
+Flags:
+  --quiet            suppress per-file OK lines
+  --require-full     also enforce 25-prompt composition (10/8/5/2)
+"""
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:
+    sys.stderr.write("error: PyYAML required (pip install pyyaml)\n")
+    sys.exit(2)
+
+QUIET = "--quiet" in sys.argv
+REQUIRE_FULL = "--require-full" in sys.argv
+
+REPO = Path(__file__).resolve().parents[2]
+CORPUS_DIR = REPO / "tests" / "eval"
+ROUTER_COVERAGE_DIR = REPO / "internal" / "bench" / "corpora" / "router-coverage"
+ROUTER_JSON = REPO / "dist" / "router.json"
+
+# Live skill directories live under every artefact root post-monorepo
+# Phase 4 (legacy + packages/*/.agent-src.uncondensed/skills/).
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _lib.agent_src import artefact_roots  # noqa: E402
+
+SKILLS_DIRS = [root / "skills" for root in artefact_roots() if (root / "skills").is_dir()]
+
+VALID_CATEGORIES = frozenset({"canonical", "ambiguous", "destructive", "long-context", "router-coverage"})
+# Non-dev corpus (pre-spec) uses legacy categories — accept them so the
+# new linter does not break that file. Migration is a follow-up.
+LEGACY_CATEGORIES = frozenset({"content", "consulting", "finance", "ops", "safety"})
+VALID_LANGUAGES = frozenset({"en", "de"})
+VALID_VERSIONS = frozenset({1})
+ID_RE = re.compile(r"^[a-z][a-z0-9-]*-\d{2}$")
+FULL_COUNTS = {"canonical": 10, "ambiguous": 8, "destructive": 5, "long-context": 2}
+
+
+def live_skills() -> set[str]:
+    slugs: set[str] = set()
+    for skills_dir in SKILLS_DIRS:
+        slugs.update(
+            p.name for p in skills_dir.iterdir()
+            if p.is_dir() and (p / "SKILL.md").exists()
+        )
+    return slugs
+
+
+def live_rule_ids() -> set[str] | None:
+    """Return all rule ids known to dist/router.json (kernel + tier_1 + tier_2).
+
+    Returns ``None`` (not an empty set) when the router is missing or
+    unparseable, signalling "cannot validate rule ids — skip the
+    unknown-trigger checks" rather than "every referenced id is unknown".
+    A missing router is expected on a fresh clone before ``task sync``;
+    returning an empty set there would falsely flag every intended /
+    opaque trigger as ``unknown_intended_trigger``.
+    """
+    if not ROUTER_JSON.exists():
+        sys.stderr.write(
+            f"warning: {ROUTER_JSON.relative_to(REPO)} missing — skipping "
+            "trigger rule-id validation (run `task sync` to generate it)\n"
+        )
+        return None
+    try:
+        data = json.loads(ROUTER_JSON.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        sys.stderr.write(
+            f"warning: {ROUTER_JSON.relative_to(REPO)} unparseable — "
+            "skipping trigger rule-id validation\n"
+        )
+        return None
+    ids: set[str] = set()
+    ids.update(data.get("kernel", []) or [])
+    for tier in ("tier_1", "tier_2"):
+        ids.update(
+            r.get("id") for r in (data.get(tier, []) or []) if r.get("id")
+        )
+    return ids
+
+
+def lint_corpus(path: Path, skills: set[str], rule_ids: set[str] | None = None) -> list[str]:
+    errors: list[str] = []
+    try:
+        data = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        return [f"{path.name}: yaml_parse_error: {exc}"]
+
+    if not isinstance(data, dict):
+        return [f"{path.name}: missing_top_level: corpus must be a mapping"]
+
+    for key in ("version", "corpus_id", "prompts"):
+        if key not in data:
+            errors.append(f"{path.name}: missing_top_level: {key}")
+
+    if data.get("version") not in VALID_VERSIONS:
+        errors.append(f"{path.name}: unsupported_version: {data.get('version')!r}")
+
+    target = data.get("selection_accuracy_target")
+    if target is not None and not (isinstance(target, (int, float)) and 0.0 <= target <= 1.0):
+        errors.append(f"{path.name}: target_out_of_range: {target!r}")
+
+    prompts = data.get("prompts") or []
+    if not isinstance(prompts, list):
+        return errors + [f"{path.name}: missing_top_level: prompts must be a list"]
+
+    seen_ids: set[str] = set()
+    bucket_counts: dict[str, int] = {}
+    is_legacy = data.get("corpus_id") == "non-dev"
+    for idx, p in enumerate(prompts):
+        loc = f"{path.name}:#{idx}"
+        if not isinstance(p, dict):
+            errors.append(f"{loc}: bad_prompt_shape")
+            continue
+        pid = p.get("id")
+        if not isinstance(pid, str) or not ID_RE.match(pid):
+            errors.append(f"{loc}: bad_id_format: {pid!r}")
+        elif pid in seen_ids:
+            errors.append(f"{loc}: duplicate_id: {pid}")
+        else:
+            seen_ids.add(pid)
+
+        cat = p.get("category")
+        if cat not in VALID_CATEGORIES and not (is_legacy and cat in LEGACY_CATEGORIES):
+            errors.append(f"{loc}: bad_category: {cat!r}")
+        bucket_counts[cat] = bucket_counts.get(cat, 0) + 1
+
+        lang = p.get("language", "en")
+        if lang not in VALID_LANGUAGES:
+            errors.append(f"{loc}: bad_language: {lang!r}")
+
+        prompt_text = p.get("prompt", "")
+        if not isinstance(prompt_text, str) or not prompt_text.strip():
+            errors.append(f"{loc}: empty_prompt")
+
+        expected = p.get("expected_skills") or []
+        if not isinstance(expected, list):
+            errors.append(f"{loc}: bad_expected_shape")
+        elif not expected and cat != "router-coverage":
+            # router-coverage corpora can have empty expected_skills —
+            # the focus is rule-trigger activation, not skill selection.
+            # The intended_triggers field is the load-bearing assertion.
+            errors.append(f"{loc}: empty_expected")
+        else:
+            for slug in expected:
+                if slug not in skills:
+                    errors.append(f"{loc}: unknown_skill: {slug}")
+
+        if cat == "destructive":
+            carve = p.get("expected_carve_outs") or []
+            if not isinstance(carve, list) or not carve:
+                errors.append(f"{loc}: missing_carve_out")
+
+        # router-coverage invariants (Council R3 honesty floor).
+        # A task's trigger prediction lives in two buckets:
+        #   intended_triggers      — deterministically replayable (keyword /
+        #                            phrase / command / path with supplied
+        #                            open_files or command context).
+        #   replay_opaque_triggers — fires at runtime only via an `intent`
+        #                            trigger (or a router coverage gap) the
+        #                            static replay cannot verify. Declared so
+        #                            the telemetry reports it separately, not
+        #                            as false `missed_intended` drift.
+        # router-coverage requires at least one bucket non-empty.
+        intended = p.get("intended_triggers")
+        opaque = p.get("replay_opaque_triggers")
+        intended_list = intended if isinstance(intended, list) else []
+        opaque_list = opaque if isinstance(opaque, list) else []
+
+        if intended is not None and not isinstance(intended, list):
+            errors.append(f"{loc}: bad_intended_triggers_shape")
+        if opaque is not None and not isinstance(opaque, list):
+            errors.append(f"{loc}: bad_replay_opaque_triggers_shape")
+
+        if cat == "router-coverage" and not intended_list and not opaque_list:
+            errors.append(f"{loc}: missing_intended_triggers")
+
+        # A rule belongs to exactly one bucket — both is a contradiction.
+        for rid in sorted(set(intended_list) & set(opaque_list)):
+            errors.append(f"{loc}: trigger_in_both_buckets: {rid}")
+
+        # Every referenced id (either bucket) must be a real router rule id.
+        if rule_ids is not None:
+            for rid in intended_list + opaque_list:
+                if rid not in rule_ids:
+                    errors.append(f"{loc}: unknown_intended_trigger: {rid}")
+
+    if REQUIRE_FULL and not is_legacy:
+        for bucket, want in FULL_COUNTS.items():
+            have = bucket_counts.get(bucket, 0)
+            if have != want:
+                errors.append(f"{path.name}: composition_drift: {bucket} have={have} want={want}")
+
+    return errors
+
+
+def main() -> int:
+    if not CORPUS_DIR.is_dir():
+        sys.stderr.write(f"error: corpus dir missing: {CORPUS_DIR}\n")
+        return 2
+    corpora = sorted(CORPUS_DIR.glob("corpus-*.yaml"))
+    # Phase 2 of road-to-corpus-expansion-evidence-based-cuts adds a second
+    # corpus tree under internal/bench/corpora/router-coverage/. Linter scans
+    # both with the same invariants — router-coverage corpora additionally
+    # require `intended_triggers` per prompt.
+    if ROUTER_COVERAGE_DIR.is_dir():
+        corpora.extend(sorted(ROUTER_COVERAGE_DIR.glob("*.yaml")))
+    if not corpora:
+        sys.stderr.write("error: no corpora found\n")
+        return 2
+
+    skills = live_skills()
+    rule_ids = live_rule_ids()
+    all_errors: list[str] = []
+    for path in corpora:
+        errs = lint_corpus(path, skills, rule_ids)
+        if errs:
+            all_errors.extend(errs)
+        elif not QUIET:
+            print(f"✅  {path.name}: contract OK")
+
+    if all_errors:
+        for err in all_errors:
+            print(f"❌  {err}", file=sys.stderr)
+        return 1
+    if not QUIET:
+        print(f"✅  lint-bench: {len(corpora)} corpora clean")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/src/scripts/lint_command_flow_coverage.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""Command → flow coverage lint (road-to-6.1.0 Step 9 — the Flows primary view).
+
+Asserts that ``src/flows/surface-map.yaml`` classifies EVERY command in the
+source-of-truth command tree (``src/domains/<pack>/**/command.md``) into exactly
+one flow / platform-surface bucket — the lintable command→flow edge that makes
+``Profile → Pack → Flow → Command → Skill → Rule`` real rather than prose.
+
+Checks (all hard — exit 1 on any violation):
+
+1. **No orphan** — every real ``src/domains`` command appears in the map.
+2. **No phantom** — every ref in the map backs a real ``src/domains`` command.
+3. **No duplicate** — no command is classified into two buckets.
+4. **Allowed buckets** — every bucket key is in ``user_work_flows`` ∪
+   ``platform_surfaces``; the four user-work flows are exactly the closed set
+   ``lint_flows.py`` enforces (``agent-admin`` is deliberately NOT a flow).
+
+Companion to ``lint_flows.py`` (which validates the rich
+``src/flows/<flow>.yaml`` definitions). This lint validates the COMPLETE flat
+classification; the two together close the flow layer.
+
+Exit codes: 0 = clean · 1 = violations · 3 = internal error.
+
+Usage:
+    python3 scripts/lint_command_flow_coverage.py
+    python3 scripts/lint_command_flow_coverage.py --quiet
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+import yaml  # type: ignore
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _lib.agent_src import _iter_domains_commands  # noqa: E402
+
+ROOT = Path(__file__).resolve().parents[2]
+SURFACE_MAP = ROOT / "src" / "flows" / "surface-map.yaml"
+
+# The closed user-work flow set — must match lint_flows.CLOSED_FLOWS and
+# flow.schema.json. agent-admin is the platform surface, NOT a flow (feedback-6).
+CLOSED_FLOWS = {"discovery", "implementation", "review", "delivery"}
+
+
+def _domains_command_refs() -> set[str]:
+    """Logical refs (``feature/plan``, ``commit/in-chunks``) for every command."""
+    return {
+        logical[len("commands/") : -len(".md")]
+        for _, logical in _iter_domains_commands()
+    }
+
+
+def _load_map() -> dict:
+    if not SURFACE_MAP.is_file():
+        raise FileNotFoundError(f"missing {SURFACE_MAP.relative_to(ROOT)}")
+    return yaml.safe_load(SURFACE_MAP.read_text(encoding="utf-8")) or {}
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--quiet", action="store_true", help="only print on failure")
+    args = ap.parse_args()
+
+    try:
+        data = _load_map()
+        real = _domains_command_refs()
+    except Exception as exc:  # noqa: BLE001
+        print(f"lint_command_flow_coverage: internal error — {exc}", file=sys.stderr)
+        return 3
+
+    user_flows = set(data.get("user_work_flows") or [])
+    surfaces = set(data.get("platform_surfaces") or [])
+    allowed = user_flows | surfaces
+    buckets = data.get("commands") or {}
+
+    violations: list[str] = []
+
+    # Check 4 — declared user-work flows match the closed set.
+    if user_flows != CLOSED_FLOWS:
+        violations.append(
+            f"user_work_flows {sorted(user_flows)} != closed set {sorted(CLOSED_FLOWS)} "
+            "(a new user-work flow is an ADR-gated governance decision)"
+        )
+
+    # Build the flat ref→bucket index; catch duplicates and bad buckets.
+    seen: dict[str, str] = {}
+    for bucket, refs in buckets.items():
+        if bucket not in allowed:
+            violations.append(
+                f"bucket '{bucket}' is not in user_work_flows ∪ platform_surfaces"
+            )
+        for ref in refs or []:
+            if ref in seen:
+                violations.append(
+                    f"command '{ref}' classified twice: '{seen[ref]}' and '{bucket}'"
+                )
+            else:
+                seen[ref] = bucket
+
+    mapped = set(seen)
+
+    # Check 1 — no orphan (real command missing from the map).
+    for ref in sorted(real - mapped):
+        violations.append(f"orphan: command '{ref}' has no flow/surface in surface-map.yaml")
+
+    # Check 2 — no phantom (map ref with no backing command).
+    for ref in sorted(mapped - real):
+        violations.append(f"phantom: surface-map ref '{ref}' backs no src/domains command")
+
+    if violations:
+        print(
+            f"lint_command_flow_coverage: {len(violations)} violation(s) "
+            f"({len(real)} commands, {len(mapped)} mapped)",
+            file=sys.stderr,
+        )
+        for v in violations:
+            print(f"  ✗ {v}", file=sys.stderr)
+        return 1
+
+    if not args.quiet:
+        per = {b: len(r or []) for b, r in buckets.items()}
+        print(
+            f"lint_command_flow_coverage: OK — {len(real)} commands fully classified "
+            f"across {len(buckets)} buckets {per}"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/src/scripts/lint_command_routing.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Routing-metadata + routing-eval linter for visible commands.
+
+6.0.0-C Phase 2 Steps 4b + 5. Every VISIBLE command (tier 0/1) must carry:
+
+  - `intent`     — non-empty one-line existence justification (Step 4b)
+  - `routes_to`  — non-empty list of skill / cluster-sub / command slugs (Step 4b)
+  - `replaces`   — a list (may be empty `[]`); the key must be present (Step 4b)
+  - a routing eval at `<command-dir>/evals/<stem>.json` with >= MIN_CASES
+    prompt→command cases (Step 5)
+
+Internal commands (tier 2 / absent) are exempt — they are the composition layer.
+Unlike the forward-only growth gates, this lint enforces on the WHOLE visible
+surface: the backfill in 6.0.0-C Phase 2 brings every existing visible command
+into compliance, so the gate is green today and stays green.
+
+Exit codes: 0 = clean, 1 = violations found, 3 = internal error.
+
+Usage:
+    python3 scripts/lint_command_routing.py
+    python3 scripts/lint_command_routing.py --quiet
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+FM_RE = re.compile(r"^---\n(.*?)\n---", re.DOTALL)
+VISIBLE_TIERS = {0, 1}
+MIN_CASES = 5  # roadmap Step 5: "5–10 example prompts"
+
+
+@dataclass
+class Violation:
+    file: str
+    reason: str
+
+
+def _command_files() -> list[Path]:
+    """Discover command sources in the post-ADR-051 layout.
+
+    Authoring lives at src/domains/<domain>/**/command.md; the legacy
+    packages/*/.agent-src.uncondensed/commands tree is kept as a
+    fallback for older checkouts.
+    """
+    domains = ROOT / "src" / "domains"
+    if domains.is_dir():
+        return sorted(domains.rglob("command.md"))
+    legacy_roots = [d for d in (ROOT / "packages").glob(
+        "*/.agent-src.uncondensed/commands") if d.is_dir()]
+    return sorted(md for root in legacy_roots for md in root.rglob("*.md"))
+
+
+# Central eval store in the post-ADR-051 layout. Eval stems use the
+# command's `name` or one of its `replaces` aliases.
+EVALS_DIR = ROOT / "src" / "agent-src" / "commands" / "evals"
+
+
+def _frontmatter(text: str) -> dict:
+    import yaml
+    m = FM_RE.search(text)
+    if not m:
+        return {}
+    try:
+        return yaml.safe_load(m.group(1)) or {}
+    except yaml.YAMLError:
+        return {}
+
+
+def check(md: Path) -> list[Violation]:
+    fm = _frontmatter(md.read_text(encoding="utf-8"))
+    tier = fm.get("tier", 2)
+    if tier not in VISIBLE_TIERS:
+        return []
+    rel = str(md.relative_to(ROOT))
+    vio: list[Violation] = []
+
+    intent = fm.get("intent")
+    if not (isinstance(intent, str) and intent.strip()):
+        vio.append(Violation(rel, "missing/empty `intent` (Step 4b)"))
+
+    routes = fm.get("routes_to")
+    if not (isinstance(routes, list) and routes):
+        vio.append(Violation(rel, "missing/empty `routes_to` list (Step 4b)"))
+
+    if "replaces" not in fm or not isinstance(fm.get("replaces"), list):
+        vio.append(Violation(rel, "missing `replaces` key (use `[]` when it "
+                                  "replaces nothing) (Step 4b)"))
+
+    eval_keys = [k for k in [fm.get("name"), *(fm.get("replaces") or [])]
+                 if isinstance(k, str) and k.strip()]
+    eval_path = next(
+        (EVALS_DIR / f"{k}.json" for k in eval_keys
+         if (EVALS_DIR / f"{k}.json").exists()),
+        md.parent / "evals" / f"{md.stem}.json",  # legacy per-dir layout
+    )
+    if not eval_path.exists():
+        vio.append(Violation(rel, f"missing routing eval under "
+                                  f"`{EVALS_DIR.relative_to(ROOT)}/` for any of "
+                                  f"{eval_keys or [md.stem]} (Step 5)"))
+    else:
+        try:
+            data = json.loads(eval_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as exc:
+            vio.append(Violation(rel, f"routing eval is invalid JSON: {exc}"))
+        else:
+            cases = data.get("cases")
+            if not (isinstance(cases, list) and len(cases) >= MIN_CASES):
+                vio.append(Violation(rel, f"routing eval has < {MIN_CASES} "
+                                          f"cases (Step 5: 5–10 prompts)"))
+            elif not all(isinstance(c, dict) and c.get("prompt")
+                         and c.get("expected") for c in cases):
+                vio.append(Violation(rel, "routing eval case missing "
+                                          "`prompt`/`expected`"))
+    return vio
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--quiet", action="store_true")
+    args = ap.parse_args()
+
+    files = _command_files()
+    if not files:
+        print("❌  No command roots found.", file=sys.stderr)
+        return 3
+
+    visible = 0
+    violations: list[Violation] = []
+    for md in files:
+        if md.name == "AGENTS.md" or "_archive" in md.parts \
+           or md.parent.name == "evals":
+            continue
+        fm = _frontmatter(md.read_text(encoding="utf-8"))
+        if fm.get("tier", 2) in VISIBLE_TIERS:
+            visible += 1
+        violations += check(md)
+
+    if violations:
+        print(f"❌  {len(violations)} routing-metadata violation(s) across "
+              f"{visible} visible command(s):")
+        for v in violations:
+            print(f"  • {v.file} — {v.reason}")
+        print("\nSee command.schema.json (intent/routes_to/replaces) and "
+              "docs/contracts/command-clusters.md § routing metadata.")
+        return 1
+    if not args.quiet:
+        print(f"✅  {visible} visible command(s) carry intent/routes_to/replaces "
+              f"+ a routing eval.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())