devrites 3.2.2 → 3.2.4

package/scripts/run-agent-contract-evals.py

@@ -273,8 +273,8 @@ def validate_matrix(data: Any) -> dict[str, Any]:
             raise ContractError("scenario.role_class is unsupported")
         if scenario["simulate"] not in {
             "accepted",
+            "dispatch-unavailable",
             "generic-fallback",
-            "inline-fallback",
             "interrupted",
             "malformed",
             "missing",
@@ -313,7 +313,7 @@ def validate_matrix(data: Any) -> dict[str, Any]:
             },
             "scenario.expect",
         )
-        if expect["execution_mode"] not in {"named", "generic", "inline"}:
+        if expect["execution_mode"] not in {"named", "generic", "none"}:
             raise ContractError("execution mode is unsupported")
         if expect["guard_strength"] not in {
             "enforced",
@@ -778,7 +778,7 @@ def validate_result(
     exact_keys(
         execution, {"mode", "guard_strength", "independence"}, "result execution"
     )
-    if execution["mode"] not in {"named", "generic", "inline"}:
+    if execution["mode"] not in {"named", "generic", "none"}:
         raise ContractError("result execution mode is unsupported")
     if execution["guard_strength"] not in {
         "enforced",

package/scripts/run-evals.sh

@@ -1,32 +1,21 @@
 #!/usr/bin/env bash
 # scripts/run-evals.sh: validate the structure of DevRites trigger evals.
 #
-# Schema check + summary. Does NOT invoke a model unless --live is passed.
-# Live execution requires CLAUDE_API_KEY and is manual-only, never CI default.
-# CI runs this script to enforce the shape and
+# Schema check + summary. CI runs this script to enforce the shape and
 # catch broken JSON, missing skills, and empty/one-sided corpora. Invocation-policy
 # shape is enforced by run-routing-evals.py in scripts/validate.sh.
 #
 # Usage:
 #   scripts/run-evals.sh                         # validate every evals/*.json
 #   scripts/run-evals.sh evals/rite-spec.json    # validate one file
-#   scripts/run-evals.sh --live evals/*.json     # execute live model evals too
 
 set -euo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 EVALS_DIR="$ROOT/evals"
-LIVE=0
-ARGS=()
-for arg in "$@"; do
-  case "$arg" in
-    --live) LIVE=1 ;;
-    *) ARGS+=("$arg") ;;
-  esac
-done
 
-if [[ ${#ARGS[@]} -gt 0 ]]; then
-  FILES=("${ARGS[@]}")
+if [[ $# -gt 0 ]]; then
+  FILES=("$@")
 else
   if [[ ! -d "$EVALS_DIR" ]]; then
     echo "No evals/ directory at $EVALS_DIR" >&2
@@ -141,23 +130,3 @@ printf 'Validated %d eval files; %d failed.\n' "$TOTAL" "$FAILED"
 if [[ $FAILED -gt 0 ]]; then
   exit 1
 fi
-
-if [[ "$LIVE" -ne 1 ]]; then
-  echo
-  echo "Live model evals disabled by default. To run them manually:"
-  echo "  CLAUDE_API_KEY=sk-... scripts/run-evals.sh --live evals/*.json"
-  exit 0
-fi
-
-if [[ -z "${CLAUDE_API_KEY:-}" ]]; then
-  echo "error: --live requires CLAUDE_API_KEY." >&2
-  exit 2
-fi
-if ! command -v python3 >/dev/null 2>&1; then
-  echo "error: --live requires python3." >&2
-  exit 2
-fi
-
-echo
-echo "Executing live trigger evals via scripts/eval-runner.py …"
-exec python3 "$ROOT/scripts/eval-runner.py" "${FILES[@]}"

package/scripts/validate-frontmatter.py

@@ -6,10 +6,12 @@ Exits non-zero if any file fails. Uses PyYAML if present, else a minimal parser
 (frontmatter here is simple key: value, no nested structures needed).
 """
 import sys
+import re
+from pathlib import Path
 
 KNOWN_SKILL_FIELDS = {
     "name", "description", "argument-hint", "user-invocable",
-    "disable-model-invocation",
+    "disable-model-invocation", "required-agent-roles",
 }
 KNOWN_AGENT_FIELDS = {
     "name", "description", "tools", "disallowedTools", "model", "permissionMode",
@@ -22,6 +24,7 @@ DESCRIPTION_WORD_LIMITS = {
     "library": 60,
     "explicit": 30,
 }
+AGENT_ROLE_RE = re.compile(r"^devrites-[a-z0-9-]+$")
 
 
 def extract_frontmatter(text):
@@ -88,6 +91,26 @@ def is_agent(path):
     return "/agents/" in path.replace("\\", "/")
 
 
+def required_agent_roles(path, value):
+    raw = str(value).strip()
+    if raw == "none":
+        return [], None
+    if not raw:
+        return [], "required-agent-roles must be 'none' or a comma-separated role list"
+    roles = [role.strip() for role in raw.split(",")]
+    if any(not AGENT_ROLE_RE.fullmatch(role) for role in roles):
+        return [], "required-agent-roles contains an invalid role"
+    if len(set(roles)) != len(roles):
+        return [], "required-agent-roles contains a duplicate role"
+    skill_path = Path(path).resolve()
+    agents_dir = skill_path.parent.parent.parent / "agents"
+    if agents_dir.is_dir():
+        missing = [role for role in roles if not (agents_dir / (role + ".md")).is_file()]
+        if missing:
+            return [], "required-agent-roles references missing agent(s): " + ", ".join(missing)
+    return roles, None
+
+
 def main(argv):
     files = argv[1:]
     if not files:
@@ -123,6 +146,16 @@ def main(argv):
                   % (path, ", ".join(unknown)))
             errors += 1
             continue
+        if not is_agent(path):
+            if "required-agent-roles" not in data:
+                print("ERROR %s: missing 'required-agent-roles' (use 'none' when no fresh agent is mandatory)" % path)
+                errors += 1
+                continue
+            _, role_err = required_agent_roles(path, data.get("required-agent-roles"))
+            if role_err:
+                print("ERROR %s: %s" % (path, role_err))
+                errors += 1
+                continue
         # description length cap is 1024 chars per Anthropic SKILL.md spec
         warn = ""
         desc = str(data.get("description", ""))

package/engine/testdata/golden/TestParityA1Guard/inline-fallback-silent.golden

@@ -1 +0,0 @@
-exit 0

package/engine/testdata/golden/TestParityDoubtCoverage/arg=inline.golden

@@ -1,3 +0,0 @@
-exit 0
-doubt-coverage: inline build (.reconcile-inline): footprint heuristic n/a; no MISSING
-verdict recorded. Verify by hand that each stood decision carries a devrites-doubt verdict.

package/engine/testdata/golden/TestParityDoubtCoverage/arg=missing-inline.golden

@@ -1,3 +0,0 @@
-exit 3
-doubt-coverage: SKIPPED: a '## Decisions stood' entry in decisions.md has doubt: MISSING.
-A decision was stood and recorded but never doubted. Re-dispatch doubt or escalate.

package/scripts/eval-runner.py

@@ -1,275 +0,0 @@
-#!/usr/bin/env python3
-"""DevRites trigger-eval runner.
-
-Wraps the Anthropic SDK to execute a trigger eval against a Claude
-model. Off by default in CI (schema-only validation lives in
-`scripts/run-evals.sh`); this runner is invoked explicitly when
-`CLAUDE_API_KEY` is set.
-
-Inputs
-------
-A trigger-eval JSON of the shape DevRites ships under `evals/`:
-
-    {
-      "skill": "rite-spec",
-      "description": "...",
-      "queries": [
-        {"text": "...", "expected": "should_trigger"|"should_not_trigger", "rationale": "..."}
-      ]
-    }
-
-Plus the pack at `pack/.claude/skills/` (so the runner can read the candidate
-skills' descriptions and present them to the model).
-
-What it does per query
-----------------------
-1. Loads each skill's `name` + `description` from `pack/.claude/skills/*/SKILL.md`.
-2. Sends the query as a `user` message to Claude with a system prompt
-   instructing it to pick exactly one skill name from the candidate list, or
-   "none" if no skill applies. We do NOT ask the model to invoke a tool: we
-   ask it to predict which skill the harness would have triggered. That keeps
-   the eval cheap (no agentic loop, one round-trip per query) and faithful to
-   what skill discovery decides.
-3. Compares the predicted skill name to the expected verdict:
-   - `should_trigger`  → predicted name must equal the eval's `skill` field.
-   - `should_not_trigger` → predicted name must NOT equal the eval's `skill`.
-
-Outputs
--------
-A per-query line and a final accuracy summary. Exit code 1 if accuracy is
-below `--min-accuracy` (default 0.90).
-
-Usage
------
-    CLAUDE_API_KEY=sk-... ./scripts/eval-runner.py evals/rite-spec.json
-    CLAUDE_API_KEY=sk-... ./scripts/eval-runner.py --min-accuracy 0.9 evals/*.json
-
-The runner is intentionally thin (~200 lines). It is not the eval-viewer in
-Anthropic's `skill-creator`: it just provides the missing "did the model
-pick the right skill" signal so we can answer that question without firing
-up the full Claude Code harness.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import re
-import sys
-from dataclasses import dataclass
-from pathlib import Path
-
-ROOT = Path(__file__).resolve().parent.parent
-SKILLS_DIR = ROOT / "pack" / ".claude" / "skills"
-
-SYSTEM_TEMPLATE = """\
-You are the DevRites skill router. Your job is to predict which DevRites
-skill (if any) should fire for a given user message.
-
-Each candidate skill is described below. Read the descriptions carefully:
-DevRites' triggers are encoded in them.
-
-For the user message at the end, respond with exactly one line of the form:
-
-    name: <skill-name>
-
-…where <skill-name> is the name of the skill that should fire, or the
-literal token `none` if no DevRites skill applies. Do not explain. Do not
-suggest. Do not output anything except that one line.
-
-# Candidate skills
-
-{skills}
-"""
-
-
-@dataclass
-class Skill:
-    name: str
-    description: str
-
-
-@dataclass
-class Query:
-    text: str
-    expected: str
-    rationale: str
-
-
-@dataclass
-class Outcome:
-    query: Query
-    predicted: str
-    correct: bool
-    false_positive: bool  # expected should_not_trigger but skill fired
-    false_negative: bool  # expected should_trigger but skill didn't fire
-
-
-def load_skills() -> list[Skill]:
-    skills: list[Skill] = []
-    for skill_dir in sorted(SKILLS_DIR.iterdir()):
-        skill_md = skill_dir / "SKILL.md"
-        if not skill_md.is_file():
-            continue
-        body = skill_md.read_text(encoding="utf-8")
-        # Lightweight YAML frontmatter parse: we only need name + description.
-        m = re.match(r"^---\n(.*?)\n---", body, re.DOTALL)
-        if not m:
-            continue
-        front = m.group(1)
-        name_m = re.search(r"^name:\s*(\S+)", front, re.MULTILINE)
-        desc_m = re.search(r"^description:\s*(.+)$", front, re.MULTILINE)
-        if not name_m or not desc_m:
-            continue
-        skills.append(Skill(name=name_m.group(1), description=desc_m.group(1).strip()))
-    return skills
-
-
-def render_system(skills: list[Skill]) -> str:
-    lines = []
-    for s in skills:
-        lines.append(f"- **{s.name}**: {s.description}")
-    return SYSTEM_TEMPLATE.format(skills="\n".join(lines))
-
-
-def load_eval(path: Path) -> tuple[str, list[Query]]:
-    data = json.loads(path.read_text(encoding="utf-8"))
-    queries = [
-        Query(text=q["text"], expected=q["expected"], rationale=q.get("rationale", ""))
-        for q in data["queries"]
-    ]
-    return data["skill"], queries
-
-
-def predict(client, model: str, system: str, query_text: str) -> str:
-    response = client.messages.create(
-        model=model,
-        max_tokens=64,
-        system=system,
-        messages=[{"role": "user", "content": query_text}],
-    )
-    text = "".join(block.text for block in response.content if getattr(block, "text", None))
-    m = re.search(r"^\s*name:\s*([A-Za-z0-9_\-]+)", text, re.MULTILINE)
-    if not m:
-        return "none"
-    name = m.group(1).strip().lower()
-    return name
-
-
-def score(target_skill: str, predicted: str, expected: str) -> bool:
-    predicted = predicted.lower()
-    target = target_skill.lower()
-    if expected == "should_trigger":
-        return predicted == target
-    if expected == "should_not_trigger":
-        return predicted != target
-    raise ValueError(f"unknown expected verdict: {expected!r}")
-
-
-def run_one(client, model: str, system: str, eval_path: Path) -> tuple[int, int, list[Outcome]]:
-    target_skill, queries = load_eval(eval_path)
-    outcomes: list[Outcome] = []
-    correct = 0
-    for q in queries:
-        predicted = predict(client, model, system, q.text)
-        ok = score(target_skill, predicted, q.expected)
-        fp = (q.expected == "should_not_trigger" and not ok)
-        fn = (q.expected == "should_trigger" and not ok)
-        outcomes.append(Outcome(query=q, predicted=predicted, correct=ok, false_positive=fp, false_negative=fn))
-        if ok:
-            correct += 1
-    return correct, len(queries), outcomes
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument("eval_files", nargs="+", type=Path, help="Eval JSON files to execute")
-    parser.add_argument("--model", default=os.environ.get("DEVRITES_EVAL_MODEL", "claude-haiku-4-5-20251001"))
-    parser.add_argument("--min-accuracy", type=float, default=0.90,
-                        help="Per-eval minimum correct/total (default: 0.90)")
-    parser.add_argument("--max-false-positives", type=int, default=2,
-                        help="Per-eval max should_not_trigger queries that fired (default: 2)")
-    parser.add_argument("--verbose", action="store_true", help="Print every per-query line")
-    parser.add_argument("--summary-file", type=Path, default=None,
-                        help="Write a machine-readable summary (one JSON line per eval) to this file")
-    args = parser.parse_args()
-
-    if not os.environ.get("CLAUDE_API_KEY"):
-        sys.stderr.write("error: CLAUDE_API_KEY is not set; cannot run live eval.\n")
-        return 2
-
-    try:
-        import anthropic
-    except ImportError:
-        sys.stderr.write(
-            "error: anthropic SDK not installed. Run `pip install anthropic` first.\n"
-        )
-        return 2
-
-    client = anthropic.Anthropic(api_key=os.environ["CLAUDE_API_KEY"])
-    skills = load_skills()
-    system = render_system(skills)
-
-    total_correct = 0
-    total_queries = 0
-    total_fp = 0
-    total_fn = 0
-    per_file_failed = 0
-    summary_lines: list[str] = []
-
-    for eval_path in args.eval_files:
-        if eval_path.name == "routing-baseline.json":
-            continue
-        if not eval_path.is_file():
-            sys.stderr.write(f"skip: {eval_path} not a file\n")
-            continue
-        print(f"== {eval_path} ==")
-        correct, n, outcomes = run_one(client, args.model, system, eval_path)
-        fp = sum(1 for o in outcomes if o.false_positive)
-        fn = sum(1 for o in outcomes if o.false_negative)
-        total_correct += correct
-        total_queries += n
-        total_fp += fp
-        total_fn += fn
-        accuracy = correct / n if n else 0.0
-        if args.verbose:
-            for o in outcomes:
-                marker = "✓" if o.correct else "✗"
-                tag = "FP" if o.false_positive else ("FN" if o.false_negative else "  ")
-                print(f"  {marker} {tag} [{o.query.expected:20s}] predicted={o.predicted:24s} : {o.query.text[:60]}")
-        print(f"  accuracy: {correct}/{n} = {accuracy:.0%}  (FP={fp}, FN={fn})")
-        failed_reasons = []
-        if accuracy < args.min_accuracy:
-            failed_reasons.append(f"accuracy {accuracy:.0%} < {args.min_accuracy:.0%}")
-        if fp > args.max_false_positives:
-            failed_reasons.append(f"false-positives {fp} > {args.max_false_positives}")
-        if failed_reasons:
-            per_file_failed += 1
-            print(f"  FAIL: {'; '.join(failed_reasons)}")
-        if args.summary_file:
-            summary_lines.append(json.dumps({
-                "file": str(eval_path),
-                "skill": load_eval(eval_path)[0],
-                "correct": correct,
-                "total": n,
-                "accuracy": round(accuracy, 4),
-                "false_positives": fp,
-                "false_negatives": fn,
-                "passed": not failed_reasons,
-            }))
-
-    overall = total_correct / total_queries if total_queries else 0.0
-    print()
-    print(f"Overall: {total_correct}/{total_queries} = {overall:.0%}  (FP={total_fp}, FN={total_fn}, model={args.model})")
-    if args.summary_file:
-        args.summary_file.write_text("\n".join(summary_lines) + "\n", encoding="utf-8")
-        print(f"Summary written to {args.summary_file}")
-    if per_file_failed:
-        print(f"{per_file_failed} eval file(s) below threshold")
-        return 1
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/scripts/eval-scorer.mjs

@@ -1,46 +0,0 @@
-#!/usr/bin/env node
-// Deterministic fixture scorer for DevRites evals: no LLM judge, no deps.
-import { readFileSync } from 'node:fs';
-
-function norm(s) {
-  return String(s || '')
-    .normalize('NFKC')
-    .toLowerCase()
-    .replace(/[\p{P}\p{S}]+/gu, ' ')
-    .replace(/\s+/g, ' ')
-    .trim();
-}
-
-function hit(text, kw) {
-  const t = norm(text);
-  const k = norm(kw);
-  if (!k) return false;
-  if (t.includes(k)) return true;
-  const parts = k.split(' ').filter(Boolean);
-  return parts.length > 1 && parts.every((p) => t.includes(p));
-}
-
-export function matchFinding(output, keywords) {
-  const kws = Array.isArray(keywords) ? keywords : [];
-  const matches = kws.filter((k) => hit(output, k)).length;
-  const needed = Math.max(2, Math.ceil(kws.length * 0.4));
-  return { matches, needed, matched: kws.length > 0 && matches >= Math.min(needed, kws.length) };
-}
-
-export function score(output, groundTruth) {
-  const findings = Array.isArray(groundTruth?.findings) ? groundTruth.findings : [];
-  const rows = findings.map((f) => ({ id: f.id || f.name || '', ...matchFinding(output, f.keywords || []) }));
-  const found = rows.filter((r) => r.matched).length;
-  return { found, total: findings.length, score: findings.length ? found / findings.length : 1, rows };
-}
-
-if (import.meta.url === `file://${process.argv[1]}`) {
-  const [, , outputPath, truthPath] = process.argv;
-  if (!outputPath || !truthPath) {
-    console.error('usage: node scripts/eval-scorer.mjs <model-output.txt> <ground-truth.json>');
-    process.exit(2);
-  }
-  const result = score(readFileSync(outputPath, 'utf8'), JSON.parse(readFileSync(truthPath, 'utf8')));
-  console.log(JSON.stringify(result, null, 2));
-  process.exit(result.found === result.total ? 0 : 1);
-}

package/scripts/render-eval-summary.py

@@ -1,48 +0,0 @@
-#!/usr/bin/env python3
-"""Render a GitHub-Actions-flavored markdown table from an eval summary JSONL.
-
-Reads eval-summary.jsonl (one JSON object per line, written by
-`scripts/eval-runner.py --summary-file`) and prints a markdown table to
-stdout. Used by `.github/workflows/evals.yml` to populate the job summary.
-"""
-from __future__ import annotations
-
-import json
-import os
-import pathlib
-import sys
-
-
-def main() -> int:
-    path = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else "eval-summary.jsonl")
-    model = os.environ.get("DEVRITES_EVAL_MODEL", "claude-haiku-4-5-20251001")
-    if not path.is_file():
-        print(f"(no summary file found at {path})")
-        return 0
-    print(f"## DevRites trigger evals\n")
-    print(f"Model: `{model}`\n")
-    print("| Skill | Correct | Accuracy | FP | FN | Passed |")
-    print("|---|---|---|---|---|---|")
-    total_correct = total_n = total_fp = total_fn = 0
-    any_failed = False
-    for line in path.read_text(encoding="utf-8").splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        r = json.loads(line)
-        tick = "✅" if r["passed"] else "❌"
-        if not r["passed"]:
-            any_failed = True
-        total_correct += r["correct"]
-        total_n += r["total"]
-        total_fp += r["false_positives"]
-        total_fn += r["false_negatives"]
-        print(f"| `{r['skill']}` | {r['correct']}/{r['total']} | {r['accuracy']:.0%} | {r['false_positives']} | {r['false_negatives']} | {tick} |")
-    if total_n:
-        overall = total_correct / total_n
-        print(f"| **overall** | **{total_correct}/{total_n}** | **{overall:.0%}** | **{total_fp}** | **{total_fn}** | {'✅' if not any_failed else '❌'} |")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())