@intentsolutions/audit-harness 1.1.5 → 1.1.6

package/scripts/fp-rate.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""
+audit-harness fp-rate — measure a gate's false-positive / false-negative rate over
+a labeled corpus.
+
+A new gate ships `enforcement: advisory`. It earns promotion to `blocking` only
+once its measured false-positive rate (clean inputs it wrongly flags) sits below a
+stated bar on a labeled corpus. This harness produces that measurement — the
+evidence an engineer cites when they pin `enforcement: blocking` in a repo's
+`tests/TESTING.md` (PP-PLAN-040 Phase 0, bead c2e; rule: docs/gate-promotion.md).
+
+Labeled corpus layout (default `tests/fixtures/conform`):
+    <corpus>/valid/<fixture>/...      → every gate that fires here SHOULD be clean
+    <corpus>/malformed/<fixture>/...  → every gate that fires here SHOULD flag
+
+Per row the gate emits on a fixture, the verdict is bucketed:
+    clean  = PASS | NOT_APPLICABLE
+    flag   = FAIL | ADVISORY(advisory_severity=error)
+    skip   = ADVISORY indeterminate (tool/schema absent) — unmeasurable, excluded
+
+    false positive (FP) = a `valid` fixture the gate flagged
+    false negative (FN) = a `malformed` fixture the gate left clean
+
+Stdlib only. Read-only. Default exit 0 (report); `--max-fp-rate X` exits 1 if any
+gate exceeds the bar (use in CI when promoting a gate).
+"""
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+CONFORM = os.path.join(HERE, "conform.py")
+DEFAULT_CORPUS = os.path.join(HERE, "..", "tests", "fixtures", "conform")
+
+
+def verdict_bucket(row):
+    r = row.get("result")
+    if r in ("PASS", "NOT_APPLICABLE"):
+        return "clean"
+    if r == "FAIL":
+        return "flag"
+    if r == "ADVISORY":
+        if row.get("metadata", {}).get("indeterminate"):
+            return "skip"
+        if row.get("advisory_severity") == "error":
+            return "flag"
+        return "skip"
+    return "skip"
+
+
+def run_conform(fixture):
+    out = subprocess.run([sys.executable, CONFORM, fixture], capture_output=True, text=True)
+    try:
+        return json.loads(out.stdout)
+    except Exception:
+        return []
+
+
+def measure(corpus):
+    # per gate_id: {valid_total, fp, malformed_total, fn, skipped}
+    stats = {}
+
+    def bump(gid, key):
+        s = stats.setdefault(gid, {"valid": 0, "fp": 0, "malformed": 0, "fn": 0, "skipped": 0})
+        s[key] += 1
+
+    for label in ("valid", "malformed"):
+        base = os.path.join(corpus, label)
+        if not os.path.isdir(base):
+            continue
+        for name in sorted(os.listdir(base)):
+            fixture = os.path.join(base, name)
+            if not os.path.isdir(fixture):
+                continue
+            for row in run_conform(fixture):
+                gid = row["gate_id"]
+                bucket = verdict_bucket(row)
+                if bucket == "skip":
+                    bump(gid, "skipped")
+                    continue
+                if label == "valid":
+                    bump(gid, "valid")
+                    if bucket == "flag":
+                        bump(gid, "fp")
+                else:
+                    bump(gid, "malformed")
+                    if bucket == "clean":
+                        bump(gid, "fn")
+    return stats
+
+
+def rate(n, d):
+    return (n / d) if d else 0.0
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Measure gate FP/FN rate over a labeled corpus")
+    ap.add_argument("--corpus", default=DEFAULT_CORPUS, help="labeled corpus root (valid/ + malformed/)")
+    ap.add_argument("--json", action="store_true", help="emit JSON report to stdout")
+    ap.add_argument("--max-fp-rate", type=float, default=None,
+                    help="exit 1 if any measured gate's FP-rate exceeds this (promotion gate)")
+    args = ap.parse_args()
+
+    corpus = os.path.abspath(args.corpus)
+    stats = measure(corpus)
+
+    report = {}
+    for gid, s in sorted(stats.items()):
+        report[gid] = {
+            "valid_samples": s["valid"],
+            "false_positives": s["fp"],
+            "fp_rate": round(rate(s["fp"], s["valid"]), 4),
+            "malformed_samples": s["malformed"],
+            "false_negatives": s["fn"],
+            "fn_rate": round(rate(s["fn"], s["malformed"]), 4),
+            "skipped_indeterminate": s["skipped"],
+        }
+
+    if args.json:
+        print(json.dumps({"corpus": os.path.relpath(corpus, os.path.join(HERE, "..")),
+                          "gates": report}, indent=2))
+    else:
+        print(f"FP/FN rate over corpus: {os.path.relpath(corpus, os.path.join(HERE, '..'))}")
+        print(f"{'gate_id':<42} {'valid':>5} {'FP':>3} {'FP%':>6} {'malf':>4} {'FN':>3} {'FN%':>6}")
+        for gid, r in report.items():
+            print(f"{gid:<42} {r['valid_samples']:>5} {r['false_positives']:>3} "
+                  f"{r['fp_rate']*100:>5.1f}% {r['malformed_samples']:>4} "
+                  f"{r['false_negatives']:>3} {r['fn_rate']*100:>5.1f}%")
+        if not report:
+            print("  (no measurable gate verdicts in corpus)")
+
+    if args.max_fp_rate is not None:
+        over = {g: r["fp_rate"] for g, r in report.items() if r["fp_rate"] > args.max_fp_rate}
+        if over:
+            sys.stderr.write(f"\nfp-rate: {len(over)} gate(s) exceed --max-fp-rate={args.max_fp_rate}:\n")
+            for g, fr in over.items():
+                sys.stderr.write(f"  {g}: {fr:.4f}\n")
+            sys.exit(1)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/gen-layer-applicability.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""
+audit-harness gen-layer-applicability — project the canonical registry datum into
+the human-readable layer-applicability matrix.
+
+`schemas/audit-profile/registry.v1.json` is THE single source of truth for "which
+gates apply to repo-type X, in which dimension, at what applicability". This
+generator renders `schemas/audit-profile/layer-applicability.md` as a PROJECTION
+of that datum so the doc can never silently drift from the registry the classifier
+actually resolves against (PP-PLAN-040 Phase 0, bead c2b).
+
+Modes:
+  (default)      print the rendered markdown to stdout
+  --write        write it to schemas/audit-profile/layer-applicability.md
+  --check        regenerate in-memory and diff against the committed file;
+                 exit 1 on drift (the CI `layer-applicability-drift` gate)
+
+Stdlib only. Read-only except in --write mode (which only writes the one doc).
+"""
+import argparse
+import difflib
+import hashlib
+import json
+import os
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+REGISTRY = os.path.join(HERE, "..", "schemas", "audit-profile", "registry.v1.json")
+DOC = os.path.join(HERE, "..", "schemas", "audit-profile", "layer-applicability.md")
+
+GLYPH = {"required": "✅", "recommended": "⭕", "conditional": "⚠", "waived": "❌"}
+
+
+def sha256_file(path):
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(65536), b""):
+            h.update(chunk)
+    return "sha256:" + h.hexdigest()
+
+
+def row(gate):
+    app = gate.get("applicability", "")
+    return "| `{gid}` | {dim} | {glyph} {app} | {enf} | {tool} |".format(
+        gid=gate["gate_id"],
+        dim=gate.get("dimension", ""),
+        glyph=GLYPH.get(app, ""),
+        app=app,
+        enf=gate.get("enforcement", "advisory"),
+        tool=("`" + gate["tool"] + "`") if gate.get("tool") else "—",
+    )
+
+
+def table(gates):
+    out = ["| Gate | Dimension | Applicability | Enforcement | Tool |",
+           "|---|---|---|---|---|"]
+    out += [row(g) for g in sorted(gates, key=lambda g: (g.get("dimension", ""), g["gate_id"]))]
+    return "\n".join(out)
+
+
+def render(registry, registry_hash):
+    lines = []
+    a = lines.append
+    a("# Layer Applicability — GENERATED from `registry.v1.json`")
+    a("")
+    a("> ⚠️ **GENERATED FILE — do not edit by hand.**")
+    a("> Source of truth: [`registry.v1.json`](registry.v1.json) "
+      "(the canonical dimension→gate datum; `classify` resolves against it).")
+    a("> Regenerate: `audit-harness gen-layer-applicability --write` "
+      "(or `python3 scripts/gen-layer-applicability.py --write`).")
+    a("> CI gate `layer-applicability-drift` fails the build if this file drifts from the registry.")
+    a(">")
+    a(f"> registry `{registry_hash}`")
+    a("")
+    a(registry.get("description", "").strip())
+    a("")
+    a("**Legend (applicability):** "
+      + " · ".join(f"{GLYPH[k]} {k}" for k in ("required", "recommended", "conditional", "waived")))
+    a("")
+    a("Every gate defaults to `enforcement: advisory`. Blocking is **earned** — "
+      "engineer-pinned in the target repo's `tests/TESTING.md`, FP-rate-gated "
+      "(see [`gate-promotion.md`](../../docs/gate-promotion.md)).")
+    a("")
+    a("## Base gates (apply to every repo)")
+    a("")
+    a(table(registry.get("base", [])))
+    a("")
+    a("## By classification")
+    a("")
+    a("A repo carries the **UNION** of every classification it matches "
+      "(`classify` never picks a single winner). Gates dedup by `gate_id`, "
+      "keeping the highest applicability.")
+    a("")
+    for kind in sorted(registry.get("classifications", {})):
+        a(f"### `{kind}`")
+        a("")
+        a(table(registry["classifications"][kind]))
+        a("")
+    overlays = registry.get("overlays", {})
+    if overlays:
+        a("## Overlays")
+        a("")
+        for name in sorted(overlays):
+            ov = overlays[name]
+            a(f"### `{name}`")
+            a("")
+            a(ov.get("description", "").strip())
+            promote = ov.get("promote_to_required", [])
+            if promote:
+                a("")
+                a("Promotes to **required**: " + ", ".join(f"`{d}`" for d in promote) + ".")
+            a("")
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Project registry.v1.json -> layer-applicability.md")
+    ap.add_argument("--write", action="store_true", help="write the doc to its canonical path")
+    ap.add_argument("--check", action="store_true", help="fail (exit 1) if the committed doc drifts")
+    ap.add_argument("--registry", default=REGISTRY)
+    ap.add_argument("--out", default=DOC)
+    args = ap.parse_args()
+
+    registry_path = os.path.abspath(args.registry)
+    with open(registry_path, "r", encoding="utf-8") as f:
+        registry = json.load(f)
+    rendered = render(registry, sha256_file(registry_path))
+
+    if args.check:
+        try:
+            with open(args.out, "r", encoding="utf-8") as f:
+                current = f.read()
+        except FileNotFoundError:
+            print(f"gen-layer-applicability: {args.out} missing — run --write", file=sys.stderr)
+            sys.exit(1)
+        if current != rendered:
+            diff = difflib.unified_diff(
+                current.splitlines(True), rendered.splitlines(True),
+                fromfile="committed", tofile="generated",
+            )
+            sys.stderr.write("".join(diff))
+            sys.stderr.write("\ngen-layer-applicability: DRIFT — regenerate with --write\n")
+            sys.exit(1)
+        print("gen-layer-applicability: layer-applicability.md matches the registry datum")
+        sys.exit(0)
+
+    if args.write:
+        with open(args.out, "w", encoding="utf-8") as f:
+            f.write(rendered)
+        print(f"gen-layer-applicability: wrote {os.path.relpath(args.out, os.path.join(HERE, '..'))}")
+        sys.exit(0)
+
+    sys.stdout.write(rendered)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/scan.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+"""
+audit-harness scan — read-only security / hygiene / skill-quality gate-runner
+(PP-PLAN-040 Phase 4 / E6).
+
+For every `dimension: security | hygiene | skill-quality` gate in a repo's
+audit-profile/v1, scan runs the right external tool with the repo present and wraps
+its exit code into a `gate-result/v1` row (JSON array, stdout). Advisory-first; a
+missing tool degrades to ADVISORY indeterminate (never a false FAIL). It NEVER
+fixes anything and NEVER reimplements a scanner.
+
+Strategies:
+  - local      hygiene-readme: deterministic README presence check (no tool).
+  - shell-out  every gate carrying a `tool` (gitleaks, osv-scanner, semgrep, syft,
+               markdownlint, lychee, ...): run it if on PATH; clean exit -> PASS;
+               findings -> ADVISORY(error) (or FAIL under --strict / blocking);
+               tool absent -> ADVISORY indeterminate.
+  - consume    skill-quality skill-behavioral (tool j-rig): CONSUME a j-rig
+               Evidence Bundle verdict row (--jrig-verdict PATH or a default
+               location). The harness does NOT run behavioral judgment itself —
+               it ingests j-rig's verdict. No verdict -> ADVISORY indeterminate.
+
+Stdlib only. No network beyond whatever the shelled-out tool does (and the only
+network-touching gates fail open to indeterminate). No filesystem mutation.
+"""
+import argparse
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import sys
+from datetime import datetime, timezone
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+if HERE not in sys.path:
+    sys.path.insert(0, HERE)
+import classify as C  # noqa: E402
+
+EMPTY_SHA = "sha256:" + hashlib.sha256(b"").hexdigest()
+SCAN_DIMENSIONS = {"security", "hygiene", "skill-quality"}
+
+# tool -> argv (run with cwd=repo). "generation" tools (syft) are PASS on exit 0,
+# INDETERMINATE on failure (they produce an artifact, they don't pass/fail policy).
+TOOL_CMD = {
+    "gitleaks": (["gitleaks", "detect", "--no-banner"], "scan"),
+    "osv-scanner": (["osv-scanner", "-r", "."], "scan"),
+    "semgrep": (["semgrep", "scan", "--error", "--quiet"], "scan"),
+    "syft": (["syft", "."], "generation"),
+    "markdownlint": (["markdownlint", "."], "scan"),
+    "lychee": (["lychee", "--offline", "--no-progress", "."], "scan"),
+}
+
+
+def sha256_str(s):
+    return "sha256:" + hashlib.sha256(s.encode("utf-8")).hexdigest()
+
+
+def make_row(gate_id, result, *, policy_hash, input_hash, commit_sha, runner,
+             metadata=None, failure_mode=None, advisory_severity=None):
+    row = {
+        "gate_id": gate_id, "result": result, "policy_hash": policy_hash,
+        "input_hash": input_hash,
+        "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "runner": runner, "commit_sha": commit_sha,
+    }
+    if metadata:
+        row["metadata"] = metadata
+    if failure_mode is not None:
+        row["failure_mode"] = failure_mode
+    if advisory_severity is not None:
+        row["advisory_severity"] = advisory_severity
+    return row
+
+
+def gate_suffix(gate_id):
+    return gate_id.rsplit(":", 1)[-1]
+
+
+def indeterminate(gate, commit_sha, runner, reason, policy):
+    return make_row(gate["gate_id"], "ADVISORY", policy_hash=sha256_str(policy),
+                    input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                    advisory_severity="warn",
+                    metadata={"indeterminate": True, "reason": reason})
+
+
+def run_readme(repo, gate, commit_sha, runner, strict):
+    enforcement = gate.get("enforcement", "advisory")
+    present = any(os.path.isfile(os.path.join(repo, n))
+                 for n in ("README.md", "README.rst", "README.txt", "README"))
+    if present:
+        return make_row(gate["gate_id"], "PASS", policy_hash=sha256_str("hygiene:readme"),
+                        input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                        metadata={"method": "local-presence", "signal": "README present"})
+    result, fm, sev = ("FAIL", "hygiene:readme-missing", None) if (strict or enforcement == "blocking") \
+        else ("ADVISORY", None, "warn")
+    return make_row(gate["gate_id"], result, policy_hash=sha256_str("hygiene:readme"),
+                    input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                    failure_mode=fm, advisory_severity=sev,
+                    metadata={"method": "local-presence", "reason": "no README found"})
+
+
+def run_tool(tool, repo, gate, commit_sha, runner, strict):
+    enforcement = gate.get("enforcement", "advisory")
+    policy = f"tool:{tool}"
+    if tool not in TOOL_CMD:
+        return indeterminate(gate, commit_sha, runner,
+                             f"no invocation wired for tool '{tool}'", policy)
+    if shutil.which(tool) is None:
+        return indeterminate(gate, commit_sha, runner,
+                             f"{tool} not on PATH — {gate.get('dimension')} unmeasured", policy)
+    argv, kind = TOOL_CMD[tool]
+    try:
+        proc = subprocess.run(argv, cwd=repo, capture_output=True, text=True, timeout=300)
+    except Exception as e:
+        return indeterminate(gate, commit_sha, runner, f"{tool} failed to run: {e}", policy)
+    if proc.returncode == 0:
+        return make_row(gate["gate_id"], "PASS", policy_hash=sha256_str(policy),
+                        input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                        metadata={"method": "shell-out", "tool": tool})
+    if kind == "generation":
+        # syft etc. failing to generate is infra, not a policy violation
+        return indeterminate(gate, commit_sha, runner,
+                             f"{tool} could not generate artifact (exit {proc.returncode})", policy)
+    detail = (proc.stdout or proc.stderr).strip()[:2000]
+    result, fm, sev = ("FAIL", f"scan:{tool}-findings", None) if (strict or enforcement == "blocking") \
+        else ("ADVISORY", None, "error")
+    return make_row(gate["gate_id"], result, policy_hash=sha256_str(policy),
+                    input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                    failure_mode=fm, advisory_severity=sev,
+                    metadata={"method": "shell-out", "tool": tool, "detail": detail})
+
+
+def consume_jrig(repo, gate, commit_sha, runner, strict, verdict_path):
+    """Ingest a j-rig Evidence Bundle verdict row — never run judgment here."""
+    policy = "consume:j-rig"
+    candidates = [verdict_path] if verdict_path else []
+    candidates += [os.path.join(repo, p) for p in
+                   (".j-rig/verdict.json", ".jrig/verdict.json", "j-rig-verdict.json")]
+    path = next((p for p in candidates if p and os.path.isfile(p)), None)
+    if path is None:
+        return indeterminate(gate, commit_sha, runner,
+                             "no j-rig verdict available — run j-rig eval and pass --jrig-verdict",
+                             policy)
+    verdict = C.read_json(path)
+    if not isinstance(verdict, dict):
+        return indeterminate(gate, commit_sha, runner, f"unreadable j-rig verdict at {path}", policy)
+    # Pass through j-rig's own result if present; otherwise interpret a boolean pass.
+    enforcement = gate.get("enforcement", "advisory")
+    jres = verdict.get("result") or ("PASS" if verdict.get("passed") else "FAIL")
+    meta = {"method": "consume-j-rig", "source": os.path.relpath(path, repo),
+            "jrig": {k: verdict.get(k) for k in ("result", "passed", "layers_passed", "baseline_delta")
+                     if k in verdict}}
+    if jres == "PASS":
+        return make_row(gate["gate_id"], "PASS", policy_hash=sha256_str(policy),
+                        input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner, metadata=meta)
+    result, fm, sev = ("FAIL", "skill-quality:jrig-fail", None) if (strict or enforcement == "blocking") \
+        else ("ADVISORY", None, "error")
+    return make_row(gate["gate_id"], result, policy_hash=sha256_str(policy),
+                    input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                    failure_mode=fm, advisory_severity=sev, metadata=meta)
+
+
+def compute_profile(repo, registry_path, profile_arg):
+    if profile_arg == "-":
+        return json.load(sys.stdin)
+    if profile_arg:
+        with open(profile_arg, "r", encoding="utf-8") as f:
+            return json.load(f)
+    out = subprocess.run([sys.executable, os.path.join(HERE, "classify.py"), repo,
+                          "--registry", registry_path], capture_output=True, text=True)
+    if out.returncode != 0:
+        sys.stderr.write(out.stderr)
+        raise SystemExit(2)
+    return json.loads(out.stdout)
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Security/hygiene/skill-quality gate-runner -> gate-result/v1")
+    ap.add_argument("repo", nargs="?", default=".")
+    ap.add_argument("--strict", action="store_true", help="treat a finding/gap as FAIL (exit 1)")
+    ap.add_argument("--registry", default=C.DEFAULT_REGISTRY)
+    ap.add_argument("--profile", default=None, help="pinned audit-profile/v1 (PATH or '-')")
+    ap.add_argument("--jrig-verdict", default=None, help="path to a j-rig Evidence Bundle verdict to consume")
+    args = ap.parse_args()
+
+    repo = os.path.abspath(args.repo)
+    runner = f"audit-harness@{C.harness_version()}"
+
+    override_path = os.path.join(repo, ".audit-harness.yml")
+    override = C.parse_override(override_path) if os.path.isfile(override_path) else {"disable": False}
+    if override.get("disable") or os.environ.get("AUDIT_HARNESS_DISABLE") == "1":
+        sys.stderr.write("audit-harness: KILL-SWITCH active — scan skipped (no rows emitted)\n")
+        print("[]")
+        sys.exit(0)
+
+    profile = compute_profile(repo, os.path.abspath(args.registry), args.profile)
+    commit_sha = profile.get("subject", {}).get("commit_sha") or C.git_short_sha(repo)
+
+    gates = [g for g in profile.get("gates", [])
+             if g.get("dimension") in SCAN_DIMENSIONS and g.get("enforcement") != "disabled"]
+
+    rows = []
+    for gate in gates:
+        suffix = gate_suffix(gate["gate_id"])
+        tool = gate.get("tool")
+        if suffix == "hygiene-readme":
+            rows.append(run_readme(repo, gate, commit_sha, runner, args.strict))
+        elif tool == "j-rig":
+            rows.append(consume_jrig(repo, gate, commit_sha, runner, args.strict, args.jrig_verdict))
+        elif tool:
+            rows.append(run_tool(tool, repo, gate, commit_sha, runner, args.strict))
+        else:
+            rows.append(indeterminate(gate, commit_sha, runner,
+                                      f"gate '{suffix}' has no tool wired in this harness version",
+                                      f"scan:{suffix}"))
+
+    print(json.dumps(rows, indent=2))
+    n_fail = sum(1 for r in rows if r["result"] == "FAIL")
+    n_adv = sum(1 for r in rows if r["result"] == "ADVISORY")
+    n_pass = sum(1 for r in rows if r["result"] == "PASS")
+    sys.stderr.write(f"audit-harness scan: {n_pass} PASS, {n_adv} ADVISORY, {n_fail} FAIL "
+                     f"across {len(rows)} gate(s)\n")
+    sys.exit(1 if n_fail else 0)
+
+
+if __name__ == "__main__":
+    main()