@intentsolutions/audit-harness 0.1.0 → 1.1.6

package/schemas/conform/v1/skillmd-frontmatter.schema.json

@@ -0,0 +1,40 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://evals.intentsolutions.io/conform/v1/skillmd-frontmatter.schema.json",
+  "title": "SKILL.md frontmatter — deterministic structural floor (conform/v1)",
+  "description": "The DETERMINISTIC STRUCTURAL FLOOR for a SKILL.md frontmatter block: it parses, carries the Anthropic-spec required pair (name + description), and the known optional fields are the right type. This is NOT the IS 100-point marketplace rubric or the 8-field enterprise required set — that grading is judgment-bearing and stays in /validate-skillmd (Tier 1) + the SAK authoring kernel. conform validates the floor; Claude adds judgment on top. additionalProperties is true because SKILL.md frontmatter is intentionally extensible (model, effort, context, hooks, metadata, user-invocable, etc.).",
+  "type": "object",
+  "required": ["name", "description"],
+  "additionalProperties": true,
+  "properties": {
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_-]*$"
+    },
+    "description": {
+      "type": "string",
+      "minLength": 1
+    },
+    "allowed-tools": {
+      "anyOf": [
+        { "type": "string" },
+        { "type": "array", "items": { "type": "string" } }
+      ]
+    },
+    "disallowed-tools": {
+      "anyOf": [
+        { "type": "string" },
+        { "type": "array", "items": { "type": "string" } }
+      ]
+    },
+    "version": { "type": "string" },
+    "author": { "type": "string" },
+    "license": { "type": "string" },
+    "compatibility": { "type": "string", "maxLength": 500 },
+    "tags": { "type": "array", "items": { "type": "string" } },
+    "user-invocable": { "type": "boolean" },
+    "disable-model-invocation": { "type": "boolean" },
+    "argument-hint": { "type": "string" }
+  }
+}

package/schemas/currency/pins.v1.json

@@ -0,0 +1,55 @@
+{
+  "pins_version": "currency-pins/v1",
+  "description": "Per-upstream-identity pin relation. Each upstream the harness/skills depend on carries ITS OWN pinned version + the date it was last verified against upstream (checked_at) + a staleness window. The `currency` advisory report reads this datum and flags pins whose checked_at is older than their window — i.e. it makes the PIN'S OWN STALENESS detectable, without ever live-fetching. Currency is advisory-only: it reports + (in /sync-testing-harness) opens PRs; it has no exit-code authority and never auto-fixes. Updating a pin (after a human re-verifies against upstream) is an engineer edit to this file + a fresh checked_at.",
+  "default_staleness_window_days": 90,
+  "pins": [
+    {
+      "identity": "mcp-spec",
+      "pinned_version": "2025-06-18",
+      "source": "https://spec.modelcontextprotocol.io/ (protocol revision)",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 90,
+      "notes": "MCP protocol spec revision the .mcp.json conform schema targets."
+    },
+    {
+      "identity": "skill-md-schema",
+      "pinned_version": "3.7.0",
+      "source": "claude-code-plugins 000-docs/SCHEMA_CHANGELOG.md",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 90,
+      "notes": "IS SKILL.md schema the conform skillmd-frontmatter floor tracks (full rubric stays in /validate-skillmd)."
+    },
+    {
+      "identity": "claude-code",
+      "pinned_version": "2.1.152",
+      "source": "https://code.claude.com/docs/en/changelog",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 60,
+      "notes": "Claude Code release (added disallowed-tools frontmatter at 2.1.152)."
+    },
+    {
+      "identity": "gate-result-predicate",
+      "pinned_version": "v1",
+      "source": "@intentsolutions/core gate-result/v1 (https://evals.intentsolutions.io/gate-result/v1)",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 180,
+      "notes": "Evidence Bundle predicate every gate emits against. Immutable URI; bump = new version, never overwrite."
+    },
+    {
+      "identity": "anthropic-sdk",
+      "pinned_version": "unverified",
+      "source": "https://github.com/anthropics/anthropic-sdk-python (+ -typescript)",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 90,
+      "notes": "Anthropic SDK surface referenced by downstream skills; pinned_version=unverified until first deliberate verification."
+    },
+    {
+      "identity": "agentskills-spec",
+      "pinned_version": "1.0.0",
+      "source": "https://agentskills.io/specification",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 90,
+      "notes": "Open SKILL.md standard (compatibility/metadata/license fields)."
+    }
+  ]
+}

package/scripts/arch-check.sh

@@ -39,7 +39,31 @@ mkdir -p "$REPORT_DIR"
 emit_result() {
   local tool="$1" status="$2" violations="$3" log="$4"
   if [[ "$JSON_OUT" -eq 1 ]]; then
-    printf '{"tool":"%s","status":"%s","violations":%s,"log":"%s"}\n' \
+    # status: pass / fail / missing-tool / not-configured
+    local result
+    case "$status" in
+      pass) result="PASS" ;;
+      fail) result="FAIL" ;;
+      missing-tool|not-configured) result="NOT_APPLICABLE" ;;
+      *) result="ADVISORY" ;;
+    esac
+    local input_hash="sha256:0000000000000000000000000000000000000000000000000000000000000000"
+    local policy_hash="sha256:0000000000000000000000000000000000000000000000000000000000000000"
+    # Best-effort: input_hash is the source tree fingerprint when running against ROOT/src
+    if [[ -d "${ROOT}/src" ]]; then
+      input_hash=$(find "${ROOT}/src" -type f \( -name "*.ts" -o -name "*.tsx" -o -name "*.js" -o -name "*.py" -o -name "*.go" -o -name "*.rs" -o -name "*.java" -o -name "*.kt" -o -name "*.cs" -o -name "*.php" \) -exec sha256sum {} \; 2>/dev/null | sort | sha256sum | awk '{print "sha256:"$1}')
+    fi
+    # Hash the architecture rule config (whichever tool's config was used)
+    for cfg in .dependency-cruiser.js .dependency-cruiser.cjs .importlinter deptrac.yaml arch-go.yml; do
+      if [[ -f "${ROOT}/${cfg}" ]]; then
+        policy_hash=$(sha256sum "${ROOT}/${cfg}" | awk '{print "sha256:"$1}')
+        break
+      fi
+    done
+    local fail_block=""
+    [[ "$result" == "FAIL" ]] && fail_block=',"failure_mode":"arch-violation"'
+    printf '{"gate_id":"audit-harness:%s:arch-check","result":"%s"%s,"input_hash":"%s","policy_hash":"%s","metadata":{"tool":"%s","status":"%s","violations":%s,"log":"%s"}}\n' \
+      "${AUDIT_HARNESS_SIDE:-ci}" "$result" "$fail_block" "$input_hash" "$policy_hash" \
       "$tool" "$status" "$violations" "$log"
   else
     echo "arch-check: tool=$tool status=$status violations=$violations"

package/scripts/audit.py

@@ -0,0 +1,386 @@
+#!/usr/bin/env python3
+"""
+audit-harness audit — read-only testing-depth gate-runner (PP-PLAN-040 Phase 3 / E5).
+
+For every `dimension: testing-depth` gate in a repo's audit-profile/v1, audit
+assesses the gate and emits a `gate-result/v1` row (JSON array, stdout). It is the
+"finish the pyramid" diagnostic: it reports which testing-depth LAYERS a repo has
+infrastructure for, advisory-first.
+
+Two assessment strategies, both read-only:
+  - crap-score    -> runs the bundled `crap` scorer (static complexity x coverage).
+  - presence      -> a per-layer static heuristic (test dirs, framework configs,
+                     dependency markers). Layer infra present -> PASS; absent ->
+                     ADVISORY(warn) "testing-depth gap"; unknowable statically ->
+                     ADVISORY indeterminate.
+
+What audit deliberately does NOT do: execute the repo's test suite. Running
+arbitrary, untrusted test suites is the job of the repo's own CI test step — the
+harness wraps that step's verdict into Evidence, it does not replace it. audit
+reports COVERAGE PRESENCE; execution stays in CI. Each row records its
+`metadata.method` so the assessment provenance is explicit.
+
+--fast (default): presence heuristics only (<10s on a reference repo).
+--deep:           presence + crap-score.
+--strict:         a testing-depth gap on an `enforcement: blocking` gate -> FAIL.
+
+Stdlib only. No network. No filesystem mutation.
+"""
+import argparse
+import hashlib
+import json
+import os
+import subprocess
+import sys
+from datetime import datetime, timezone
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+if HERE not in sys.path:
+    sys.path.insert(0, HERE)
+import classify as C  # noqa: E402
+
+EMPTY_SHA = "sha256:" + hashlib.sha256(b"").hexdigest()
+
+SKIP_DIRS = ("node_modules", ".git", ".venv", "dist", "build", "vendor", "target")
+
+# Gates assessed quickly (presence heuristics) belong to the fast tier; crap-score
+# is deep-only because it shells out to radon/gocyclo and can be slow.
+DEEP_ONLY = {"crap-score"}
+
+
+def sha256_str(s):
+    return "sha256:" + hashlib.sha256(s.encode("utf-8")).hexdigest()
+
+
+# --------------------------------------------------------------------------- #
+# repo signal collectors
+# --------------------------------------------------------------------------- #
+def collect_node_deps(repo):
+    deps = {}
+    for pkgpath in [os.path.join(repo, "package.json")] + [
+        os.path.join(s, "package.json") for s in C.list_pkg_subdirs(repo)
+    ]:
+        pkg = C.read_json(pkgpath)
+        if isinstance(pkg, dict):
+            for k in ("dependencies", "devDependencies"):
+                if isinstance(pkg.get(k), dict):
+                    deps.update(pkg[k])
+    return deps
+
+
+def node_test_script(repo):
+    pkg = C.read_json(os.path.join(repo, "package.json")) or {}
+    scripts = pkg.get("scripts") if isinstance(pkg, dict) else None
+    return isinstance(scripts, dict) and bool(scripts.get("test"))
+
+
+def py_dep_text(repo):
+    txt = ""
+    for f in ("requirements.txt", "pyproject.toml", "Pipfile", "setup.cfg", "tox.ini"):
+        p = os.path.join(repo, f)
+        if os.path.isfile(p):
+            try:
+                txt += open(p, "r", encoding="utf-8").read().lower()
+            except Exception:
+                pass
+    return txt
+
+
+def walk_names(repo, max_depth=4):
+    """Yield (dirpath_rel, dirnames, filenames) skipping vendor/build dirs."""
+    repo = os.path.abspath(repo)
+    for root, dirs, files in os.walk(repo):
+        dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
+        depth = root[len(repo):].count(os.sep)
+        if depth > max_depth:
+            dirs[:] = []
+            continue
+        yield root, dirs, files
+
+
+def has_dir(repo, *names):
+    targets = set(names)
+    for _root, dirs, _files in walk_names(repo):
+        if targets & set(dirs):
+            return True
+    return False
+
+
+def has_file_matching(repo, predicate):
+    for _root, _dirs, files in walk_names(repo):
+        if any(predicate(f) for f in files):
+            return True
+    return False
+
+
+def has_glob_suffix(repo, *suffixes):
+    return has_file_matching(repo, lambda f: any(f.endswith(s) for s in suffixes))
+
+
+# --------------------------------------------------------------------------- #
+# per-layer presence detectors -> (present: bool|None, signal: str)
+#   present True  -> infra detected (PASS)
+#   present False -> no infra detected (ADVISORY gap)
+#   present None  -> not assessable statically (ADVISORY indeterminate)
+# --------------------------------------------------------------------------- #
+def d_unit(repo, deps):
+    if node_test_script(repo):
+        return True, "package.json scripts.test"
+    if any(x in deps for x in ("vitest", "jest", "mocha", "ava", "@jest/core", "node:test")):
+        return True, "node test framework dep"
+    txt = py_dep_text(repo)
+    if any(x in txt for x in ("pytest", "unittest", "nose")):
+        return True, "python test framework"
+    if has_glob_suffix(repo, "_test.go"):
+        return True, "go *_test.go"
+    if has_dir(repo, "tests", "test", "__tests__") or \
+       has_glob_suffix(repo, ".test.ts", ".test.js", ".spec.ts", ".spec.js"):
+        return True, "test dir / *.test|spec file"
+    if has_file_matching(repo, lambda f: f.startswith("test_") and f.endswith(".py")):
+        return True, "python test_*.py"
+    return False, "no unit test infrastructure detected"
+
+
+def d_integration(repo, deps):
+    if has_dir(repo, "integration") or \
+       has_glob_suffix(repo, ".integration.test.ts", ".integration.test.js", ".int.test.ts"):
+        return True, "integration test dir/files"
+    if any(x in deps for x in ("testcontainers", "supertest")):
+        return True, "integration tooling dep"
+    if "tests/integration" in py_dep_text(repo):
+        return True, "python integration tests"
+    return False, "no integration test infrastructure detected"
+
+
+def d_e2e(repo, deps):
+    if any(x in deps for x in ("@playwright/test", "playwright", "cypress", "puppeteer", "@testing-library/react")):
+        return True, "e2e framework dep"
+    cfgs = ("playwright.config.ts", "cypress.config.ts", "cypress.config.js")
+    if has_dir(repo, "e2e") or any(os.path.isfile(os.path.join(repo, c)) for c in cfgs):
+        return True, "e2e config/dir"
+    return False, "no e2e test infrastructure detected"
+
+
+def d_smoke(repo, deps):
+    pkg = C.read_json(os.path.join(repo, "package.json")) or {}
+    scripts = pkg.get("scripts") if isinstance(pkg, dict) else {}
+    if isinstance(scripts, dict) and any("smoke" in k for k in scripts):
+        return True, "package.json smoke script"
+    if has_dir(repo, "smoke") or has_file_matching(repo, lambda f: "smoke" in f.lower()):
+        return True, "smoke test dir/file"
+    return False, "no smoke test detected"
+
+
+def d_perf(repo, deps):
+    if any(x in deps for x in ("benchmark", "tinybench", "vitest-bench", "k6", "autocannon")):
+        return True, "perf/bench dep"
+    if has_dir(repo, "bench", "benchmark", "benchmarks", "perf") or \
+       has_glob_suffix(repo, ".bench.ts", ".bench.js", "_bench.go"):
+        return True, "bench dir/files"
+    return False, "no performance test infrastructure detected"
+
+
+def d_a11y(repo, deps):
+    if any(x in deps for x in ("axe-core", "@axe-core/playwright", "jest-axe", "pa11y")):
+        return True, "a11y tooling dep"
+    return False, "no accessibility test infrastructure detected"
+
+
+def d_contract(repo, deps):
+    if any(x in deps for x in ("@pact-foundation/pact", "pact")):
+        return True, "contract testing dep (pact)"
+    if has_dir(repo, "contract", "contracts", "pacts"):
+        return True, "contract test dir"
+    return False, "no contract test infrastructure detected"
+
+
+def d_migration(repo, deps):
+    if has_dir(repo, "migrations", "migration"):
+        return True, "migrations dir"
+    if any(x in deps for x in ("prisma", "knex", "typeorm", "drizzle-kit")):
+        return True, "migration tooling dep"
+    if any(x in py_dep_text(repo) for x in ("alembic", "django")):
+        return True, "python migration tooling"
+    return False, "no migration test infrastructure detected"
+
+
+def d_property(repo, deps):
+    if any(x in deps for x in ("fast-check", "jsverify")):
+        return True, "property-based dep (fast-check)"
+    if any(x in py_dep_text(repo) for x in ("hypothesis",)):
+        return True, "python hypothesis"
+    if "proptest" in py_dep_text(repo) or has_glob_suffix(repo, "_proptest.rs"):
+        return True, "rust proptest"
+    return False, "no property-based test infrastructure detected"
+
+
+def d_fuzz(repo, deps):
+    if has_dir(repo, "fuzz") or has_glob_suffix(repo, "_fuzz.go", "fuzz_target.rs"):
+        return True, "fuzz dir/targets"
+    if any(x in deps for x in ("@jazzer.js/core", "jazzer")) or "atheris" in py_dep_text(repo):
+        return True, "fuzz tooling dep"
+    return False, "no fuzz test infrastructure detected"
+
+
+def d_sanitizers(repo, deps):
+    for f in ("Makefile", "CMakeLists.txt"):
+        p = os.path.join(repo, f)
+        if os.path.isfile(p):
+            try:
+                if "-fsanitize" in open(p, "r", encoding="utf-8").read():
+                    return True, "-fsanitize in build config"
+            except Exception:
+                pass
+    return False, "no sanitizer configuration detected"
+
+
+DETECTORS = {
+    "unit": d_unit, "integration": d_integration, "e2e": d_e2e, "smoke": d_smoke,
+    "perf": d_perf, "a11y": d_a11y, "contract": d_contract, "migration": d_migration,
+    "property-based": d_property, "fuzz": d_fuzz, "sanitizers": d_sanitizers,
+}
+
+
+# --------------------------------------------------------------------------- #
+def make_row(gate_id, result, *, policy_hash, input_hash, commit_sha, runner,
+             metadata=None, failure_mode=None, advisory_severity=None):
+    row = {
+        "gate_id": gate_id, "result": result, "policy_hash": policy_hash,
+        "input_hash": input_hash,
+        "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "runner": runner, "commit_sha": commit_sha,
+    }
+    if metadata:
+        row["metadata"] = metadata
+    if failure_mode is not None:
+        row["failure_mode"] = failure_mode
+    if advisory_severity is not None:
+        row["advisory_severity"] = advisory_severity
+    return row
+
+
+def gate_suffix(gate_id):
+    return gate_id.rsplit(":", 1)[-1]
+
+
+def run_crap(repo, gate, commit_sha, runner, strict):
+    enforcement = gate.get("enforcement", "advisory")
+    try:
+        proc = subprocess.run([sys.executable, os.path.join(HERE, "crap-score.py")],
+                              cwd=repo, capture_output=True, text=True, timeout=120)
+        ok = proc.returncode == 0
+        detail = (proc.stdout or proc.stderr).strip().splitlines()[-1:] if (proc.stdout or proc.stderr) else []
+    except Exception as e:
+        return make_row(gate["gate_id"], "ADVISORY", policy_hash=sha256_str("crap:default"),
+                        input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                        advisory_severity="warn",
+                        metadata={"method": "crap-static", "indeterminate": True, "reason": str(e)})
+    if ok:
+        return make_row(gate["gate_id"], "PASS", policy_hash=sha256_str("crap:default"),
+                        input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                        metadata={"method": "crap-static", "detail": detail})
+    result, fm, sev = ("FAIL", "testing-depth:crap-threshold", None) if (strict or enforcement == "blocking") \
+        else ("ADVISORY", None, "error")
+    return make_row(gate["gate_id"], result, policy_hash=sha256_str("crap:default"),
+                    input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                    failure_mode=fm, advisory_severity=sev,
+                    metadata={"method": "crap-static", "detail": detail})
+
+
+def run_presence(suffix, repo, deps, gate, commit_sha, runner, strict):
+    enforcement = gate.get("enforcement", "advisory")
+    present, signal = DETECTORS[suffix](repo, deps)
+    if present is True:
+        return make_row(gate["gate_id"], "PASS", policy_hash=sha256_str(f"presence:{suffix}"),
+                        input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                        metadata={"method": "presence-heuristic", "layer": suffix, "signal": signal})
+    if present is None:
+        return make_row(gate["gate_id"], "ADVISORY", policy_hash=sha256_str(f"presence:{suffix}"),
+                        input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                        advisory_severity="warn",
+                        metadata={"method": "presence-heuristic", "layer": suffix,
+                                  "indeterminate": True, "reason": signal})
+    # gap
+    result, fm, sev = ("FAIL", f"testing-depth:{suffix}-gap", None) if (strict or enforcement == "blocking") \
+        else ("ADVISORY", None, "warn")
+    return make_row(gate["gate_id"], result, policy_hash=sha256_str(f"presence:{suffix}"),
+                    input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                    failure_mode=fm, advisory_severity=sev,
+                    metadata={"method": "presence-heuristic", "layer": suffix, "reason": signal})
+
+
+def compute_profile(repo, registry_path, profile_arg):
+    if profile_arg == "-":
+        return json.load(sys.stdin)
+    if profile_arg:
+        with open(profile_arg, "r", encoding="utf-8") as f:
+            return json.load(f)
+    out = subprocess.run([sys.executable, os.path.join(HERE, "classify.py"), repo,
+                          "--registry", registry_path], capture_output=True, text=True)
+    if out.returncode != 0:
+        sys.stderr.write(out.stderr)
+        raise SystemExit(2)
+    return json.loads(out.stdout)
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Read-only testing-depth gate-runner -> gate-result/v1 rows")
+    ap.add_argument("repo", nargs="?", default=".")
+    ap.add_argument("--fast", action="store_true", help="presence heuristics only (default tier)")
+    ap.add_argument("--deep", action="store_true", help="presence + crap-score")
+    ap.add_argument("--strict", action="store_true", help="treat a testing-depth gap as FAIL (exit 1)")
+    ap.add_argument("--registry", default=C.DEFAULT_REGISTRY)
+    ap.add_argument("--profile", default=None, help="pinned audit-profile/v1 (PATH or '-')")
+    args = ap.parse_args()
+
+    deep = args.deep and not args.fast
+    repo = os.path.abspath(args.repo)
+    runner = f"audit-harness@{C.harness_version()}"
+
+    override_path = os.path.join(repo, ".audit-harness.yml")
+    override = C.parse_override(override_path) if os.path.isfile(override_path) else {"disable": False}
+    if override.get("disable") or os.environ.get("AUDIT_HARNESS_DISABLE") == "1":
+        sys.stderr.write("audit-harness: KILL-SWITCH active — audit skipped (no rows emitted)\n")
+        print("[]")
+        sys.exit(0)
+
+    profile = compute_profile(repo, os.path.abspath(args.registry), args.profile)
+    commit_sha = profile.get("subject", {}).get("commit_sha") or C.git_short_sha(repo)
+    deps = collect_node_deps(repo)
+
+    gates = [g for g in profile.get("gates", [])
+             if g.get("dimension") == "testing-depth" and g.get("enforcement") != "disabled"]
+
+    rows = []
+    for gate in gates:
+        suffix = gate_suffix(gate["gate_id"])
+        if suffix == "crap-score":
+            if not deep:
+                rows.append(make_row(gate["gate_id"], "ADVISORY", policy_hash=sha256_str("crap:default"),
+                                     input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                                     advisory_severity="info",
+                                     metadata={"method": "crap-static", "skipped": "deep-only (run with --deep)"}))
+            else:
+                rows.append(run_crap(repo, gate, commit_sha, runner, args.strict))
+        elif suffix in DETECTORS:
+            rows.append(run_presence(suffix, repo, deps, gate, commit_sha, runner, args.strict))
+        else:
+            # e.g. per-package-classify — assessment delegated, not a static signal
+            rows.append(make_row(gate["gate_id"], "ADVISORY", policy_hash=sha256_str(f"audit:{suffix}"),
+                                 input_hash=EMPTY_SHA, commit_sha=commit_sha, runner=runner,
+                                 advisory_severity="info",
+                                 metadata={"method": "delegated", "indeterminate": True,
+                                           "reason": f"'{suffix}' has no static testing-depth heuristic "
+                                                     f"in this harness version"}))
+
+    print(json.dumps(rows, indent=2))
+    n_fail = sum(1 for r in rows if r["result"] == "FAIL")
+    n_gap = sum(1 for r in rows if r["result"] == "ADVISORY" and r.get("advisory_severity") == "warn")
+    n_pass = sum(1 for r in rows if r["result"] == "PASS")
+    sys.stderr.write(f"audit-harness audit ({'deep' if deep else 'fast'}): {n_pass} PASS, "
+                     f"{n_gap} gap(s), {n_fail} FAIL across {len(rows)} testing-depth gate(s)\n")
+    sys.exit(1 if n_fail else 0)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/bias-count.sh

@@ -1,20 +1,48 @@
 #!/usr/bin/env bash
 # Quick test bias pattern counter
-# Usage: bash bias-count.sh [test-directory]
+# Usage: bash bias-count.sh [test-directory] [--json]
 #
 # Scans test files for common bias patterns that weaken test suites.
 # See references/test-quality-deep-audit.md Section 1 for full details.
+#
+# JSON mode:
+#   stdout = single JSON object suitable for piping to `audit-harness emit-evidence`
+#   stderr = unchanged human-readable summary (preserves backward-compat)
+#   exit code unchanged (always 0; advisory gate)
 
 set -euo pipefail
 
-TEST_DIR="${1:-tests}"
+JSON_OUT=0
+TEST_DIR="tests"
+
+# Peel --json from anywhere; first non-flag positional is TEST_DIR.
+_pos=()
+for arg in "$@"; do
+  case "$arg" in
+    --json) JSON_OUT=1 ;;
+    *) _pos+=("$arg") ;;
+  esac
+done
+[[ "${#_pos[@]}" -gt 0 ]] && TEST_DIR="${_pos[0]}"
 
 if [ ! -d "$TEST_DIR" ]; then
-  echo "ERROR: Test directory '$TEST_DIR' not found"
-  echo "Usage: bash bias-count.sh [test-directory]"
+  if [[ "$JSON_OUT" -eq 1 ]]; then
+    printf '{"gate_id":"audit-harness:%s:bias-count","result":"NOT_APPLICABLE","input_hash":"sha256:0000000000000000000000000000000000000000000000000000000000000000","policy_hash":"sha256:0000000000000000000000000000000000000000000000000000000000000000","metadata":{"reason":"test directory not found","path":"%s"}}\n' \
+      "${AUDIT_HARNESS_SIDE:-ci}" "$TEST_DIR"
+  fi
+  echo "ERROR: Test directory '$TEST_DIR' not found" >&2
+  echo "Usage: bash bias-count.sh [test-directory] [--json]" >&2
   exit 1
 fi
 
+# Hash the test directory tree as the "input"
+INPUT_HASH=$(find "$TEST_DIR" -type f \( -name "*.py" -o -name "*.ts" -o -name "*.js" -o -name "*.tsx" -o -name "*.jsx" -o -name "*.go" -o -name "*.rs" -o -name "*.java" -o -name "*.kt" -o -name "*.cs" -o -name "*.php" -o -name "*.rb" \) -exec sha256sum {} + 2>/dev/null | sort | sha256sum | awk '{print "sha256:"$1}')
+
+if [[ "$JSON_OUT" -eq 1 ]]; then
+  exec 3>&1   # save stdout for the JSON object
+  exec 1>&2   # redirect human-readable to stderr
+fi
+
 echo "═══════════════════════════════════════"
 echo "  TEST BIAS SCAN — $TEST_DIR"
 echo "═══════════════════════════════════════"
@@ -75,14 +103,32 @@ printf "  %-30s %s\n" "Per-100-tests rate" "$RATE"
 echo
 
 # Grade
+GRADE="LOW"
 if [ "$(echo "$RATE <= 5" | bc)" -eq 1 ]; then
+  GRADE="LOW"
   echo "  Grade: LOW — no action needed"
 elif [ "$(echo "$RATE <= 15" | bc)" -eq 1 ]; then
+  GRADE="MODERATE"
   echo "  Grade: MODERATE — review flagged tests"
 elif [ "$(echo "$RATE <= 30" | bc)" -eq 1 ]; then
+  GRADE="HIGH"
   echo "  Grade: HIGH — systematic remediation needed"
 else
+  GRADE="CRITICAL"
   echo "  Grade: CRITICAL — full rewrite of flagged tests"
 fi
 echo
 echo "═══════════════════════════════════════"
+
+if [[ "$JSON_OUT" -eq 1 ]]; then
+  # Restore stdout for JSON emission
+  exec 1>&3 3>&-
+  # bias-count is advisory — never FAILs, severity rises with grade
+  case "$GRADE" in
+    LOW) sev="info" ;;
+    MODERATE) sev="warn" ;;
+    HIGH|CRITICAL) sev="error" ;;
+  esac
+  printf '{"gate_id":"audit-harness:%s:bias-count","result":"ADVISORY","advisory_severity":"%s","input_hash":"%s","policy_hash":"sha256:0000000000000000000000000000000000000000000000000000000000000000","metadata":{"test_count":%d,"assertion_count":%d,"assertion_density":"%s","bias_total":%d,"per_100_rate":"%s","grade":"%s"}}\n' \
+    "${AUDIT_HARNESS_SIDE:-ci}" "$sev" "$INPUT_HASH" "$TEST_COUNT" "$ASSERT_COUNT" "$DENSITY" "$TOTAL_BIAS" "$RATE" "$GRADE"
+fi