@profoundry-us/highball 0.3.2 → 0.4.0

package/README.md

@@ -76,6 +76,10 @@ checks:
     exec: host                         # host-side tool, opts out
     fast: true                         # cheap → runs on every agent edit
 
+  - id: architecture-quality
+    name: Architecture & naming (AI)
+    rubric: .highball/packs/rails/rubrics/architecture.md
+
   - id: coverage-ratchet
     name: Coverage never decreases
     todo: true                         # declared, tracked, not yet built
@@ -85,6 +89,38 @@ The runner computes the branch's changed-file list once (it owns git) and
 hands it to every rule via `HIGHBALL_CHANGED_FILES` — check scripts stay pure
 analyzers and need no git in their execution context.
 
+## AI-judged rules
+
+A rule with `rubric:` instead of `run:` is judged by headless Claude rather
+than by a script: the runner bundles the changed files the rubric asks for,
+applies the rubric, and turns the verdict into the same pass/fail contract
+every other rule uses.
+
+The rubric is markdown with optional YAML front matter, which carries the only
+language-specific part:
+
+```markdown
+---
+include: "**/*.rb"          # default: every changed file
+exclude: [db/, config/]     # path prefixes
+model: claude-haiku-4-5-20251001
+---
+
+A comment VIOLATES this rubric when it restates what the code already says.
+```
+
+Three properties are enforced by the runner rather than left to each repo:
+
+- **Never on the fast path.** Rubric rules are dropped from `--fast` runs even
+  if marked `fast: true` — otherwise you pay model latency on every edit.
+- **Always host-side.** The judge needs the `claude` CLI, so `exec.via` never
+  wraps it and no `exec: host` annotation is required.
+- **No evidence, no call.** When nothing in the changed set matches `include`,
+  the rule passes without spawning the model at all.
+
+Rubrics live with the opinions they express: a framework pack such as
+`@profoundry-us/highball-rails` ships them, and the runner supplies the engine.
+
 ## The MCP dashboard widget
 
 `highball mcp` serves the journal over MCP (stdio) with three tools —
@@ -129,8 +165,8 @@ attribution — it's never required to see what happened.
 
 ## Roadmap
 
-AI-judged rules (`rubric:` — headless Claude applying a markdown rubric to
-changed files) land here as a first-class rule type in a future release.
 Built-in generic rules (spec pairing, focused-spec detection, diff budgets)
-likewise, along with per-framework starter packs (`highball-rails`,
-`highball-python`, `highball-go`) carrying recommended check scripts.
+land here in a future release, along with more per-framework starter packs
+(`highball-python`, `highball-go`) carrying recommended check scripts. The
+Rails pack ([`@profoundry-us/highball-rails`](https://github.com/profoundry-us/highball-rails))
+and AI-judged `rubric:` rules have shipped.

package/lib/config.js

@@ -21,6 +21,14 @@ export function loadConfig(root = process.cwd()) {
   if (!Array.isArray(config.checks)) {
     throw new Error(`${CONFIG_PATH} is missing its \`checks:\` list.`);
   }
+  for (const rule of config.checks) {
+    if (!rule.run && !rule.rubric && !rule.todo) {
+      throw new Error(
+        `${CONFIG_PATH}: rule \`${rule.id ?? "(unnamed)"}\` needs ` +
+          "`run:`, `rubric:`, or `todo: true`."
+      );
+    }
+  }
   return config;
 }
 
@@ -45,6 +53,11 @@ export function readCredentials(path = CREDENTIALS_PATH) {
 // rule runs through it unless it opts out with `exec: host`. No declared
 // context means everything runs on the host unchanged.
 export function commandFor(rule, config) {
+  // Rubric rules never become shell commands: the runner executes them
+  // in-process and host-side, because the `claude` CLI lives on the machine
+  // rather than in a project's container (see lib/judge.js).
+  if (rule.rubric) return null;
+
   const via = config.exec?.via;
   if (!via || rule.exec === "host") return rule.run;
   return `${via} ${rule.run}`;

package/lib/judge.js

@@ -0,0 +1,205 @@
+// The AI judge: hands a rubric plus the files this branch touched to
+// headless Claude and turns the verdict into the same pass/fail contract
+// the deterministic rules use.
+//
+// This lives in the runner, not in a language pack, because almost none of
+// it is language-specific: bundling, the prompt contract, the recursion
+// guard, verdict extraction and exit codes are identical whether the repo
+// is Rails, Django or Next. Packs own the *rubrics* — the actual opinions,
+// which are entirely framework-specific — and declare their language policy
+// in rubric front matter. Before this split, a JS-only shop needed Ruby
+// installed to run an AI rule, and the recursion guard lived in this file's
+// repo while the thing it guarded lived in another.
+import { readFileSync, existsSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import YAML from "yaml";
+
+// Judging against a narrow rubric is exactly the fast-and-cheap tier's job.
+// A rubric that needs deeper reasoning overrides this in its front matter —
+// which the previous hardcoded implementation only aspired to.
+const DEFAULT_MODEL = "claude-haiku-4-5-20251001";
+
+// Caps the evidence bundle so the judge stays fast and cheap. Anything
+// dropped is reported, never silently skipped — a cap that lies reads as
+// "covered everything" when it didn't.
+const DEFAULT_MAX_BYTES = 48_000;
+
+// Rubrics are markdown with optional YAML front matter:
+//
+//   ---
+//   include: "**/*.rb"
+//   exclude: [db/, config/]
+//   model: claude-haiku-4-5-20251001
+//   ---
+//
+// Everything after the fence is the rubric prose sent to the judge.
+export function parseRubric(text) {
+  const match = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/.exec(text);
+  if (!match) return { meta: {}, body: text };
+  return { meta: YAML.parse(match[1]) ?? {}, body: text.slice(match[0].length) };
+}
+
+// Minimal glob support — `**`, `*`, `?` — rather than a dependency. Rubric
+// patterns are file-extension filters in practice ("**/*.rb"), not the kind
+// of brace/extglob expressions that would justify pulling in picomatch.
+export function globToRegExp(pattern) {
+  let out = "";
+  for (let i = 0; i < pattern.length; i++) {
+    const char = pattern[i];
+    if (char === "*" && pattern[i + 1] === "*") {
+      i += 1;
+      // `**/` spans zero or more directories; a trailing `**` matches the rest.
+      if (pattern[i + 1] === "/") {
+        i += 1;
+        out += "(?:.*/)?";
+      } else {
+        out += ".*";
+      }
+    } else if (char === "*") {
+      out += "[^/]*";
+    } else if (char === "?") {
+      out += "[^/]";
+    } else {
+      out += char.replace(/[.+^${}()|[\]\\]/g, "\\$&");
+    }
+  }
+  return new RegExp(`^${out}$`);
+}
+
+// The runner already computed the changed list once (ADR 202608) — the judge
+// consumes it rather than shelling out to git again, which is what let the
+// old Ruby implementation drift out of step with its own sibling checks.
+export function selectFiles(changed, meta = {}, exists = existsSync) {
+  const include = [ meta.include ?? "**/*" ].flat().map(globToRegExp);
+  const exclude = [ meta.exclude ?? [] ].flat();
+
+  return [ ...new Set(String(changed).split("\n").map((line) => line.trim())) ]
+    .filter(Boolean)
+    .filter((path) => include.some((pattern) => pattern.test(path)))
+    .filter((path) => !exclude.some((prefix) => path.startsWith(prefix)))
+    .filter((path) => exists(path));
+}
+
+export function buildBundle(files, maxBytes, read) {
+  let bundle = "";
+  const included = [];
+  for (const file of files) {
+    const source = read(file);
+    if (Buffer.byteLength(bundle) + Buffer.byteLength(source) > maxBytes) break;
+    bundle += `\n===== ${file} =====\n${source}`;
+    included.push(file);
+  }
+  return { bundle, included, skipped: files.length - included.length };
+}
+
+export function buildPrompt(rubric, bundle) {
+  return `You are a code-review judge. Apply ONLY the rubric below to the files
+provided. Do not invent rules the rubric does not state. When uncertain,
+pass — report only clear violations.
+
+RUBRIC:
+${rubric}
+
+Respond with ONLY a JSON object, no markdown fences, in this shape:
+{"status":"passed","offenses":[]}
+or
+{"status":"failed","offenses":[{"file":"path","line":1,"message":"..."}]}
+
+FILES:
+${bundle}
+`;
+}
+
+// Models occasionally wrap the verdict in fences or append a prose recap
+// despite the "ONLY a JSON object" instruction — extract the outermost object
+// instead of trusting the envelope to be bare JSON.
+export function extractVerdict(stdout) {
+  let result;
+  try {
+    result = JSON.parse(stdout).result;
+  } catch {
+    throw new Error(`AI judge returned unreadable output:\n${stdout}`);
+  }
+  const json = String(result ?? "").match(/\{[\s\S]*\}/);
+  if (!json) throw new Error(`AI judge returned no JSON verdict:\n${result}`);
+  try {
+    return JSON.parse(json[0]);
+  } catch {
+    throw new Error(`AI judge returned an unparseable verdict:\n${result}`);
+  }
+}
+
+// Always runs host-side: it needs the `claude` CLI, which lives on the
+// machine rather than in a project's container. Being a runner builtin makes
+// that structural instead of an `exec: host` annotation every adopter has to
+// remember to write.
+export function judge(options) {
+  const {
+    rubricPath,
+    changed,
+    spawn = spawnSync,
+    exists = existsSync,
+    read = (path) => readFileSync(path, "utf8")
+  } = options;
+
+  if (!exists(rubricPath)) {
+    return { passed: false, output: `rubric not found: ${rubricPath}` };
+  }
+
+  const { meta, body } = parseRubric(read(rubricPath));
+  const files = selectFiles(changed, meta, exists);
+  if (files.length === 0) {
+    return { passed: true, output: `AI-judged 0 file(s) against ${rubricPath}` };
+  }
+
+  const { bundle, included, skipped } =
+    buildBundle(files, meta.max_bytes ?? DEFAULT_MAX_BYTES, read);
+
+  // HIGHBALL_JUDGE guards recursion: the judge session inherits this repo's
+  // Claude Code hooks, and the runner exits immediately when it sees this
+  // variable — otherwise the judge's own Stop hook would spawn another judge,
+  // forever. Guard and guarded now live in the same package.
+  const child = spawn(
+    "claude",
+    [ "-p", "--model", meta.model ?? DEFAULT_MODEL, "--output-format", "json" ],
+    {
+      input: buildPrompt(body, bundle),
+      encoding: "utf8",
+      env: { ...process.env, HIGHBALL_JUDGE: "1" },
+      maxBuffer: 32 * 1024 * 1024
+    }
+  );
+
+  if (child.error?.code === "ENOENT") {
+    return {
+      passed: false,
+      output: "AI judge needs the `claude` CLI on PATH, and it wasn't found."
+    };
+  }
+  if (child.status !== 0) {
+    return { passed: false, output: `AI judge failed to run: ${child.stderr ?? ""}` };
+  }
+
+  let verdict;
+  try {
+    verdict = extractVerdict(child.stdout ?? "");
+  } catch (error) {
+    return { passed: false, output: error.message };
+  }
+
+  const note = skipped > 0 ? ` (${skipped} file(s) skipped for size)` : "";
+  const header = `AI-judged ${included.length} file(s)${note} against ${rubricPath}`;
+  const offenses = verdict.offenses ?? [];
+
+  if (verdict.status === "passed" || offenses.length === 0) {
+    return { passed: true, output: `${header}\n0 offense(s)` };
+  }
+
+  const lines = offenses.map(
+    (offense) => `${offense.file}:${offense.line}: ${offense.message}`
+  );
+  return {
+    passed: false,
+    output: `${header}\n${lines.join("\n")}\n${offenses.length} offense(s)`
+  };
+}

package/lib/run.js

@@ -7,6 +7,7 @@ import { execSync, spawnSync } from "node:child_process";
 import { loadConfig, resolveReporting, commandFor } from "./config.js";
 import { appendRun } from "./journal.js";
 import { git, report } from "./report.js";
+import { judge } from "./judge.js";
 import { latestUserPrompt } from "./transcript.js";
 
 export async function run(args) {
@@ -25,7 +26,12 @@ export async function run(args) {
     return 1;
   }
 
-  const rules = fastOnly ? config.checks.filter((rule) => rule.fast) : config.checks;
+  // Rubric rules never join a fast run, even if a config marks one `fast`:
+  // LLM latency and cost would be paid on every edit. That belongs at turn
+  // end, and the invariant is enforced here rather than left to each repo.
+  const rules = fastOnly
+    ? config.checks.filter((rule) => rule.fast && !rule.rubric)
+    : config.checks;
   const hook = await readHookPayload();
   const changed = changedFiles();
 
@@ -48,6 +54,19 @@ export async function run(args) {
       continue;
     }
 
+    // Rubric rules run in-process instead of shelling out: the judge needs
+    // the `claude` CLI, which lives on the host, so it bypasses `exec.via`
+    // by construction rather than by annotation.
+    if (rule.rubric) {
+      const t0 = process.hrtime.bigint();
+      const { passed, output } = judge({ rubricPath: rule.rubric, changed });
+      const durationMs = Math.round(Number(process.hrtime.bigint() - t0) / 1e6);
+
+      console.log(`${passed ? "passed" : "FAILED"} (${(durationMs / 1000).toFixed(1)}s)`);
+      results.push({ rule, passed, todo: false, durationMs, output });
+      continue;
+    }
+
     const command = commandFor(rule, config);
     const t0 = process.hrtime.bigint();
     // The runner owns git (ADR 202608): check scripts get the changed
@@ -111,7 +130,9 @@ export async function run(args) {
         // command is the one detail every real rule can always show. todo
         // rules have no command — nothing ran — which is exactly what makes
         // them inert rather than falsely clickable.
-        command: result.rule.run ?? null,
+        command:
+          result.rule.run ??
+          (result.rule.rubric ? `judge ${result.rule.rubric}` : null),
         // Unlike the dashboard (failure tails only), the journal keeps
         // every rule's output GitHub-Actions-style — it's the user's own
         // disk, and `highball runs <n> --logs` is the payoff.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profoundry-us/highball",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "Highball runner — local CI for AI coding agents: runs a repo's .highball/checks.yml rules, blocks the agent on failure, and reports runs to a Highball dashboard.",
   "keywords": [
     "ai",