@sawbuck/reins 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,66 @@
-# Temporary Holding Version
+# reins (engine)
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+The engine behind reins: measure how much you steer your AI coding agent. It reads
+a Claude Code session transcript, scores each of your turns for **steering
+pressure** (0 passive → 1 light refine → 2 redirect → 3 hard yank) with a small LLM,
+and prints a seismograph of the session plus the distribution.
+
+This is the LLM half of reins. The deterministic brake-slam counter (`sawbuck yank`)
+lives in sawbuck's offline core; the continuous pressure judge needs a model, so it
+lives here, keeping sawbuck's core model-free. (The reins *plugin* — the Claude Code
+surface — lives alongside this in `plugins/sawbuck-reins/`.)
+
+```
+$ reins
+  reins · steering this session   (judge: ollama:qwen2.5:7b)
+  ..--+-+---+---.--.--..+--+.--
+    . passive   - refine   + redirect   # yank
+
+  you steered 22 of 29 turns (76%)   ·   mean pressure 0.93/3
+  17 light refine · 5 redirect · 0 hard yank · 7 passive
+  the check guards the 7 turns you didn't steer - a miss there is the risk.
+```
+
+## Backend (SAWBUCK_JUDGE)
+
+The judge model is a socket. Set `SAWBUCK_JUDGE` (default `ollama:qwen2.5:7b`):
+
+- `ollama:<model>` — local, free, private (nothing leaves your box). `OLLAMA_HOST`
+  overrides the endpoint. Benchmarked default `qwen2.5:7b`: fastest, smallest, and
+  tied-best accuracy against an opus-labeled gold — model size did not help.
+- `haiku` — Anthropic `claude-haiku-5-5` (needs `ANTHROPIC_API_KEY`); zero VRAM,
+  about $0.00005 per turn.
+- `anthropic:<model>` — any Anthropic model.
+- `openai:<model>` — any OpenAI-compatible endpoint (`OPENAI_BASE_URL`,
+  `OPENAI_API_KEY`).
+
+Output is constrained to a JSON schema where the backend supports it (ollama
+structured outputs), so a thinking model can't break the parse.
+
+## Live on the statusline
+
+`--write [path]` writes the summary JSON (default `.sawbuck/steer.json`). The
+offline `sawbuck statusline` reads that file and shows `🤠 steer 76%` next to its
+own segment — the LLM work happens here, sawbuck's core stays model-free (it only
+reads a file). Keep it fresh by running `reins --write` on a cadence (a `/loop`, a
+watch, or a post-turn hook):
+
+```
+reins --write          # writes .sawbuck/steer.json; the statusline picks it up
+```
+
+## Usage
+
+```
+reins [--session <path.jsonl>] [--judge <backend>] [-n N] [--json] [--write [path]]
+```
+
+Defaults to your newest Claude Code session. `-n` scores only the most recent N
+turns. `--json` is machine-readable. Zero-dep, Node 18+.
+
+## Honest limits
+
+A single-turn judge: it sees the agent action and your reply, not the whole thread.
+Models agree on the *shape* (pressure within one level ~95% of the time) more than
+on exact per-turn scores, so read the seismograph and the distribution, not any one
+cell.

package/bin/reins.mjs

@@ -0,0 +1,127 @@
+#!/usr/bin/env node
+// reins: how much did you steer your agent this session? Reads your Claude Code
+// transcript, scores each turn's steering pressure via SAWBUCK_JUDGE, and prints
+// the seismograph + distribution. The check guards the turns you did not steer.
+
+import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { scan, resolveSessionPath, pairs } from "../scan.mjs";
+import { score, backend } from "../judge.mjs";
+
+const argv = process.argv.slice(2);
+const flag = (n) => { const i = argv.indexOf(n); return i > -1 ? argv[i + 1] : null; };
+const has = (n) => argv.includes(n);
+
+if (has("--help") || has("-h")) {
+  process.stdout.write(`reins: measure how much you steer your agent this session
+
+Usage: reins [--session <path.jsonl>] [--judge <backend>] [-n N] [--json] [--write [path]] [--share [--label <term>]]
+
+  --session <path>   a Claude Code transcript (default: your newest session)
+  --judge <backend>  override SAWBUCK_JUDGE: ollama:<model> | haiku | anthropic:<model> | openai:<model>
+  -n <N>             score only the most recent N turns
+  --json             machine-readable output
+  --write [path]     also write the summary JSON (default .sawbuck/steer.json) for the
+                     offline sawbuck statusline to read - run it on a cadence for a live readout
+  --share            print a paste-ready card for posting (LinkedIn / X / Reddit)
+  --label <term>     the word for the stat on the share card (default "steer rate";
+                     e.g. "yank rate" for a playful channel - keep it neutral for LinkedIn)
+
+Backend is SAWBUCK_JUDGE (default ollama:qwen2.5:7b). Local = free + private.
+`);
+  process.exit(0);
+}
+
+if (flag("--judge")) process.env.SAWBUCK_JUDGE = flag("--judge");
+const sess = resolveSessionPath(flag("--session"));
+if (!sess) { process.stderr.write("reins: no session transcript found (pass --session <path.jsonl>)\n"); process.exit(2); }
+
+let text = "";
+try { text = readFileSync(sess, "utf8"); } catch { process.stderr.write("reins: cannot read " + sess + "\n"); process.exit(2); }
+
+// --append: incremental, BOUNDED per-turn scoring for the live watcher. Reads the
+// prior tally from .sawbuck/steer.json, scores only the NEW turn-pairs since the last
+// run, merges, rewrites. One new pair per turn - not a full re-scan (which would be
+// quadratic over a session). The sawbuck statusline reads the same steer.json offline.
+if (has("--append")) {
+  const out = join(process.cwd(), ".sawbuck", "steer.json");
+  let prior = { lastPairs: 0, scored: 0, steering: 0, wsum: 0, distribution: { 0: 0, 1: 0, 2: 0, 3: 0 } };
+  try {
+    const p = JSON.parse(readFileSync(out, "utf8"));
+    if (p && p.lastPairs != null) prior = { lastPairs: p.lastPairs, scored: p.scored || 0, steering: p.steering || 0, wsum: p._wsum || 0, distribution: p.distribution || prior.distribution };
+  } catch { /* no prior tally = fresh */ }
+  const all = pairs(text);
+  let added = 0;
+  for (const { prior: a, user: u } of all.slice(prior.lastPairs)) {
+    let s = null; try { s = await score(a, u); } catch {}
+    if (!s) continue;
+    prior.distribution[s.pressure] = (prior.distribution[s.pressure] || 0) + 1;
+    prior.scored++; prior.wsum += s.pressure; added++;
+    if (s.pressure >= 1) prior.steering++;
+  }
+  const rate = prior.scored ? prior.steering / prior.scored : 0;
+  const rec = { at: new Date().toISOString(), backend: backend(), lastPairs: all.length, turns: all.length, scored: prior.scored, steering: prior.steering, steeringRate: rate, meanPressure: prior.scored ? prior.wsum / prior.scored : 0, distribution: prior.distribution, _wsum: prior.wsum };
+  try { mkdirSync(dirname(out), { recursive: true }); writeFileSync(out, JSON.stringify(rec) + "\n"); } catch { process.stderr.write("reins: could not write " + out + "\n"); }
+  if (has("--json")) process.stdout.write(JSON.stringify(rec) + "\n");
+  else process.stdout.write(`reins: +${added} new turn(s), steer rate ${Math.round(rate * 100)}% over ${prior.scored} turns\n`);
+  process.exit(0);
+}
+
+const n = parseInt(flag("-n"), 10) || 0;
+const r = await scan(text, score, { limit: n });
+
+// --write [path]: persist the summary so an OFFLINE reader (the sawbuck statusline)
+// can display the live steering number without ever calling a model itself.
+if (has("--write")) {
+  const wf = flag("--write");
+  const out = wf && !wf.startsWith("--") ? wf : join(process.cwd(), ".sawbuck", "steer.json");
+  try {
+    mkdirSync(dirname(out), { recursive: true });
+    writeFileSync(out, JSON.stringify({ at: new Date().toISOString(), backend: backend(), turns: r.turns, scored: r.scored, steering: r.steering, steeringRate: r.steeringRate, meanPressure: r.meanPressure, distribution: r.distribution }) + "\n");
+  } catch { process.stderr.write("reins: could not write " + out + "\n"); }
+}
+
+if (has("--json")) { process.stdout.write(JSON.stringify({ backend: backend(), ...r }) + "\n"); process.exit(0); }
+
+// --share: a tasteful, paste-anywhere card (plain text works on LinkedIn / X / Reddit).
+// --label swaps the stat's word; default stays neutral ("steer rate") so it's safe for a
+// professional feed, e.g. --label "yank rate" for a playful channel.
+if (has("--share")) {
+  if (r.scored === 0) { process.stdout.write("reins: nothing to share yet (no turns scored).\n"); process.exit(0); }
+  const lf = flag("--label");
+  const label = lf && !lf.startsWith("--") ? lf : "steer rate";
+  const pct = Math.round(r.steeringRate * 100);
+  const d = r.distribution;
+  const texture = d[1] >= d[2] + d[3] ? "mostly light refinements" : d[3] > 0 ? "a few hard pulls" : "real redirects";
+  const grip = pct >= 60 ? "hands-on" : pct >= 30 ? "a steady hand" : "a loose rein";
+  process.stdout.write(
+`my ${label} this session 🐴
+
+  ${r.seismograph}
+  · let it ride   - nudge   + redirect   # hard pull
+
+I steered ${r.steering} of ${r.scored} turns (${pct}%) — ${grip}, ${texture}.
+The ${r.scored - r.steering} I let ride, the check had my back.
+
+— measured with reins · sawbuck
+`);
+  process.exit(0);
+}
+
+const pct = Math.round(r.steeringRate * 100);
+const d = r.distribution;
+process.stdout.write(`\n  reins · steering this session   (judge: ${backend()})\n`);
+if (r.scored === 0) {
+  process.stdout.write(`  no turns scored${r.fails ? ` (${r.fails} judge errors - is the backend reachable?)` : ""}.\n\n`);
+  process.exit(r.fails ? 2 : 0);
+}
+process.stdout.write(`  ${r.seismograph}\n`);
+process.stdout.write(`    . passive   - refine   + redirect   # yank\n\n`);
+process.stdout.write(`  you steered ${r.steering} of ${r.scored} turns (${pct}%)   ·   mean pressure ${r.meanPressure.toFixed(2)}/3\n`);
+process.stdout.write(`  ${d[1]} light refine · ${d[2]} redirect · ${d[3]} hard yank · ${d[0]} passive${r.fails ? ` · ${r.fails} skipped` : ""}\n`);
+process.stdout.write(`  the check guards the ${r.scored - r.steering} turns you didn't steer - a miss there is the risk.\n`);
+if (r.recent.length) {
+  process.stdout.write(`\n  recent steers:\n`);
+  for (const s of r.recent) process.stdout.write(`    ${["", "-", "+", "#"][s.pressure]}${s.pressure} ${(s.axis || "").padEnd(12)} ${s.nudge}\n`);
+}
+process.stdout.write("\n");

package/judge.mjs

@@ -0,0 +1,104 @@
+// judge.mjs - the steering-PRESSURE judge's backend socket.
+//
+// score(prior, user) -> {pressure 0-3, axis, nudge}, via the backend SAWBUCK_JUDGE
+// selects. This file does network/LLM calls ON PURPOSE - it is a STANDALONE tool,
+// never imported into sawbuck's offline core (which is stabilizer-enforced
+// model-free). The deterministic yank meter lives in sawbuck; this LLM judge lives
+// here, the way steering lives in the harness layer, not the git core.
+//
+// Backends (SAWBUCK_JUDGE):
+//   ollama:<model>     local, free, private - the default (ollama:qwen2.5:7b),
+//                      "put your GPU to work"; OLLAMA_HOST overrides the endpoint.
+//   haiku              Anthropic claude-haiku-5-5 (needs ANTHROPIC_API_KEY); cheap
+//                      hosted, zero VRAM (~$0.00005/turn).
+//   anthropic:<model>  any Anthropic model.
+//   openai:<model>     any OpenAI-compatible endpoint (OPENAI_BASE_URL + OPENAI_API_KEY).
+//
+// Benchmarked default = qwen2.5:7b: tied-best accuracy vs an opus gold, fastest
+// (0.2s/turn), smallest, 0 parse-fails; model size did not help (see the bench).
+
+// JSON Schema that constrains the model's output (ollama structured outputs). This
+// is the robust fix for valid JSON - a grammar, not a prayer - so no backend
+// parse-fails the way free-form prompting did (qwen3's <think> truncation, etc.).
+const PRESSURE_SCHEMA = {
+  type: "object",
+  properties: {
+    pressure: { type: "integer", enum: [0, 1, 2, 3] },
+    axis: { type: "string" },
+    nudge: { type: "string" },
+  },
+  required: ["pressure", "axis", "nudge"],
+};
+
+const PROMPT = (a, u) =>
+`You rate how much a developer's message STEERS an AI coding agent's current work.
+Output ONLY one JSON object: {"pressure":0-3,"axis":"one-word","nudge":"<=8 words"}
+pressure: 0 passive (approval like "yes/go/do it", a pure question, or a brand-new unrelated task; NOT steering current work). 1 light refine (small adjustment to the SAME work: tone, length, scope, emphasis, "a bit more X", "also Y"). 2 redirect (change the approach/direction of current work). 3 hard-stop ("no/stop/undo/revert", reject and restart).
+AGENT just did: ${a}
+USER replied: ${u}
+JSON:`;
+
+// lenient parse: strip reasoning-model <think> blocks, take the last JSON object.
+// (With a constrained-output backend the response is already clean; this is a net.)
+export function parse(txt) {
+  txt = String(txt).replace(/<think>[\s\S]*?<\/think>/g, "");
+  const cands = txt.match(/\{[^{}]*\}/g) || [];
+  for (const c of cands.reverse()) {
+    try {
+      const j = JSON.parse(c);
+      if (j && j.pressure != null) {
+        return {
+          pressure: Math.max(0, Math.min(3, parseInt(j.pressure, 10) || 0)),
+          axis: String(j.axis || "").slice(0, 16),
+          nudge: String(j.nudge || "").slice(0, 48),
+        };
+      }
+    } catch { /* keep scanning */ }
+  }
+  return null;
+}
+
+async function ollama(model, a, u) {
+  const host = process.env.OLLAMA_HOST || "http://localhost:11434";
+  const res = await fetch(host + "/api/generate", {
+    method: "POST", headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ model, prompt: PROMPT(a, u), stream: false, format: PRESSURE_SCHEMA, options: { temperature: 0, num_predict: 800 } }),
+  });
+  const j = await res.json();
+  return parse(j.response || "");
+}
+
+async function anthropic(model, a, u) {
+  const key = process.env.ANTHROPIC_API_KEY;
+  if (!key) throw new Error("ANTHROPIC_API_KEY not set (SAWBUCK_JUDGE=haiku needs it)");
+  const res = await fetch("https://api.anthropic.com/v1/messages", {
+    method: "POST", headers: { "Content-Type": "application/json", "x-api-key": key, "anthropic-version": "2023-06-01" },
+    body: JSON.stringify({ model, max_tokens: 200, messages: [{ role: "user", content: PROMPT(a, u) }] }),
+  });
+  const j = await res.json();
+  const t = (j.content || []).filter((b) => b.type === "text").map((b) => b.text).join("");
+  return parse(t);
+}
+
+async function openai(model, a, u) {
+  const base = process.env.OPENAI_BASE_URL || "https://api.openai.com/v1";
+  const key = process.env.OPENAI_API_KEY;
+  if (!key) throw new Error("OPENAI_API_KEY not set");
+  const res = await fetch(base.replace(/\/$/, "") + "/chat/completions", {
+    method: "POST", headers: { "Content-Type": "application/json", Authorization: "Bearer " + key },
+    body: JSON.stringify({ model, temperature: 0, max_tokens: 200, response_format: { type: "json_object" }, messages: [{ role: "user", content: PROMPT(a, u) }] }),
+  });
+  const j = await res.json();
+  return parse(j.choices?.[0]?.message?.content || "");
+}
+
+export function backend() { return process.env.SAWBUCK_JUDGE || "ollama:qwen2.5:7b"; }
+
+export async function score(prior, user) {
+  const b = backend();
+  if (b === "haiku") return anthropic("claude-haiku-5-5", prior, user);
+  if (b.startsWith("anthropic:")) return anthropic(b.slice(10), prior, user);
+  if (b.startsWith("ollama:")) return ollama(b.slice(7), prior, user);
+  if (b.startsWith("openai:")) return openai(b.slice(7), prior, user);
+  throw new Error("unknown SAWBUCK_JUDGE backend: " + b + " (use ollama:<model> | haiku | anthropic:<model> | openai:<model>)");
+}

package/package.json

@@ -1,6 +1,19 @@
 {
   "name": "@sawbuck/reins",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Reins: measure how much you steer your AI coding agent. A per-turn steering-pressure judge over your session transcript, pluggable backend (Haiku / local ollama / any OpenAI-compatible). The LLM half of the reins steering layer; keeps sawbuck's core offline.",
+  "type": "module",
+  "license": "MIT",
+  "bin": {
+    "reins": "bin/reins.mjs"
+  },
+  "scripts": {
+    "test": "node --test"
+  },
+  "files": [
+    "bin/reins.mjs",
+    "judge.mjs",
+    "scan.mjs",
+    "README.md"
+  ]
+}

package/scan.mjs

@@ -0,0 +1,103 @@
+// scan.mjs - turn a Claude Code session transcript into a steering-pressure
+// reading: score each of your turns (given the agent action it followed) and
+// aggregate into a distribution + a seismograph. The scorer is injected so the
+// aggregation is testable without a live model.
+
+import { readFileSync, readdirSync, statSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+
+function humanText(o) {
+  if (!o || o.type !== "user" || o.isMeta) return null;
+  if (o.userType && o.userType !== "external") return null;
+  const c = o.message && o.message.content;
+  if (typeof c === "string") return c;
+  if (Array.isArray(c)) {
+    if (c.some((b) => b && b.type === "tool_result")) return null;
+    const t = c.filter((b) => b && b.type === "text").map((b) => b.text || "").join(" ");
+    return t || null;
+  }
+  return null;
+}
+function assistantText(o) {
+  const c = o.message && o.message.content;
+  if (Array.isArray(c)) return c.filter((b) => b && b.type === "text").map((b) => b.text || "").join(" ").trim();
+  if (typeof c === "string") return c.trim();
+  return "";
+}
+const noise = (t) => {
+  const s = t.trim();
+  if (s.includes("<task-notification>") || s.startsWith("<task-id>")) return true;
+  if (s.replace(/\[Image #\d+\]/g, "").trim().length < 3) return true;
+  return false;
+};
+
+// Extract the (prior-agent, user-turn) pairs to score: every non-opener human turn
+// that followed an agent action.
+export function pairs(text) {
+  const out = [];
+  let prev = "", seen = false;
+  for (const line of String(text).split("\n")) {
+    if (!line.trim()) continue;
+    let o; try { o = JSON.parse(line); } catch { continue; }
+    if (o.type === "assistant") { prev = assistantText(o); continue; }
+    const t = humanText(o);
+    if (t === null || noise(t)) continue;
+    if (!seen) { seen = true; continue; }       // opener = a request, not a steer
+    if (prev) out.push({ prior: prev.slice(0, 600), user: t.replace(/\s+/g, " ").slice(0, 800) });
+    prev = "";
+  }
+  return out;
+}
+
+const GLYPH = { 0: ".", 1: "-", 2: "+", 3: "#" };
+
+// scan(text, scoreFn) -> aggregate. scoreFn(prior, user) -> {pressure, axis, nudge} | null.
+export async function scan(text, scoreFn, { limit = 0 } = {}) {
+  let ps = pairs(text);
+  if (limit > 0 && ps.length > limit) ps = ps.slice(-limit);   // most recent N
+  const dist = { 0: 0, 1: 0, 2: 0, 3: 0 };
+  let scored = 0, fails = 0, wsum = 0, spark = "";
+  const recent = [];
+  for (const { prior, user } of ps) {
+    let r = null;
+    try { r = await scoreFn(prior, user); } catch { r = null; }
+    if (!r) { fails++; spark += "?"; continue; }
+    dist[r.pressure]++; scored++; wsum += r.pressure; spark += GLYPH[r.pressure];
+    if (r.pressure >= 1) recent.push({ pressure: r.pressure, axis: r.axis, nudge: r.nudge, user: user.slice(0, 60) });
+  }
+  const turns = ps.length;
+  const steering = dist[1] + dist[2] + dist[3];
+  return {
+    turns, scored, fails,
+    distribution: dist,
+    steering, steeringRate: scored ? steering / scored : 0,
+    meanPressure: scored ? wsum / scored : 0,
+    seismograph: spark,
+    recent: recent.slice(-6),
+  };
+}
+
+// Resolve the newest session transcript for a cwd (Claude Code: ~/.claude/projects/
+// <encoded-cwd>/<id>.jsonl). Best-effort; prefer the cwd's dir, else newest overall.
+export function resolveSessionPath(explicit, cwd = process.cwd()) {
+  if (explicit) return existsSync(explicit) ? explicit : null;
+  const root = join(homedir(), ".claude", "projects");
+  if (!existsSync(root)) return null;
+  const key = cwd.replace(/[^A-Za-z0-9]/g, "-");
+  let dirs = [];
+  try { dirs = readdirSync(root).map((d) => join(root, d)).filter((p) => { try { return statSync(p).isDirectory(); } catch { return false; } }); }
+  catch { return null; }
+  const matched = dirs.filter((p) => p.endsWith("/" + key));
+  const pool = matched.length ? matched : dirs;
+  let best = null, bestT = 0;
+  for (const full of pool) {
+    let files = []; try { files = readdirSync(full); } catch { continue; }
+    for (const f of files) {
+      if (!f.endsWith(".jsonl")) continue;
+      const p = join(full, f);
+      try { const st = statSync(p); if (st.mtimeMs > bestT) { bestT = st.mtimeMs; best = p; } } catch {}
+    }
+  }
+  return best;
+}