@sawbuck/reins 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/reins.mjs +16 -9
- package/judge.mjs +13 -3
- package/package.json +1 -1
- package/wizard.mjs +6 -1
package/bin/reins.mjs
CHANGED
|
@@ -4,13 +4,17 @@
|
|
|
4
4
|
// the seismograph + distribution. The check guards the turns you did not steer.
|
|
5
5
|
|
|
6
6
|
import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
|
7
|
-
import { join, dirname } from "node:path";
|
|
7
|
+
import { join, dirname, relative } from "node:path";
|
|
8
8
|
import * as claudeHarness from "../scan.mjs";
|
|
9
9
|
import * as codexHarness from "../codex.mjs";
|
|
10
10
|
import { scan } from "../scan.mjs";
|
|
11
11
|
import { score, backend, harvestScore } from "../judge.mjs";
|
|
12
12
|
import { harvest, aggregateCues } from "../wizard.mjs";
|
|
13
13
|
|
|
14
|
+
// show a path relative to the cwd when it lives under it (clean output, no leaked home dir);
|
|
15
|
+
// fall back to the absolute path for anything outside the tree.
|
|
16
|
+
const rel = (p) => { const r = relative(process.cwd(), p); return r && !r.startsWith("..") ? r : p; };
|
|
17
|
+
|
|
14
18
|
const argv = process.argv.slice(2);
|
|
15
19
|
const flag = (n) => { const i = argv.indexOf(n); return i > -1 ? argv[i + 1] : null; };
|
|
16
20
|
const has = (n) => argv.includes(n);
|
|
@@ -60,14 +64,16 @@ It reads every turn-pair with a model. On the local ollama default that is free;
|
|
|
60
64
|
if (has("--json")) { process.stdout.write(JSON.stringify({ backend: b, ...rw }, null, 2) + "\n"); process.exit(0); }
|
|
61
65
|
const pct = rw.scored ? Math.round((rw.steers / rw.scored) * 100) : 0;
|
|
62
66
|
process.stdout.write(`\n reins · wizard (judge: ${b})\n`);
|
|
63
|
-
process.stdout.write(` read ${rw.pairs} turn-pairs from ${rw.sessionsRead} session(s) · ${rw.steers} steers (${pct}%)\n
|
|
67
|
+
process.stdout.write(` read ${rw.pairs} turn-pairs from ${rw.sessionsRead} session(s) · ${rw.steers} steers (${pct}%)\n`);
|
|
68
|
+
if (rw.axes && rw.axes.length) process.stdout.write(` you steer most on: ${rw.axes.slice(0, 5).map((a) => `${a.axis} (${a.count})`).join(" · ")}\n`);
|
|
69
|
+
process.stdout.write("\n");
|
|
64
70
|
const top = rw.cues.filter((c) => c.count >= 2 || c.gen > 0).slice(0, 12);
|
|
65
|
-
if (!top.length) { process.stdout.write(" no repeated cues
|
|
71
|
+
if (!top.length) { process.stdout.write(" no repeated specific cues yet (the axis fingerprint above is the durable signal) - harvest more with --sessions / --since.\n\n"); process.exit(0); }
|
|
66
72
|
process.stdout.write(" your top cues (candidates from your history, review before trusting):\n");
|
|
67
73
|
for (const c of top) {
|
|
68
74
|
process.stdout.write(` ${("[" + c.axis + "]").padEnd(13)} ${(c.cue || "(unnamed)").padEnd(34)} · ${c.sessions} sess · gen ${c.genShare} · ${c.count}x\n`);
|
|
69
75
|
}
|
|
70
|
-
process.stdout.write(`\n wrote ${out}\n these are LEADS. Promote the ones that are really you; the check still guards the rest.\n\n`);
|
|
76
|
+
process.stdout.write(`\n wrote ${rel(out)}\n these are LEADS. Promote the ones that are really you; the check still guards the rest.\n\n`);
|
|
71
77
|
process.exit(0);
|
|
72
78
|
}
|
|
73
79
|
|
|
@@ -115,9 +121,10 @@ Free on the local ollama default. Seed from your whole history once with: reins
|
|
|
115
121
|
try { mkdirSync(dirname(cout), { recursive: true }); writeFileSync(cout, JSON.stringify(rec, null, 2) + "\n"); } catch { process.stderr.write("reins cues: could not write " + cout + "\n"); }
|
|
116
122
|
if (has("--json")) { const { items: _omit, ...pub } = rec; process.stdout.write(JSON.stringify(pub, null, 2) + "\n"); process.exit(0); }
|
|
117
123
|
const top = agg.cues.filter((c) => c.count >= 2 || c.gen > 0).slice(0, 10);
|
|
118
|
-
process.stdout.write(`\n reins · cues (judge: ${b})\n ${agg.steers} steers over ${agg.scored} scored turns
|
|
119
|
-
|
|
120
|
-
process.stdout.write(`
|
|
124
|
+
process.stdout.write(`\n reins · cues (judge: ${b})\n ${agg.steers} steers over ${agg.scored} scored turns\n`);
|
|
125
|
+
if (agg.axes && agg.axes.length) process.stdout.write(` you steer most on: ${agg.axes.slice(0, 4).map((a) => `${a.axis} (${a.count})`).join(" · ")}\n`);
|
|
126
|
+
if (top.length) { process.stdout.write(` candidate cues (review before trusting):\n`); for (const c of top) process.stdout.write(` ${("[" + c.axis + "]").padEnd(12)} ${(c.cue || "(unnamed)").padEnd(32)} · ${c.count}x\n`); }
|
|
127
|
+
process.stdout.write(` wrote ${rel(cout)}\n\n`);
|
|
121
128
|
process.exit(0);
|
|
122
129
|
}
|
|
123
130
|
|
|
@@ -217,10 +224,10 @@ if (has("--share")) {
|
|
|
217
224
|
${r.seismograph}
|
|
218
225
|
· let it ride - nudge + redirect # hard pull
|
|
219
226
|
|
|
220
|
-
I steered ${r.steering} of ${r.scored} turns (${pct}%)
|
|
227
|
+
I steered ${r.steering} of ${r.scored} turns (${pct}%): ${grip}, ${texture}.
|
|
221
228
|
The ${r.scored - r.steering} I let ride, the check had my back.
|
|
222
229
|
|
|
223
|
-
|
|
230
|
+
measured with reins · sawbuck
|
|
224
231
|
`);
|
|
225
232
|
process.exit(0);
|
|
226
233
|
}
|
package/judge.mjs
CHANGED
|
@@ -95,15 +95,25 @@ async function complete(promptStr, schema) {
|
|
|
95
95
|
throw new Error("unknown SAWBUCK_JUDGE backend: " + b + " (use ollama:<model> | haiku | anthropic:<model> | openai:<model>)");
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
-
async function
|
|
99
|
-
const
|
|
98
|
+
async function ollamaCall(host, model, promptStr, format) {
|
|
99
|
+
const body = { model, prompt: promptStr, stream: false, options: { temperature: 0, num_predict: 800 } };
|
|
100
|
+
if (format) body.format = format;
|
|
100
101
|
const res = await fetch(host + "/api/generate", {
|
|
101
102
|
method: "POST", headers: { "Content-Type": "application/json" },
|
|
102
|
-
body: JSON.stringify(
|
|
103
|
+
body: JSON.stringify(body),
|
|
103
104
|
});
|
|
104
105
|
const j = await res.json();
|
|
105
106
|
return j.response || "";
|
|
106
107
|
}
|
|
108
|
+
async function ollama(model, promptStr, schema) {
|
|
109
|
+
const host = process.env.OLLAMA_HOST || "http://localhost:11434";
|
|
110
|
+
// Grammar-constrained first: clean JSON on most models (qwen2.5, llama). But reasoning models
|
|
111
|
+
// (gpt-oss, deepseek-r1) spend their budget in a hidden channel and return an EMPTY response
|
|
112
|
+
// under a format grammar - so on empty we retry free-form; the parser lifts JSON from raw text.
|
|
113
|
+
let out = await ollamaCall(host, model, promptStr, schema);
|
|
114
|
+
if (!out.trim()) out = await ollamaCall(host, model, promptStr, null);
|
|
115
|
+
return out;
|
|
116
|
+
}
|
|
107
117
|
async function anthropic(model, promptStr) {
|
|
108
118
|
const key = process.env.ANTHROPIC_API_KEY;
|
|
109
119
|
if (!key) throw new Error("ANTHROPIC_API_KEY not set (SAWBUCK_JUDGE=haiku needs it)");
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sawbuck/reins",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "Reins: measure how much you steer your AI coding agent. A per-turn steering-pressure judge over your session transcript, pluggable backend (Haiku / local ollama / any OpenAI-compatible). The LLM half of the reins steering layer; keeps sawbuck's core offline.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
package/wizard.mjs
CHANGED
|
@@ -15,12 +15,16 @@ import { pairs, resolveSessions } from "./scan.mjs";
|
|
|
15
15
|
// one-session spike never outranks a cue you give across many sessions - the study's lesson).
|
|
16
16
|
export function aggregateCues(items) {
|
|
17
17
|
const clusters = new Map();
|
|
18
|
+
const axisTally = new Map(); // the robust signal: WHICH axes you steer on (research: the fingerprint
|
|
19
|
+
// replicates across judges; exact cue text does not, so it rarely clusters)
|
|
18
20
|
let scored = 0, steers = 0, fails = 0;
|
|
19
21
|
for (const { session, result } of items) {
|
|
20
22
|
if (!result) { fails++; continue; }
|
|
21
23
|
scored++;
|
|
22
24
|
if (!result.steer) continue;
|
|
23
25
|
steers++;
|
|
26
|
+
const ax = result.axis || "other";
|
|
27
|
+
axisTally.set(ax, (axisTally.get(ax) || 0) + 1);
|
|
24
28
|
const key = (result.axis || "other") + "|" + String(result.cue || "").toLowerCase().replace(/\s+/g, " ").trim().slice(0, 40);
|
|
25
29
|
const c = clusters.get(key) || { axis: result.axis || "other", cue: result.cue || "", count: 0, gen: 0, sessions: new Set(), mag: { refine: 0, redirect: 0, "hard-stop": 0 } };
|
|
26
30
|
c.count++;
|
|
@@ -34,7 +38,8 @@ export function aggregateCues(items) {
|
|
|
34
38
|
const cues = [...clusters.values()]
|
|
35
39
|
.map((c) => ({ axis: c.axis, cue: c.cue, count: c.count, gen: c.gen, genShare: c.count ? +(c.gen / c.count).toFixed(2) : 0, sessions: c.sessions.size, magnitude: c.mag }))
|
|
36
40
|
.sort((a, b) => (b.sessions - a.sessions) || (b.gen - a.gen) || (b.count - a.count));
|
|
37
|
-
|
|
41
|
+
const axes = [...axisTally.entries()].map(([axis, count]) => ({ axis, count })).sort((a, b) => b.count - a.count);
|
|
42
|
+
return { scored, steers, fails, steerRate: scored ? +(steers / scored).toFixed(3) : 0, axes, cues };
|
|
38
43
|
}
|
|
39
44
|
|
|
40
45
|
// harvest(scoreFn, { sessions, maxPairs }) -> aggregateCues over the pairs in those session
|