@bglocation/tune-context 2.5.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +80 -0
- package/README.md +138 -22
- package/bin/tune-context +1 -1
- package/bin/tune-context.mjs +167 -51
- package/cli/archive.mjs +322 -0
- package/cli/entry-point.mjs +73 -0
- package/cli/experiment.mjs +33 -1
- package/cli/version.mjs +118 -0
- package/eval/adoption-report.mjs +11 -5
- package/eval/analyze.mjs +141 -38
- package/eval/prompt-metrics.mjs +39 -18
- package/eval/report.mjs +56 -45
- package/eval/verdict.mjs +357 -33
- package/package.json +1 -1
- package/skills/token-efficiency/SKILL.md +1 -1
- package/skills/tune-context/SKILL.md +4 -2
package/eval/verdict.mjs
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
1
2
|
// verdict — the step analyze.mjs deliberately stops before: may this run say
|
|
2
3
|
// "one arm is cheaper"? See tasks/TASK-090.
|
|
3
4
|
//
|
|
4
|
-
// Pure predicates over the TASK-089 aggregate. No disk, no clock, no stdout
|
|
5
|
-
//
|
|
5
|
+
// Pure predicates over the TASK-089 aggregate. No disk, no clock, no stdout —
|
|
6
|
+
// an aggregate in, a decision out. TASK-091 renders it as HTML; formatVerdict()
|
|
7
|
+
// renders it as text for a terminal. The CLI section at the bottom of this file
|
|
8
|
+
// (TASK-096) is the ONLY place in it that reads a file, stdin or the process,
|
|
9
|
+
// or writes anything: everything above it stays a function of its arguments.
|
|
6
10
|
//
|
|
7
11
|
// The failure this file exists to prevent is not a wrong number; it is a right
|
|
8
12
|
// number about the wrong thing. An aggregate that adds up perfectly still
|
|
@@ -35,20 +39,43 @@
|
|
|
35
39
|
// constant — it is background for asking whether the paired result points the
|
|
36
40
|
// same way, and it is never evidence (analyze.mjs keeps the two un-addable by
|
|
37
41
|
// shape; this file simply never reaches for the second one).
|
|
42
|
+
import { readFileSync } from 'node:fs';
|
|
43
|
+
import { parseArgs } from 'node:util';
|
|
44
|
+
import { endQuietlyOnClosedPipe, isEntryPoint, readStdin } from '../cli/entry-point.mjs';
|
|
45
|
+
import { meterStamp, stampProblem } from '../cli/version.mjs';
|
|
38
46
|
import { ARMS } from './analyze.mjs';
|
|
39
47
|
import { formatUsd } from './pricing.mjs';
|
|
40
48
|
|
|
41
49
|
/**
|
|
42
|
-
*
|
|
50
|
+
* An input this module refuses: an aggregate that is not one, a run journal of
|
|
51
|
+
* the wrong shape, and — in the CLI — a file that is missing or is not JSON.
|
|
52
|
+
* One class for all of them, so a CLI can answer exactly these with a message
|
|
53
|
+
* and exit 1, and let anything else go up as the bug it is.
|
|
54
|
+
*
|
|
55
|
+
* A TypeError, because the pure functions threw TypeError at this boundary
|
|
56
|
+
* before the class existed, and a caller matching on that keeps working.
|
|
57
|
+
*/
|
|
58
|
+
export class VerdictInputError extends TypeError {
|
|
59
|
+
constructor(message) {
|
|
60
|
+
super(message);
|
|
61
|
+
this.name = 'VerdictInputError';
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Frozen with the PO on 2026-07-30 (COST_EVALUATION_PROCEDURE §10, §11); the
|
|
67
|
+
* seventh gate, meter_mixed, frozen by the PO on 2026-10-15 (EPIC-010).
|
|
43
68
|
*
|
|
44
69
|
* Each entry carries its own `frozen` date next to its value, not just the
|
|
45
70
|
* table: a later change to one threshold must be visible as a change to THAT
|
|
46
71
|
* line. A threshold moved after seeing the data is a threshold fitted to the
|
|
47
72
|
* result — if it has to happen, it lands in `amended` with a new date and the
|
|
48
|
-
* admission that it happened after the run.
|
|
73
|
+
* admission that it happened after the run. A gate ADDED before the next run is
|
|
74
|
+
* not that, which is why meter_mixed carries its own date and `amended` stays
|
|
75
|
+
* empty; the top-level `frozen` stays the date the table was first frozen.
|
|
49
76
|
*/
|
|
50
77
|
export const VERDICT_THRESHOLDS = {
|
|
51
|
-
schema: 'tune-context/verdict-thresholds@
|
|
78
|
+
schema: 'tune-context/verdict-thresholds@2',
|
|
52
79
|
frozen: '2026-07-30',
|
|
53
80
|
frozen_by: 'PO',
|
|
54
81
|
source: 'COST_EVALUATION_PROCEDURE.md §10 and §11 (tasks/TASK-090)',
|
|
@@ -107,6 +134,17 @@ export const VERDICT_THRESHOLDS = {
|
|
|
107
134
|
' rejected run exceeds it. That is the point of the threshold, not a flaw in it: at four sessions there is' +
|
|
108
135
|
' nothing to spare. A broken run is repeated (§7), not absorbed.',
|
|
109
136
|
},
|
|
137
|
+
meter_mixed: {
|
|
138
|
+
requires: 'single identifiable stamp across the chain',
|
|
139
|
+
frozen: '2026-10-15',
|
|
140
|
+
note:
|
|
141
|
+
'Chosen by the PO (EPIC-010 decision 2), not by the model — the TASK-062 anti-bias rule. The meter is the whole' +
|
|
142
|
+
' chain: rows (prompt-metrics), the aggregate (analyze) and this verdict each carry the stamp of the code that' +
|
|
143
|
+
' computed them, and in the EPIC-009 series the measured tasks changed analyze and verdict themselves. So' +
|
|
144
|
+
' exactly one stamp, and one that names code: `.dirty` was computed from uncommitted changes and `+unknown`' +
|
|
145
|
+
' from a checkout git could not read — neither can be rebuilt, so neither can be checked. EPIC-009 was voided' +
|
|
146
|
+
' because its rows came from a meter nobody could name.',
|
|
147
|
+
},
|
|
110
148
|
},
|
|
111
149
|
};
|
|
112
150
|
|
|
@@ -136,28 +174,34 @@ export const VERDICT_THRESHOLDS = {
|
|
|
136
174
|
*/
|
|
137
175
|
function normalizeJournalRejects(input) {
|
|
138
176
|
if (input === undefined || input === null) return { reported: false, rows: 0, runs: [] };
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
);
|
|
149
|
-
}
|
|
150
|
-
return {
|
|
151
|
-
session: run.session ?? null,
|
|
152
|
-
task: run.task ?? null,
|
|
153
|
-
arm: run.arm ?? null,
|
|
154
|
-
prompts,
|
|
155
|
-
reason: run.reason ?? null,
|
|
156
|
-
};
|
|
157
|
-
});
|
|
177
|
+
const problem = journalShapeProblem(input);
|
|
178
|
+
if (problem !== null) throw new VerdictInputError(`verdict: ${problem}`);
|
|
179
|
+
const runs = input.map((run) => ({
|
|
180
|
+
session: run.session ?? null,
|
|
181
|
+
task: run.task ?? null,
|
|
182
|
+
arm: run.arm ?? null,
|
|
183
|
+
prompts: run.prompts,
|
|
184
|
+
reason: run.reason ?? null,
|
|
185
|
+
}));
|
|
158
186
|
return { reported: true, rows: runs.reduce((n, r) => n + r.prompts, 0), runs };
|
|
159
187
|
}
|
|
160
188
|
|
|
189
|
+
/**
|
|
190
|
+
* What is wrong with a supplied journal, or null when nothing is. A message
|
|
191
|
+
* rather than a throw, because two layers report it in their own words — the
|
|
192
|
+
* pure function above, and the CLI reading a `--rejected-runs` file — and the
|
|
193
|
+
* check itself must exist once.
|
|
194
|
+
*/
|
|
195
|
+
function journalShapeProblem(input) {
|
|
196
|
+
if (!Array.isArray(input)) return 'rejectedRuns must be an array of journal entries, or omitted to say nobody asked';
|
|
197
|
+
const i = input.findIndex((run) => !Number.isInteger(run?.prompts) || run.prompts < 0);
|
|
198
|
+
if (i === -1) return null;
|
|
199
|
+
return (
|
|
200
|
+
`rejectedRuns[${i}].prompts must be a non-negative integer — this gate counts rows, and a` +
|
|
201
|
+
' rejected run whose prompt count is a guess cannot be counted into a threshold'
|
|
202
|
+
);
|
|
203
|
+
}
|
|
204
|
+
|
|
161
205
|
/**
|
|
162
206
|
* Reject reasons that mean "a run that was TRYING to be part of the paired
|
|
163
207
|
* measurement failed" — the §7 invalidations, counted in rows because that is
|
|
@@ -282,10 +326,10 @@ function gate(id, question, passed, measured, threshold, { reason = null, missin
|
|
|
282
326
|
return { id, question, passed, measured, threshold, reason, missing };
|
|
283
327
|
}
|
|
284
328
|
|
|
285
|
-
// ─── the
|
|
329
|
+
// ─── the gates ──────────────────────────────────────────────────────────────
|
|
286
330
|
//
|
|
287
331
|
// Each one holds the verdict on its own and for its own reason. They are
|
|
288
|
-
// written as
|
|
332
|
+
// written as independent functions over the same context rather than one
|
|
289
333
|
// chain of ifs, because a gate that can only fire when the previous one passed
|
|
290
334
|
// cannot report a run's second problem — and then the run gets repeated to
|
|
291
335
|
// discover it.
|
|
@@ -553,12 +597,76 @@ function rejectedGate({ agg, arms, t, journal }) {
|
|
|
553
597
|
});
|
|
554
598
|
}
|
|
555
599
|
|
|
556
|
-
|
|
600
|
+
/**
|
|
601
|
+
* 7. More than one meter computed this chain, or one that names no code
|
|
602
|
+
* (EPIC-010, PO decision 2, frozen 2026-10-15).
|
|
603
|
+
*
|
|
604
|
+
* The stamps read are the paired pool's rows (`paired.meter_versions`), the
|
|
605
|
+
* aggregate's `analyzed_by` and this verdict's own `judged_by` — the whole
|
|
606
|
+
* chain, because in the EPIC-009 series the measured tasks changed analyze and
|
|
607
|
+
* verdict themselves, and rows from one version judged by another is the same
|
|
608
|
+
* error one floor up. Natural work never reaches the verdict, so its stamps are
|
|
609
|
+
* not read.
|
|
610
|
+
*
|
|
611
|
+
* analyze does not refuse mixed stamps (decision 2): the per-pair costs must
|
|
612
|
+
* stay visible to plan a repeat. So the refusal lives here, and it says which
|
|
613
|
+
* step carried which stamp — "two meters" alone does not say what to recompute.
|
|
614
|
+
*/
|
|
615
|
+
function meterGate({ agg, paired, judgedBy, t }) {
|
|
616
|
+
const q = 'Did one identifiable meter compute the whole chain, from the rows to this verdict?';
|
|
617
|
+
const carriers = new Map();
|
|
618
|
+
const carry = (stamp, what) => {
|
|
619
|
+
if (!carriers.has(stamp)) carriers.set(stamp, []);
|
|
620
|
+
carriers.get(stamp).push(what);
|
|
621
|
+
};
|
|
622
|
+
const rows = paired.meter_versions ?? {};
|
|
623
|
+
for (const [stamp, n] of Object.entries(rows)) carry(stamp, `rows ×${n}`);
|
|
624
|
+
// A step with no stamp is not a stamp of its own: it is a step nobody can
|
|
625
|
+
// name, and it holds the gate under that description.
|
|
626
|
+
const unstamped = [];
|
|
627
|
+
if (agg.analyzed_by) carry(agg.analyzed_by, 'analyze');
|
|
628
|
+
else unstamped.push('analyze');
|
|
629
|
+
if (judgedBy) carry(judgedBy, 'verdict');
|
|
630
|
+
else unstamped.push('verdict');
|
|
631
|
+
|
|
632
|
+
const stamps = [...carriers.keys()].sort();
|
|
633
|
+
const unidentifiable = Object.fromEntries(
|
|
634
|
+
stamps.map((stamp) => [stamp, stampProblem(stamp)]).filter(([, problem]) => problem !== null),
|
|
635
|
+
);
|
|
636
|
+
const measured = {
|
|
637
|
+
stamps,
|
|
638
|
+
carried_by: Object.fromEntries(stamps.map((stamp) => [stamp, carriers.get(stamp)])),
|
|
639
|
+
rows_by_stamp: { ...rows },
|
|
640
|
+
analyzed_by: agg.analyzed_by ?? null,
|
|
641
|
+
judged_by: judgedBy ?? null,
|
|
642
|
+
unidentifiable,
|
|
643
|
+
unstamped_steps: unstamped,
|
|
644
|
+
};
|
|
645
|
+
if (stamps.length === 1 && Object.keys(unidentifiable).length === 0 && unstamped.length === 0) {
|
|
646
|
+
return gate('meter_mixed', q, true, measured, t.meter_mixed);
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
const where = (stamp) => `${stamp} (${carriers.get(stamp).join(', ')})`;
|
|
650
|
+
const reasons = [];
|
|
651
|
+
if (stamps.length > 1) reasons.push(`the chain was computed by ${stamps.length} meters: ${stamps.map(where).join('; ')}`);
|
|
652
|
+
for (const [stamp, problem] of Object.entries(unidentifiable)) reasons.push(`${where(stamp)} names no code: ${problem}`);
|
|
653
|
+
if (unstamped.length > 0) reasons.push(`${unstamped.join(' and ')} left no stamp`);
|
|
654
|
+
return gate('meter_mixed', q, false, measured, t.meter_mixed, {
|
|
655
|
+
reason: reasons.join('; '),
|
|
656
|
+
missing:
|
|
657
|
+
'the whole chain from one meter that names its code — recompute the rows from the transcripts or from an' +
|
|
658
|
+
' archive (`tune-context experiment archive`) with the current version, or run every step with the version' +
|
|
659
|
+
' that computed the rows (`npx -p @bglocation/tune-context@<version> tune-context eval …`); a series runs' +
|
|
660
|
+
' from a release or a clean commit, never from a .dirty tree',
|
|
661
|
+
});
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
const GATES = [sampleGate, homogeneityGate, delegationGate, orderGate, unpricedGate, rejectedGate, meterGate];
|
|
557
665
|
|
|
558
666
|
/**
|
|
559
667
|
* Aggregate in, decision out.
|
|
560
668
|
*
|
|
561
|
-
* @param agg the object analyze() returned (schema tune-context/analyze@
|
|
669
|
+
* @param agg the object analyze() returned (schema tune-context/analyze@2).
|
|
562
670
|
* @param thresholds the frozen table. Injectable so a test can prove a gate
|
|
563
671
|
* READS it rather than repeating the number in code — which
|
|
564
672
|
* is the only way to tell a frozen threshold from a
|
|
@@ -568,6 +676,11 @@ const GATES = [sampleGate, homogeneityGate, delegationGate, orderGate, unpricedG
|
|
|
568
676
|
* not the same as passing `[]` — see
|
|
569
677
|
* normalizeJournalRejects(). A paired pool cannot be ruled
|
|
570
678
|
* on while it is omitted.
|
|
679
|
+
* @param judgedBy the stamp of the code running this function, from the
|
|
680
|
+
* caller — the same arrangement as report's `generatedAt`,
|
|
681
|
+
* so this stays free of ambient state. Omitted, the
|
|
682
|
+
* meter_mixed gate holds: a verdict nobody can trace to code
|
|
683
|
+
* is the thing EPIC-010 exists to stop.
|
|
571
684
|
*
|
|
572
685
|
* Throws on an input that is not an analyze aggregate: the same layer-boundary
|
|
573
686
|
* rule classifyRow() applies to unannotated rows. A shape this does not
|
|
@@ -575,9 +688,15 @@ const GATES = [sampleGate, homogeneityGate, delegationGate, orderGate, unpricedG
|
|
|
575
688
|
* rejects — i.e. as a withheld verdict, which is the one wrong answer nobody
|
|
576
689
|
* would look at twice.
|
|
577
690
|
*/
|
|
578
|
-
export function verdict(agg, { thresholds = VERDICT_THRESHOLDS, rejectedRuns } = {}) {
|
|
579
|
-
if (!agg || typeof agg !== 'object' || agg.schema !== 'tune-context/analyze@
|
|
580
|
-
|
|
691
|
+
export function verdict(agg, { thresholds = VERDICT_THRESHOLDS, rejectedRuns, judgedBy = null } = {}) {
|
|
692
|
+
if (!agg || typeof agg !== 'object' || agg.schema !== 'tune-context/analyze@2') {
|
|
693
|
+
// An @1 aggregate is refused, not upgraded: its rows carry no stamp, and the
|
|
694
|
+
// only way back to that data is through the transcripts (EPIC-010 decision 4).
|
|
695
|
+
const got = agg && typeof agg === 'object' && typeof agg.schema === 'string' ? ` — got ${agg.schema}` : '';
|
|
696
|
+
throw new VerdictInputError(
|
|
697
|
+
`verdict: expected an aggregate from analyze() (schema tune-context/analyze@2)${got}; recompute it with` +
|
|
698
|
+
' `tune-context eval analyze` from stamped rows (EPIC-010: unstamped data is not used)',
|
|
699
|
+
);
|
|
581
700
|
}
|
|
582
701
|
const paired = agg.paired;
|
|
583
702
|
const arms = paired.arms;
|
|
@@ -585,7 +704,7 @@ export function verdict(agg, { thresholds = VERDICT_THRESHOLDS, rejectedRuns } =
|
|
|
585
704
|
const effects = pairEffects(paired.pairs);
|
|
586
705
|
const journal = normalizeJournalRejects(rejectedRuns);
|
|
587
706
|
|
|
588
|
-
const ctx = { agg, paired, arms, effects, t, journal };
|
|
707
|
+
const ctx = { agg, paired, arms, effects, t, journal, judgedBy };
|
|
589
708
|
const gates = GATES.map((fn) => fn(ctx));
|
|
590
709
|
const holds = gates.filter((g) => !g.passed);
|
|
591
710
|
// Three outcomes, named, because two of them are not "cheaper arm" and
|
|
@@ -603,8 +722,11 @@ export function verdict(agg, { thresholds = VERDICT_THRESHOLDS, rejectedRuns } =
|
|
|
603
722
|
const ruled = finding !== 'withheld';
|
|
604
723
|
|
|
605
724
|
return {
|
|
606
|
-
schema: 'tune-context/verdict@
|
|
725
|
+
schema: 'tune-context/verdict@2',
|
|
607
726
|
pool: 'paired',
|
|
727
|
+
// Which code judged — read by the meter_mixed gate above, and kept here so
|
|
728
|
+
// a verdict read a year later still names it.
|
|
729
|
+
judged_by: judgedBy,
|
|
608
730
|
// The table travels with the decision: a verdict read six months later has
|
|
609
731
|
// to show which thresholds it was judged against, without anyone trusting
|
|
610
732
|
// that the file still says the same thing.
|
|
@@ -637,3 +759,205 @@ export function verdict(agg, { thresholds = VERDICT_THRESHOLDS, rejectedRuns } =
|
|
|
637
759
|
observation_only: true,
|
|
638
760
|
};
|
|
639
761
|
}
|
|
762
|
+
|
|
763
|
+
// ─── text ───────────────────────────────────────────────────────────────────
|
|
764
|
+
|
|
765
|
+
const pctText = (x) => (Number.isFinite(x) ? `${(x * 100).toFixed(1)}%` : '—');
|
|
766
|
+
|
|
767
|
+
/**
|
|
768
|
+
* Per gate: what was measured, against what. One entry per gate in GATES — a
|
|
769
|
+
* test holds the two lists equal, so a new gate cannot reach the terminal as a
|
|
770
|
+
* bare id. Kept here rather than in the gates, because a gate's job is to
|
|
771
|
+
* decide and this is only how a decision reads at 80 columns.
|
|
772
|
+
*/
|
|
773
|
+
const GATE_SUMMARIES = {
|
|
774
|
+
sample: (m, t) =>
|
|
775
|
+
`${m.n_pairs_complete} complete pair(s), sessions off ${m.sessions_per_arm.off} / on ${m.sessions_per_arm.on}; floor ${t.min_pairs}`,
|
|
776
|
+
homogeneity: (m) => `model ${m.distinct_models.join(', ') || '—'}, effort ${m.distinct_efforts.join(', ') || '—'}; must be identical`,
|
|
777
|
+
delegation: (m, t) =>
|
|
778
|
+
`spawns off ${m.spawns_by_arm.off} / on ${m.spawns_by_arm.on}` +
|
|
779
|
+
(m.direction_known ? `, cheaper arm ${m.cheaper_arm} +${m.diff}; max +${t.max_diff}` : ', no cheaper arm'),
|
|
780
|
+
order_effect: (m, t) =>
|
|
781
|
+
m.arm_effect_usd === null || m.order_effect_usd === null
|
|
782
|
+
? `no usable pair; needs |arm| >= ${t.min_arm_over_order}x |order|`
|
|
783
|
+
: `arm ${formatUsd(m.arm_effect_usd)}, order ${formatUsd(m.order_effect_usd)}, ratio ${m.ratio ?? '—'}x; needs >= ${t.min_arm_over_order}x`,
|
|
784
|
+
unpriced: (m, t) => `${m.unpriced_rows} unpriced row(s); max ${t.max_rows}`,
|
|
785
|
+
rejected: (m, t) =>
|
|
786
|
+
`${pctText(m.share)} rejected (${m.rejected_rows} of ${m.rows_in_pool} row(s)), journal ` +
|
|
787
|
+
`${m.operator_rejects === 'NOT REPORTED' ? 'NOT REPORTED' : `${m.operator_rejects} row(s)`}; max ${pctText(t.max_share)}`,
|
|
788
|
+
meter_mixed: (m) =>
|
|
789
|
+
`${m.stamps.map((stamp) => `${stamp} (${m.carried_by[stamp].join(', ')})`).join('; ') || 'no stamp'}` +
|
|
790
|
+
`${m.unstamped_steps.length > 0 ? `, unstamped: ${m.unstamped_steps.join(', ')}` : ''}; one identifiable stamp`,
|
|
791
|
+
};
|
|
792
|
+
|
|
793
|
+
/**
|
|
794
|
+
* The verdict as text, for stderr: one line per gate — passed or held, what it
|
|
795
|
+
* measured against its threshold, and why it holds — then the finding in one
|
|
796
|
+
* word. Every gate, always, in the order verdict() evaluated them, for the same
|
|
797
|
+
* reason verdict() evaluates them all: a reader who fixes one problem and
|
|
798
|
+
* reruns has paid a run to learn about the second.
|
|
799
|
+
*
|
|
800
|
+
* Pure: a verdict in, a string out.
|
|
801
|
+
*/
|
|
802
|
+
export function formatVerdict(v) {
|
|
803
|
+
const L = [`verdict: paired pool, ${v.n_pairs_used} usable pair(s) — an observation, never proof`];
|
|
804
|
+
for (const g of v.gates) {
|
|
805
|
+
const summary = GATE_SUMMARIES[g.id]?.(g.measured, g.threshold) ?? '';
|
|
806
|
+
L.push(` ${g.passed ? '✓' : '✗'} ${g.id.padEnd(12)} ${summary}${g.passed ? '' : ` — ${g.reason}`}`);
|
|
807
|
+
}
|
|
808
|
+
// Said on its own line, because it is the one hold the operator causes by
|
|
809
|
+
// omission: the flag was not passed, so nobody asked the journal.
|
|
810
|
+
const journal = v.gates.find((g) => g.id === 'rejected');
|
|
811
|
+
if (journal && !journal.passed && journal.measured.operator_rejects === 'NOT REPORTED') {
|
|
812
|
+
L.push(
|
|
813
|
+
' ! no --rejected-runs: nobody asked the run journal, so gate `rejected` holds by construction —' +
|
|
814
|
+
' pass its §7 rejections, or a file containing [] once the journal has been read',
|
|
815
|
+
);
|
|
816
|
+
}
|
|
817
|
+
L.push(
|
|
818
|
+
`finding: ${
|
|
819
|
+
v.finding === 'withheld'
|
|
820
|
+
? `withheld: ${v.holds.length} gate(s) hold — ${v.holds.map((g) => g.id).join(', ')}`
|
|
821
|
+
: v.finding === 'no-difference'
|
|
822
|
+
? 'no-difference'
|
|
823
|
+
: `cheaper-arm: ${v.cheaper_arm}`
|
|
824
|
+
}`,
|
|
825
|
+
);
|
|
826
|
+
return L.join('\n');
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
// ─── CLI ────────────────────────────────────────────────────────────────────
|
|
830
|
+
//
|
|
831
|
+
// The only I/O in this file (see the header). `eval verdict` and `eval report`
|
|
832
|
+
// both judge, so both read their input through readVerdictInput() below.
|
|
833
|
+
|
|
834
|
+
const USAGE = `
|
|
835
|
+
verdict — may this run say "one arm is cheaper"? The gates over an analyze aggregate (TASK-090).
|
|
836
|
+
|
|
837
|
+
Usage:
|
|
838
|
+
verdict [--aggregate <path>] [--rejected-runs <path>]
|
|
839
|
+
|
|
840
|
+
Options:
|
|
841
|
+
--aggregate <path> JSON written by analyze (default: read stdin)
|
|
842
|
+
--rejected-runs <path> JSON array of the run journal's §7 rejections,
|
|
843
|
+
[{ session, task, arm, prompts, reason }]. OMITTING it
|
|
844
|
+
is not the same as passing a file containing []: the
|
|
845
|
+
first says nobody asked the journal, and gate 6 holds
|
|
846
|
+
the verdict for it. Same flag, same meaning as report.
|
|
847
|
+
-h, --help Show this help
|
|
848
|
+
|
|
849
|
+
Writes the verdict as JSON to stdout — the same object report embeds in its
|
|
850
|
+
HTML — and one line per gate to stderr. Exit 0 for every verdict, a withheld one
|
|
851
|
+
included: withholding is an answer, not a failure. Exit 1 only when the input
|
|
852
|
+
cannot be read.
|
|
853
|
+
`.trim();
|
|
854
|
+
|
|
855
|
+
/**
|
|
856
|
+
* A file the CLI was pointed at, as text — or the VerdictInputError naming the
|
|
857
|
+
* flag: missing, a directory, unreadable. Asking existsSync first let a
|
|
858
|
+
* directory through to readFileSync, and up as an EISDIR stacktrace.
|
|
859
|
+
*/
|
|
860
|
+
function readInputFile(path, flag) {
|
|
861
|
+
try {
|
|
862
|
+
return readFileSync(path, 'utf8');
|
|
863
|
+
} catch (e) {
|
|
864
|
+
throw new VerdictInputError(e.code === 'ENOENT' ? `no file at ${path} (${flag})` : `cannot read ${path} (${flag}): ${e.message}`);
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
function parseJson(text, what) {
|
|
869
|
+
try {
|
|
870
|
+
return JSON.parse(text);
|
|
871
|
+
} catch (e) {
|
|
872
|
+
throw new VerdictInputError(`${what} is not JSON: ${e.message}`);
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
/**
|
|
877
|
+
* The aggregate and the run journal, read and checked in ONE place for both
|
|
878
|
+
* steps that judge. Two copies would drift, and the drift nobody notices is the
|
|
879
|
+
* report accepting an input the verdict refuses.
|
|
880
|
+
*
|
|
881
|
+
* @param aggregate a path, or undefined to read stdin
|
|
882
|
+
* @param rejectedRuns a path, or undefined — and undefined STAYS undefined. It
|
|
883
|
+
* means nobody asked the journal, which gate 6 holds on;
|
|
884
|
+
* defaulting it to [] here would pass that gate on a share
|
|
885
|
+
* whose denominator nobody supplied.
|
|
886
|
+
* @returns {Promise<{agg: object, rejectedRuns: object[]|undefined}>}
|
|
887
|
+
* @throws {VerdictInputError} a file that is missing, a directory or
|
|
888
|
+
* unreadable, a file that is not JSON, or a journal of the wrong shape —
|
|
889
|
+
* the message names the flag and the path.
|
|
890
|
+
* Whether the aggregate IS an aggregate is verdict()'s check, the same
|
|
891
|
+
* one a module caller gets.
|
|
892
|
+
*/
|
|
893
|
+
export async function readVerdictInput({ aggregate, rejectedRuns }) {
|
|
894
|
+
// Both files before stdin: a mistyped --rejected-runs is reported at once,
|
|
895
|
+
// not after the aggregate has been piped in.
|
|
896
|
+
const aggText = aggregate === undefined ? undefined : readInputFile(aggregate, '--aggregate');
|
|
897
|
+
const runsText = rejectedRuns === undefined ? undefined : readInputFile(rejectedRuns, '--rejected-runs');
|
|
898
|
+
const agg =
|
|
899
|
+
aggText === undefined ? parseJson(await readStdin(), 'the aggregate on stdin') : parseJson(aggText, `--aggregate ${aggregate}`);
|
|
900
|
+
if (runsText === undefined) return { agg, rejectedRuns: undefined };
|
|
901
|
+
const runs = parseJson(runsText, `--rejected-runs ${rejectedRuns}`);
|
|
902
|
+
const problem = journalShapeProblem(runs);
|
|
903
|
+
if (problem !== null) throw new VerdictInputError(`--rejected-runs ${rejectedRuns}: ${problem}`);
|
|
904
|
+
return { agg, rejectedRuns: runs };
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
/**
|
|
908
|
+
* The line an input error prints: the step's name, then the message — unless
|
|
909
|
+
* the message already names this step (verdict()'s own errors start with
|
|
910
|
+
* `verdict:`), so `eval verdict` never stutters while `eval report` still says
|
|
911
|
+
* which layer refused.
|
|
912
|
+
*/
|
|
913
|
+
export function inputErrorLine(step, error) {
|
|
914
|
+
return error.message.startsWith(`${step}: `) ? error.message : `${step}: ${error.message}`;
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
/**
|
|
918
|
+
* The CLI. `args` is everything after the script name, or after
|
|
919
|
+
* `tune-context eval verdict` — one parser behind both doors (EPIC-010).
|
|
920
|
+
*/
|
|
921
|
+
export async function main(args = process.argv.slice(2)) {
|
|
922
|
+
endQuietlyOnClosedPipe();
|
|
923
|
+
let values;
|
|
924
|
+
try {
|
|
925
|
+
({ values } = parseArgs({
|
|
926
|
+
args,
|
|
927
|
+
options: {
|
|
928
|
+
aggregate: { type: 'string' },
|
|
929
|
+
'rejected-runs': { type: 'string' },
|
|
930
|
+
help: { type: 'boolean', short: 'h', default: false },
|
|
931
|
+
},
|
|
932
|
+
strict: true,
|
|
933
|
+
}));
|
|
934
|
+
} catch (e) {
|
|
935
|
+
process.stderr.write(`verdict: ${e.message}\n`);
|
|
936
|
+
process.exitCode = 1;
|
|
937
|
+
return;
|
|
938
|
+
}
|
|
939
|
+
if (values.help) {
|
|
940
|
+
process.stdout.write(`${USAGE}\n`);
|
|
941
|
+
return;
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
let v;
|
|
945
|
+
try {
|
|
946
|
+
const { agg, rejectedRuns } = await readVerdictInput({ aggregate: values.aggregate, rejectedRuns: values['rejected-runs'] });
|
|
947
|
+
v = verdict(agg, { rejectedRuns, judgedBy: meterStamp() });
|
|
948
|
+
} catch (e) {
|
|
949
|
+
// Exactly the input boundary. A TypeError from inside a gate is a bug and
|
|
950
|
+
// keeps its stacktrace.
|
|
951
|
+
if (!(e instanceof VerdictInputError)) throw e;
|
|
952
|
+
process.stderr.write(`${inputErrorLine('verdict', e)}\n`);
|
|
953
|
+
process.exitCode = 1;
|
|
954
|
+
return;
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
process.stdout.write(`${JSON.stringify(v, null, 2)}\n`);
|
|
958
|
+
process.stderr.write(`${formatVerdict(v)}\n`);
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
if (isEntryPoint(import.meta.url)) {
|
|
962
|
+
await main();
|
|
963
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bglocation/tune-context",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.0.0",
|
|
4
4
|
"description": "Context-efficiency configurator for Claude Code — token-saving doctrine packaged as skills, plus a tune-context configurator that scaffolds a lean CLAUDE.md, doctrine skills, subagents, settings, hooks and MCP wiring.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -45,7 +45,7 @@ comments in the domain vocabulary someone would *search* with — better contrac
|
|
|
45
45
|
## 6. Measure, don't believe
|
|
46
46
|
If the tool logs usage (`.rag/usage.jsonl` → `rag-usage`), review it
|
|
47
47
|
periodically: is semantic search actually replacing grep? Promote or drop levers
|
|
48
|
-
on data, not faith. If `tune-context` is set up, `
|
|
48
|
+
on data, not faith. If `tune-context` is set up, `tune-context eval adoption-report`
|
|
49
49
|
covers the rest of this doctrine the same way — search lever, skills, subagents,
|
|
50
50
|
re-read churn — as a behavioral proxy (tool calls), not token cost.
|
|
51
51
|
|
|
@@ -156,8 +156,10 @@ covering the detected build/test/lint commands.
|
|
|
156
156
|
they fire their own event carrying `agent_type`). It appends one line per
|
|
157
157
|
event to `~/.claude/tune-context/adoption.jsonl` by default — which lever the
|
|
158
158
|
model reached for (RAG vs grep, Skill, subagent, re-reads). Read it with
|
|
159
|
-
`eval
|
|
160
|
-
|
|
159
|
+
`tune-context eval adoption-report` (module `eval/adoption-report.mjs`; the
|
|
160
|
+
`rag-usage` analog). Behavioral proxy only:
|
|
161
|
+
no token cost (that is `tune-context eval prompt-metrics`, from the session
|
|
162
|
+
transcripts; OTEL `claude_code.cost.usage` is Anthropic's own figure) and no
|
|
161
163
|
caveman (an output style, not a tool call). Doctrine-as-method (narrow reads,
|
|
162
164
|
contracts-first edits) is likewise not `Skill`-call-visible — the verdict
|
|
163
165
|
reports zero `Skill` calls as a neutral note, not "unused", and points at the
|