@skill-harness/core 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +1 -2
  2. package/dist/adapters/types.d.ts +0 -12
  3. package/dist/capture-trace-types.d.ts +0 -96
  4. package/dist/capture-trace-types.js +0 -4
  5. package/dist/command-cost.d.ts +1 -1
  6. package/dist/command-cost.js +0 -5
  7. package/dist/execution-trace.js +1 -1
  8. package/dist/index.d.ts +3 -23
  9. package/dist/index.js +3 -23
  10. package/dist/instruction-coverage.d.ts +0 -7
  11. package/dist/instruction-coverage.js +5 -7
  12. package/dist/journal.d.ts +0 -8
  13. package/dist/metrics.d.ts +18 -0
  14. package/dist/metrics.js +28 -0
  15. package/dist/redaction.d.ts +6 -0
  16. package/dist/redaction.js +55 -0
  17. package/dist/regate.js +6 -71
  18. package/dist/regrade.d.ts +1 -3
  19. package/dist/regrade.js +1 -21
  20. package/dist/report.d.ts +1 -15
  21. package/dist/report.js +1 -13
  22. package/dist/reps.d.ts +1 -2
  23. package/dist/reps.js +0 -2
  24. package/dist/results.d.ts +0 -9
  25. package/dist/results.js +2 -19
  26. package/dist/run.js +11 -72
  27. package/dist/seeded.d.ts +1 -7
  28. package/dist/seeded.js +5 -11
  29. package/dist/sources.js +4 -9
  30. package/dist/spec.d.ts +0 -13
  31. package/dist/spec.js +6 -48
  32. package/dist/trajectory-events.d.ts +58 -0
  33. package/dist/trajectory-events.js +119 -0
  34. package/package.json +1 -1
  35. package/schemas/results-v2.schema.json +376 -75
  36. package/schemas/specification-v1.schema.json +128 -174
  37. package/dist/adjudication.d.ts +0 -219
  38. package/dist/adjudication.js +0 -422
  39. package/dist/adoption.d.ts +0 -128
  40. package/dist/adoption.js +0 -102
  41. package/dist/affected.d.ts +0 -88
  42. package/dist/affected.js +0 -222
  43. package/dist/capture.d.ts +0 -193
  44. package/dist/capture.js +0 -344
  45. package/dist/comparison.d.ts +0 -97
  46. package/dist/comparison.js +0 -189
  47. package/dist/factory-calibration.d.ts +0 -63
  48. package/dist/factory-calibration.js +0 -115
  49. package/dist/intervention-results.d.ts +0 -6
  50. package/dist/intervention-results.js +0 -52
  51. package/dist/intervention.d.ts +0 -123
  52. package/dist/intervention.js +0 -218
  53. package/dist/investigation.d.ts +0 -94
  54. package/dist/investigation.js +0 -148
  55. package/dist/judge-agreement.d.ts +0 -19
  56. package/dist/judge-agreement.js +0 -31
  57. package/dist/qualification-capture.d.ts +0 -44
  58. package/dist/qualification-capture.js +0 -163
  59. package/dist/qualification-config.d.ts +0 -233
  60. package/dist/qualification-config.js +0 -694
  61. package/dist/qualification-lock.d.ts +0 -15
  62. package/dist/qualification-lock.js +0 -87
  63. package/dist/qualification-oauth-directory.d.ts +0 -79
  64. package/dist/qualification-oauth-directory.js +0 -444
  65. package/dist/qualification-panel-store.d.ts +0 -14
  66. package/dist/qualification-panel-store.js +0 -188
  67. package/dist/qualification-panels.d.ts +0 -99
  68. package/dist/qualification-panels.js +0 -91
  69. package/dist/qualification-process.d.ts +0 -13
  70. package/dist/qualification-process.js +0 -190
  71. package/dist/qualification-runner.d.ts +0 -203
  72. package/dist/qualification-runner.js +0 -2037
  73. package/dist/qualification-store.d.ts +0 -187
  74. package/dist/qualification-store.js +0 -833
  75. package/dist/qualification-validation.d.ts +0 -12
  76. package/dist/qualification-validation.js +0 -8
  77. package/dist/screen.d.ts +0 -31
  78. package/dist/screen.js +0 -105
  79. package/dist/trajectory-gates.d.ts +0 -137
  80. package/dist/trajectory-gates.js +0 -576
  81. package/dist/trusted-host-supervision.d.ts +0 -25
  82. package/dist/trusted-host-supervision.js +0 -94
  83. package/dist/work-capture-schema.d.ts +0 -38
  84. package/dist/work-capture-schema.js +0 -33
  85. package/dist/work-capture.d.ts +0 -43
  86. package/dist/work-capture.js +0 -81
  87. package/dist/work-signals.d.ts +0 -55
  88. package/dist/work-signals.js +0 -71
  89. package/schemas/qualification-cell-result-v1.schema.json +0 -32
  90. package/schemas/qualification-config-v1.schema.json +0 -244
  91. package/schemas/qualification-invocation-request-v1.schema.json +0 -59
  92. package/schemas/qualification-judge-panel-result-v1.schema.json +0 -42
  93. package/schemas/trajectory-event-v1.schema.json +0 -80
@@ -1,422 +0,0 @@
1
- import { existsSync, readFileSync, writeFileSync } from "node:fs";
2
- import { join } from "node:path";
3
- import { judgeRawPath, writeResults, scoreContextFor, findTranscriptFiles, repIndexOf, rebuildScenarioResult, mergeScenarioMetrics, effectiveThreshold, completeCriterionVotes, } from "./results.js";
4
- import { score } from "./score.js";
5
- import { buildJudgePrompt, judgeInWorkspace, parseVerdict } from "./grade.js";
6
- import { appendJournal } from "./journal.js";
7
- import { collapseVotePanel } from "./vote-panel.js";
8
- /** Maximum judgments per cell, first wave included. Hard cap, not a default. */
9
- export const MAX_JUDGMENTS = 3;
10
- /**
11
- * Classify triggers and compute the spend ceiling. Pure — calls nothing.
12
- *
13
- * Triggers are computed from the COMPLETE first wave, never incrementally. A
14
- * `ship_deciding` cell can only be identified once every other cell's verdict is
15
- * known, and computing it mid-wave would make the trigger set depend on the order
16
- * scenarios happened to finish in.
17
- */
18
- export function planAdjudication(input) {
19
- const enabled = new Set(input.enabled ?? ["ambiguous", "contradictory", "non_unanimous", "ship_deciding"]);
20
- const decisions = [];
21
- for (const cell of input.cells) {
22
- // Delivery-gated schema-3 cells carry no behavioral claim to adjudicate.
23
- if (cell.deliveryStatus && cell.deliveryStatus !== "PASS")
24
- continue;
25
- const triggers = [];
26
- // `JUDGE-AMBIGUOUS` is what the parser emits when a judge's verdict blocks
27
- // disagree. `ERROR` is what it emits when nothing parsed at all — and that
28
- // was matched by no trigger here, so the least readable judgments in the run
29
- // were the ones never asked again. Both are "the first wave produced no
30
- // usable answer", which is precisely what a second opinion is for.
31
- if (enabled.has("ambiguous") && (cell.verdict === "JUDGE-AMBIGUOUS" || cell.verdict === "ERROR")) {
32
- triggers.push("ambiguous");
33
- }
34
- // A misfire IS the contradiction: the overall verdict disagrees with the
35
- // AND of the per-item grades. `detectMisfire` already found it; this is the
36
- // decision about what to do with it.
37
- if (enabled.has("contradictory") && cell.suspect && cell.verdict !== "JUDGE-AMBIGUOUS" && cell.verdict !== "ERROR") {
38
- triggers.push("contradictory");
39
- }
40
- if (enabled.has("non_unanimous") && isNonUnanimous(cell))
41
- triggers.push("non_unanimous");
42
- if (enabled.has("ship_deciding") && flipsShipDecision(cell, input))
43
- triggers.push("ship_deciding");
44
- decisions.push({ id: cell.id, triggers });
45
- }
46
- const suspectById = new Map(input.cells.map((c) => [c.id, c.suspect]));
47
- const fired = decisions.filter((d) => d.triggers.length > 0);
48
- // A misfired first-wave judgment is not a clean vote, so with only one extra
49
- // judge the cell can never reach the two clean votes a collapse needs. The call
50
- // is still worth making — it records a second opinion for the author — but the
51
- // buyer has to be told it cannot settle the verdict.
52
- const needsTieBreak = input.tieBreakAvailable ? [] : fired.filter((d) => suspectById.get(d.id)).map((d) => d.id);
53
- const triggered = fired.map((d) => d.id);
54
- // Per triggered cell: one secondary call always, plus one tie-break call only
55
- // if a third judge is available to make it.
56
- const perCell = input.tieBreakAvailable ? 2 : 1;
57
- return { decisions, triggered, needsTieBreak, maxAdditionalCalls: triggered.length * perCell };
58
- }
59
- /** A rep set containing both a PASS and a non-PASS is not a settled result. */
60
- function isNonUnanimous(cell) {
61
- const reps = cell.repVerdicts ?? [];
62
- if (reps.length < 2)
63
- return false;
64
- const passes = reps.filter((v) => v === "PASS").length;
65
- return passes > 0 && passes < reps.length;
66
- }
67
- /**
68
- * Would flipping this one cell change SHIP / NOT READY?
69
- *
70
- * Counterfactual against the REAL scorer, not a reimplementation of the ship
71
- * rules. min-pass, critical and B-series all move the answer, and a second copy
72
- * of that logic would be a copy that drifts. Both directions are tested: a cell
73
- * that would break a shipping run and one that would rescue a blocked one are
74
- * equally worth a second opinion.
75
- */
76
- function flipsShipDecision(cell, input) {
77
- // The target cell's `suspect` is cleared on BOTH sides. It has to be: a suspect
78
- // cell blocks SHIP on its own, so leaving the flag on the baseline would make
79
- // the two scores differ because of the flag rather than the verdict — and every
80
- // suspect cell would report as ship-deciding. Clearing it on both sides asks the
81
- // question actually being asked: holding everything else fixed, does THIS
82
- // CELL'S VERDICT decide the ship?
83
- const verdictsWith = (targetVerdict) => input.cells.map((c) => c.id === cell.id
84
- ? { id: c.id, verdict: targetVerdict, suspect: false }
85
- : { id: c.id, verdict: c.verdict, suspect: c.suspect });
86
- const opts = { shipBar: input.shipBar, critical: input.critical };
87
- const flipped = cell.verdict === "PASS" ? "FAIL" : "PASS";
88
- return score(verdictsWith(cell.verdict), opts).ship !== score(verdictsWith(flipped), opts).ship;
89
- }
90
- /**
91
- * Collapse the judgments recorded for one cell.
92
- *
93
- * A judgment is a CLEAN vote only if it is a plain PASS or FAIL from a judge that
94
- * did not misfire. Ambiguous, suspect and unreadable judgments are recorded in
95
- * full but never counted — a malformed answer is not evidence, and letting one
96
- * cast a deciding vote would launder exactly the unreliability being measured.
97
- */
98
- export function collapseJudgments(judgments, trigger) {
99
- const collapsed = collapseVotePanel(judgments);
100
- return {
101
- trigger,
102
- judgments,
103
- state: collapsed.state,
104
- ...(collapsed.verdict ? { verdict: collapsed.verdict } : {}),
105
- };
106
- }
107
- /**
108
- * Project an adjudication onto the compatibility fields the scorer reads.
109
- *
110
- * `unresolved` becomes `suspect: true`, which the existing ship bar already
111
- * blocks on. The raw judgments survive on the result either way — an author
112
- * resolving this needs to see what each judge actually said, not just the
113
- * collapsed answer.
114
- */
115
- export function boundAdjudicationToRepetitions(result, scenario, adj) {
116
- const criticalAggregate = (result.reps ?? 1) > 1 && effectiveThreshold(result, scenario) === 1;
117
- // A cell-level re-judge reads one transcript. It can confirm the aggregate,
118
- // but cannot erase another clean repetition's failure.
119
- return criticalAggregate && result.judge_verdict !== "PASS" && adj.verdict === "PASS"
120
- ? { ...adj, state: "unresolved", verdict: undefined }
121
- : adj;
122
- }
123
- export function projectAdjudication(result, adj) {
124
- if (adj.state === "unresolved") {
125
- return {
126
- ...result,
127
- // Verdict is left as recorded rather than forced to FAIL: `suspect` is what
128
- // blocks the ship, and overwriting the verdict would destroy the
129
- // first-wave answer an author needs in order to adjudicate.
130
- judge_reason: `${adj.judgments.length} judgments disagree (${adj.trigger}) — resolve or re-judge`,
131
- suspect: true,
132
- adjudication: adj,
133
- };
134
- }
135
- return {
136
- ...result,
137
- judge_verdict: adj.verdict ?? result.judge_verdict,
138
- judge_reason: reasonFor(adj),
139
- // A confirmed or tie-broken cell is no longer untrustworthy — that is the
140
- // entire point of having asked again.
141
- suspect: false,
142
- adjudication: adj,
143
- };
144
- }
145
- /** Apply the one-transcript bound and label it only when the bound changed the panel result. */
146
- export function projectAdjudicationForScenario(result, scenario, adj) {
147
- const bounded = boundAdjudicationToRepetitions(result, scenario, adj);
148
- const projected = projectAdjudication(result, bounded);
149
- if (bounded !== adj) {
150
- projected.judge_reason = `${adj.judgments.length} judgments on one transcript cannot replace a critical all-repetitions aggregate`;
151
- }
152
- return projected;
153
- }
154
- function reasonFor(adj) {
155
- const n = adj.judgments.length;
156
- const verb = adj.state === "confirmed" ? "confirmed by" : "resolved by majority of";
157
- return `${adj.verdict} ${verb} ${n} judgments (${adj.trigger})`;
158
- }
159
- /**
160
- * Execute an adjudication plan.
161
- *
162
- * ONE implementation, called from `run`, `grade` and the review server's rejudge
163
- * path. The plan's three collapse policies were the most likely thing to be
164
- * reimplemented three times and to disagree — a cell that shipped from the CLI
165
- * and blocked in the browser would be worse than no feature.
166
- *
167
- * The tie-break call is made only when the secondary genuinely disagreed. A third
168
- * opinion on a settled cell is spend with no decision attached to it.
169
- */
170
- export async function runAdjudication(opts) {
171
- const byId = new Map();
172
- const log = opts.log ?? (() => { });
173
- let callsMade = 0;
174
- for (const decision of opts.plan.decisions) {
175
- if (decision.triggers.length === 0)
176
- continue;
177
- const cell = opts.cells.find((c) => c.id === decision.id);
178
- if (!cell)
179
- continue;
180
- // A cell can satisfy several triggers; the first is recorded as the reason,
181
- // in the fixed order `planAdjudication` evaluates them.
182
- const trigger = decision.triggers[0];
183
- const judgments = [
184
- cell.primaryJudgment
185
- ? { ...cell.primaryJudgment, ordinal: 1 }
186
- : { ordinal: 1, judge: { ...opts.primaryJudge }, verdict: cell.verdict, reason: cell.reason, suspect: cell.suspect, criteria: cell.criteria ?? [] },
187
- ];
188
- const second = await opts.rejudge(decision.id, opts.secondaryJudge);
189
- callsMade++;
190
- judgments.push({ ordinal: 2, judge: { ...opts.secondaryJudge }, ...second });
191
- let collapsed = collapseJudgments(judgments, trigger);
192
- if (collapsed.state === "unresolved" && opts.tieBreakJudge) {
193
- const third = await opts.rejudge(decision.id, opts.tieBreakJudge);
194
- callsMade++;
195
- judgments.push({ ordinal: 3, judge: { ...opts.tieBreakJudge }, ...third });
196
- collapsed = collapseJudgments(judgments, trigger);
197
- }
198
- log(` ${decision.id}: ${collapsed.state}${collapsed.verdict ? ` → ${collapsed.verdict}` : ""} (${judgments.length} judgments)`);
199
- byId.set(decision.id, collapsed);
200
- }
201
- return { byId, callsMade };
202
- }
203
- /**
204
- * Adjudicate a completed run: plan, disclose the ceiling, execute, persist.
205
- *
206
- * Re-judges the SAME saved transcripts — no subject re-run, so the model's
207
- * behavior is held fixed and any movement is the judge. That is what makes this
208
- * a measurement of judge reliability rather than another sample of the model.
209
- */
210
- export async function adjudicateRun(opts) {
211
- const log = opts.log ?? (() => { });
212
- const mode = opts.results.mode;
213
- const cells = cellsFromResults(opts.runDir, opts.results);
214
- const plan = planAdjudication({
215
- cells,
216
- scenarios: opts.spec.scenarios,
217
- shipBar: opts.spec.ship_bar,
218
- critical: opts.spec.critical,
219
- tieBreakAvailable: opts.tieBreakJudge !== undefined,
220
- });
221
- // Disclosed before the first extra call, always — including the zero case, so
222
- // "nothing triggered" is visibly different from "the feature did not run".
223
- log(formatAdjudicationPlan(plan, { secondary: opts.secondaryJudge, tieBreak: opts.tieBreakJudge }));
224
- if (plan.triggered.length === 0)
225
- return opts.results;
226
- const byIdScenario = new Map(opts.spec.scenarios.map((s) => [s.id, s]));
227
- const { byId, callsMade } = await runAdjudication({
228
- plan,
229
- cells,
230
- primaryJudge: opts.primaryJudge,
231
- secondaryJudge: opts.secondaryJudge,
232
- tieBreakJudge: opts.tieBreakJudge,
233
- log,
234
- rejudge: async (id, judge) => {
235
- const scenario = byIdScenario.get(id);
236
- if (!scenario)
237
- throw new Error(`adjudication: scenario \`${id}\` is not in the spec`);
238
- return judgeCell({ ...opts, scenario, judge, mode });
239
- },
240
- });
241
- const scenarios = opts.results.scenarios.map((s) => {
242
- const adj = byId.get(s.id);
243
- // Adjudication asks judges again and re-measures nothing else, so the run's
244
- // objective evidence carries. Overrides survive too — a judge panel does not
245
- // outvote the author.
246
- if (!adj)
247
- return s;
248
- const scenario = byIdScenario.get(s.id);
249
- const withRepetition = { ...adj, repetition: s.rep_judgments?.[0]?.repetition ?? 0 };
250
- const projected = scenario ? projectAdjudicationForScenario(s, scenario, withRepetition) : projectAdjudication(s, withRepetition);
251
- const extraCalls = adj.judgments.filter((judgment) => judgment.ordinal > 1).length;
252
- projected.metrics = mergeScenarioMetrics(s.metrics, {
253
- wall_time_ms: 0,
254
- judge_calls: extraCalls,
255
- judge_rejudge_calls: extraCalls,
256
- subject_metrics_reps: 0,
257
- total_reps: s.metrics?.total_reps ?? s.reps ?? 1,
258
- });
259
- return rebuildScenarioResult(projected, s, { objective: "carry", adjudication: "fresh" });
260
- });
261
- appendJournal(opts.runDir, {
262
- event: "adjudication",
263
- ts: opts.now(),
264
- triggered: plan.triggered,
265
- judge_calls: callsMade,
266
- unresolved: [...byId.entries()].filter(([, a]) => a.state === "unresolved").map(([id]) => id),
267
- });
268
- const ctx = scoreContextFor(opts.results, opts.spec);
269
- return writeResults(opts.runDir, { ...opts.results, scenarios }, ctx);
270
- }
271
- /**
272
- * Ask one judge about one cell, reading the SAME saved transcript the first wave
273
- * was graded from.
274
- *
275
- * A multi-rep cell is adjudicated on its FIRST rep's transcript under one
276
- * documented policy, rather than whichever rep would move the headline. Picking
277
- * the convenient rep is how a "second opinion" becomes a way to get the answer
278
- * you wanted, which would make the whole feature worse than not having it.
279
- *
280
- * The extra judgment is written to `.judge2.txt` / `.judge3.txt`, leaving the
281
- * first-wave `.judge.txt` untouched — the audit trail is the point.
282
- */
283
- async function judgeCell(opts) {
284
- const files = findTranscriptFiles(opts.runDir, opts.scenario.id, opts.mode);
285
- if (files.length === 0) {
286
- throw new Error(`adjudication: no ${opts.mode} transcript for \`${opts.scenario.id}\` in ${opts.runDir} — ` +
287
- `transcripts are gitignored, so this needs the run dir that produced them`);
288
- }
289
- const transcript = readFileSync(join(opts.runDir, files[0]), "utf8");
290
- const prompt = buildJudgePrompt({
291
- skill: opts.spec.skill, persona: opts.spec.judge_persona, scenario: opts.scenario, transcript,
292
- });
293
- const g = await judgeInWorkspace(opts.adapter, opts.judge, prompt, opts.specDir);
294
- // Ordinal 2 and 3 get their own artifacts; the first wave's stays as recorded.
295
- const rep = repIndexOf(files[0]) ?? undefined;
296
- const base = judgeRawPath(opts.runDir, opts.scenario.id, opts.mode, rep);
297
- const nth = existsSync(base.replace(/\.judge\.txt$/, ".judge2.txt")) ? 3 : 2;
298
- writeFileSync(base.replace(/\.judge\.txt$/, `.judge${nth}.txt`), g.raw, "utf8");
299
- appendJournal(opts.runDir, {
300
- event: "judge-verdict", ts: opts.now(), id: opts.scenario.id,
301
- verdict: g.verdict, reason: g.reason, suspect: g.suspect,
302
- });
303
- return { verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, opts.scenario.checklist.length) };
304
- }
305
- /**
306
- * Build the plan's input cells from a run.
307
- *
308
- * EXPORTED so the CLI, the review server and the pi extension all price the work
309
- * exactly as `adjudicateRun` will perform it. When the preflight built cells
310
- * without `repVerdicts` and the executor built them with, the browser could show
311
- * "no cell triggered" and then spend on non-unanimous cells nobody was told
312
- * about — inverting the one invariant this feature actually promises.
313
- */
314
- export function cellsFromResults(runDir, results) {
315
- return results.scenarios.map((s) => {
316
- const primaryJudgment = s.rep_judgments?.find(panel => panel.repetition === 0)?.judgments[0];
317
- const deliveryStatus = s.objective?.assertions.find(assertion => assertion.kind === "skill_delivered")?.status;
318
- return {
319
- id: s.id,
320
- verdict: s.judge_verdict,
321
- reason: s.judge_reason,
322
- suspect: s.suspect,
323
- repVerdicts: repVerdictsOf(runDir, s, results.mode),
324
- criteria: primaryJudgment?.criteria,
325
- ...(deliveryStatus === "PASS" || deliveryStatus === "NOT-MEASURED" || deliveryStatus === "ERROR" ? { deliveryStatus } : {}),
326
- ...(primaryJudgment ? { primaryJudgment } : {}),
327
- };
328
- });
329
- }
330
- /** Per-rep verdicts from saved judge artifacts, for the non-unanimous trigger. */
331
- function repVerdictsOf(runDir, s, mode) {
332
- if (!s.reps || s.reps < 2)
333
- return undefined;
334
- const out = [];
335
- // 0-based: `run.ts` writes rep0..repN-1. Looping from 1 dropped rep0 and probed
336
- // a repN that never exists, so a 3-rep FAIL/PASS/PASS read as unanimous PASS and
337
- // a 2-rep cell returned undefined — `non_unanimous` could never fire correctly.
338
- for (let rep = 0; rep < s.reps; rep++) {
339
- const path = judgeRawPath(runDir, s.id, mode, rep);
340
- if (!existsSync(path)) {
341
- // A rep with no judge artifact is an ABSENT vote, not a vote to skip.
342
- // `run.ts` deliberately does not call the judge for a rep blocked by a gate
343
- // or ending in ERROR — so the missing files are exactly the reps that
344
- // failed hardest, and dropping them made `[FAIL, PASS, PASS]` read as
345
- // `[PASS, PASS]`: unanimous. The cell whose split was caused by a forbidden
346
- // tool call was the one cell adjudication declined to look at, and the
347
- // preflight told the buyer "no cell triggered".
348
- out.push("ERROR");
349
- continue;
350
- }
351
- out.push(parseVerdict(readFileSync(path, "utf8")).verdict);
352
- }
353
- return out.length >= 2 ? out : undefined;
354
- }
355
- /**
356
- * Resolve and validate the extra judges.
357
- *
358
- * Every configured judge passes the SAME two gates the primary does: the metered
359
- * refusal and the judge≠subject check. A secondary judge is still a judge — a
360
- * feature that multiplies judge calls is the last place to let one slip past the
361
- * policy that exists because a default once billed a corpus by accident.
362
- *
363
- * Returns null when adjudication was not authorized, so callers can treat "not
364
- * enabled" and "enabled with no triggers" as different things.
365
- */
366
- export function resolveAdjudicationJudges(opts) {
367
- if (!opts.enabled)
368
- return null;
369
- // An unreadable subject skips the resemblance warning rather than failing the
370
- // run. The judge≠subject check is advice; the metered refusal below is the gate.
371
- let subject = null;
372
- try {
373
- subject = opts.parseRef(opts.subjectToken);
374
- }
375
- catch {
376
- opts.warn(` ⚠ cannot read the run's model (\`${opts.subjectToken}\`) — skipping the judge≠subject check`);
377
- }
378
- // With no explicit secondary the primary judge is asked again as an independent
379
- // draw. That is a real measurement — the judge-variance study found a ~2%
380
- // disagreement rate on identical transcripts — not a no-op.
381
- const secondary = opts.secondaryToken ? opts.parseRef(opts.secondaryToken) : opts.primary;
382
- const tieBreak = opts.tieBreakToken ? opts.parseRef(opts.tieBreakToken) : undefined;
383
- opts.assertAllowed(secondary, "--secondary-judge");
384
- if (tieBreak)
385
- opts.assertAllowed(tieBreak, "--tie-break-judge");
386
- if (subject) {
387
- for (const [label, judge] of [["secondary", secondary], ["tie-break", tieBreak]]) {
388
- if (judge && opts.resemblesSubject(judge, subject)) {
389
- opts.warn(` ⚠ ${label} judge (${judge.provider}:${judge.model}) resembles the model under test ` +
390
- `(${subject.provider}:${subject.model}) — same-family grading inflates scores.`);
391
- }
392
- }
393
- }
394
- return tieBreak ? { secondary, tieBreak } : { secondary };
395
- }
396
- /** Human-readable preflight line. Counts, never dollars — see `maxAdditionalCalls`. */
397
- export function formatAdjudicationPlan(plan, judges) {
398
- const stuck = plan.needsTieBreak.length
399
- ? [
400
- ` ${plan.needsTieBreak.length} of those cannot be SETTLED by this plan: ${plan.needsTieBreak.join(", ")}`,
401
- " (their first judgment misfired, so one more judge cannot reach two clean votes — the call",
402
- " buys a second opinion to resolve by hand; add a tie-break judge to settle them outright)",
403
- ]
404
- : [];
405
- if (plan.triggered.length === 0) {
406
- return ["adjudication: no cell triggered — no additional judge calls", ...stuck].join("\n");
407
- }
408
- const lines = [
409
- `adjudication: ${plan.triggered.length} cell(s) triggered — up to ${plan.maxAdditionalCalls} additional judge call(s)`,
410
- ` secondary judge: ${judges.secondary.provider}:${judges.secondary.model}`,
411
- ];
412
- if (judges.tieBreak)
413
- lines.push(` tie-break judge: ${judges.tieBreak.provider}:${judges.tieBreak.model}`);
414
- else
415
- lines.push(" no tie-break judge — a disagreement stays unresolved and blocks SHIP");
416
- for (const d of plan.decisions) {
417
- if (d.triggers.length)
418
- lines.push(` ${d.id}: ${d.triggers.join(", ")}`);
419
- }
420
- return [...lines, ...stuck].join("\n");
421
- }
422
- //# sourceMappingURL=adjudication.js.map
@@ -1,128 +0,0 @@
1
- export interface AdoptionInput {
2
- hypothesisDigest: string;
3
- experimentDigest: string;
4
- candidateDigest: string;
5
- scopeDigest: string;
6
- assessmentPolicyDigest: string;
7
- rollbackCandidateDigest: string;
8
- activationBoundary: "next-orders";
9
- expiresAt: number;
10
- }
11
- export interface AdoptionBinding extends AdoptionInput {
12
- version: "adoption-binding-v1";
13
- id: string;
14
- }
15
- export interface AdoptionAuthority {
16
- id: string;
17
- adoptions: readonly string[];
18
- rollbacks: readonly string[];
19
- }
20
- export interface AdoptionFacts {
21
- experimentDigest: string;
22
- candidateDigest: string;
23
- scopeDigest: string;
24
- assessmentPolicyDigest: string;
25
- eligible: boolean;
26
- }
27
- export declare function buildAdoptionBinding(input: AdoptionInput): AdoptionBinding;
28
- /** Independent host declarations and independently verified eligible facts are required again at use. */
29
- export declare function authorizeAdoption(binding: AdoptionBinding, authority: AdoptionAuthority | null, facts: AdoptionFacts, now: number): Readonly<{
30
- id: string;
31
- version: "adoption-receipt-v1";
32
- bindingId: string;
33
- authorityId: string;
34
- hypothesisDigest: string;
35
- experimentDigest: string;
36
- candidateDigest: string;
37
- scopeDigest: string;
38
- assessmentPolicyDigest: string;
39
- rollbackCandidateDigest: string;
40
- activationBoundary: "next-orders";
41
- grantExpansion: false;
42
- expiresAt: number;
43
- }>;
44
- export type AdoptionReceipt = ReturnType<typeof authorizeAdoption>;
45
- export declare function validateAdoptionReceipt(receipt: AdoptionReceipt, binding: AdoptionBinding, authority: AdoptionAuthority | null, facts: AdoptionFacts, now: number): void;
46
- export interface ProductionObservation {
47
- id: string;
48
- adoptionId: string;
49
- candidateDigest: string;
50
- scopeDigest: string;
51
- originalRequirementDigest: string;
52
- currentRequirementDigest: string;
53
- /** Only an independent host may supply confirmed outcomes; observer/model claims are not confirmation. */
54
- outcome: "confirmed-defect" | "success" | "unknown";
55
- acceptanceDigest: string | null;
56
- acceptedArtifactDigest: string | null;
57
- observedArtifactDigest: string;
58
- evidence: string[];
59
- }
60
- export declare function classifyProductionObservation(adoption: AdoptionReceipt, input: ProductionObservation): Readonly<{
61
- state: "unknown" | "success" | "out-of-scope" | "changed-requirements" | "changed-artifact" | "caught-defect" | "escape";
62
- adoptionId: string;
63
- hypothesisDigest: string;
64
- experimentDigest: string;
65
- observationId: string;
66
- acceptanceDigest: string | null;
67
- acceptedArtifactDigest: string | null;
68
- observedArtifactDigest: string;
69
- automaticRollback: false;
70
- }>;
71
- export interface RollbackRequest {
72
- version: "rollback-request-v1";
73
- id: string;
74
- adoptionId: string;
75
- scopeDigest: string;
76
- restoreCandidateDigest: string;
77
- reason: "operator-request" | "confirmed-escape";
78
- evidence: readonly string[];
79
- expiresAt: number;
80
- }
81
- export declare function buildRollbackRequest(adoption: AdoptionReceipt, reason: RollbackRequest["reason"], evidence: string[], expiresAt?: number): RollbackRequest;
82
- /** This validates a recorded choice, not an automatic rollback action or a caller-created authority flag. */
83
- export declare function authorizeRollback(adoption: AdoptionReceipt, request: RollbackRequest, authority: AdoptionAuthority | null, now: number, current: {
84
- adoptionId: string;
85
- candidateDigest: string;
86
- scopeDigest: string;
87
- }): Readonly<{
88
- requestId: string;
89
- adoptionId: string;
90
- restoreCandidateDigest: string;
91
- scopeDigest: string;
92
- grantExpansion: false;
93
- application: "not-performed";
94
- }>;
95
- export interface LeanOutcome {
96
- population: string;
97
- complete: boolean;
98
- accepted: boolean | null;
99
- repairAttempts: number | null;
100
- escape: "confirmed" | "none-observed" | "unknown";
101
- requirementsChanged: boolean;
102
- decisionWaitMs: number | null;
103
- questions: number | null;
104
- }
105
- export declare function summarizeLeanOutcomes(population: string, rows: readonly LeanOutcome[]): {
106
- population: string;
107
- firstPassAccepted: number;
108
- firstPassDenominator: number;
109
- incomplete: number;
110
- confirmedEscapes: number;
111
- changedRequirements: number;
112
- recovery: {
113
- known: number;
114
- observed: number;
115
- total: number;
116
- };
117
- decisionWait: {
118
- known: number;
119
- observed: number;
120
- total: number;
121
- };
122
- questions: {
123
- known: number;
124
- observed: number;
125
- total: number;
126
- };
127
- universalScore: null;
128
- };
package/dist/adoption.js DELETED
@@ -1,102 +0,0 @@
1
- import { createHash } from "node:crypto";
2
- const SHA = /^[a-f0-9]{64}$/;
3
- const text = (x) => typeof x === "string" && x.length > 0 && x.length <= 512 && !/[\u0000-\u001f\u007f]/.test(x);
4
- const closed = (value, names) => {
5
- if (!value || ![Object.prototype, null].includes(Object.getPrototypeOf(value)))
6
- return false;
7
- const fields = Object.getOwnPropertyDescriptors(value);
8
- return Reflect.ownKeys(fields).length === names.length && Reflect.ownKeys(fields).every(key => typeof key === "string" && names.includes(key) && fields[key].enumerable && Object.hasOwn(fields[key], "value"));
9
- };
10
- const hash = (value) => createHash("sha256").update(JSON.stringify(Object.keys(value).sort().map(k => [k, value[k]]))).digest("hex");
11
- export function buildAdoptionBinding(input) {
12
- if (!input || !closed(input, ["hypothesisDigest", "experimentDigest", "candidateDigest", "scopeDigest", "assessmentPolicyDigest", "rollbackCandidateDigest", "activationBoundary", "expiresAt"])
13
- || ![input.hypothesisDigest, input.experimentDigest, input.candidateDigest, input.scopeDigest, input.assessmentPolicyDigest, input.rollbackCandidateDigest].every(s => typeof s === "string" && SHA.test(s))
14
- || input.activationBoundary !== "next-orders" || !Number.isSafeInteger(input.expiresAt) || input.expiresAt < 0)
15
- throw new Error("invalid scoped adoption binding");
16
- const body = { version: "adoption-binding-v1", ...input };
17
- return Object.freeze({ ...body, id: hash(body) });
18
- }
19
- function bindingValid(binding) {
20
- const { version, id, ...input } = binding;
21
- if (version !== "adoption-binding-v1" || buildAdoptionBinding(input).id !== id)
22
- throw new Error("adoption binding changed");
23
- }
24
- function authorityValid(authority) {
25
- return !!authority && text(authority.id) && Array.isArray(authority.adoptions) && Array.isArray(authority.rollbacks)
26
- && authority.adoptions.length <= 256 && authority.rollbacks.length <= 256 && [...authority.adoptions, ...authority.rollbacks].every(id => typeof id === "string" && SHA.test(id));
27
- }
28
- /** Independent host declarations and independently verified eligible facts are required again at use. */
29
- export function authorizeAdoption(binding, authority, facts, now) {
30
- bindingValid(binding);
31
- if (!authorityValid(authority) || !authority.adoptions.includes(binding.id))
32
- throw new Error("independent adoption authority required");
33
- if (!Number.isSafeInteger(now) || now < 0 || now >= binding.expiresAt || !facts || !closed(facts, ["experimentDigest", "candidateDigest", "scopeDigest", "assessmentPolicyDigest", "eligible"]) || facts.eligible !== true
34
- || facts.experimentDigest !== binding.experimentDigest || facts.candidateDigest !== binding.candidateDigest
35
- || facts.scopeDigest !== binding.scopeDigest || facts.assessmentPolicyDigest !== binding.assessmentPolicyDigest)
36
- throw new Error("adoption expired, ineligible or outside verified scope");
37
- const body = { version: "adoption-receipt-v1", bindingId: binding.id, authorityId: authority.id, hypothesisDigest: binding.hypothesisDigest,
38
- experimentDigest: binding.experimentDigest, candidateDigest: binding.candidateDigest, scopeDigest: binding.scopeDigest,
39
- assessmentPolicyDigest: binding.assessmentPolicyDigest, rollbackCandidateDigest: binding.rollbackCandidateDigest,
40
- activationBoundary: "next-orders", grantExpansion: false, expiresAt: binding.expiresAt };
41
- return Object.freeze({ ...body, id: hash(body) });
42
- }
43
- export function validateAdoptionReceipt(receipt, binding, authority, facts, now) {
44
- if (authorizeAdoption(binding, authority, facts, now).id !== receipt.id)
45
- throw new Error("adoption receipt does not match current independent authority/facts");
46
- receiptValid(receipt);
47
- }
48
- function receiptValid(receipt) {
49
- const { id, ...body } = receipt;
50
- if (!closed(receipt, ["version", "bindingId", "authorityId", "hypothesisDigest", "experimentDigest", "candidateDigest", "scopeDigest", "assessmentPolicyDigest", "rollbackCandidateDigest", "activationBoundary", "grantExpansion", "expiresAt", "id"])
51
- || body.version !== "adoption-receipt-v1" || !text(body.authorityId) || body.activationBoundary !== "next-orders" || body.grantExpansion !== false || hash(body) !== id)
52
- throw new Error("adoption receipt changed");
53
- const binding = buildAdoptionBinding({ hypothesisDigest: body.hypothesisDigest, experimentDigest: body.experimentDigest, candidateDigest: body.candidateDigest,
54
- scopeDigest: body.scopeDigest, assessmentPolicyDigest: body.assessmentPolicyDigest, rollbackCandidateDigest: body.rollbackCandidateDigest, activationBoundary: body.activationBoundary, expiresAt: body.expiresAt });
55
- if (binding.id !== body.bindingId)
56
- throw new Error("adoption receipt binding mismatch");
57
- }
58
- export function classifyProductionObservation(adoption, input) {
59
- receiptValid(adoption);
60
- if (!input || !text(input.id) || input.evidence.length > 256 || input.evidence.some(h => !SHA.test(h))
61
- || ![input.originalRequirementDigest, input.currentRequirementDigest, input.candidateDigest, input.scopeDigest, input.observedArtifactDigest].every(h => typeof h === "string" && SHA.test(h))
62
- || ![input.acceptanceDigest, input.acceptedArtifactDigest].every(h => h === null || (typeof h === "string" && SHA.test(h)))
63
- || (input.acceptanceDigest === null) !== (input.acceptedArtifactDigest === null)
64
- || !["confirmed-defect", "success", "unknown"].includes(input.outcome))
65
- throw new Error("invalid production observation");
66
- const state = input.adoptionId !== adoption.id || input.candidateDigest !== adoption.candidateDigest || input.scopeDigest !== adoption.scopeDigest ? "out-of-scope"
67
- : input.originalRequirementDigest !== input.currentRequirementDigest ? "changed-requirements"
68
- : input.acceptedArtifactDigest !== null && input.acceptedArtifactDigest !== input.observedArtifactDigest ? "changed-artifact"
69
- : input.outcome === "confirmed-defect" && input.evidence.length ? input.acceptanceDigest === null ? "caught-defect" : "escape" : input.outcome === "success" ? "success" : "unknown";
70
- return Object.freeze({ state, adoptionId: adoption.id, hypothesisDigest: adoption.hypothesisDigest, experimentDigest: adoption.experimentDigest, observationId: input.id, acceptanceDigest: input.acceptanceDigest, acceptedArtifactDigest: input.acceptedArtifactDigest, observedArtifactDigest: input.observedArtifactDigest, automaticRollback: false });
71
- }
72
- export function buildRollbackRequest(adoption, reason, evidence, expiresAt = adoption.expiresAt) {
73
- receiptValid(adoption);
74
- if (!Number.isSafeInteger(expiresAt) || expiresAt < 0 || !["operator-request", "confirmed-escape"].includes(reason) || !evidence.length || evidence.length > 256 || evidence.some(h => !SHA.test(h)))
75
- throw new Error("invalid rollback request");
76
- const body = { version: "rollback-request-v1", adoptionId: adoption.id, scopeDigest: adoption.scopeDigest, restoreCandidateDigest: adoption.rollbackCandidateDigest, reason, expiresAt, evidence: Object.freeze([...new Set(evidence)].sort()) };
77
- return Object.freeze({ ...body, id: hash(body) });
78
- }
79
- /** This validates a recorded choice, not an automatic rollback action or a caller-created authority flag. */
80
- export function authorizeRollback(adoption, request, authority, now, current) {
81
- receiptValid(adoption);
82
- const { id, ...body } = request;
83
- if (!authorityValid(authority) || authority.id !== adoption.authorityId || !authority.rollbacks.includes(id))
84
- throw new Error("independent rollback authority required");
85
- if (hash(body) !== id || request.version !== "rollback-request-v1" || request.adoptionId !== adoption.id || request.scopeDigest !== adoption.scopeDigest
86
- || request.restoreCandidateDigest !== adoption.rollbackCandidateDigest || !Number.isSafeInteger(now) || now < 0 || !Number.isSafeInteger(request.expiresAt) || now >= request.expiresAt
87
- || !current || current.adoptionId !== adoption.id || current.candidateDigest !== adoption.candidateDigest || current.scopeDigest !== adoption.scopeDigest)
88
- throw new Error("rollback binding changed or expired");
89
- return Object.freeze({ requestId: id, adoptionId: adoption.id, restoreCandidateDigest: request.restoreCandidateDigest, scopeDigest: adoption.scopeDigest, grantExpansion: false, application: "not-performed" });
90
- }
91
- export function summarizeLeanOutcomes(population, rows) {
92
- if (!text(population) || rows.length > 10000 || rows.some(r => r.population !== population || typeof r.complete !== "boolean" || typeof r.requirementsChanged !== "boolean"
93
- || ![true, false, null].includes(r.accepted) || !["confirmed", "none-observed", "unknown"].includes(r.escape)
94
- || [r.repairAttempts, r.decisionWaitMs, r.questions].some(n => n !== null && (!Number.isSafeInteger(n) || n < 0))))
95
- throw new Error("invalid or incomparable lean population");
96
- const complete = rows.filter(r => r.complete && r.accepted !== null && r.repairAttempts !== null);
97
- const observed = (field) => ({ known: rows.reduce((n, r) => n + (r[field] ?? 0), 0), observed: rows.filter(r => r[field] !== null).length, total: rows.length });
98
- return { population, firstPassAccepted: complete.filter(r => r.accepted && r.repairAttempts === 0).length, firstPassDenominator: complete.length,
99
- incomplete: rows.length - complete.length, confirmedEscapes: rows.filter(r => r.escape === "confirmed" && !r.requirementsChanged).length,
100
- changedRequirements: rows.filter(r => r.requirementsChanged).length, recovery: observed("repairAttempts"), decisionWait: observed("decisionWaitMs"), questions: observed("questions"), universalScore: null };
101
- }
102
- //# sourceMappingURL=adoption.js.map