@dzhechkov/harness-core 0.3.118 → 0.3.120
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bto-optimize.d.ts +97 -0
- package/dist/bto-optimize.d.ts.map +1 -0
- package/dist/bto-optimize.js +250 -0
- package/dist/bto-optimize.js.map +1 -0
- package/dist/discrimination-gate.d.ts +81 -0
- package/dist/discrimination-gate.d.ts.map +1 -0
- package/dist/discrimination-gate.js +164 -0
- package/dist/discrimination-gate.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +2 -0
- package/dist/registry.js.map +1 -1
- package/package.json +2 -2
- package/src/bto-optimize.ts +270 -0
- package/src/discrimination-gate.ts +238 -0
- package/src/index.ts +2 -0
- package/src/registry.ts +1 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hold-out validated skill optimization (feature bto-optimize-holdout, ADR-001).
|
|
3
|
+
*
|
|
4
|
+
* Strengthens the EXISTING `/bto-optimize` skill from within with dspy-MIPROv2 rigor (grounded in the shipped
|
|
5
|
+
* `dspy.ts/src/optimize/miprov2.ts`: propose → minibatch-tune → validate on held-out → best). The current BTO
|
|
6
|
+
* loop selects the highest score on the SAME eval it tuned on — which games the LLM judge panel (Goodhart).
|
|
7
|
+
* This engine adds what it lacks: a deterministic tune/holdout split, a hard budget cap, a weakest-dimension
|
|
8
|
+
* objective, and a **no-regress-on-HOLD-OUT** winner selector. All functions are PURE over injected judge
|
|
9
|
+
* scores + text (no LLM, no clock, no random); candidate-prose generation and judging stay skill-side.
|
|
10
|
+
*
|
|
11
|
+
* SAFETY PROPERTIES (ADR-001, load-bearing, each pinned by a test):
|
|
12
|
+
* 1. §2 — `selectWinner` accepts a candidate ONLY on the HOLD-OUT scores (weakest dim improves + no other
|
|
13
|
+
* dim / the aggregate regresses beyond `tolerance`). A candidate that wins on `tune` but regresses on
|
|
14
|
+
* `holdout` is REJECTED — this defeats judge-gaming. `tune` never decides acceptance.
|
|
15
|
+
* 2. §? — `budgetPlan` NEVER returns a plan exceeding `maxJudgeRuns`: it shrinks candidates/rounds to fit
|
|
16
|
+
* and reports what it trimmed, so the loop can never run a surprise-cost number of judge passes.
|
|
17
|
+
* 3. §3 — the engine NEVER writes a file (only renders a diff + returns a decision); `proseScopeOk` rejects
|
|
18
|
+
* a candidate that touches frontmatter or structural headings (augment-not-clobber).
|
|
19
|
+
*/
|
|
20
|
+
export type BtoDimension = 'METHODOLOGY' | 'DEPTH' | 'CORRECTNESS' | 'USABILITY' | 'ROBUSTNESS';
|
|
21
|
+
export declare const BTO_DIMENSIONS: readonly BtoDimension[];
|
|
22
|
+
export type DimScores = Record<BtoDimension, number>;
|
|
23
|
+
export interface ScenarioSplit {
|
|
24
|
+
readonly tune: readonly string[];
|
|
25
|
+
readonly holdout: readonly string[];
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Partition scenario ids into tune/holdout DETERMINISTICALLY (no clock/random): sort by id, then interleave —
|
|
29
|
+
* every k-th id (k = round(1/holdoutRatio)) goes to holdout. Guarantees ≥1 on each side when ≥2 ids exist.
|
|
30
|
+
* Same input → byte-identical split.
|
|
31
|
+
*/
|
|
32
|
+
export declare function splitScenarios(ids: readonly string[], holdoutRatio?: number): ScenarioSplit;
|
|
33
|
+
export interface BudgetInput {
|
|
34
|
+
readonly candidates: number;
|
|
35
|
+
readonly rounds: number;
|
|
36
|
+
readonly tuneCount: number;
|
|
37
|
+
readonly holdoutCount: number;
|
|
38
|
+
}
|
|
39
|
+
export interface BudgetPlan {
|
|
40
|
+
readonly candidates: number;
|
|
41
|
+
readonly rounds: number;
|
|
42
|
+
readonly tuneRuns: number;
|
|
43
|
+
readonly holdoutRuns: number;
|
|
44
|
+
readonly totalRuns: number;
|
|
45
|
+
readonly cap: number;
|
|
46
|
+
readonly withinCap: boolean;
|
|
47
|
+
readonly trimmed?: string;
|
|
48
|
+
}
|
|
49
|
+
/** Documented default cap (configurable — NOT a magic constant): a single L2 judge pass per scenario per
|
|
50
|
+
* candidate is the unit; 24 keeps a 5-candidate × ~4-scenario tune + holdout validation comfortably bounded. */
|
|
51
|
+
export declare const DEFAULT_MAX_JUDGE_RUNS = 24;
|
|
52
|
+
/**
|
|
53
|
+
* Judge-run accounting: tune scoring = candidates × rounds × tuneCount; holdout validation = 1 (top-1) ×
|
|
54
|
+
* holdoutCount. If the total exceeds `maxJudgeRuns`, SHRINK — first `rounds` to 1, then `candidates` — until it
|
|
55
|
+
* fits, and report what was trimmed. Never returns an over-cap plan (safety property 2).
|
|
56
|
+
*/
|
|
57
|
+
export declare function budgetPlan(input: BudgetInput, maxJudgeRuns?: number): BudgetPlan;
|
|
58
|
+
/** Every one of the 5 BTO dimensions must be a valid score for a DimScores to be usable. */
|
|
59
|
+
export declare function validDimScores(s: unknown): s is DimScores;
|
|
60
|
+
export declare function aggregate(s: DimScores): number;
|
|
61
|
+
/** The lowest-scoring dimension (the bottleneck to lift). Deterministic tie-break by BTO_DIMENSIONS order. */
|
|
62
|
+
export declare function weakestDimension(s: DimScores): BtoDimension;
|
|
63
|
+
export interface Candidate {
|
|
64
|
+
readonly id: string;
|
|
65
|
+
readonly prose: string;
|
|
66
|
+
readonly tune: DimScores;
|
|
67
|
+
readonly holdout: DimScores;
|
|
68
|
+
}
|
|
69
|
+
export interface WinnerResult {
|
|
70
|
+
readonly winner: string | null;
|
|
71
|
+
readonly reason: string;
|
|
72
|
+
readonly weakest: BtoDimension;
|
|
73
|
+
readonly deltas?: Record<string, number>;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Accept a candidate ONLY on the HOLD-OUT scores (ADR §2): the baseline's weakest dimension must improve by > 0
|
|
77
|
+
* AND no other dimension AND the aggregate may regress by more than `tolerance` (default 0 = strict). Candidates
|
|
78
|
+
* are considered in descending TUNE-aggregate order (the minibatch rank — dspy's search), but acceptance reads
|
|
79
|
+
* only holdout, so a tune-winner that regresses on holdout is rejected. Deterministic; returns the first
|
|
80
|
+
* candidate that passes, or null with a reason.
|
|
81
|
+
*/
|
|
82
|
+
export declare function selectWinner(baseline: {
|
|
83
|
+
readonly holdout: DimScores;
|
|
84
|
+
}, candidates: readonly Candidate[], opts?: {
|
|
85
|
+
tolerance?: number;
|
|
86
|
+
}): WinnerResult;
|
|
87
|
+
/** Only directive PROSE may change (Phase-1): frontmatter + the set of ALL markdown structural markers must be
|
|
88
|
+
* identical. A candidate that alters them (incl. CRLF, tab-ATX, empty-ATX, setext) is rejected (augment-not-clobber). */
|
|
89
|
+
export declare function proseScopeOk(original: string, candidate: string): {
|
|
90
|
+
ok: boolean;
|
|
91
|
+
reason: string;
|
|
92
|
+
};
|
|
93
|
+
/** A minimal deterministic unified-ish line diff for the confirm gate (pure). */
|
|
94
|
+
export declare function renderProseDiff(original: string, candidate: string): string;
|
|
95
|
+
/** Read a scenario-id list: JSON array, or newline/comma-separated. Absent/unreadable → []. */
|
|
96
|
+
export declare function readScenarioIds(path: string): string[];
|
|
97
|
+
//# sourceMappingURL=bto-optimize.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bto-optimize.d.ts","sourceRoot":"","sources":["../src/bto-optimize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,MAAM,MAAM,YAAY,GAAG,aAAa,GAAG,OAAO,GAAG,aAAa,GAAG,WAAW,GAAG,YAAY,CAAC;AAChG,eAAO,MAAM,cAAc,EAAE,SAAS,YAAY,EAAsF,CAAC;AACzI,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;AAMrD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CACrC;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,EAAE,YAAY,SAAO,GAAG,aAAa,CAazF;AAID,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AACD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;iHACiH;AACjH,eAAO,MAAM,sBAAsB,KAAK,CAAC;AAEzC;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,YAAY,GAAE,MAA+B,GAAG,UAAU,CAmBxG;AAYD,4FAA4F;AAC5F,wBAAgB,cAAc,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,SAAS,CAIzD;AACD,wBAAgB,SAAS,CAAC,CAAC,EAAE,SAAS,GAAG,MAAM,CAE9C;AACD,8GAA8G;AAC9G,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,SAAS,GAAG,YAAY,CAI3D;AAID,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC;CAC7B;AACD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1C;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE;IAAE,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAA;CAAE,EAAE,UAAU,EAAE,SAAS,SAAS,EAAE,EAAE,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,YAAY,CAiCzJ;AA0CD;0HAC0H;AAC1H,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAMjG;AAED,iFAAiF;AACjF,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAa3E;AAID,+FAA+F;AAC/F,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAatD"}
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hold-out validated skill optimization (feature bto-optimize-holdout, ADR-001).
|
|
3
|
+
*
|
|
4
|
+
* Strengthens the EXISTING `/bto-optimize` skill from within with dspy-MIPROv2 rigor (grounded in the shipped
|
|
5
|
+
* `dspy.ts/src/optimize/miprov2.ts`: propose → minibatch-tune → validate on held-out → best). The current BTO
|
|
6
|
+
* loop selects the highest score on the SAME eval it tuned on — which games the LLM judge panel (Goodhart).
|
|
7
|
+
* This engine adds what it lacks: a deterministic tune/holdout split, a hard budget cap, a weakest-dimension
|
|
8
|
+
* objective, and a **no-regress-on-HOLD-OUT** winner selector. All functions are PURE over injected judge
|
|
9
|
+
* scores + text (no LLM, no clock, no random); candidate-prose generation and judging stay skill-side.
|
|
10
|
+
*
|
|
11
|
+
* SAFETY PROPERTIES (ADR-001, load-bearing, each pinned by a test):
|
|
12
|
+
* 1. §2 — `selectWinner` accepts a candidate ONLY on the HOLD-OUT scores (weakest dim improves + no other
|
|
13
|
+
* dim / the aggregate regresses beyond `tolerance`). A candidate that wins on `tune` but regresses on
|
|
14
|
+
* `holdout` is REJECTED — this defeats judge-gaming. `tune` never decides acceptance.
|
|
15
|
+
* 2. §? — `budgetPlan` NEVER returns a plan exceeding `maxJudgeRuns`: it shrinks candidates/rounds to fit
|
|
16
|
+
* and reports what it trimmed, so the loop can never run a surprise-cost number of judge passes.
|
|
17
|
+
* 3. §3 — the engine NEVER writes a file (only renders a diff + returns a decision); `proseScopeOk` rejects
|
|
18
|
+
* a candidate that touches frontmatter or structural headings (augment-not-clobber).
|
|
19
|
+
*/
|
|
20
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
21
|
+
export const BTO_DIMENSIONS = Object.freeze(['METHODOLOGY', 'DEPTH', 'CORRECTNESS', 'USABILITY', 'ROBUSTNESS']);
|
|
22
|
+
const byStr = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
23
|
+
/**
|
|
24
|
+
* Partition scenario ids into tune/holdout DETERMINISTICALLY (no clock/random): sort by id, then interleave —
|
|
25
|
+
* every k-th id (k = round(1/holdoutRatio)) goes to holdout. Guarantees ≥1 on each side when ≥2 ids exist.
|
|
26
|
+
* Same input → byte-identical split.
|
|
27
|
+
*/
|
|
28
|
+
export function splitScenarios(ids, holdoutRatio = 0.34) {
|
|
29
|
+
const sorted = [...new Set(ids.map((s) => String(s)))].filter((s) => s !== '').sort(byStr);
|
|
30
|
+
if (sorted.length === 0)
|
|
31
|
+
return { tune: [], holdout: [] };
|
|
32
|
+
if (sorted.length === 1)
|
|
33
|
+
return { tune: sorted, holdout: [] };
|
|
34
|
+
const ratio = holdoutRatio > 0 && holdoutRatio < 1 ? holdoutRatio : 0.34;
|
|
35
|
+
const k = Math.max(2, Math.round(1 / ratio)); // every k-th → holdout
|
|
36
|
+
const tune = [];
|
|
37
|
+
const holdout = [];
|
|
38
|
+
sorted.forEach((id, i) => ((i + 1) % k === 0 ? holdout : tune).push(id));
|
|
39
|
+
// guarantee ≥1 each side
|
|
40
|
+
if (holdout.length === 0)
|
|
41
|
+
holdout.push(tune.pop());
|
|
42
|
+
if (tune.length === 0)
|
|
43
|
+
tune.push(holdout.pop());
|
|
44
|
+
return { tune, holdout };
|
|
45
|
+
}
|
|
46
|
+
/** Documented default cap (configurable — NOT a magic constant): a single L2 judge pass per scenario per
|
|
47
|
+
* candidate is the unit; 24 keeps a 5-candidate × ~4-scenario tune + holdout validation comfortably bounded. */
|
|
48
|
+
export const DEFAULT_MAX_JUDGE_RUNS = 24;
|
|
49
|
+
/**
|
|
50
|
+
* Judge-run accounting: tune scoring = candidates × rounds × tuneCount; holdout validation = 1 (top-1) ×
|
|
51
|
+
* holdoutCount. If the total exceeds `maxJudgeRuns`, SHRINK — first `rounds` to 1, then `candidates` — until it
|
|
52
|
+
* fits, and report what was trimmed. Never returns an over-cap plan (safety property 2).
|
|
53
|
+
*/
|
|
54
|
+
export function budgetPlan(input, maxJudgeRuns = DEFAULT_MAX_JUDGE_RUNS) {
|
|
55
|
+
const cap = Number.isFinite(maxJudgeRuns) && maxJudgeRuns > 0 ? Math.floor(maxJudgeRuns) : DEFAULT_MAX_JUDGE_RUNS;
|
|
56
|
+
const tuneCount = Math.max(0, Math.floor(input.tuneCount));
|
|
57
|
+
const holdoutCount = Math.max(0, Math.floor(input.holdoutCount));
|
|
58
|
+
let candidates = Math.max(1, Math.floor(input.candidates));
|
|
59
|
+
let rounds = Math.max(1, Math.floor(input.rounds));
|
|
60
|
+
const trims = [];
|
|
61
|
+
const total = (c, r) => c * r * tuneCount + holdoutCount;
|
|
62
|
+
if (total(candidates, rounds) > cap && rounds > 1) {
|
|
63
|
+
rounds = 1;
|
|
64
|
+
trims.push('rounds→1');
|
|
65
|
+
}
|
|
66
|
+
while (total(candidates, rounds) > cap && candidates > 1) {
|
|
67
|
+
candidates -= 1;
|
|
68
|
+
}
|
|
69
|
+
if (candidates < Math.max(1, Math.floor(input.candidates)))
|
|
70
|
+
trims.push('candidates→' + candidates);
|
|
71
|
+
const tuneRuns = candidates * rounds * tuneCount;
|
|
72
|
+
const holdoutRuns = holdoutCount; // validate the top-1 candidate on holdout
|
|
73
|
+
const totalRuns = tuneRuns + holdoutRuns;
|
|
74
|
+
const plan = {
|
|
75
|
+
candidates, rounds, tuneRuns, holdoutRuns, totalRuns, cap, withinCap: totalRuns <= cap,
|
|
76
|
+
...(trims.length ? { trimmed: trims.join(', ') } : {}),
|
|
77
|
+
};
|
|
78
|
+
return plan;
|
|
79
|
+
}
|
|
80
|
+
// ── FR-3 / aggregate ────────────────────────────────────────────────────────────────────────────────────
|
|
81
|
+
const dimVal = (s, d) => {
|
|
82
|
+
const v = s[d];
|
|
83
|
+
return typeof v === 'number' && Number.isFinite(v) ? v : 0;
|
|
84
|
+
};
|
|
85
|
+
/** A judge score is a finite number in the rubric range [0, 10]. Anything else (missing, string, NaN,
|
|
86
|
+
* Infinity, negative, >10) is MALFORMED — it must never coerce silently to 0 and fabricate a winner (QE). */
|
|
87
|
+
const isValidScore = (v) => typeof v === 'number' && Number.isFinite(v) && v >= 0 && v <= 10;
|
|
88
|
+
/** Every one of the 5 BTO dimensions must be a valid score for a DimScores to be usable. */
|
|
89
|
+
export function validDimScores(s) {
|
|
90
|
+
if (s === null || typeof s !== 'object')
|
|
91
|
+
return false;
|
|
92
|
+
const r = s;
|
|
93
|
+
return BTO_DIMENSIONS.every((d) => isValidScore(r[d]));
|
|
94
|
+
}
|
|
95
|
+
export function aggregate(s) {
|
|
96
|
+
return BTO_DIMENSIONS.reduce((sum, d) => sum + dimVal(s, d), 0) / BTO_DIMENSIONS.length;
|
|
97
|
+
}
|
|
98
|
+
/** The lowest-scoring dimension (the bottleneck to lift). Deterministic tie-break by BTO_DIMENSIONS order. */
|
|
99
|
+
export function weakestDimension(s) {
|
|
100
|
+
let best = BTO_DIMENSIONS[0];
|
|
101
|
+
for (const d of BTO_DIMENSIONS)
|
|
102
|
+
if (dimVal(s, d) < dimVal(s, best))
|
|
103
|
+
best = d;
|
|
104
|
+
return best;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Accept a candidate ONLY on the HOLD-OUT scores (ADR §2): the baseline's weakest dimension must improve by > 0
|
|
108
|
+
* AND no other dimension AND the aggregate may regress by more than `tolerance` (default 0 = strict). Candidates
|
|
109
|
+
* are considered in descending TUNE-aggregate order (the minibatch rank — dspy's search), but acceptance reads
|
|
110
|
+
* only holdout, so a tune-winner that regresses on holdout is rejected. Deterministic; returns the first
|
|
111
|
+
* candidate that passes, or null with a reason.
|
|
112
|
+
*/
|
|
113
|
+
export function selectWinner(baseline, candidates, opts = {}) {
|
|
114
|
+
// Reject malformed input BEFORE any comparison (QE): coercing missing/junk dims to 0 fabricates a winner.
|
|
115
|
+
if (!baseline || !validDimScores(baseline.holdout)) {
|
|
116
|
+
return { winner: null, reason: 'invalid baseline holdout scores (every dimension must be a number in [0,10])', weakest: BTO_DIMENSIONS[0] };
|
|
117
|
+
}
|
|
118
|
+
// tolerance must itself be a finite non-negative number (Infinity would disable the no-regress guard).
|
|
119
|
+
const tolerance = typeof opts.tolerance === 'number' && Number.isFinite(opts.tolerance) && opts.tolerance >= 0 ? opts.tolerance : 0;
|
|
120
|
+
const weakest = weakestDimension(baseline.holdout);
|
|
121
|
+
// only candidates with VALID holdout scores are eligible; a malformed candidate can never win.
|
|
122
|
+
const eligible = candidates.filter((c) => validDimScores(c.holdout) && validDimScores(c.tune));
|
|
123
|
+
const ranked = [...eligible].sort((a, b) => aggregate(b.tune) - aggregate(a.tune) || byStr(a.id, b.id));
|
|
124
|
+
let lastReason = `no candidate improved the weakest dimension (${weakest}) on the holdout without regression`;
|
|
125
|
+
for (const c of ranked) {
|
|
126
|
+
const weakDelta = dimVal(c.holdout, weakest) - dimVal(baseline.holdout, weakest);
|
|
127
|
+
if (weakDelta <= 0) {
|
|
128
|
+
lastReason = `${c.id}: weakest ${weakest} did not improve on holdout (Δ=${weakDelta.toFixed(2)})`;
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
// no-regress on every OTHER dimension + the aggregate
|
|
132
|
+
let regressed = null;
|
|
133
|
+
for (const d of BTO_DIMENSIONS) {
|
|
134
|
+
if (d === weakest)
|
|
135
|
+
continue;
|
|
136
|
+
const delta = dimVal(c.holdout, d) - dimVal(baseline.holdout, d);
|
|
137
|
+
if (delta < -tolerance) {
|
|
138
|
+
regressed = `${d} (Δ=${delta.toFixed(2)})`;
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
const aggDelta = aggregate(c.holdout) - aggregate(baseline.holdout);
|
|
143
|
+
if (!regressed && aggDelta < -tolerance)
|
|
144
|
+
regressed = `aggregate (Δ=${aggDelta.toFixed(2)})`;
|
|
145
|
+
if (regressed) {
|
|
146
|
+
lastReason = `${c.id}: lifted ${weakest} but regressed ${regressed} on holdout`;
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
return {
|
|
150
|
+
winner: c.id,
|
|
151
|
+
reason: `accepted on holdout: ${weakest} +${weakDelta.toFixed(2)}, aggregate ${aggDelta >= 0 ? '+' : ''}${aggDelta.toFixed(2)}, no regression`,
|
|
152
|
+
weakest,
|
|
153
|
+
deltas: { [weakest]: weakDelta, aggregate: aggDelta },
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
return { winner: null, reason: lastReason, weakest };
|
|
157
|
+
}
|
|
158
|
+
// ── FR-7: prose-scope guard + diff ──────────────────────────────────────────────────────────────────────
|
|
159
|
+
// Normalize CRLF/CR → LF first, else a `\r\n` file evades the `\n`-anchored frontmatter/heading regexes (QE bypass).
|
|
160
|
+
const normEol = (t) => t.replace(/\r\n?/g, '\n');
|
|
161
|
+
const frontmatter = (t) => {
|
|
162
|
+
const m = /^---\n([\s\S]*?)\n---/.exec(normEol(t));
|
|
163
|
+
return m ? m[1] : '';
|
|
164
|
+
};
|
|
165
|
+
/** The document BODY (after the frontmatter block) — so the frontmatter's own closing `---` is never misread
|
|
166
|
+
* as a setext underline, and heading detection runs only on prose. */
|
|
167
|
+
const bodyAfterFrontmatter = (t) => {
|
|
168
|
+
const n = normEol(t);
|
|
169
|
+
const m = /^---\n[\s\S]*?\n---\n?/.exec(n);
|
|
170
|
+
return m ? n.slice(m[0].length) : n;
|
|
171
|
+
};
|
|
172
|
+
/**
|
|
173
|
+
* ALL structural markers on the BODY, not just space-delimited ATX (QE: `##\tNEW`, bare `##`, and setext
|
|
174
|
+
* `Title\n===` / `Title\n---` evaded the old regex). Fenced code blocks (``` / ~~~) are SKIPPED so a `===`/`---`
|
|
175
|
+
* line INSIDE code is not misread as a heading (QE false-positive). Collect ATX (`#`..`######` + any/no ws)
|
|
176
|
+
* AND setext underlines (a non-empty line immediately followed by `=+`/`-+`).
|
|
177
|
+
*/
|
|
178
|
+
const headings = (t) => {
|
|
179
|
+
const lines = bodyAfterFrontmatter(t).split('\n');
|
|
180
|
+
const out = [];
|
|
181
|
+
let inFence = false;
|
|
182
|
+
for (let i = 0; i < lines.length; i++) {
|
|
183
|
+
const line = lines[i];
|
|
184
|
+
if (/^\s*(```|~~~)/.test(line)) {
|
|
185
|
+
inFence = !inFence;
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
if (inFence)
|
|
189
|
+
continue;
|
|
190
|
+
const atx = /^(#{1,6})(?:\s.*)?$/.exec(line.replace(/\s+$/, ''));
|
|
191
|
+
if (atx) {
|
|
192
|
+
out.push('ATX:' + line.trim());
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
const next = lines[i + 1];
|
|
196
|
+
if (line.trim() !== '' && !/^\s*(```|~~~)/.test(next ?? '') && next !== undefined && /^(=+|-+)\s*$/.test(next)) {
|
|
197
|
+
out.push('SETEXT:' + line.trim() + '|' + next.trim());
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
return out;
|
|
201
|
+
};
|
|
202
|
+
/** Only directive PROSE may change (Phase-1): frontmatter + the set of ALL markdown structural markers must be
|
|
203
|
+
* identical. A candidate that alters them (incl. CRLF, tab-ATX, empty-ATX, setext) is rejected (augment-not-clobber). */
|
|
204
|
+
export function proseScopeOk(original, candidate) {
|
|
205
|
+
if (frontmatter(original) !== frontmatter(candidate))
|
|
206
|
+
return { ok: false, reason: 'candidate changed the YAML frontmatter (out of scope: prose only)' };
|
|
207
|
+
const ho = headings(original);
|
|
208
|
+
const hc = headings(candidate);
|
|
209
|
+
if (ho.length !== hc.length || ho.some((h, i) => h !== hc[i]))
|
|
210
|
+
return { ok: false, reason: 'candidate changed the section headings/structure (out of scope: prose only)' };
|
|
211
|
+
return { ok: true, reason: 'prose-only change' };
|
|
212
|
+
}
|
|
213
|
+
/** A minimal deterministic unified-ish line diff for the confirm gate (pure). */
|
|
214
|
+
export function renderProseDiff(original, candidate) {
|
|
215
|
+
const a = original.split('\n');
|
|
216
|
+
const b = candidate.split('\n');
|
|
217
|
+
const out = ['--- current', '+++ candidate'];
|
|
218
|
+
const max = Math.max(a.length, b.length);
|
|
219
|
+
for (let i = 0; i < max; i++) {
|
|
220
|
+
const l = a[i];
|
|
221
|
+
const r = b[i];
|
|
222
|
+
if (l === r)
|
|
223
|
+
continue;
|
|
224
|
+
if (l !== undefined)
|
|
225
|
+
out.push('- ' + l);
|
|
226
|
+
if (r !== undefined)
|
|
227
|
+
out.push('+ ' + r);
|
|
228
|
+
}
|
|
229
|
+
return out.length > 2 ? out.join('\n') : '(no textual difference)';
|
|
230
|
+
}
|
|
231
|
+
// ── thin I/O (top-level ESM fs; never throws) ───────────────────────────────────────────────────────────
|
|
232
|
+
/** Read a scenario-id list: JSON array, or newline/comma-separated. Absent/unreadable → []. */
|
|
233
|
+
export function readScenarioIds(path) {
|
|
234
|
+
try {
|
|
235
|
+
if (!existsSync(path))
|
|
236
|
+
return [];
|
|
237
|
+
const raw = readFileSync(path, 'utf8').trim();
|
|
238
|
+
if (raw === '')
|
|
239
|
+
return [];
|
|
240
|
+
if (raw.startsWith('[')) {
|
|
241
|
+
const arr = JSON.parse(raw);
|
|
242
|
+
return Array.isArray(arr) ? arr.map((x) => String(x)).filter((s) => s !== '') : [];
|
|
243
|
+
}
|
|
244
|
+
return raw.split(/[\n,]/).map((s) => s.trim()).filter((s) => s !== '');
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
return [];
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
//# sourceMappingURL=bto-optimize.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bto-optimize.js","sourceRoot":"","sources":["../src/bto-optimize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAGnD,MAAM,CAAC,MAAM,cAAc,GAA4B,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC;AAGzI,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAS7E;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,GAAsB,EAAE,YAAY,GAAG,IAAI;IACxE,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3F,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC1D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC9D,MAAM,KAAK,GAAG,YAAY,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IACzE,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,uBAAuB;IACrE,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IACzE,yBAAyB;IACzB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAG,CAAC,CAAC;IACpD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAG,CAAC,CAAC;IACjD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAC3B,CAAC;AAqBD;iHACiH;AACjH,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAEzC;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,KAAkB,EAAE,eAAuB,sBAAsB;IAC1F,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC;IAClH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;IAC3D,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;IACjE,IAAI,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3D,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,SAAS,GAAG,YAAY,CAAC;IACjF,IAAI,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,GAAG,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAAC,MAAM,GAAG,CAAC,CAAC;QAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAAC,CAAC;IAC1F,OAAO,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,GAAG,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QAAC,UAAU,IAAI,CAAC,CAAC;IAAC,CAAC;IAC9E,IAAI,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,CAAC;IACnG,MAAM,QAAQ,GAAG,UAAU,GAAG,MAAM,GAAG,SAAS,CAAC;IACjD,MAAM,WAAW,GAAG,YAAY,CAAC,CAAC,0CAA0C;IAC5E,MAAM,SAAS,GAAG,QAAQ,GAAG,WAAW,CAAC;IACzC,MAAM,IAAI,GAAe;QACvB,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,IAAI,GAAG;QACtF,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvD,CAAC;IACF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,2GAA2G;AAE3G,MAAM,MAAM,GAAG,CAAC,CAAY,EAAE,CAAe,EAAU,EAAE;IACvD,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACf,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7D,CAAC,CAAC;AAEF;8GAC8G;AAC9G,MAAM,YAAY,GAAG,CAAC,CAAU,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;AACnH,4FAA4F;AAC5F,MAAM,UAAU,cAAc,CAAC,CAAU;IACvC,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,CAAC,GAAG,CAA4B,CAAC;IACvC,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC;AACD,MAAM,UAAU,SAAS,CAAC,CAAY;IACpC,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;AAC1F,CAAC;AACD,8GAA8G;AAC9G,MAAM,UAAU,gBAAgB,CAAC,CAAY;IAC3C,IAAI,IAAI,GAAiB,cAAc,CAAC,CAAC,CAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,cAAc;QAAE,IAAI,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC;YAAE,IAAI,GAAG,CAAC,CAAC;IAC7E,OAAO,IAAI,CAAC;AACd,CAAC;AAiBD;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,QAAyC,EAAE,UAAgC,EAAE,OAA+B,EAAE;IACzI,0GAA0G;IAC1G,IAAI,CAAC,QAAQ,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,8EAA8E,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC,CAAE,EAAE,CAAC;IAC/I,CAAC;IACD,uGAAuG;IACvG,MAAM,SAAS,GAAG,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACpI,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACnD,+FAA+F;IAC/F,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/F,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxG,IAAI,UAAU,GAAG,gDAAgD,OAAO,qCAAqC,CAAC;IAC9G,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACjF,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YAAC,UAAU,GAAG,GAAG,CAAC,CAAC,EAAE,aAAa,OAAO,kCAAkC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;YAAC,SAAS;QAAC,CAAC;QACpI,sDAAsD;QACtD,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,KAAK,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC,KAAK,OAAO;gBAAE,SAAS;YAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjE,IAAI,KAAK,GAAG,CAAC,SAAS,EAAE,CAAC;gBAAC,SAAS,GAAG,GAAG,CAAC,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;gBAAC,MAAM;YAAC,CAAC;QAChF,CAAC;QACD,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,IAAI,QAAQ,GAAG,CAAC,SAAS;YAAE,SAAS,GAAG,gBAAgB,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5F,IAAI,SAAS,EAAE,CAAC;YAAC,UAAU,GAAG,GAAG,CAAC,CAAC,EAAE,YAAY,OAAO,kBAAkB,SAAS,aAAa,CAAC;YAAC,SAAS;QAAC,CAAC;QAC7G,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,EAAE;YACZ,MAAM,EAAE,wBAAwB,OAAO,KAAK,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB;YAC9I,OAAO;YACP,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE;SACtD,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;AACvD,CAAC;AAED,2GAA2G;AAE3G,qHAAqH;AACrH,MAAM,OAAO,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AACjE,MAAM,WAAW,GAAG,CAAC,CAAS,EAAU,EAAE;IACxC,MAAM,CAAC,GAAG,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACnD,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AACxB,CAAC,CAAC;AACF;uEACuE;AACvE,MAAM,oBAAoB,GAAG,CAAC,CAAS,EAAU,EAAE;IACjD,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACrB,MAAM,CAAC,GAAG,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3C,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAY,EAAE;IACvC,MAAM,KAAK,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QACvB,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,OAAO,GAAG,CAAC,OAAO,CAAC;YAAC,SAAS;QAAC,CAAC;QACjE,IAAI,OAAO;YAAE,SAAS;QACtB,MAAM,GAAG,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QACjE,IAAI,GAAG,EAAE,CAAC;YAAC,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QACtD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1B,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,IAAI,KAAK,SAAS,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/G,GAAG,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF;0HAC0H;AAC1H,MAAM,UAAU,YAAY,CAAC,QAAgB,EAAE,SAAiB;IAC9D,IAAI,WAAW,CAAC,QAAQ,CAAC,KAAK,WAAW,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mEAAmE,EAAE,CAAC;IACxJ,MAAM,EAAE,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,MAAM,KAAK,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,6EAA6E,EAAE,CAAC;IAC3K,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;AACnD,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,eAAe,CAAC,QAAgB,EAAE,SAAiB;IACjE,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,GAAG,GAAa,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACf,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACf,IAAI,CAAC,KAAK,CAAC;YAAE,SAAS;QACtB,IAAI,CAAC,KAAK,SAAS;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,SAAS;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC;AACrE,CAAC;AAED,2GAA2G;AAE3G,+FAA+F;AAC/F,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,IAAI,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9C,IAAI,GAAG,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QAC1B,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAY,CAAC;YACvC,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACrF,CAAC;QACD,OAAO,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/** The outcome of running one property test in the base (pre-feature) worktree. */
|
|
2
|
+
export type BaseOutcome = 'pass' | 'fail' | 'error';
|
|
3
|
+
/** Per-property verdict after classification. */
|
|
4
|
+
export type DiscriminationVerdict = 'DISCRIMINATES' | 'DISCRIMINATES_VIA_ERROR' | 'NON_DISCRIMINATING' | 'CANNOT_ISOLATE';
|
|
5
|
+
/** A property test mapped from the ADR Confirmation `Required automated check`. */
|
|
6
|
+
export interface PropertyTestRef {
|
|
7
|
+
/** repo-relative path to the NEW test file written for the feature. */
|
|
8
|
+
readonly file: string;
|
|
9
|
+
/** optional test-name filter (e.g. vitest `-t`) to run just the property case. */
|
|
10
|
+
readonly name?: string;
|
|
11
|
+
}
|
|
12
|
+
export interface DiscriminationPlanInput {
|
|
13
|
+
/** the git ref of pre-feature HEAD — the "base" the property test must fail against. */
|
|
14
|
+
readonly baseRef: string;
|
|
15
|
+
/** property test(s) mapped from the ADR Confirmation. Empty ⇒ CANNOT_ISOLATE. */
|
|
16
|
+
readonly propertyTests: readonly PropertyTestRef[];
|
|
17
|
+
/** test-runner command template; sanitized. Default `npx vitest run`. */
|
|
18
|
+
readonly runner?: string;
|
|
19
|
+
}
|
|
20
|
+
export interface DiscriminationPlan {
|
|
21
|
+
/** false ⇒ nothing to run (no isolable test, or every ref rejected as unsafe). */
|
|
22
|
+
readonly runnable: boolean;
|
|
23
|
+
/** why not runnable, when `runnable` is false. */
|
|
24
|
+
readonly reason?: string;
|
|
25
|
+
/** the sanitized base ref actually used. */
|
|
26
|
+
readonly baseRef: string;
|
|
27
|
+
/** the accepted, sanitized targets. */
|
|
28
|
+
readonly targets: readonly PropertyTestRef[];
|
|
29
|
+
/** refs rejected by sanitation, with the reason — surfaced so a rejection is never silent. */
|
|
30
|
+
readonly rejected: readonly {
|
|
31
|
+
readonly file: string;
|
|
32
|
+
readonly reason: string;
|
|
33
|
+
}[];
|
|
34
|
+
/**
|
|
35
|
+
* Ordered shell steps the caller runs: add a detached worktree at baseRef, copy each NEW property test
|
|
36
|
+
* file into it (they do not exist at base), run the runner over the targets, then remove the worktree.
|
|
37
|
+
* `{{WORKTREE}}` is a placeholder the caller substitutes with a fresh temp dir path it owns — the engine
|
|
38
|
+
* never invents a filesystem path. Commands use only sanitized tokens.
|
|
39
|
+
*/
|
|
40
|
+
readonly commands: readonly string[];
|
|
41
|
+
}
|
|
42
|
+
export interface ClassifyInput {
|
|
43
|
+
readonly propertyTests: readonly PropertyTestRef[];
|
|
44
|
+
/** observed base outcome per target file (+optional name), same identity the plan used. */
|
|
45
|
+
readonly results: readonly {
|
|
46
|
+
readonly file: string;
|
|
47
|
+
readonly name?: string;
|
|
48
|
+
readonly outcome: BaseOutcome;
|
|
49
|
+
}[];
|
|
50
|
+
}
|
|
51
|
+
export interface PerTestVerdict {
|
|
52
|
+
readonly file: string;
|
|
53
|
+
readonly name?: string;
|
|
54
|
+
readonly verdict: DiscriminationVerdict;
|
|
55
|
+
}
|
|
56
|
+
export interface DiscriminationFinding {
|
|
57
|
+
/** high only for a proven false green; the rest are advisory. Never a hard blocker (owner decides). */
|
|
58
|
+
readonly severity: 'high' | 'info';
|
|
59
|
+
readonly title: string;
|
|
60
|
+
readonly detail: string;
|
|
61
|
+
}
|
|
62
|
+
export interface DiscriminationResult {
|
|
63
|
+
readonly perTest: readonly PerTestVerdict[];
|
|
64
|
+
/** the gate's overall verdict (worst case across targets). */
|
|
65
|
+
readonly aggregate: DiscriminationVerdict;
|
|
66
|
+
/** the Step-8 finding to fold into 08_qe_report.md — null when everything discriminates cleanly. */
|
|
67
|
+
readonly finding: DiscriminationFinding | null;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Plan the discrimination check. Pure: validates inputs, drops unsafe refs (never silently), and emits the
|
|
71
|
+
* ordered worktree commands. Returns `runnable:false` with a reason when there is nothing safe to run.
|
|
72
|
+
*/
|
|
73
|
+
export declare function planDiscriminationCheck(input: DiscriminationPlanInput): DiscriminationPlan;
|
|
74
|
+
/**
|
|
75
|
+
* Classify observed base outcomes into per-test and aggregate verdicts + a Step-8 finding. A green-at-base
|
|
76
|
+
* property test is the load-bearing failure this gate exists to catch: it is surfaced as a HIGH finding but
|
|
77
|
+
* NEVER auto-aborts (dz's rule — a false gate kills trust; the owner decides). CANNOT_ISOLATE folds into the
|
|
78
|
+
* existing "property untested" finding rather than minting a new severity concept.
|
|
79
|
+
*/
|
|
80
|
+
export declare function classifyDiscrimination(input: ClassifyInput): DiscriminationResult;
|
|
81
|
+
//# sourceMappingURL=discrimination-gate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"discrimination-gate.d.ts","sourceRoot":"","sources":["../src/discrimination-gate.ts"],"names":[],"mappings":"AAcA,mFAAmF;AACnF,MAAM,MAAM,WAAW,GACnB,MAAM,GACN,MAAM,GACN,OAAO,CAAC;AAEZ,iDAAiD;AACjD,MAAM,MAAM,qBAAqB,GAC7B,eAAe,GACf,yBAAyB,GACzB,oBAAoB,GACpB,gBAAgB,CAAC;AAErB,mFAAmF;AACnF,MAAM,WAAW,eAAe;IAC9B,uEAAuE;IACvE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,kFAAkF;IAClF,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,uBAAuB;IACtC,wFAAwF;IACxF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,iFAAiF;IACjF,QAAQ,CAAC,aAAa,EAAE,SAAS,eAAe,EAAE,CAAC;IACnD,yEAAyE;IACzE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAkB;IACjC,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,kDAAkD;IAClD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,4CAA4C;IAC5C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,uCAAuC;IACvC,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;IAC7C,8FAA8F;IAC9F,QAAQ,CAAC,QAAQ,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjF;;;;;OAKG;IACH,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,aAAa,EAAE,SAAS,eAAe,EAAE,CAAC;IACnD,2FAA2F;IAC3F,QAAQ,CAAC,OAAO,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAA;KAAE,EAAE,CAAC;CAC/G;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAC;CACzC;AAED,MAAM,WAAW,qBAAqB;IACpC,uGAAuG;IACvG,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,SAAS,cAAc,EAAE,CAAC;IAC5C,8DAA8D;IAC9D,QAAQ,CAAC,SAAS,EAAE,qBAAqB,CAAC;IAC1C,oGAAoG;IACpG,QAAQ,CAAC,OAAO,EAAE,qBAAqB,GAAG,IAAI,CAAC;CAChD;AAsBD;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,uBAAuB,GAAG,kBAAkB,CA0C1F;AAiBD;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,aAAa,GAAG,oBAAoB,CAyDjF"}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// §42 test-discrimination gate for feature-adr Step-8.
|
|
2
|
+
//
|
|
3
|
+
// Grounded in cve-bench/scripts/evaluate.mjs (RuvNet), whose §42 loop proves a regression test actually
|
|
4
|
+
// CATCHES the defect: `base+test → FAIL`, `gold+test → PASS`. feature-adr Step-8 already asserts the ADR's
|
|
5
|
+
// named safety property *has* a test; this gate asserts that test *discriminates* — run the property test(s)
|
|
6
|
+
// in an isolated git worktree at pre-feature HEAD (the "base"), and they MUST go red. A test that stays green
|
|
7
|
+
// without the feature diff does not exercise the property; it is a false green.
|
|
8
|
+
//
|
|
9
|
+
// This module is PURE: it plans the check (which worktree, which targets, what commands) and classifies an
|
|
10
|
+
// observed result. The worktree creation + test execution is I/O performed by the caller (the CLI / the
|
|
11
|
+
// sandboxed workflow), which feeds the observed per-target outcome back into `classifyDiscrimination`. Same
|
|
12
|
+
// pure-engine + injected-data shape as challenge-panel.ts and routing-outcomes.ts, so it is deterministic and
|
|
13
|
+
// unit-testable without a git repo or a test runner.
|
|
14
|
+
/** git-ref-safe characters only; rejects shell metacharacters and traversal that would break out of a token. */
|
|
15
|
+
const SAFE_REF = /^[A-Za-z0-9_][A-Za-z0-9_./~^@{}-]{0,199}$/;
|
|
16
|
+
/**
|
|
17
|
+
* A repo-relative test path. Rejects: absolute / drive / `~`; a `..` segment; a segment starting with `-`
|
|
18
|
+
* (would become an argv option like `--config` injected into the runner); whitespace (word-splitting); and any
|
|
19
|
+
* NUL/backtick/quote/backslash/shell metacharacter. Deliberately strict — a property test path is a plain
|
|
20
|
+
* relative file, and anything exotic is safer rejected (and surfaced) than quoted-and-hoped.
|
|
21
|
+
*/
|
|
22
|
+
const UNSAFE_PATH = /(^\/)|(^[A-Za-z]:)|(^~)|(^-)|(\/-)|(\.\.(\/|\\|$))|[\0`$;&|<>*?"'\n\r\t \\]/;
|
|
23
|
+
const DEFAULT_RUNNER = 'npx vitest run';
|
|
24
|
+
/** a runner must be a plain command with flags — no shell metacharacters that could chain a second command. */
|
|
25
|
+
const UNSAFE_RUNNER = /[\0`$;&|<>()\n\r]/;
|
|
26
|
+
function sanitizeName(name) {
|
|
27
|
+
// a test-name filter is passed as a single-quoted argv; forbid the quote/metacharacters that would escape it.
|
|
28
|
+
if (name.length === 0 || name.length > 200)
|
|
29
|
+
return null;
|
|
30
|
+
if (/[\0`$;&|<>*?"'\n\r\\]/.test(name))
|
|
31
|
+
return null;
|
|
32
|
+
return name;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Plan the discrimination check. Pure: validates inputs, drops unsafe refs (never silently), and emits the
|
|
36
|
+
* ordered worktree commands. Returns `runnable:false` with a reason when there is nothing safe to run.
|
|
37
|
+
*/
|
|
38
|
+
export function planDiscriminationCheck(input) {
|
|
39
|
+
const baseRef = typeof input.baseRef === 'string' ? input.baseRef.trim() : '';
|
|
40
|
+
const rejected = [];
|
|
41
|
+
if (!SAFE_REF.test(baseRef)) {
|
|
42
|
+
return { runnable: false, reason: 'unsafe-or-missing-base-ref', baseRef, targets: [], rejected, commands: [] };
|
|
43
|
+
}
|
|
44
|
+
const runnerRaw = typeof input.runner === 'string' && input.runner.trim() ? input.runner.trim() : DEFAULT_RUNNER;
|
|
45
|
+
const runner = UNSAFE_RUNNER.test(runnerRaw) ? DEFAULT_RUNNER : runnerRaw;
|
|
46
|
+
const targets = [];
|
|
47
|
+
const seen = new Set();
|
|
48
|
+
for (const t of Array.isArray(input.propertyTests) ? input.propertyTests : []) {
|
|
49
|
+
const file = t && typeof t.file === 'string' ? t.file.trim() : '';
|
|
50
|
+
if (!file) {
|
|
51
|
+
rejected.push({ file: String(t?.file ?? ''), reason: 'empty-path' });
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (UNSAFE_PATH.test(file)) {
|
|
55
|
+
rejected.push({ file, reason: 'unsafe-path' });
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const name = t && typeof t.name === 'string' ? sanitizeName(t.name) : null;
|
|
59
|
+
if (t && typeof t.name === 'string' && name === null) {
|
|
60
|
+
rejected.push({ file, reason: 'unsafe-test-name' });
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
const key = `${file}|${name ?? ''}`;
|
|
64
|
+
if (seen.has(key))
|
|
65
|
+
continue;
|
|
66
|
+
seen.add(key);
|
|
67
|
+
targets.push(name !== null ? { file, name } : { file });
|
|
68
|
+
}
|
|
69
|
+
if (targets.length === 0) {
|
|
70
|
+
return { runnable: false, reason: 'no-isolable-test', baseRef, targets, rejected, commands: [] };
|
|
71
|
+
}
|
|
72
|
+
// `{{WORKTREE}}` is substituted by the caller with a temp dir IT owns; the engine never invents a path.
|
|
73
|
+
// Paths are already metacharacter-free (UNSAFE_PATH), but quote them + use `--` so a leading-dash or spaced
|
|
74
|
+
// path can never become a runner option or split a word — belt-and-suspenders over the sanitation above.
|
|
75
|
+
const commands = [`git worktree add --detach {{WORKTREE}} ${baseRef}`];
|
|
76
|
+
for (const t of targets) {
|
|
77
|
+
commands.push(`mkdir -p "{{WORKTREE}}/$(dirname -- '${t.file}')" && cp -- '${t.file}' "{{WORKTREE}}/${t.file}"`);
|
|
78
|
+
}
|
|
79
|
+
const fileArgs = [...new Set(targets.map((t) => t.file))].map((f) => `'${f}'`).join(' ');
|
|
80
|
+
const nameFilters = targets.filter((t) => t.name).map((t) => `-t '${t.name}'`).join(' ');
|
|
81
|
+
commands.push(`( cd {{WORKTREE}} && ${runner}${nameFilters ? ' ' + nameFilters : ''} -- ${fileArgs} )`);
|
|
82
|
+
commands.push(`git worktree remove --force {{WORKTREE}}`);
|
|
83
|
+
return { runnable: true, baseRef, targets, rejected, commands };
|
|
84
|
+
}
|
|
85
|
+
function verdictFor(outcome) {
|
|
86
|
+
if (outcome === 'fail')
|
|
87
|
+
return 'DISCRIMINATES';
|
|
88
|
+
if (outcome === 'error')
|
|
89
|
+
return 'DISCRIMINATES_VIA_ERROR';
|
|
90
|
+
if (outcome === 'pass')
|
|
91
|
+
return 'NON_DISCRIMINATING';
|
|
92
|
+
return 'CANNOT_ISOLATE'; // no observed result for this target
|
|
93
|
+
}
|
|
94
|
+
/** Severity rank for aggregation — a proven false green is the worst outcome the gate reports. */
|
|
95
|
+
const RANK = {
|
|
96
|
+
NON_DISCRIMINATING: 3,
|
|
97
|
+
CANNOT_ISOLATE: 2,
|
|
98
|
+
DISCRIMINATES_VIA_ERROR: 1,
|
|
99
|
+
DISCRIMINATES: 0,
|
|
100
|
+
};
|
|
101
|
+
/**
|
|
102
|
+
* Classify observed base outcomes into per-test and aggregate verdicts + a Step-8 finding. A green-at-base
|
|
103
|
+
* property test is the load-bearing failure this gate exists to catch: it is surfaced as a HIGH finding but
|
|
104
|
+
* NEVER auto-aborts (dz's rule — a false gate kills trust; the owner decides). CANNOT_ISOLATE folds into the
|
|
105
|
+
* existing "property untested" finding rather than minting a new severity concept.
|
|
106
|
+
*/
|
|
107
|
+
export function classifyDiscrimination(input) {
|
|
108
|
+
const results = Array.isArray(input.results) ? input.results : [];
|
|
109
|
+
const byKey = new Map();
|
|
110
|
+
for (const r of results) {
|
|
111
|
+
if (r && typeof r.file === 'string')
|
|
112
|
+
byKey.set(`${r.file}|${r.name ?? ''}`, r.outcome);
|
|
113
|
+
}
|
|
114
|
+
const propertyTests = Array.isArray(input.propertyTests) ? input.propertyTests : [];
|
|
115
|
+
if (propertyTests.length === 0) {
|
|
116
|
+
return {
|
|
117
|
+
perTest: [],
|
|
118
|
+
aggregate: 'CANNOT_ISOLATE',
|
|
119
|
+
finding: {
|
|
120
|
+
severity: 'info',
|
|
121
|
+
title: 'discrimination gate: no property test to check',
|
|
122
|
+
detail: 'No test was mapped to the ADR safety property, so discrimination could not be evaluated — this is the existing "property untested" finding.',
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
const perTest = propertyTests.map((t) => {
|
|
127
|
+
const file = typeof t.file === 'string' ? t.file : '';
|
|
128
|
+
const name = typeof t.name === 'string' ? t.name : undefined;
|
|
129
|
+
const outcome = byKey.get(`${file}|${name ?? ''}`);
|
|
130
|
+
const verdict = verdictFor(outcome);
|
|
131
|
+
return name !== undefined ? { file, name, verdict } : { file, verdict };
|
|
132
|
+
});
|
|
133
|
+
let aggregate = 'DISCRIMINATES';
|
|
134
|
+
for (const p of perTest)
|
|
135
|
+
if (RANK[p.verdict] > RANK[aggregate])
|
|
136
|
+
aggregate = p.verdict;
|
|
137
|
+
const falseGreens = perTest.filter((p) => p.verdict === 'NON_DISCRIMINATING');
|
|
138
|
+
let finding = null;
|
|
139
|
+
if (falseGreens.length > 0) {
|
|
140
|
+
finding = {
|
|
141
|
+
severity: 'high',
|
|
142
|
+
title: 'non-discriminating property test (false green)',
|
|
143
|
+
detail: `${falseGreens.length} property test(s) PASS at pre-feature base — they do not exercise the ADR safety property and would stay green even if the fix regressed: ` +
|
|
144
|
+
falseGreens.map((p) => (p.name ? `${p.file} (${p.name})` : p.file)).join(', ') +
|
|
145
|
+
'. Strengthen the assertion so it fails without the feature diff. (Advisory — the pipeline continues; the owner decides.)',
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
else if (aggregate === 'CANNOT_ISOLATE') {
|
|
149
|
+
finding = {
|
|
150
|
+
severity: 'info',
|
|
151
|
+
title: 'discrimination gate: could not run a mapped test',
|
|
152
|
+
detail: 'One or more mapped property tests produced no observed base outcome — folds into the existing "property untested" finding.',
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
else if (aggregate === 'DISCRIMINATES_VIA_ERROR') {
|
|
156
|
+
finding = {
|
|
157
|
+
severity: 'info',
|
|
158
|
+
title: 'discrimination inferred via load error',
|
|
159
|
+
detail: 'The property test could not load at pre-feature base (it references feature code that does not exist yet), so discrimination is INFERRED, not proven by an assertion. Acceptable, but a green-path assertion would be stronger.',
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
return { perTest, aggregate, finding };
|
|
163
|
+
}
|
|
164
|
+
//# sourceMappingURL=discrimination-gate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"discrimination-gate.js","sourceRoot":"","sources":["../src/discrimination-gate.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,EAAE;AACF,wGAAwG;AACxG,2GAA2G;AAC3G,6GAA6G;AAC7G,8GAA8G;AAC9G,gFAAgF;AAChF,EAAE;AACF,2GAA2G;AAC3G,wGAAwG;AACxG,4GAA4G;AAC5G,8GAA8G;AAC9G,qDAAqD;AA+ErD,gHAAgH;AAChH,MAAM,QAAQ,GAAG,2CAA2C,CAAC;AAC7D;;;;;GAKG;AACH,MAAM,WAAW,GAAG,6EAA6E,CAAC;AAClG,MAAM,cAAc,GAAG,gBAAgB,CAAC;AACxC,+GAA+G;AAC/G,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAE1C,SAAS,YAAY,CAAC,IAAY;IAChC,8GAA8G;IAC9G,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,IAAI,CAAC;IACxD,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACpD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAA8B;IACpE,MAAM,OAAO,GAAG,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9E,MAAM,QAAQ,GAAuC,EAAE,CAAC;IAExD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,4BAA4B,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACjH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC;IACjH,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC;IAE1E,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC9E,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,IAAI,CAAC,IAAI,EAAE,CAAC;YAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAC9F,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QACzF,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3E,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QACxH,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACnG,CAAC;IAED,wGAAwG;IACxG,4GAA4G;IAC5G,yGAAyG;IACzG,MAAM,QAAQ,GAAa,CAAC,0CAA0C,OAAO,EAAE,CAAC,CAAC;IACjF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC,IAAI,iBAAiB,CAAC,CAAC,IAAI,mBAAmB,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;IACnH,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzF,QAAQ,CAAC,IAAI,CAAC,wBAAwB,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,QAAQ,IAAI,CAAC,CAAC;IACxG,QAAQ,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;IAE1D,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,UAAU,CAAC,OAAgC;IAClD,IAAI,OAAO,KAAK,MAAM;QAAE,OAAO,eAAe,CAAC;IAC/C,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,yBAAyB,CAAC;IAC1D,IAAI,OAAO,KAAK,MAAM;QAAE,OAAO,oBAAoB,CAAC;IACpD,OAAO,gBAAgB,CAAC,CAAC,qCAAqC;AAChE,CAAC;AAED,kGAAkG;AAClG,MAAM,IAAI,GAA0C;IAClD,kBAAkB,EAAE,CAAC;IACrB,cAAc,EAAE,CAAC;IACjB,uBAAuB,EAAE,CAAC;IAC1B,aAAa,EAAE,CAAC;CACjB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,KAAoB;IACzD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;YAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IACzF,CAAC;IAED,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO;YACL,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,gBAAgB;YAC3B,OAAO,EAAE;gBACP,QAAQ,EAAE,MAAM;gBAChB,KAAK,EAAE,gDAAgD;gBACvD,MAAM,EAAE,6IAA6I;aACtJ;SACF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAqB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxD,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7D,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;QACpC,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,IAAI,SAAS,GAA0B,eAAe,CAAC;IACvD,KAAK,MAAM,CAAC,IAAI,OAAO;QAAE,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;YAAE,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC;IAEtF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,oBAAoB,CAAC,CAAC;IAC9E,IAAI,OAAO,GAAiC,IAAI,CAAC;IACjD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,GAAG;YACR,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,gDAAgD;YACvD,MAAM,EACJ,GAAG,WAAW,CAAC,MAAM,4IAA4I;gBACjK,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC9E,0HAA0H;SAC7H,CAAC;IACJ,CAAC;SAAM,IAAI,SAAS,KAAK,gBAAgB,EAAE,CAAC;QAC1C,OAAO,GAAG;YACR,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,kDAAkD;YACzD,MAAM,EAAE,4HAA4H;SACrI,CAAC;IACJ,CAAC;SAAM,IAAI,SAAS,KAAK,yBAAyB,EAAE,CAAC;QACnD,OAAO,GAAG;YACR,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,wCAAwC;YAC/C,MAAM,EAAE,iOAAiO;SAC1O,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;AACzC,CAAC"}
|