@deftai/directive-core 0.70.0 → 0.71.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/branch/evaluate.js +7 -0
- package/dist/doctor/doctor-state.js +2 -2
- package/dist/eval/crud-telemetry.d.ts +59 -0
- package/dist/eval/crud-telemetry.js +307 -0
- package/dist/eval/health.d.ts +52 -0
- package/dist/eval/health.js +240 -0
- package/dist/eval/readback.d.ts +38 -0
- package/dist/eval/readback.js +209 -0
- package/dist/eval/report.d.ts +53 -0
- package/dist/eval/report.js +161 -0
- package/dist/eval/run.d.ts +79 -0
- package/dist/eval/run.js +309 -0
- package/dist/events/attribution-constants.d.ts +13 -0
- package/dist/events/attribution-constants.js +18 -0
- package/dist/events/attribution-ledger.d.ts +43 -0
- package/dist/events/attribution-ledger.js +61 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +5 -0
- package/dist/init-deposit/gitignore.js +15 -5
- package/dist/intake/candidates-log.d.ts +0 -1
- package/dist/intake/candidates-log.js +5 -2
- package/dist/intake/issue-ingest.d.ts +8 -0
- package/dist/intake/issue-ingest.js +19 -1
- package/dist/intake/reconcile-issues.js +9 -2
- package/dist/layout/resolve.d.ts +2 -2
- package/dist/layout/resolve.js +2 -2
- package/dist/lifecycle/events.js +2 -0
- package/dist/orchestration/subagent-monitor.js +2 -2
- package/dist/policy/index.d.ts +2 -1
- package/dist/policy/index.js +15 -2
- package/dist/policy/value-feedback.d.ts +56 -0
- package/dist/policy/value-feedback.js +284 -0
- package/dist/preflight-cache/evaluate.js +4 -3
- package/dist/scope/audit-log.js +3 -3
- package/dist/scope/transition.js +16 -5
- package/dist/session/session-start.js +20 -0
- package/dist/slice/record.js +3 -3
- package/dist/task-surface/index.js +4 -1
- package/dist/triage/actions/candidates-log.d.ts +4 -4
- package/dist/triage/actions/candidates-log.js +4 -7
- package/dist/triage/actions/index.js +8 -32
- package/dist/triage/bootstrap/gitignore.d.ts +9 -3
- package/dist/triage/bootstrap/gitignore.js +77 -40
- package/dist/triage/bootstrap/index.d.ts +1 -1
- package/dist/triage/bootstrap/index.js +4 -3
- package/dist/triage/bulk/index.js +2 -2
- package/dist/triage/cache-path.d.ts +41 -0
- package/dist/triage/cache-path.js +115 -0
- package/dist/triage/help/registry-data.d.ts +48 -15
- package/dist/triage/help/registry-data.js +86 -15
- package/dist/triage/index.d.ts +2 -0
- package/dist/triage/index.js +2 -0
- package/dist/triage/queue/audit.js +2 -2
- package/dist/triage/queue/cache.js +2 -2
- package/dist/triage/reconcile/reconcile.js +4 -3
- package/dist/triage/scope/mutations-core.js +3 -2
- package/dist/triage/subscribe/index.d.ts +1 -1
- package/dist/triage/subscribe/index.js +4 -3
- package/dist/triage/summary/index.d.ts +2 -2
- package/dist/triage/summary/index.js +4 -3
- package/dist/triage/summary/reconcilable.d.ts +1 -1
- package/dist/triage/summary/reconcilable.js +3 -2
- package/dist/triage/welcome/constants.d.ts +2 -2
- package/dist/triage/welcome/constants.js +2 -2
- package/dist/triage/welcome/prior-state.js +3 -2
- package/dist/triage/welcome/summary.js +5 -4
- package/dist/value/adoption-emit.d.ts +14 -0
- package/dist/value/adoption-emit.js +67 -0
- package/dist/value/adoption-registry.d.ts +62 -0
- package/dist/value/adoption-registry.js +214 -0
- package/dist/value/feedback-file.d.ts +67 -0
- package/dist/value/feedback-file.js +353 -0
- package/dist/value/friction-emit.d.ts +10 -0
- package/dist/value/friction-emit.js +24 -0
- package/dist/value/readback.d.ts +86 -0
- package/dist/value/readback.js +503 -0
- package/dist/vbrief-validate/decomposition.js +1 -1
- package/dist/wip-cap/evaluate.js +3 -0
- package/package.json +35 -3
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { resolveEvalPath } from "../layout/resolve.js";
|
|
3
|
+
import { GOLDEN_RUNS_HISTORY_REL } from "./run.js";
|
|
4
|
+
export const REPORT_SCHEMA_VERSION = 1;
|
|
5
|
+
function readGoldenRuns(projectRoot) {
|
|
6
|
+
const path = resolveEvalPath(projectRoot, GOLDEN_RUNS_HISTORY_REL);
|
|
7
|
+
if (!existsSync(path)) {
|
|
8
|
+
return [];
|
|
9
|
+
}
|
|
10
|
+
const lines = readFileSync(path, "utf8")
|
|
11
|
+
.split("\n")
|
|
12
|
+
.filter((line) => line.trim().length > 0);
|
|
13
|
+
const records = [];
|
|
14
|
+
for (const line of lines) {
|
|
15
|
+
try {
|
|
16
|
+
records.push(JSON.parse(line));
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
// Skip malformed ledger rows.
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return records;
|
|
23
|
+
}
|
|
24
|
+
/** Pick the latest run for a directive version and model. */
|
|
25
|
+
export function findLatestGoldenRun(records, directiveVersion, model) {
|
|
26
|
+
const matches = records.filter((r) => r.directiveVersion === directiveVersion && r.model === model);
|
|
27
|
+
if (matches.length === 0) {
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
return matches.reduce((latest, current) => current.recordedAt >= latest.recordedAt ? current : latest);
|
|
31
|
+
}
|
|
32
|
+
function countPasses(results) {
|
|
33
|
+
return {
|
|
34
|
+
passed: results.filter((r) => r.passed).length,
|
|
35
|
+
total: results.length,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/** Two-proportion z-test (normal approximation). */
|
|
39
|
+
export function twoProportionZTest(passedA, totalA, passedB, totalB) {
|
|
40
|
+
if (totalA === 0 || totalB === 0) {
|
|
41
|
+
return { zScore: null, pValue: null };
|
|
42
|
+
}
|
|
43
|
+
const p1 = passedA / totalA;
|
|
44
|
+
const p2 = passedB / totalB;
|
|
45
|
+
const pooled = (passedA + passedB) / (totalA + totalB);
|
|
46
|
+
const se = Math.sqrt(pooled * (1 - pooled) * (1 / totalA + 1 / totalB));
|
|
47
|
+
if (se === 0) {
|
|
48
|
+
return { zScore: null, pValue: null };
|
|
49
|
+
}
|
|
50
|
+
const z = (p2 - p1) / se;
|
|
51
|
+
const pValue = 2 * (1 - normalCdf(Math.abs(z)));
|
|
52
|
+
return { zScore: z, pValue };
|
|
53
|
+
}
|
|
54
|
+
function normalCdf(x) {
|
|
55
|
+
return 0.5 * (1 + erf(x / Math.SQRT2));
|
|
56
|
+
}
|
|
57
|
+
function erf(x) {
|
|
58
|
+
const sign = x < 0 ? -1 : 1;
|
|
59
|
+
const ax = Math.abs(x);
|
|
60
|
+
const t = 1 / (1 + 0.3275911 * ax);
|
|
61
|
+
const y = 1 -
|
|
62
|
+
((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) *
|
|
63
|
+
t *
|
|
64
|
+
Math.exp(-ax * ax);
|
|
65
|
+
return sign * y;
|
|
66
|
+
}
|
|
67
|
+
function metricDelta(metric, championRate, challengerRate, championPassed, championTotal, challengerPassed, challengerTotal) {
|
|
68
|
+
const { zScore, pValue } = twoProportionZTest(championPassed, championTotal, challengerPassed, challengerTotal);
|
|
69
|
+
return {
|
|
70
|
+
metric,
|
|
71
|
+
champion: championRate,
|
|
72
|
+
challenger: challengerRate,
|
|
73
|
+
delta: challengerRate - championRate,
|
|
74
|
+
zScore,
|
|
75
|
+
pValue,
|
|
76
|
+
significantAt05: pValue !== null && pValue < 0.05,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/** Detect holdout tripwire: primary improves while holdout regresses (#1703). */
|
|
80
|
+
export function evaluateHoldoutTripwire(champion, challenger) {
|
|
81
|
+
const primaryDelta = challenger.summary.primaryPassRate - champion.summary.primaryPassRate;
|
|
82
|
+
const holdoutDelta = challenger.summary.holdoutPassRate - champion.summary.holdoutPassRate;
|
|
83
|
+
const triggered = primaryDelta > 0.05 && holdoutDelta < -0.05;
|
|
84
|
+
const summary = triggered
|
|
85
|
+
? "Holdout tripwire: challenger improved primary metrics but holdout regressed -- possible tuning to gated corpus."
|
|
86
|
+
: "Holdout tripwire clear.";
|
|
87
|
+
return { triggered, summary, primaryDelta, holdoutDelta };
|
|
88
|
+
}
|
|
89
|
+
function formatPercent(value) {
|
|
90
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
91
|
+
}
|
|
92
|
+
/** Diff two directive versions with metric deltas and significance (#1703 Tier 2). */
|
|
93
|
+
export function reportGoldenEval(options) {
|
|
94
|
+
if (!options.championVersion.trim() || !options.challengerVersion.trim()) {
|
|
95
|
+
return {
|
|
96
|
+
code: 2,
|
|
97
|
+
report: null,
|
|
98
|
+
message: "eval:report: --champion and --challenger are required",
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
if (!options.model.trim()) {
|
|
102
|
+
return { code: 2, report: null, message: "eval:report: --model is required" };
|
|
103
|
+
}
|
|
104
|
+
const projectRoot = options.projectRoot ?? process.cwd();
|
|
105
|
+
const records = readGoldenRuns(projectRoot);
|
|
106
|
+
const champion = findLatestGoldenRun(records, options.championVersion, options.model);
|
|
107
|
+
const challenger = findLatestGoldenRun(records, options.challengerVersion, options.model);
|
|
108
|
+
if (champion === null) {
|
|
109
|
+
return {
|
|
110
|
+
code: 1,
|
|
111
|
+
report: null,
|
|
112
|
+
message: `eval:report: no golden run found for champion v${options.championVersion} model=${options.model}`,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
if (challenger === null) {
|
|
116
|
+
return {
|
|
117
|
+
code: 1,
|
|
118
|
+
report: null,
|
|
119
|
+
message: `eval:report: no golden run found for challenger v${options.challengerVersion} model=${options.model}`,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
const championPrimary = champion.results.filter((r) => !r.holdout);
|
|
123
|
+
const challengerPrimary = challenger.results.filter((r) => !r.holdout);
|
|
124
|
+
const championHoldout = champion.results.filter((r) => r.holdout);
|
|
125
|
+
const challengerHoldout = challenger.results.filter((r) => r.holdout);
|
|
126
|
+
const championPrimaryCount = countPasses(championPrimary);
|
|
127
|
+
const challengerPrimaryCount = countPasses(challengerPrimary);
|
|
128
|
+
const championHoldoutCount = countPasses(championHoldout);
|
|
129
|
+
const challengerHoldoutCount = countPasses(challengerHoldout);
|
|
130
|
+
const deltas = [
|
|
131
|
+
metricDelta("primaryPassRate", champion.summary.primaryPassRate, challenger.summary.primaryPassRate, championPrimaryCount.passed, championPrimaryCount.total, challengerPrimaryCount.passed, challengerPrimaryCount.total),
|
|
132
|
+
metricDelta("holdoutPassRate", champion.summary.holdoutPassRate, challenger.summary.holdoutPassRate, championHoldoutCount.passed, championHoldoutCount.total, challengerHoldoutCount.passed, challengerHoldoutCount.total),
|
|
133
|
+
metricDelta("overallPassRate", champion.summary.passRate, challenger.summary.passRate, countPasses(champion.results).passed, countPasses(champion.results).total, countPasses(challenger.results).passed, countPasses(challenger.results).total),
|
|
134
|
+
];
|
|
135
|
+
const holdoutTripwire = evaluateHoldoutTripwire(champion, challenger);
|
|
136
|
+
const report = {
|
|
137
|
+
schemaVersion: REPORT_SCHEMA_VERSION,
|
|
138
|
+
championVersion: options.championVersion,
|
|
139
|
+
challengerVersion: options.challengerVersion,
|
|
140
|
+
model: options.model,
|
|
141
|
+
championRunId: champion.runId,
|
|
142
|
+
challengerRunId: challenger.runId,
|
|
143
|
+
deltas,
|
|
144
|
+
holdoutTripwire,
|
|
145
|
+
};
|
|
146
|
+
const lines = [
|
|
147
|
+
`eval:report champion=v${options.championVersion} challenger=v${options.challengerVersion} model=${options.model}`,
|
|
148
|
+
...deltas.map((d) => {
|
|
149
|
+
const sig = d.pValue === null
|
|
150
|
+
? "n/a"
|
|
151
|
+
: d.significantAt05
|
|
152
|
+
? `p=${d.pValue.toFixed(4)} *`
|
|
153
|
+
: `p=${d.pValue.toFixed(4)}`;
|
|
154
|
+
return ` ${d.metric}: ${formatPercent(d.champion)} -> ${formatPercent(d.challenger)} (delta ${(d.delta * 100).toFixed(1)}pp, ${sig})`;
|
|
155
|
+
}),
|
|
156
|
+
` ${holdoutTripwire.summary}`,
|
|
157
|
+
];
|
|
158
|
+
const code = holdoutTripwire.triggered ? 1 : 0;
|
|
159
|
+
return { code, report, message: lines.join("\n") };
|
|
160
|
+
}
|
|
161
|
+
//# sourceMappingURL=report.js.map
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
export declare const GOLDEN_RUNS_HISTORY_REL = "results/golden-runs.jsonl";
|
|
2
|
+
export declare const GOLDEN_RUN_SCHEMA_VERSION: 1;
|
|
3
|
+
export declare const GOLDEN_CORPUS_VERSION = "2026-07-05-tier2-fixture-v1";
|
|
4
|
+
/** One synthetic golden-task definition with an objective grader. */
|
|
5
|
+
export interface GoldenTaskDefinition {
|
|
6
|
+
readonly id: string;
|
|
7
|
+
readonly title: string;
|
|
8
|
+
readonly holdout: boolean;
|
|
9
|
+
readonly grade: (context: GoldenTaskContext) => GoldenTaskGrade;
|
|
10
|
+
}
|
|
11
|
+
/** Runtime context passed to golden-task graders. */
|
|
12
|
+
export interface GoldenTaskContext {
|
|
13
|
+
readonly tempDir: string;
|
|
14
|
+
readonly seed: number;
|
|
15
|
+
readonly directiveVersion: string;
|
|
16
|
+
readonly model: string;
|
|
17
|
+
}
|
|
18
|
+
/** Objective grader output for a single task × seed. */
|
|
19
|
+
export interface GoldenTaskGrade {
|
|
20
|
+
readonly passed: boolean;
|
|
21
|
+
readonly metrics: Readonly<Record<string, number | boolean>>;
|
|
22
|
+
}
|
|
23
|
+
/** One graded task outcome within a golden run. */
|
|
24
|
+
export interface GoldenTaskResult {
|
|
25
|
+
readonly taskId: string;
|
|
26
|
+
readonly seed: number;
|
|
27
|
+
readonly passed: boolean;
|
|
28
|
+
readonly holdout: boolean;
|
|
29
|
+
readonly metrics: Readonly<Record<string, number | boolean>>;
|
|
30
|
+
}
|
|
31
|
+
/** Versioned golden-run record persisted to the `.eval` ledger (#1703 Tier 2). */
|
|
32
|
+
export interface GoldenRunRecord {
|
|
33
|
+
readonly schemaVersion: typeof GOLDEN_RUN_SCHEMA_VERSION;
|
|
34
|
+
readonly runId: string;
|
|
35
|
+
readonly directiveVersion: string;
|
|
36
|
+
readonly model: string;
|
|
37
|
+
readonly harness: string;
|
|
38
|
+
readonly seeds: readonly number[];
|
|
39
|
+
readonly corpusVersion: string;
|
|
40
|
+
readonly recordedAt: string;
|
|
41
|
+
readonly results: readonly GoldenTaskResult[];
|
|
42
|
+
readonly summary: GoldenRunSummary;
|
|
43
|
+
}
|
|
44
|
+
/** Aggregate metrics for one golden run. */
|
|
45
|
+
export interface GoldenRunSummary {
|
|
46
|
+
readonly primaryPassRate: number;
|
|
47
|
+
readonly holdoutPassRate: number;
|
|
48
|
+
readonly passRate: number;
|
|
49
|
+
readonly primaryTotal: number;
|
|
50
|
+
readonly holdoutTotal: number;
|
|
51
|
+
}
|
|
52
|
+
export interface RunGoldenEvalOptions {
|
|
53
|
+
readonly projectRoot?: string;
|
|
54
|
+
readonly model: string;
|
|
55
|
+
readonly seeds?: readonly number[];
|
|
56
|
+
readonly directiveVersion?: string;
|
|
57
|
+
readonly harness?: string;
|
|
58
|
+
readonly persist?: boolean;
|
|
59
|
+
readonly now?: () => Date;
|
|
60
|
+
readonly mkTempDir?: () => string;
|
|
61
|
+
}
|
|
62
|
+
export interface RunGoldenEvalResult {
|
|
63
|
+
readonly code: 0 | 1 | 2;
|
|
64
|
+
readonly record: GoldenRunRecord | null;
|
|
65
|
+
readonly message: string;
|
|
66
|
+
}
|
|
67
|
+
/** Fixed golden corpus with objective graders (#1703 Tier 2). */
|
|
68
|
+
export declare const GOLDEN_CORPUS: readonly GoldenTaskDefinition[];
|
|
69
|
+
/** Absolute path to the golden-run results ledger. */
|
|
70
|
+
export declare function goldenRunsHistoryPath(projectRoot: string): string;
|
|
71
|
+
/** Append one golden run to the versioned ledger (#1703 Tier 2). */
|
|
72
|
+
export declare function persistGoldenRun(projectRoot: string, record: GoldenRunRecord): void;
|
|
73
|
+
/** Stable hash for rotating holdout selection (#1703 Goodhart mitigation). */
|
|
74
|
+
export declare function holdoutRotationIndex(directiveVersion: string, model: string, holdoutCount: number): number;
|
|
75
|
+
/** Select the active holdout task for this version × model tuple. */
|
|
76
|
+
export declare function selectRotatingHoldoutTask(directiveVersion: string, model: string): GoldenTaskDefinition | null;
|
|
77
|
+
/** Execute the golden corpus for one model × seed set and optionally persist (#1703 Tier 2). */
|
|
78
|
+
export declare function runGoldenEval(options: RunGoldenEvalOptions): RunGoldenEvalResult;
|
|
79
|
+
//# sourceMappingURL=run.d.ts.map
|
package/dist/eval/run.js
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { appendFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import { readCorePackageVersion } from "../engine-version.js";
|
|
6
|
+
import { resolveEvalPath } from "../layout/resolve.js";
|
|
7
|
+
import { BYTE_DIFF_WHOLE_FILE_THRESHOLD, InstrumentedVbriefCrud } from "./crud-telemetry.js";
|
|
8
|
+
import { evaluateHealth } from "./health.js";
|
|
9
|
+
export const GOLDEN_RUNS_HISTORY_REL = "results/golden-runs.jsonl";
|
|
10
|
+
export const GOLDEN_RUN_SCHEMA_VERSION = 1;
|
|
11
|
+
export const GOLDEN_CORPUS_VERSION = "2026-07-05-tier2-fixture-v1";
|
|
12
|
+
const VALID_VBRIEF = `{
|
|
13
|
+
"vBRIEFInfo": { "version": "0.6", "description": "golden fixture" },
|
|
14
|
+
"plan": {
|
|
15
|
+
"id": "golden-fixture",
|
|
16
|
+
"title": "Golden fixture",
|
|
17
|
+
"status": "pending",
|
|
18
|
+
"narratives": { "Description": "Valid golden corpus document." },
|
|
19
|
+
"items": [
|
|
20
|
+
{
|
|
21
|
+
"id": "golden-a1",
|
|
22
|
+
"title": "Item",
|
|
23
|
+
"status": "pending",
|
|
24
|
+
"narrative": { "Acceptance": "Schema valid." }
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}
|
|
28
|
+
}`;
|
|
29
|
+
const INVENTED_KEY_VBRIEF = VALID_VBRIEF.replace('"items": [', '"agentInventedField": "bad",\n "items": [');
|
|
30
|
+
function passRate(results, holdout) {
|
|
31
|
+
const subset = results.filter((r) => r.holdout === holdout);
|
|
32
|
+
if (subset.length === 0) {
|
|
33
|
+
return 0;
|
|
34
|
+
}
|
|
35
|
+
return subset.filter((r) => r.passed).length / subset.length;
|
|
36
|
+
}
|
|
37
|
+
function summarizeResults(results) {
|
|
38
|
+
const primary = results.filter((r) => !r.holdout);
|
|
39
|
+
const holdout = results.filter((r) => r.holdout);
|
|
40
|
+
const primaryPassRate = passRate(results, false);
|
|
41
|
+
const holdoutPassRate = passRate(results, true);
|
|
42
|
+
return {
|
|
43
|
+
primaryPassRate,
|
|
44
|
+
holdoutPassRate,
|
|
45
|
+
passRate: results.length === 0 ? 0 : results.filter((r) => r.passed).length / results.length,
|
|
46
|
+
primaryTotal: primary.length,
|
|
47
|
+
holdoutTotal: holdout.length,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
function gradeCrudValidCreate(context) {
|
|
51
|
+
const path = join(context.tempDir, "create.json");
|
|
52
|
+
const crud = new InstrumentedVbriefCrud({ directiveVersion: context.directiveVersion });
|
|
53
|
+
const result = crud.create(path, VALID_VBRIEF);
|
|
54
|
+
const metric = crud.getMetrics()[0];
|
|
55
|
+
return {
|
|
56
|
+
passed: result.ok && metric?.schemaValid === true,
|
|
57
|
+
metrics: {
|
|
58
|
+
schemaValid: metric?.schemaValid === true,
|
|
59
|
+
fieldInventionCount: metric?.fieldInventionCount ?? -1,
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
function gradeCrudRejectInvention(context) {
|
|
64
|
+
const path = join(context.tempDir, "invented.json");
|
|
65
|
+
const crud = new InstrumentedVbriefCrud({ directiveVersion: context.directiveVersion });
|
|
66
|
+
crud.create(path, INVENTED_KEY_VBRIEF);
|
|
67
|
+
const metric = crud.getMetrics()[0];
|
|
68
|
+
const inventedKeys = metric?.inventedKeys.length ?? 0;
|
|
69
|
+
return {
|
|
70
|
+
passed: metric?.schemaValid === true && inventedKeys > 0 && (metric?.fieldInventionCount ?? 0) > 0,
|
|
71
|
+
metrics: {
|
|
72
|
+
detectedInvention: inventedKeys > 0,
|
|
73
|
+
fieldInventionCount: metric?.fieldInventionCount ?? -1,
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
function gradeCrudSurgicalUpdate(context) {
|
|
78
|
+
const path = join(context.tempDir, "update.json");
|
|
79
|
+
const crud = new InstrumentedVbriefCrud({ directiveVersion: context.directiveVersion });
|
|
80
|
+
crud.create(path, VALID_VBRIEF);
|
|
81
|
+
const updated = VALID_VBRIEF.replace('"pending"', '"running"');
|
|
82
|
+
crud.update(path, updated);
|
|
83
|
+
const updateMetric = crud.getMetrics().find((m) => m.operation === "update");
|
|
84
|
+
return {
|
|
85
|
+
passed: updateMetric?.schemaValid === true &&
|
|
86
|
+
updateMetric.byteDiffMinimality === "surgical" &&
|
|
87
|
+
(updateMetric.byteDiffChangedRatio ?? 1) < BYTE_DIFF_WHOLE_FILE_THRESHOLD,
|
|
88
|
+
metrics: {
|
|
89
|
+
schemaValid: updateMetric?.schemaValid === true,
|
|
90
|
+
byteDiffMinimalitySurgical: updateMetric?.byteDiffMinimality === "surgical",
|
|
91
|
+
changedRatio: updateMetric?.byteDiffChangedRatio ?? -1,
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
function seedHealthFixture(tempDir) {
|
|
96
|
+
writeFileSync(join(tempDir, "xbrief", "PROJECT-DEFINITION.xbrief.json"), JSON.stringify({
|
|
97
|
+
xBRIEFInfo: { version: "0.6" },
|
|
98
|
+
plan: {
|
|
99
|
+
title: "Golden health fixture",
|
|
100
|
+
status: "running",
|
|
101
|
+
items: [],
|
|
102
|
+
"x-directive/policy": { triageScope: [{ rule: "all-open" }] },
|
|
103
|
+
},
|
|
104
|
+
}), "utf8");
|
|
105
|
+
writeFileSync(join(tempDir, "AGENTS.md"), "<!-- deft:managed-section v3 -->\n<!-- /deft:managed-section -->\n", "utf8");
|
|
106
|
+
}
|
|
107
|
+
function gradeHealthFixture(context) {
|
|
108
|
+
seedHealthFixture(context.tempDir);
|
|
109
|
+
const health = evaluateHealth({
|
|
110
|
+
projectRoot: context.tempDir,
|
|
111
|
+
persist: false,
|
|
112
|
+
frameworkSource: false,
|
|
113
|
+
});
|
|
114
|
+
return {
|
|
115
|
+
passed: health.report !== null && typeof health.report.score === "number",
|
|
116
|
+
metrics: {
|
|
117
|
+
score: health.report?.score ?? -1,
|
|
118
|
+
gateCount: health.report?.gates.length ?? -1,
|
|
119
|
+
},
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
function gradeHoldoutSchemaRoundtrip(context) {
|
|
123
|
+
const path = join(context.tempDir, "roundtrip.json");
|
|
124
|
+
const crud = new InstrumentedVbriefCrud({ directiveVersion: context.directiveVersion });
|
|
125
|
+
crud.create(path, VALID_VBRIEF);
|
|
126
|
+
const read = crud.read(path);
|
|
127
|
+
const roundtripOk = read.ok && read.content === VALID_VBRIEF;
|
|
128
|
+
const noise = (context.seed % 997) / 997;
|
|
129
|
+
return {
|
|
130
|
+
passed: roundtripOk && noise < 0.99,
|
|
131
|
+
metrics: { roundtripOk, seedNoise: noise },
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
function gradeHoldoutByteDiffTripwire(context) {
|
|
135
|
+
const path = join(context.tempDir, "rewrite.json");
|
|
136
|
+
const crud = new InstrumentedVbriefCrud({ directiveVersion: context.directiveVersion });
|
|
137
|
+
crud.create(path, VALID_VBRIEF);
|
|
138
|
+
const rewritten = JSON.stringify(JSON.parse(VALID_VBRIEF), null, 2);
|
|
139
|
+
crud.update(path, rewritten);
|
|
140
|
+
const updateMetric = crud.getMetrics().find((m) => m.operation === "update");
|
|
141
|
+
const detectedRewrite = updateMetric?.byteDiffMinimality === "whole-file-rewrite";
|
|
142
|
+
const tripwire = (context.seed % 991) / 991;
|
|
143
|
+
return {
|
|
144
|
+
passed: detectedRewrite && tripwire < 0.99,
|
|
145
|
+
metrics: {
|
|
146
|
+
detectedRewrite,
|
|
147
|
+
tripwire,
|
|
148
|
+
},
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
/** Fixed golden corpus with objective graders (#1703 Tier 2). */
|
|
152
|
+
export const GOLDEN_CORPUS = [
|
|
153
|
+
{
|
|
154
|
+
id: "crud-valid-create",
|
|
155
|
+
title: "Instrumented create accepts valid vBRIEF",
|
|
156
|
+
holdout: false,
|
|
157
|
+
grade: gradeCrudValidCreate,
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
id: "crud-reject-invention",
|
|
161
|
+
title: "Instrumented create rejects invented keys",
|
|
162
|
+
holdout: false,
|
|
163
|
+
grade: gradeCrudRejectInvention,
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
id: "crud-surgical-update",
|
|
167
|
+
title: "Instrumented update classifies surgical edits",
|
|
168
|
+
holdout: false,
|
|
169
|
+
grade: gradeCrudSurgicalUpdate,
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
id: "health-fixture-score",
|
|
173
|
+
title: "Tier-0 health eval runs on fixture repo",
|
|
174
|
+
holdout: false,
|
|
175
|
+
grade: gradeHealthFixture,
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
id: "holdout-schema-roundtrip",
|
|
179
|
+
title: "Holdout: read-after-create roundtrip",
|
|
180
|
+
holdout: true,
|
|
181
|
+
grade: gradeHoldoutSchemaRoundtrip,
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
id: "holdout-byte-diff-tripwire",
|
|
185
|
+
title: "Holdout: whole-file rewrite detection tripwire",
|
|
186
|
+
holdout: true,
|
|
187
|
+
grade: gradeHoldoutByteDiffTripwire,
|
|
188
|
+
},
|
|
189
|
+
];
|
|
190
|
+
/** Absolute path to the golden-run results ledger. */
|
|
191
|
+
export function goldenRunsHistoryPath(projectRoot) {
|
|
192
|
+
return resolveEvalPath(projectRoot, GOLDEN_RUNS_HISTORY_REL);
|
|
193
|
+
}
|
|
194
|
+
/** Append one golden run to the versioned ledger (#1703 Tier 2). */
|
|
195
|
+
export function persistGoldenRun(projectRoot, record) {
|
|
196
|
+
const path = goldenRunsHistoryPath(projectRoot);
|
|
197
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
198
|
+
appendFileSync(path, `${JSON.stringify(record)}\n`, "utf8");
|
|
199
|
+
}
|
|
200
|
+
/** Stable hash for rotating holdout selection (#1703 Goodhart mitigation). */
|
|
201
|
+
export function holdoutRotationIndex(directiveVersion, model, holdoutCount) {
|
|
202
|
+
if (holdoutCount <= 0) {
|
|
203
|
+
return 0;
|
|
204
|
+
}
|
|
205
|
+
const digest = createHash("sha256")
|
|
206
|
+
.update(`${directiveVersion}\0${model}\0${GOLDEN_CORPUS_VERSION}`)
|
|
207
|
+
.digest("hex");
|
|
208
|
+
return Number.parseInt(digest.slice(0, 8), 16) % holdoutCount;
|
|
209
|
+
}
|
|
210
|
+
/** Select the active holdout task for this version × model tuple. */
|
|
211
|
+
export function selectRotatingHoldoutTask(directiveVersion, model) {
|
|
212
|
+
const holdouts = GOLDEN_CORPUS.filter((task) => task.holdout);
|
|
213
|
+
if (holdouts.length === 0) {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
return holdouts[holdoutRotationIndex(directiveVersion, model, holdouts.length)] ?? null;
|
|
217
|
+
}
|
|
218
|
+
function seedTempDir(taskId, seed) {
|
|
219
|
+
const base = mkdtempSync(join(tmpdir(), `deft-golden-${taskId}-seed-${seed}-`));
|
|
220
|
+
mkdirSync(join(base, "xbrief"), { recursive: true });
|
|
221
|
+
return base;
|
|
222
|
+
}
|
|
223
|
+
function runIdFor(directiveVersion, model, harness, seeds) {
|
|
224
|
+
const digest = createHash("sha256")
|
|
225
|
+
.update(`${directiveVersion}\0${model}\0${harness}\0${seeds.join(",")}\0${GOLDEN_CORPUS_VERSION}`)
|
|
226
|
+
.digest("hex");
|
|
227
|
+
return digest.slice(0, 12);
|
|
228
|
+
}
|
|
229
|
+
/** Execute the golden corpus for one model × seed set and optionally persist (#1703 Tier 2). */
|
|
230
|
+
export function runGoldenEval(options) {
|
|
231
|
+
if (!options.model.trim()) {
|
|
232
|
+
return { code: 2, record: null, message: "eval:run: --model is required" };
|
|
233
|
+
}
|
|
234
|
+
const projectRoot = options.projectRoot ?? process.cwd();
|
|
235
|
+
const directiveVersion = options.directiveVersion ?? readCorePackageVersion();
|
|
236
|
+
const harness = options.harness ?? "deterministic-fixture";
|
|
237
|
+
const seeds = options.seeds ?? [1, 2, 3];
|
|
238
|
+
const now = options.now ?? (() => new Date());
|
|
239
|
+
const persist = options.persist ?? true;
|
|
240
|
+
if (seeds.length === 0) {
|
|
241
|
+
return { code: 2, record: null, message: "eval:run: at least one seed is required" };
|
|
242
|
+
}
|
|
243
|
+
const primaryTasks = GOLDEN_CORPUS.filter((task) => !task.holdout);
|
|
244
|
+
const rotatingHoldout = selectRotatingHoldoutTask(directiveVersion, options.model);
|
|
245
|
+
const tasksToRun = [...primaryTasks];
|
|
246
|
+
if (rotatingHoldout !== null) {
|
|
247
|
+
tasksToRun.push(rotatingHoldout);
|
|
248
|
+
}
|
|
249
|
+
const results = [];
|
|
250
|
+
const scratchDirs = [];
|
|
251
|
+
for (const task of tasksToRun) {
|
|
252
|
+
for (const seed of seeds) {
|
|
253
|
+
const tempDir = seedTempDir(task.id, seed);
|
|
254
|
+
scratchDirs.push(tempDir);
|
|
255
|
+
const grade = task.grade({
|
|
256
|
+
tempDir,
|
|
257
|
+
seed,
|
|
258
|
+
directiveVersion,
|
|
259
|
+
model: options.model,
|
|
260
|
+
});
|
|
261
|
+
results.push({
|
|
262
|
+
taskId: task.id,
|
|
263
|
+
seed,
|
|
264
|
+
passed: grade.passed,
|
|
265
|
+
holdout: task.holdout,
|
|
266
|
+
metrics: grade.metrics,
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
for (const dir of scratchDirs) {
|
|
271
|
+
rmSync(dir, { recursive: true, force: true });
|
|
272
|
+
}
|
|
273
|
+
const summary = summarizeResults(results);
|
|
274
|
+
const record = {
|
|
275
|
+
schemaVersion: GOLDEN_RUN_SCHEMA_VERSION,
|
|
276
|
+
runId: runIdFor(directiveVersion, options.model, harness, seeds),
|
|
277
|
+
directiveVersion,
|
|
278
|
+
model: options.model,
|
|
279
|
+
harness,
|
|
280
|
+
seeds,
|
|
281
|
+
corpusVersion: GOLDEN_CORPUS_VERSION,
|
|
282
|
+
recordedAt: now()
|
|
283
|
+
.toISOString()
|
|
284
|
+
.replace(/\.\d{3}Z$/, "Z"),
|
|
285
|
+
results,
|
|
286
|
+
summary,
|
|
287
|
+
};
|
|
288
|
+
if (persist) {
|
|
289
|
+
try {
|
|
290
|
+
persistGoldenRun(projectRoot, record);
|
|
291
|
+
}
|
|
292
|
+
catch (err) {
|
|
293
|
+
return {
|
|
294
|
+
code: 2,
|
|
295
|
+
record,
|
|
296
|
+
message: `eval:run: failed to persist golden run: ${String(err)}`,
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
const lines = [
|
|
301
|
+
`eval:run v${record.directiveVersion} model=${record.model} harness=${record.harness} seeds=[${record.seeds.join(",")}]`,
|
|
302
|
+
` primary pass rate: ${(summary.primaryPassRate * 100).toFixed(1)}% (${summary.primaryTotal} trials)`,
|
|
303
|
+
` holdout pass rate: ${(summary.holdoutPassRate * 100).toFixed(1)}% (${summary.holdoutTotal} trials)`,
|
|
304
|
+
` rotating holdout task: ${rotatingHoldout?.id ?? "none"}`,
|
|
305
|
+
` runId=${record.runId}`,
|
|
306
|
+
];
|
|
307
|
+
return { code: 0, record, message: lines.join("\n") };
|
|
308
|
+
}
|
|
309
|
+
//# sourceMappingURL=run.js.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Canonical behavioral event names for the attribution ledger (#1709). */
|
|
2
|
+
export declare const ATTRIBUTION_EVENT_NAMES: {
|
|
3
|
+
readonly valueGateCatch: "value:gate-catch";
|
|
4
|
+
readonly valueWipCapProtect: "value:wip-cap-protect";
|
|
5
|
+
readonly bypassOffFlow: "bypass:off-flow";
|
|
6
|
+
readonly adoptionUnusedCapability: "adoption:unused-capability";
|
|
7
|
+
readonly frictionDirectiveGap: "friction:directive-gap";
|
|
8
|
+
};
|
|
9
|
+
export type AttributionEventName = (typeof ATTRIBUTION_EVENT_NAMES)[keyof typeof ATTRIBUTION_EVENT_NAMES];
|
|
10
|
+
/** Required payload keys per attribution event (merged into lifecycle/events). */
|
|
11
|
+
export declare const ATTRIBUTION_REQUIRED_PAYLOAD: Readonly<Record<string, readonly string[]>>;
|
|
12
|
+
export declare const ALL_ATTRIBUTION_EVENT_NAMES: ("value:gate-catch" | "value:wip-cap-protect" | "bypass:off-flow" | "adoption:unused-capability" | "friction:directive-gap")[];
|
|
13
|
+
//# sourceMappingURL=attribution-constants.d.ts.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** Canonical behavioral event names for the attribution ledger (#1709). */
|
|
2
|
+
export const ATTRIBUTION_EVENT_NAMES = {
|
|
3
|
+
valueGateCatch: "value:gate-catch",
|
|
4
|
+
valueWipCapProtect: "value:wip-cap-protect",
|
|
5
|
+
bypassOffFlow: "bypass:off-flow",
|
|
6
|
+
adoptionUnusedCapability: "adoption:unused-capability",
|
|
7
|
+
frictionDirectiveGap: "friction:directive-gap",
|
|
8
|
+
};
|
|
9
|
+
/** Required payload keys per attribution event (merged into lifecycle/events). */
|
|
10
|
+
export const ATTRIBUTION_REQUIRED_PAYLOAD = {
|
|
11
|
+
[ATTRIBUTION_EVENT_NAMES.valueGateCatch]: ["signal_class", "source"],
|
|
12
|
+
[ATTRIBUTION_EVENT_NAMES.valueWipCapProtect]: ["signal_class", "source", "count", "cap"],
|
|
13
|
+
[ATTRIBUTION_EVENT_NAMES.bypassOffFlow]: ["signal_class", "source"],
|
|
14
|
+
[ATTRIBUTION_EVENT_NAMES.adoptionUnusedCapability]: ["signal_class", "source", "capability"],
|
|
15
|
+
[ATTRIBUTION_EVENT_NAMES.frictionDirectiveGap]: ["signal_class", "source"],
|
|
16
|
+
};
|
|
17
|
+
export const ALL_ATTRIBUTION_EVENT_NAMES = Object.values(ATTRIBUTION_EVENT_NAMES);
|
|
18
|
+
//# sourceMappingURL=attribution-constants.js.map
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { type BehavioralEventRecord } from "../lifecycle/events.js";
|
|
2
|
+
import { type ValueFeedbackResolved } from "../policy/value-feedback.js";
|
|
3
|
+
import { type AttributionEventName } from "./attribution-constants.js";
|
|
4
|
+
export { ALL_ATTRIBUTION_EVENT_NAMES, ATTRIBUTION_EVENT_NAMES, ATTRIBUTION_REQUIRED_PAYLOAD, type AttributionEventName, } from "./attribution-constants.js";
|
|
5
|
+
/** Four value-attribution signal classes (#1709 RFC). */
|
|
6
|
+
export type SignalClass = "value" | "bypass" | "adoption" | "friction";
|
|
7
|
+
export interface EmitAttributionOptions {
|
|
8
|
+
readonly projectRoot: string;
|
|
9
|
+
readonly logPath?: string | null;
|
|
10
|
+
/** Test hook: skip disk policy read. */
|
|
11
|
+
readonly policyOverride?: ValueFeedbackResolved;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Append an attribution ledger entry when valueFeedback emitEvents is allowed.
|
|
15
|
+
* Returns null when gated OFF (no disk write).
|
|
16
|
+
*/
|
|
17
|
+
export declare function emitAttributionSignal(name: AttributionEventName, payload: Record<string, unknown>, options: EmitAttributionOptions): BehavioralEventRecord | null;
|
|
18
|
+
/** Record a detection-bound gate catch (value class). */
|
|
19
|
+
export declare function recordGateCatch(projectRoot: string, source: string, detail: string, options?: {
|
|
20
|
+
logPath?: string | null;
|
|
21
|
+
policyOverride?: ValueFeedbackResolved;
|
|
22
|
+
}): BehavioralEventRecord | null;
|
|
23
|
+
/** Record a WIP-cap protect refusal (value class). */
|
|
24
|
+
export declare function recordWipCapProtect(projectRoot: string, count: number, cap: number, options?: {
|
|
25
|
+
logPath?: string | null;
|
|
26
|
+
policyOverride?: ValueFeedbackResolved;
|
|
27
|
+
}): BehavioralEventRecord | null;
|
|
28
|
+
/** Record a bypass/off-flow signal. */
|
|
29
|
+
export declare function recordBypassSignal(projectRoot: string, source: string, detail: string, options?: {
|
|
30
|
+
logPath?: string | null;
|
|
31
|
+
policyOverride?: ValueFeedbackResolved;
|
|
32
|
+
}): BehavioralEventRecord | null;
|
|
33
|
+
/** Record an adoption/unused-capability signal. */
|
|
34
|
+
export declare function recordAdoptionSignal(projectRoot: string, capability: string, detail: string, options?: {
|
|
35
|
+
logPath?: string | null;
|
|
36
|
+
policyOverride?: ValueFeedbackResolved;
|
|
37
|
+
}): BehavioralEventRecord | null;
|
|
38
|
+
/** Record a friction/directive-gap signal. */
|
|
39
|
+
export declare function recordFrictionSignal(projectRoot: string, source: string, detail: string, options?: {
|
|
40
|
+
logPath?: string | null;
|
|
41
|
+
policyOverride?: ValueFeedbackResolved;
|
|
42
|
+
}): BehavioralEventRecord | null;
|
|
43
|
+
//# sourceMappingURL=attribution-ledger.d.ts.map
|