@ngockhoale/ukit 3.5.3 → 3.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/manifests/documentation.yaml +12 -1
- package/manifests/platform.full.yaml +12 -0
- package/package.json +29 -2
- package/src/cli/commands/doctor.js +41 -0
- package/src/cli/commands/indexTools.js +10 -4
- package/src/cli/commands/memory.js +88 -6
- package/src/core/memory/episodes.js +106 -7
- package/src/core/memory/memoryCheck.js +241 -0
- package/src/core/memory/memoryCheckRun.js +39 -0
- package/src/core/memory/memoryHit.js +31 -15
- package/src/core/memory/mutateMemory.js +37 -1
- package/src/core/memory/observations.js +192 -0
- package/src/core/memory/recallEngine.js +159 -10
- package/src/core/memory/recallLanes.js +182 -0
- package/src/core/memory/records.js +23 -0
- package/src/core/memory/retrieval.js +13 -1
- package/src/core/memory/topicCommand.js +75 -0
- package/src/core/memory/topicPages.js +545 -0
- package/src/core/memory/writeGate.js +137 -0
- package/src/core/runtimeConfig.js +56 -1
- package/src/decision/registry.js +17 -0
- package/src/index/buildIndex.js +312 -5
- package/src/index/languageTools.js +100 -0
- package/src/index/paths.js +4 -1
- package/src/index/queryIndex.js +356 -14
- package/src/index/resolveContext.js +1 -1
- package/template_project/.claude/hooks/edit-hint.mjs +64 -17
- package/template_project/.claude/hooks/session-episode-start.mjs +35 -0
- package/template_project/.claude/hooks/session-episode.sh +9 -4
- package/template_project/.claude/settings.json +2 -2
- package/template_project/.claude/ukit/index/lib/index-core.mjs +769 -18
- package/template_project/.claude/ukit/index/query-index.mjs +14 -3
- package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
- package/template_project/.claude/ukit/index/route-task.mjs +4 -1
- package/template_project/.claude/ukit/runtime/memory-check.mjs +241 -0
- package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +159 -10
- package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
- package/template_project/.claude/ukit/runtime/memory-validity.mjs +28 -1
- package/template_project/.claude/ukit/runtime/self-improve-trigger.mjs +212 -5
- package/template_project/.omp/hooks/pre/ukit-bridge.js +2 -1
- package/template_project/ukit/storage/config.json +21 -2
- package/manifests/instructionRules.yaml +0 -442
- package/src/core/agentRuntime/completionGate.js +0 -179
- package/src/core/agentRuntime/context.js +0 -571
- package/src/core/agentRuntime/qualityComparison.js +0 -112
- package/src/core/agentRuntime/resourcePolicy.js +0 -78
- package/src/core/agentRuntime/runtimeSupport.js +0 -255
- package/src/core/codeintel/manifest.js +0 -176
- package/src/core/experiments/deliberation.js +0 -327
- package/src/core/experiments/dynamicWorkflow.js +0 -507
- package/src/core/handoffDocValidator.js +0 -154
- package/src/core/memory/deltaOverlays.js +0 -493
- package/src/core/memory/learningCandidates.js +0 -347
- package/src/core/observability/emit/sessionBoot.js +0 -407
- package/src/core/observability/evaluation/replay.js +0 -149
- package/src/core/observability/evaluation/scorecard.js +0 -465
- package/src/core/observability/schema/compatibility.js +0 -180
- package/src/core/observability/support/renderer.js +0 -130
- package/src/core/reviewPanelAggregate.js +0 -204
- package/src/core/taskBudgetValidator.js +0 -251
- package/src/core/taskProgressGuard.js +0 -227
- package/src/decision/reviewVerdict.js +0 -339
- package/src/decision/runtimeFilter.js +0 -150
- package/src/decision/runtimeScheduler.js +0 -239
- package/src/index/contextPack.js +0 -285
- package/src/index/impactContext.js +0 -360
- package/src/index/reviewPolicy.js +0 -368
- package/src/index/sensitivePath.js +0 -21
|
@@ -1,465 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* scorecard.js (TASK-014, SPEC §5 DF-FR08/DF-FR09/DF-FR12, §8) — golden
|
|
3
|
-
* corpus evaluation.
|
|
4
|
-
*
|
|
5
|
-
* evaluateVariant(corpus, policy): Scorecard
|
|
6
|
-
* compareScorecards(a, b, { seed }): Comparison
|
|
7
|
-
*
|
|
8
|
-
* Quality is the guardrail: golden cases declare evidence / actions /
|
|
9
|
-
* forbidden mistakes / verification / acceptable bounds — never exact-prose
|
|
10
|
-
* equality. Correction semantics are honest: when a user correction follows
|
|
11
|
-
* the first answer, that first answer is NOT an accepted success and
|
|
12
|
-
* tokens_to_success / time_to_success_ms include the observed retry.
|
|
13
|
-
*
|
|
14
|
-
* Scorecard (SPEC §8):
|
|
15
|
-
* { metric_version, policy, quality: { met, unmet, violated, unknown,
|
|
16
|
-
* forbidden, bounds }, latency, tokens, cohorts, provenance, fidelity,
|
|
17
|
-
* sample_size, verdict: 'pass'|'fail'|'inconclusive', cases }
|
|
18
|
-
*
|
|
19
|
-
* `cohorts` blocks are keyed by `task_type` first (falling back to the
|
|
20
|
-
* case's `cohort` string, then 'unknown') so a mixed-task corpus can never
|
|
21
|
-
* hide a per-task regression inside a pooled aggregate (Simpson's-paradox
|
|
22
|
-
* guard, G5-FR05).
|
|
23
|
-
* Everything here is offline and deterministic — no model calls, no IO, no
|
|
24
|
-
* clock, no randomness outside the seeded bootstrap in compareScorecards.
|
|
25
|
-
*/
|
|
26
|
-
|
|
27
|
-
import { summarizeTrace, percentileOf } from '../analytics/summary.js';
|
|
28
|
-
import { replayCase } from './replay.js';
|
|
29
|
-
|
|
30
|
-
export const EVALUATION_METRIC_VERSION = 'df-m1';
|
|
31
|
-
export const DEFAULT_MIN_COHORT_N = 3;
|
|
32
|
-
const BOOTSTRAP_RESAMPLES = 2000;
|
|
33
|
-
|
|
34
|
-
function isPlainObject(value) {
|
|
35
|
-
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
/** Deterministic PRNG (mulberry32) — same construction as the bench corpus. */
|
|
39
|
-
function mulberry32(seed) {
|
|
40
|
-
let a = seed >>> 0;
|
|
41
|
-
return () => {
|
|
42
|
-
a |= 0;
|
|
43
|
-
a = (a + 0x6d2b79f5) | 0;
|
|
44
|
-
let t = Math.imul(a ^ (a >>> 15), 1 | a);
|
|
45
|
-
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
46
|
-
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
47
|
-
};
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
function resolvePolicy(corpus, policy) {
|
|
51
|
-
if (typeof policy === 'string') {
|
|
52
|
-
const registered = isPlainObject(corpus.pre_registered)
|
|
53
|
-
&& isPlainObject(corpus.pre_registered.policies)
|
|
54
|
-
? corpus.pre_registered.policies[policy]
|
|
55
|
-
: null;
|
|
56
|
-
return { id: policy, ...(isPlainObject(registered) ? registered : {}) };
|
|
57
|
-
}
|
|
58
|
-
if (isPlainObject(policy)) {
|
|
59
|
-
return { id: typeof policy.id === 'string' ? policy.id : 'custom', ...policy };
|
|
60
|
-
}
|
|
61
|
-
return { id: 'custom' };
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
function semanticNames(records) {
|
|
65
|
-
const names = new Set();
|
|
66
|
-
for (const r of records) {
|
|
67
|
-
if (isPlainObject(r) && typeof r.semantic_name === 'string') names.add(r.semantic_name);
|
|
68
|
-
}
|
|
69
|
-
return names;
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
/** Human-reviewable action labels: operation/tool names on span records. */
|
|
73
|
-
function actionLabels(records) {
|
|
74
|
-
const actions = new Set();
|
|
75
|
-
for (const r of records) {
|
|
76
|
-
if (!isPlainObject(r) || !isPlainObject(r.payload)) continue;
|
|
77
|
-
if (typeof r.payload.operation === 'string') actions.add(r.payload.operation);
|
|
78
|
-
if (typeof r.payload.tool === 'string') actions.add(r.payload.tool);
|
|
79
|
-
}
|
|
80
|
-
return actions;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
function executionOutcome(records) {
|
|
84
|
-
for (const r of records) {
|
|
85
|
-
if (!isPlainObject(r) || !isPlainObject(r.payload)) continue;
|
|
86
|
-
if ((r.semantic_name === 'execution.completed' || r.semantic_name === 'execution.failed'
|
|
87
|
-
|| r.semantic_name === 'execution.blocked') && typeof r.payload.outcome === 'string') {
|
|
88
|
-
return r.payload.outcome;
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
return null;
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
function correctionObserved(records) {
|
|
95
|
-
return records.some((r) => isPlainObject(r) && isPlainObject(r.payload)
|
|
96
|
-
&& (r.payload.correction_observed === true || r.payload.kind === 'user_correction'));
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
const BOUND_METRICS = Object.freeze(['tokens_to_success', 'time_to_success_ms']);
|
|
100
|
-
|
|
101
|
-
function checkBounds(bounds, metrics) {
|
|
102
|
-
const result = { checked: 0, violated: 0, unknown: 0, detail: {} };
|
|
103
|
-
if (!isPlainObject(bounds)) return result;
|
|
104
|
-
for (const metric of BOUND_METRICS) {
|
|
105
|
-
const bound = bounds[metric];
|
|
106
|
-
if (!isPlainObject(bound)) continue;
|
|
107
|
-
const value = metrics[metric];
|
|
108
|
-
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
109
|
-
result.unknown += 1;
|
|
110
|
-
result.detail[metric] = 'unknown';
|
|
111
|
-
continue;
|
|
112
|
-
}
|
|
113
|
-
result.checked += 1;
|
|
114
|
-
const overMax = typeof bound.max === 'number' && value > bound.max;
|
|
115
|
-
const underMin = typeof bound.min === 'number' && value < bound.min;
|
|
116
|
-
if (overMax || underMin) {
|
|
117
|
-
result.violated += 1;
|
|
118
|
-
result.detail[metric] = 'violated';
|
|
119
|
-
} else {
|
|
120
|
-
result.detail[metric] = 'met';
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
return result;
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
/**
|
|
127
|
-
* Evaluate one replayed case against its rubric. Quality precedence:
|
|
128
|
-
* violated > unmet > unknown > met. SIMULATED replays can never claim a
|
|
129
|
-
* quality label — user acceptance is not observable.
|
|
130
|
-
*/
|
|
131
|
-
function evaluateCase(goldenCase, policyId) {
|
|
132
|
-
const replay = replayCase(goldenCase, { policyId });
|
|
133
|
-
const base = {
|
|
134
|
-
case_id: replay.case_id,
|
|
135
|
-
cohort: typeof goldenCase.cohort === 'string' ? goldenCase.cohort : 'default',
|
|
136
|
-
task_type: typeof goldenCase.task_type === 'string' && goldenCase.task_type.length > 0
|
|
137
|
-
? goldenCase.task_type
|
|
138
|
-
: 'unknown',
|
|
139
|
-
fidelity: { tag: replay.tag, limitations: replay.limitations },
|
|
140
|
-
};
|
|
141
|
-
if (!replay.ok) {
|
|
142
|
-
return { ...base, ok: false, error: replay.reason, quality: 'unknown', metrics: {} };
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
const records = replay.records;
|
|
146
|
-
const summary = summarizeTrace(records);
|
|
147
|
-
const names = semanticNames(records);
|
|
148
|
-
const actions = actionLabels(records);
|
|
149
|
-
const expect = isPlainObject(goldenCase.expect) ? goldenCase.expect : {};
|
|
150
|
-
|
|
151
|
-
const evidenceRequired = Array.isArray(expect.evidence) ? expect.evidence : [];
|
|
152
|
-
const evidenceMissing = evidenceRequired.filter((name) => !names.has(name));
|
|
153
|
-
const actionsRequired = Array.isArray(expect.actions) ? expect.actions : [];
|
|
154
|
-
const actionsMissing = actionsRequired.filter((name) => !actions.has(name));
|
|
155
|
-
const forbiddenList = Array.isArray(expect.forbidden_mistakes) ? expect.forbidden_mistakes : [];
|
|
156
|
-
const forbiddenHits = forbiddenList.filter((name) => names.has(name));
|
|
157
|
-
|
|
158
|
-
const outcome = executionOutcome(records);
|
|
159
|
-
const expectedOutcome = isPlainObject(expect.verification) && typeof expect.verification.outcome === 'string'
|
|
160
|
-
? expect.verification.outcome
|
|
161
|
-
: null;
|
|
162
|
-
const verification = expectedOutcome === null
|
|
163
|
-
? (outcome === null ? 'unknown' : 'met')
|
|
164
|
-
: (outcome === null ? 'unknown' : (outcome === expectedOutcome ? 'met' : 'violated'));
|
|
165
|
-
|
|
166
|
-
const succeeded = outcome === 'success'
|
|
167
|
-
|| (outcome === null && summary.coverage.open_spans === 0 && summary.retries.failed_spans === 0);
|
|
168
|
-
const corrected = correctionObserved(records);
|
|
169
|
-
const metrics = {
|
|
170
|
-
tokens_to_success: succeeded ? summary.resource.total_value : null,
|
|
171
|
-
time_to_success_ms: succeeded ? summary.critical_path_ms : null,
|
|
172
|
-
model_attempts: summary.retries.model_attempts,
|
|
173
|
-
correction_observed: corrected,
|
|
174
|
-
// one accepted success per successful episode; a correction means the
|
|
175
|
-
// first answer was NOT it.
|
|
176
|
-
accepted_success: succeeded ? 1 : 0,
|
|
177
|
-
first_answer_accepted: succeeded && !corrected && summary.retries.model_attempts <= 1,
|
|
178
|
-
succeeded,
|
|
179
|
-
};
|
|
180
|
-
const bounds = checkBounds(expect.bounds, metrics);
|
|
181
|
-
|
|
182
|
-
let quality;
|
|
183
|
-
if (replay.tag === 'SIMULATED') {
|
|
184
|
-
quality = 'unknown';
|
|
185
|
-
} else if (forbiddenHits.length > 0 || verification === 'violated' || bounds.violated > 0) {
|
|
186
|
-
quality = 'violated';
|
|
187
|
-
} else if (evidenceMissing.length > 0 || actionsMissing.length > 0) {
|
|
188
|
-
quality = 'unmet';
|
|
189
|
-
} else if (verification === 'unknown' || bounds.unknown > 0) {
|
|
190
|
-
quality = 'unknown';
|
|
191
|
-
} else {
|
|
192
|
-
quality = 'met';
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
return {
|
|
196
|
-
...base,
|
|
197
|
-
ok: true,
|
|
198
|
-
quality,
|
|
199
|
-
evidence: { required: evidenceRequired, missing: evidenceMissing },
|
|
200
|
-
actions: { required: actionsRequired, missing: actionsMissing },
|
|
201
|
-
forbidden_mistakes: { forbidden: forbiddenList, hits: forbiddenHits },
|
|
202
|
-
verification: { expected: expectedOutcome, observed: outcome, status: verification },
|
|
203
|
-
bounds,
|
|
204
|
-
metrics,
|
|
205
|
-
provenance: replay.provenance,
|
|
206
|
-
};
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
function meanOf(values) {
|
|
210
|
-
if (values.length === 0) return null;
|
|
211
|
-
return values.reduce((a, b) => a + b, 0) / values.length;
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
function distribution(values) {
|
|
215
|
-
const sorted = values.filter((v) => typeof v === 'number' && Number.isFinite(v)).sort((a, b) => a - b);
|
|
216
|
-
return {
|
|
217
|
-
n: sorted.length,
|
|
218
|
-
mean: meanOf(sorted),
|
|
219
|
-
p50: percentileOf(sorted, 50),
|
|
220
|
-
p95: percentileOf(sorted, 95),
|
|
221
|
-
};
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
function emptyQuality() {
|
|
225
|
-
return {
|
|
226
|
-
met: 0, unmet: 0, violated: 0, unknown: 0, forbidden: 0,
|
|
227
|
-
bounds: { checked: 0, violated: 0, unknown: 0 },
|
|
228
|
-
};
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
function accumulateQuality(bucket, result) {
|
|
232
|
-
if (result.quality === 'met') bucket.met += 1;
|
|
233
|
-
else if (result.quality === 'unmet') bucket.unmet += 1;
|
|
234
|
-
else if (result.quality === 'violated') bucket.violated += 1;
|
|
235
|
-
else bucket.unknown += 1;
|
|
236
|
-
if (result.forbidden_mistakes) bucket.forbidden += result.forbidden_mistakes.hits.length;
|
|
237
|
-
if (result.bounds) {
|
|
238
|
-
bucket.bounds.checked += result.bounds.checked;
|
|
239
|
-
bucket.bounds.violated += result.bounds.violated;
|
|
240
|
-
bucket.bounds.unknown += result.bounds.unknown;
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
function cohortVerdict(results, minN) {
|
|
245
|
-
if (results.length < minN) return 'inconclusive';
|
|
246
|
-
if (results.some((r) => r.quality === 'violated' || r.quality === 'unmet')) return 'fail';
|
|
247
|
-
if (results.every((r) => r.quality === 'unknown')) return 'inconclusive';
|
|
248
|
-
return 'pass';
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
/**
|
|
252
|
-
* Stratification block key: `task_type` is the first-class axis; cases
|
|
253
|
-
* without one fall back to their declared `cohort` string, then 'unknown'.
|
|
254
|
-
* Mixed-task corpora never pool into a bare global block.
|
|
255
|
-
*/
|
|
256
|
-
function blockKeyOf(result) {
|
|
257
|
-
return typeof result.task_type === 'string' && result.task_type !== 'unknown'
|
|
258
|
-
? result.task_type
|
|
259
|
-
: (typeof result.cohort === 'string' && result.cohort.length > 0
|
|
260
|
-
? result.cohort
|
|
261
|
-
: 'unknown');
|
|
262
|
-
}
|
|
263
|
-
/**
|
|
264
|
-
* @param {object} corpus — `{ cases: GoldenCase[], pre_registered?, metric_version? }`.
|
|
265
|
-
* @param {string|object} policy — policy id (resolved against
|
|
266
|
-
* `corpus.pre_registered.policies`) or an inline policy object.
|
|
267
|
-
* @returns {object} Scorecard
|
|
268
|
-
*/
|
|
269
|
-
export function evaluateVariant(corpus, policy) {
|
|
270
|
-
const source = isPlainObject(corpus) ? corpus : {};
|
|
271
|
-
const cases = Array.isArray(source.cases) ? source.cases : [];
|
|
272
|
-
const pre = isPlainObject(source.pre_registered) ? source.pre_registered : {};
|
|
273
|
-
const minN = Number.isInteger(pre.min_cohort_n) && pre.min_cohort_n > 0
|
|
274
|
-
? pre.min_cohort_n
|
|
275
|
-
: DEFAULT_MIN_COHORT_N;
|
|
276
|
-
const policyInfo = resolvePolicy(source, policy);
|
|
277
|
-
|
|
278
|
-
const results = cases.map((c) => evaluateCase(isPlainObject(c) ? c : {}, policyInfo.id));
|
|
279
|
-
|
|
280
|
-
const quality = emptyQuality();
|
|
281
|
-
const fidelityTags = {};
|
|
282
|
-
const fidelityLimitations = new Set();
|
|
283
|
-
const cohorts = new Map();
|
|
284
|
-
const latencyValues = [];
|
|
285
|
-
const tokenValues = [];
|
|
286
|
-
|
|
287
|
-
for (const result of results) {
|
|
288
|
-
accumulateQuality(quality, result);
|
|
289
|
-
fidelityTags[result.fidelity.tag] = (fidelityTags[result.fidelity.tag] || 0) + 1;
|
|
290
|
-
for (const lim of result.fidelity.limitations) fidelityLimitations.add(lim);
|
|
291
|
-
if (typeof result.metrics.time_to_success_ms === 'number') {
|
|
292
|
-
latencyValues.push(result.metrics.time_to_success_ms);
|
|
293
|
-
}
|
|
294
|
-
if (typeof result.metrics.tokens_to_success === 'number') {
|
|
295
|
-
tokenValues.push(result.metrics.tokens_to_success);
|
|
296
|
-
}
|
|
297
|
-
const blockKey = blockKeyOf(result);
|
|
298
|
-
if (!cohorts.has(blockKey)) cohorts.set(blockKey, []);
|
|
299
|
-
cohorts.get(blockKey).push(result);
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
const cohortReport = {};
|
|
303
|
-
for (const [key, members] of [...cohorts.entries()].sort(([a], [b]) => a.localeCompare(b))) {
|
|
304
|
-
const q = emptyQuality();
|
|
305
|
-
for (const m of members) accumulateQuality(q, m);
|
|
306
|
-
cohortReport[key] = {
|
|
307
|
-
n: members.length,
|
|
308
|
-
min_n: minN,
|
|
309
|
-
verdict: cohortVerdict(members, minN),
|
|
310
|
-
quality: q,
|
|
311
|
-
latency: distribution(members.map((m) => m.metrics.time_to_success_ms)),
|
|
312
|
-
tokens: distribution(members.map((m) => m.metrics.tokens_to_success)),
|
|
313
|
-
cases: members.map((m) => m.case_id),
|
|
314
|
-
};
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
const cohortVerdicts = Object.values(cohortReport).map((c) => c.verdict);
|
|
318
|
-
const verdict = cohortVerdicts.includes('fail')
|
|
319
|
-
? 'fail'
|
|
320
|
-
: (cohortVerdicts.length === 0 || cohortVerdicts.includes('inconclusive') ? 'inconclusive' : 'pass');
|
|
321
|
-
|
|
322
|
-
return {
|
|
323
|
-
metric_version: typeof source.metric_version === 'string'
|
|
324
|
-
? source.metric_version
|
|
325
|
-
: EVALUATION_METRIC_VERSION,
|
|
326
|
-
policy: policyInfo,
|
|
327
|
-
quality,
|
|
328
|
-
latency: distribution(latencyValues),
|
|
329
|
-
tokens: distribution(tokenValues),
|
|
330
|
-
cohorts: cohortReport,
|
|
331
|
-
provenance: {
|
|
332
|
-
policy: policyInfo,
|
|
333
|
-
versions: {
|
|
334
|
-
metric_version: typeof source.metric_version === 'string'
|
|
335
|
-
? source.metric_version
|
|
336
|
-
: EVALUATION_METRIC_VERSION,
|
|
337
|
-
...(isPlainObject(pre.versions) ? pre.versions : {}),
|
|
338
|
-
},
|
|
339
|
-
pre_registered: {
|
|
340
|
-
min_cohort_n: minN,
|
|
341
|
-
cohorts: Array.isArray(pre.cohorts) ? pre.cohorts : Object.keys(cohortReport),
|
|
342
|
-
policies: isPlainObject(pre.policies) ? Object.keys(pre.policies).sort() : [policyInfo.id],
|
|
343
|
-
},
|
|
344
|
-
corpus_cases: cases.length,
|
|
345
|
-
},
|
|
346
|
-
fidelity: {
|
|
347
|
-
tags: fidelityTags,
|
|
348
|
-
limitations: [...fidelityLimitations].sort(),
|
|
349
|
-
},
|
|
350
|
-
sample_size: results.length,
|
|
351
|
-
verdict,
|
|
352
|
-
cases: results,
|
|
353
|
-
};
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
/**
|
|
357
|
-
* Paired bootstrap CI for per-case deltas (fixed seed → deterministic).
|
|
358
|
-
* Returns null when the pair count cannot support an interval.
|
|
359
|
-
*/
|
|
360
|
-
function bootstrapCi95(deltas, seed) {
|
|
361
|
-
if (deltas.length < 2) return null;
|
|
362
|
-
const rand = mulberry32(seed);
|
|
363
|
-
const means = [];
|
|
364
|
-
for (let i = 0; i < BOOTSTRAP_RESAMPLES; i += 1) {
|
|
365
|
-
let sum = 0;
|
|
366
|
-
for (let j = 0; j < deltas.length; j += 1) {
|
|
367
|
-
sum += deltas[Math.floor(rand() * deltas.length)];
|
|
368
|
-
}
|
|
369
|
-
means.push(sum / deltas.length);
|
|
370
|
-
}
|
|
371
|
-
means.sort((a, b) => a - b);
|
|
372
|
-
return [percentileOf(means, 2.5), percentileOf(means, 97.5)];
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
function metricDelta(pairs, key, seed) {
|
|
376
|
-
const deltas = [];
|
|
377
|
-
let skipped = 0;
|
|
378
|
-
for (const [a, b] of pairs) {
|
|
379
|
-
const va = a.metrics[key];
|
|
380
|
-
const vb = b.metrics[key];
|
|
381
|
-
if (typeof va === 'number' && Number.isFinite(va)
|
|
382
|
-
&& typeof vb === 'number' && Number.isFinite(vb)) {
|
|
383
|
-
deltas.push(vb - va);
|
|
384
|
-
} else {
|
|
385
|
-
skipped += 1;
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
|
-
return {
|
|
389
|
-
n_paired: deltas.length,
|
|
390
|
-
skipped,
|
|
391
|
-
mean_delta: meanOf(deltas),
|
|
392
|
-
ci95: bootstrapCi95(deltas, seed),
|
|
393
|
-
};
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
function directionOf(metric) {
|
|
397
|
-
if (!metric.ci95) return 'inconclusive';
|
|
398
|
-
if (metric.ci95[1] < 0) return 'improvement';
|
|
399
|
-
if (metric.ci95[0] > 0) return 'regression';
|
|
400
|
-
return 'no_difference';
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
/**
|
|
404
|
-
* Compare two scorecards over the same golden case set. Paired by case_id
|
|
405
|
-
* within each cohort; differences carry a deterministic bootstrap
|
|
406
|
-
* confidence band. Token/latency deltas are proxy evidence — the report
|
|
407
|
-
* never upgrades a proxy-only delta into a user-visible improvement claim.
|
|
408
|
-
*/
|
|
409
|
-
export function compareScorecards(a, b, { seed = 42 } = {}) {
|
|
410
|
-
const aById = new Map((a.cases || []).map((c) => [c.case_id, c]));
|
|
411
|
-
const bById = new Map((b.cases || []).map((c) => [c.case_id, c]));
|
|
412
|
-
const cohortKeys = new Set([...Object.keys(a.cohorts || {}), ...Object.keys(b.cohorts || {})]);
|
|
413
|
-
|
|
414
|
-
const cohorts = {};
|
|
415
|
-
for (const key of [...cohortKeys].sort()) {
|
|
416
|
-
const pairs = [];
|
|
417
|
-
for (const [id, ca] of aById) {
|
|
418
|
-
const cb = bById.get(id);
|
|
419
|
-
if (cb && blockKeyOf(ca) === key && blockKeyOf(cb) === key && ca.ok && cb.ok) pairs.push([ca, cb]);
|
|
420
|
-
}
|
|
421
|
-
const tokens = metricDelta(pairs, 'tokens_to_success', seed);
|
|
422
|
-
const time = metricDelta(pairs, 'time_to_success_ms', seed + 1);
|
|
423
|
-
const directions = [directionOf(tokens), directionOf(time)];
|
|
424
|
-
const verdict = directions.includes('regression')
|
|
425
|
-
? 'regression'
|
|
426
|
-
: directions.includes('improvement')
|
|
427
|
-
? 'improvement'
|
|
428
|
-
: directions.every((d) => d === 'inconclusive') ? 'inconclusive' : 'no_difference';
|
|
429
|
-
cohorts[key] = {
|
|
430
|
-
n_paired: pairs.length,
|
|
431
|
-
tokens_to_success: tokens,
|
|
432
|
-
time_to_success_ms: time,
|
|
433
|
-
quality: {
|
|
434
|
-
met_delta: (b.cohorts?.[key]?.quality.met ?? 0) - (a.cohorts?.[key]?.quality.met ?? 0),
|
|
435
|
-
violated_delta: (b.cohorts?.[key]?.quality.violated ?? 0)
|
|
436
|
-
- (a.cohorts?.[key]?.quality.violated ?? 0),
|
|
437
|
-
},
|
|
438
|
-
verdict,
|
|
439
|
-
};
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
const verdicts = Object.values(cohorts).map((c) => c.verdict);
|
|
443
|
-
const verdict = verdicts.includes('regression')
|
|
444
|
-
? 'regression'
|
|
445
|
-
: verdicts.includes('improvement')
|
|
446
|
-
? 'improvement'
|
|
447
|
-
: verdicts.includes('inconclusive') ? 'inconclusive' : 'no_difference';
|
|
448
|
-
|
|
449
|
-
return {
|
|
450
|
-
metric_version: EVALUATION_METRIC_VERSION,
|
|
451
|
-
seed,
|
|
452
|
-
baseline: a.policy?.id ?? 'a',
|
|
453
|
-
variant: b.policy?.id ?? 'b',
|
|
454
|
-
cohorts,
|
|
455
|
-
verdict,
|
|
456
|
-
basis: 'proxy_metrics_plus_quality_labels',
|
|
457
|
-
notes: [
|
|
458
|
-
'tokens_to_success and time_to_success_ms are proxy metrics: a proxy-only '
|
|
459
|
-
+ 'delta does not claim user-visible improvement — human-reviewable '
|
|
460
|
-
+ 'quality labels (met/unmet/violated) are reported separately.',
|
|
461
|
-
'confidence bands are paired bootstrap intervals at a fixed seed; '
|
|
462
|
-
+ 'cohorts below the pre-registered minimum are inconclusive, never failed.',
|
|
463
|
-
],
|
|
464
|
-
};
|
|
465
|
-
}
|
|
@@ -1,180 +0,0 @@
|
|
|
1
|
-
// Registry compatibility — SPEC W1 G1-FR02, §4.
|
|
2
|
-
//
|
|
3
|
-
// checkRegistryCompatibility(prev, next, options?) → { ok, violations }
|
|
4
|
-
// validateEnvelopeSchemaVersion(record) → { ok } | { ok, reason }
|
|
5
|
-
//
|
|
6
|
-
// Enforces the frozen registry lifecycle rules mechanically, not by
|
|
7
|
-
// convention: entries are append-only. New keys and new OPTIONAL metadata
|
|
8
|
-
// fields are additive and tolerated; mutating an existing entry's meaning
|
|
9
|
-
// (definition, unit, privacy floor) in place, removing a key, resurrecting
|
|
10
|
-
// a deprecated entry, or re-dating an entry's introduced_version is a
|
|
11
|
-
// breaking change that requires a new SCHEMA_VERSION major — never an
|
|
12
|
-
// in-place edit.
|
|
13
|
-
//
|
|
14
|
-
// Violation codes are a static closed set:
|
|
15
|
-
// meaning_changed — definition/unit/privacy_class mutated in place
|
|
16
|
-
// removed — a key present in prev is absent in next
|
|
17
|
-
// deprecated_resurrected — a deprecated entry lost its deprecation marker
|
|
18
|
-
// major_required — introduced_version was rewritten on an
|
|
19
|
-
// existing entry (re-dating history) or the
|
|
20
|
-
// registry's own version marker regressed
|
|
21
|
-
|
|
22
|
-
import { SCHEMA_VERSION } from './constants.js';
|
|
23
|
-
|
|
24
|
-
export const COMPATIBILITY_VIOLATION_CODES = Object.freeze([
|
|
25
|
-
'meaning_changed',
|
|
26
|
-
'removed',
|
|
27
|
-
'deprecated_resurrected',
|
|
28
|
-
'major_required',
|
|
29
|
-
]);
|
|
30
|
-
|
|
31
|
-
// Fields whose mutation changes what an entry MEANS. New optional fields
|
|
32
|
-
// not in this list (e.g. additional hints) are additive and tolerated.
|
|
33
|
-
const MEANING_FIELDS = ['definition', 'unit', 'privacy_class'];
|
|
34
|
-
|
|
35
|
-
function isPlainObject(value) {
|
|
36
|
-
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
function isDeprecated(entry) {
|
|
40
|
-
return entry.deprecated !== undefined && entry.deprecated !== null;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* Compare two registry snapshots ({ key: entry }) under the additive-only
|
|
45
|
-
* lifecycle contract. `prev`/`next` may be SEMANTIC_REGISTRY, REASON_CODES,
|
|
46
|
-
* or any same-shaped registry map.
|
|
47
|
-
*
|
|
48
|
-
* @param {object} prev — earlier registry snapshot.
|
|
49
|
-
* @param {object} next — candidate registry snapshot.
|
|
50
|
-
* @param {{ schemaVersion?: number }} [options] — the schema major the next
|
|
51
|
-
* snapshot ships under. Breaking changes are only legitimate when this is
|
|
52
|
-
* a NEW major (> SCHEMA_VERSION); defaults to the current SCHEMA_VERSION,
|
|
53
|
-
* so callers must opt into a major bump explicitly.
|
|
54
|
-
* @returns {{ ok: boolean, violations: Array<{ code: string, key: string }> }}
|
|
55
|
-
*/
|
|
56
|
-
export function checkRegistryCompatibility(prev, next, options = {}) {
|
|
57
|
-
const violations = [];
|
|
58
|
-
const prevRegistry = isPlainObject(prev) ? prev : {};
|
|
59
|
-
const nextRegistry = isPlainObject(next) ? next : {};
|
|
60
|
-
const schemaVersion = options.schemaVersion === undefined ? SCHEMA_VERSION : options.schemaVersion;
|
|
61
|
-
|
|
62
|
-
// A declared major bump opens a new registry lineage — breaking changes
|
|
63
|
-
// are legitimate there and there is nothing to enforce on this contract.
|
|
64
|
-
if (Number.isInteger(schemaVersion) && schemaVersion > SCHEMA_VERSION) {
|
|
65
|
-
return { ok: true, violations: [] };
|
|
66
|
-
}
|
|
67
|
-
// A regressed major marker is always invalid.
|
|
68
|
-
if (schemaVersion !== SCHEMA_VERSION) {
|
|
69
|
-
return { ok: false, violations: [{ code: 'major_required', key: 'schema_version' }] };
|
|
70
|
-
}
|
|
71
|
-
// Same-major snapshot pair: enforce the additive-only lifecycle rules.
|
|
72
|
-
|
|
73
|
-
for (const key of Object.keys(prevRegistry)) {
|
|
74
|
-
const before = prevRegistry[key];
|
|
75
|
-
const after = nextRegistry[key];
|
|
76
|
-
|
|
77
|
-
if (after === undefined) {
|
|
78
|
-
violations.push({ code: 'removed', key });
|
|
79
|
-
continue;
|
|
80
|
-
}
|
|
81
|
-
if (!isPlainObject(before) || !isPlainObject(after)) {
|
|
82
|
-
violations.push({ code: 'meaning_changed', key });
|
|
83
|
-
continue;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
// Re-dating an existing entry's introduced_version rewrites registry
|
|
87
|
-
// history — always a major boundary, never an in-place edit.
|
|
88
|
-
if (before.introduced_version !== after.introduced_version) {
|
|
89
|
-
violations.push({ code: 'major_required', key });
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
if (isDeprecated(before) && !isDeprecated(after)) {
|
|
93
|
-
violations.push({ code: 'deprecated_resurrected', key });
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
for (const field of MEANING_FIELDS) {
|
|
97
|
-
if (before[field] === undefined && after[field] === undefined) continue;
|
|
98
|
-
// A missing field on either side is not a meaning change — older
|
|
99
|
-
// snapshots simply did not carry the metadata.
|
|
100
|
-
if (before[field] === undefined || after[field] === undefined) continue;
|
|
101
|
-
if (before[field] !== after[field]) {
|
|
102
|
-
violations.push({ code: 'meaning_changed', key });
|
|
103
|
-
break;
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
return { ok: violations.length === 0, violations };
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
/**
|
|
112
|
-
* Gate a semantic envelope on its declared schema major. The version gate
|
|
113
|
-
* itself never inspects fields — compatibility is judged ONLY on
|
|
114
|
-
* `schema_version` (envelope-field strictness lives in
|
|
115
|
-
* validateSemanticRecord).
|
|
116
|
-
*
|
|
117
|
-
* @param {object} record — candidate semantic envelope.
|
|
118
|
-
* @returns {{ ok: true } | { ok: false, reason: string }}
|
|
119
|
-
*/
|
|
120
|
-
export function validateEnvelopeSchemaVersion(record) {
|
|
121
|
-
if (!isPlainObject(record)) {
|
|
122
|
-
return { ok: false, reason: 'record must be a plain object' };
|
|
123
|
-
}
|
|
124
|
-
const version = record.schema_version;
|
|
125
|
-
if (!Number.isInteger(version)) {
|
|
126
|
-
return { ok: false, reason: `schema_version must be an integer (got ${JSON.stringify(version)})` };
|
|
127
|
-
}
|
|
128
|
-
if (version !== SCHEMA_VERSION) {
|
|
129
|
-
return {
|
|
130
|
-
ok: false,
|
|
131
|
-
reason: `unsupported schema_version ${version} — this major requires schema ${version} support; supported: ${SCHEMA_VERSION}`,
|
|
132
|
-
};
|
|
133
|
-
}
|
|
134
|
-
return { ok: true };
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
/**
|
|
138
|
-
* Reader tolerance check (DF2-FR01): a v1 reader can consume records that
|
|
139
|
-
* carry v1.1 additive envelope fields (agent_id, project_instance_id,
|
|
140
|
-
* sampling) because the additive contract keeps schema_version at 1.
|
|
141
|
-
* The check gates on schema_version and REPORTS which fields were newer
|
|
142
|
-
* than the reader's known set — tolerance is observed, not assumed.
|
|
143
|
-
*
|
|
144
|
-
* @param {object} record — candidate semantic envelope.
|
|
145
|
-
* @returns {{ ok: true, newerFields: string[] } | { ok: false, reason: string }}
|
|
146
|
-
*/
|
|
147
|
-
export function checkReaderTolerance(record) {
|
|
148
|
-
const gate = validateEnvelopeSchemaVersion(record);
|
|
149
|
-
if (!gate.ok) return gate;
|
|
150
|
-
const newerFields = Object.keys(record).filter((key) => !V1_0_ENVELOPE_FIELDS.has(key));
|
|
151
|
-
return { ok: true, newerFields };
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
// The v1.0 baseline envelope — what a reader frozen before the v1.1
|
|
155
|
-
// additive release knows about. `redaction_version` is the stamp
|
|
156
|
-
// sanitizeObserved adds before persistence; v1.1's agent_id,
|
|
157
|
-
// project_instance_id and sampling are intentionally ABSENT so the check
|
|
158
|
-
// reports them as tolerated-but-newer.
|
|
159
|
-
const V1_0_ENVELOPE_FIELDS = new Set([
|
|
160
|
-
'record_type',
|
|
161
|
-
'semantic_name',
|
|
162
|
-
'schema_version',
|
|
163
|
-
'record_id',
|
|
164
|
-
'trace_id',
|
|
165
|
-
'span_id',
|
|
166
|
-
'parent_span_id',
|
|
167
|
-
'execution_id',
|
|
168
|
-
'session_id',
|
|
169
|
-
'project_ref',
|
|
170
|
-
'boot_id',
|
|
171
|
-
'writer_id',
|
|
172
|
-
'sequence',
|
|
173
|
-
'wall_time_utc',
|
|
174
|
-
'monotonic_ns',
|
|
175
|
-
'importance',
|
|
176
|
-
'privacy_class',
|
|
177
|
-
'origin',
|
|
178
|
-
'payload',
|
|
179
|
-
'redaction_version',
|
|
180
|
-
]);
|