@warmdrift/kgauto-compiler 2.0.0-alpha.74 → 2.0.0-alpha.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-EB2YR7HL.mjs → chunk-WP22F3CX.mjs} +1 -1
- package/dist/index.d.mts +174 -8
- package/dist/index.d.ts +174 -8
- package/dist/index.js +250 -25
- package/dist/index.mjs +240 -24
- package/dist/key-health.js +1 -1
- package/dist/key-health.mjs +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -30,6 +30,7 @@ __export(index_exports, {
|
|
|
30
30
|
DEFAULT_MEASURED_FAILURE_ENDPOINT: () => DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
31
31
|
DEFAULT_PROMOTIONS_ENDPOINT: () => DEFAULT_PROMOTIONS_ENDPOINT,
|
|
32
32
|
DIALECT_VERSION: () => DIALECT_VERSION,
|
|
33
|
+
DISCIPLINE_GATES_V1_ALT_HEADER: () => DISCIPLINE_GATES_V1_ALT_HEADER,
|
|
33
34
|
FamilyResolutionError: () => FamilyResolutionError,
|
|
34
35
|
INTENT_ARCHETYPES: () => INTENT_ARCHETYPES,
|
|
35
36
|
JUDGE_RUBRICS: () => JUDGE_RUBRICS,
|
|
@@ -39,13 +40,17 @@ __export(index_exports, {
|
|
|
39
40
|
MEASURED_GROUNDING_MIN_N: () => MEASURED_GROUNDING_MIN_N,
|
|
40
41
|
PRODUCER_OWNED_RULE_CODES: () => PRODUCER_OWNED_RULE_CODES,
|
|
41
42
|
PROVIDER_ENV_KEYS: () => PROVIDER_ENV_KEYS,
|
|
43
|
+
RULE_DISCIPLINE_GATES_V1: () => RULE_DISCIPLINE_GATES_V1,
|
|
44
|
+
RULE_DISCIPLINE_GATES_V1_STRUCTURED: () => RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
42
45
|
RULE_SEQUENTIAL_TOOL_CLIFF: () => RULE_SEQUENTIAL_TOOL_CLIFF,
|
|
46
|
+
STRATEGY_AUTHORSHIP_LIMITATION: () => STRATEGY_AUTHORSHIP_LIMITATION,
|
|
43
47
|
TRANSLATOR_FLOOR: () => TRANSLATOR_FLOOR,
|
|
44
48
|
_testResetMeasuredFailure: () => _testResetMeasuredFailure,
|
|
45
49
|
_testResetPromotions: () => _testResetPromotions,
|
|
46
50
|
_testWaitForMeasuredFailureRefresh: () => _testWaitForMeasuredFailureRefresh,
|
|
47
51
|
_testWaitForPromotionsRefresh: () => _testWaitForPromotionsRefresh,
|
|
48
52
|
allProfiles: () => allProfiles,
|
|
53
|
+
altGatesBlockFor: () => altGatesBlockFor,
|
|
49
54
|
applyArchetypeConvention: () => applyArchetypeConvention,
|
|
50
55
|
applySectionRewrites: () => applySectionRewrites,
|
|
51
56
|
attachCacheControlToStreamTextInput: () => attachCacheControlToStreamTextInput,
|
|
@@ -60,6 +65,7 @@ __export(index_exports, {
|
|
|
60
65
|
buildShadowProbeRow: () => buildShadowProbeRow,
|
|
61
66
|
call: () => call,
|
|
62
67
|
captureGoldenIr: () => captureGoldenIr,
|
|
68
|
+
classifyStrategyOutcome: () => classifyStrategyOutcome,
|
|
63
69
|
clearBrain: () => clearBrain,
|
|
64
70
|
combineOrderSwappedVerdicts: () => combineOrderSwappedVerdicts,
|
|
65
71
|
compile: () => compile2,
|
|
@@ -137,10 +143,13 @@ __export(index_exports, {
|
|
|
137
143
|
rubricFor: () => rubricFor,
|
|
138
144
|
runAdvisor: () => runAdvisor,
|
|
139
145
|
runGoldenEval: () => runGoldenEval,
|
|
146
|
+
runStrategyEvalWithAttribution: () => runStrategyEvalWithAttribution,
|
|
140
147
|
setTokenizer: () => setTokenizer,
|
|
141
148
|
shouldCaptureGolden: () => shouldCaptureGolden,
|
|
142
149
|
tryGetProfile: () => tryGetProfile,
|
|
143
|
-
wilsonLowerBound: () => wilsonLowerBound
|
|
150
|
+
wilsonLowerBound: () => wilsonLowerBound,
|
|
151
|
+
withAltDisciplineContract: () => withAltDisciplineContract,
|
|
152
|
+
withDisciplineContract: () => withDisciplineContract
|
|
144
153
|
});
|
|
145
154
|
module.exports = __toCommonJS(index_exports);
|
|
146
155
|
|
|
@@ -2934,7 +2943,7 @@ function passScoreTargets(ir, opts) {
|
|
|
2934
2943
|
}
|
|
2935
2944
|
const measuredGate = opts.measuredFailureGates?.get(modelId);
|
|
2936
2945
|
if (measuredGate) qualityGatePenalty = QUALITY_GATE_PENALTY;
|
|
2937
|
-
const isPromoted = promotion?.promotedModel === modelId;
|
|
2946
|
+
const isPromoted = promotion?.mode === "downswap" && promotion.promotedModel === modelId;
|
|
2938
2947
|
if (isPromoted && promotion.suppressQualityGate && !measuredGate) {
|
|
2939
2948
|
qualityGatePenalty = 0;
|
|
2940
2949
|
}
|
|
@@ -4447,6 +4456,16 @@ var DISCIPLINE_GATES_V1_NO_TOOLS = `Work through these gates at every judgment p
|
|
|
4447
4456
|
3. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.
|
|
4448
4457
|
4. Label each claim: mark it observed, inferred, or assumed.
|
|
4449
4458
|
5. A surfaced gap beats a guessed answer: flag what you cannot determine rather than fabricating past it.`;
|
|
4459
|
+
var RULE_DISCIPLINE_GATES_V1_STRUCTURED = "discipline-gates-v1-structured";
|
|
4460
|
+
var DISCIPLINE_GATES_V1_STRUCTURED_WITH_TOOLS = `Work through these gates at every judgment point, explicitly:
|
|
4461
|
+
1. Evidence before reasoning: cite what you observed before concluding from it.
|
|
4462
|
+
2. One extra signal: when a finding feels conclusive, check one more adjacent signal before stating it.
|
|
4463
|
+
3. Expand, don't guess: resolve a compressed or referenced item by looking it up rather than inferring its contents.
|
|
4464
|
+
4. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.`;
|
|
4465
|
+
var DISCIPLINE_GATES_V1_STRUCTURED_NO_TOOLS = `Work through these gates at every judgment point, explicitly:
|
|
4466
|
+
1. Evidence before reasoning: cite what you observed before concluding from it.
|
|
4467
|
+
2. One extra signal: when a finding feels conclusive, check one more adjacent signal before stating it.
|
|
4468
|
+
3. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.`;
|
|
4450
4469
|
var DISCIPLINE_ELIGIBLE_ARCHETYPES = /* @__PURE__ */ new Set([
|
|
4451
4470
|
"hunt",
|
|
4452
4471
|
"summarize",
|
|
@@ -4457,7 +4476,12 @@ var DISCIPLINE_ELIGIBLE_ARCHETYPES = /* @__PURE__ */ new Set([
|
|
|
4457
4476
|
function matchRule(kind, profile, archetype, ctx) {
|
|
4458
4477
|
if (kind === "discipline_contract") {
|
|
4459
4478
|
if (!DISCIPLINE_ELIGIBLE_ARCHETYPES.has(archetype)) return null;
|
|
4460
|
-
if (ctx.outputMode !== "text")
|
|
4479
|
+
if (ctx.outputMode !== "text") {
|
|
4480
|
+
return {
|
|
4481
|
+
id: RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
4482
|
+
preamble: ctx.hasTools ? DISCIPLINE_GATES_V1_STRUCTURED_WITH_TOOLS : DISCIPLINE_GATES_V1_STRUCTURED_NO_TOOLS
|
|
4483
|
+
};
|
|
4484
|
+
}
|
|
4461
4485
|
return {
|
|
4462
4486
|
id: RULE_DISCIPLINE_GATES_V1,
|
|
4463
4487
|
preamble: ctx.hasTools ? DISCIPLINE_GATES_V1_WITH_TOOLS : DISCIPLINE_GATES_V1_NO_TOOLS
|
|
@@ -4550,9 +4574,13 @@ function mapRowsToPromotions(rows) {
|
|
|
4550
4574
|
if (!isRawPromotionRow(row)) continue;
|
|
4551
4575
|
const id = coerceId(row.id);
|
|
4552
4576
|
if (id === null) continue;
|
|
4577
|
+
const mode = row.mode === "strategy" ? "strategy" : row.mode === "downswap" || row.mode === void 0 ? "downswap" : null;
|
|
4578
|
+
if (mode === null) continue;
|
|
4553
4579
|
out.push({
|
|
4554
4580
|
id,
|
|
4555
4581
|
archetype: row.intent_archetype,
|
|
4582
|
+
mode,
|
|
4583
|
+
strategy: typeof row.strategy === "string" ? row.strategy : null,
|
|
4556
4584
|
promotedModel: row.promoted_model,
|
|
4557
4585
|
incumbentModel: row.incumbent_model,
|
|
4558
4586
|
evalRunId: coerceId(row.eval_run_id ?? null),
|
|
@@ -4583,7 +4611,7 @@ function getApplicablePromotion(opts) {
|
|
|
4583
4611
|
const rt = runtime5;
|
|
4584
4612
|
if (!rt) return void 0;
|
|
4585
4613
|
const appId = opts.appId;
|
|
4586
|
-
if (!appId || !opts.archetype) return void 0;
|
|
4614
|
+
if (!appId || !opts.archetype || !opts.mode) return void 0;
|
|
4587
4615
|
let snap = snapshots4.get(appId);
|
|
4588
4616
|
if (!snap) {
|
|
4589
4617
|
snap = { data: [], expiresAt: 0, refreshing: false };
|
|
@@ -4595,7 +4623,9 @@ function getApplicablePromotion(opts) {
|
|
|
4595
4623
|
snap.refreshing = true;
|
|
4596
4624
|
void asyncRefresh5(rt, appId);
|
|
4597
4625
|
}
|
|
4598
|
-
return snap.data.find(
|
|
4626
|
+
return snap.data.find(
|
|
4627
|
+
(p) => p.archetype === opts.archetype && p.mode === opts.mode
|
|
4628
|
+
);
|
|
4599
4629
|
}
|
|
4600
4630
|
var pendingRefreshes4 = /* @__PURE__ */ new Map();
|
|
4601
4631
|
async function asyncRefresh5(rt, appId) {
|
|
@@ -4881,10 +4911,12 @@ function compile(ir, opts = {}) {
|
|
|
4881
4911
|
];
|
|
4882
4912
|
const activePromotion = getApplicablePromotion({
|
|
4883
4913
|
appId: ir.appId,
|
|
4884
|
-
archetype: ir.intent.archetype
|
|
4914
|
+
archetype: ir.intent.archetype,
|
|
4915
|
+
mode: "downswap"
|
|
4885
4916
|
});
|
|
4886
4917
|
const promotion = activePromotion ? {
|
|
4887
4918
|
id: activePromotion.id,
|
|
4919
|
+
mode: activePromotion.mode,
|
|
4888
4920
|
promotedModel: activePromotion.promotedModel,
|
|
4889
4921
|
evalRunId: activePromotion.evalRunId,
|
|
4890
4922
|
suppressQualityGate: activePromotion.suppressQualityGate
|
|
@@ -4935,6 +4967,24 @@ function compile(ir, opts = {}) {
|
|
|
4935
4967
|
structuredOutput: ir.constraints?.structuredOutput,
|
|
4936
4968
|
toolCount: ir.tools?.length ?? 0
|
|
4937
4969
|
});
|
|
4970
|
+
const strategyPromotion = getApplicablePromotion({
|
|
4971
|
+
appId: ir.appId,
|
|
4972
|
+
archetype: ir.intent.archetype,
|
|
4973
|
+
mode: "strategy"
|
|
4974
|
+
});
|
|
4975
|
+
if (strategyPromotion?.strategy === "discipline-gates-v1" && !(workingIR.sections ?? []).some((s) => s.kind === "discipline_contract")) {
|
|
4976
|
+
workingIR = {
|
|
4977
|
+
...workingIR,
|
|
4978
|
+
sections: [
|
|
4979
|
+
...workingIR.sections ?? [],
|
|
4980
|
+
{
|
|
4981
|
+
id: `__kgauto_strategy_promotion_${strategyPromotion.id}__`,
|
|
4982
|
+
kind: "discipline_contract",
|
|
4983
|
+
text: ""
|
|
4984
|
+
}
|
|
4985
|
+
]
|
|
4986
|
+
};
|
|
4987
|
+
}
|
|
4938
4988
|
const translated = applySectionRewrites({
|
|
4939
4989
|
ir: workingIR,
|
|
4940
4990
|
profile,
|
|
@@ -4943,6 +4993,16 @@ function compile(ir, opts = {}) {
|
|
|
4943
4993
|
});
|
|
4944
4994
|
workingIR = translated.rewrittenIR;
|
|
4945
4995
|
const sectionRewritesApplied = translated.rewrites;
|
|
4996
|
+
if (strategyPromotion && translated.rewrites.some(
|
|
4997
|
+
(rw) => rw.kind === "discipline_contract" && rw.sectionId === `__kgauto_strategy_promotion_${strategyPromotion.id}__`
|
|
4998
|
+
)) {
|
|
4999
|
+
accumulatedMutations.push({
|
|
5000
|
+
id: `strategy-promotion-applied-${strategyPromotion.id}`,
|
|
5001
|
+
source: "strategy_promotion",
|
|
5002
|
+
passName: "translator",
|
|
5003
|
+
description: `Strategy promotion #${strategyPromotion.id} (${strategyPromotion.strategy}) enabled the discipline gates on ${ir.appId}/${ir.intent.archetype} \u2014 measured gates-on verdict behind the 7-day rollback guard.`
|
|
5004
|
+
});
|
|
5005
|
+
}
|
|
4946
5006
|
const disciplineRewrite = sectionRewritesApplied.find(
|
|
4947
5007
|
(rw) => rw.kind === "discipline_contract"
|
|
4948
5008
|
);
|
|
@@ -8228,6 +8288,58 @@ function hashForGolden(s) {
|
|
|
8228
8288
|
}
|
|
8229
8289
|
return `g${h.toString(36)}`;
|
|
8230
8290
|
}
|
|
8291
|
+
var STRATEGY_GATES_SECTION_ID = "__kgauto_strategy_eval_gates__";
|
|
8292
|
+
function withDisciplineContract(ir) {
|
|
8293
|
+
return {
|
|
8294
|
+
...ir,
|
|
8295
|
+
sections: [
|
|
8296
|
+
...ir.sections ?? [],
|
|
8297
|
+
{ id: STRATEGY_GATES_SECTION_ID, kind: "discipline_contract", text: "" }
|
|
8298
|
+
]
|
|
8299
|
+
};
|
|
8300
|
+
}
|
|
8301
|
+
var DISCIPLINE_GATES_V1_ALT_HEADER = "Before stating any conclusion, run this check:";
|
|
8302
|
+
var ALT_BULLETS = {
|
|
8303
|
+
observe: "- What did I actually observe? Quote it before reasoning from it.",
|
|
8304
|
+
extraSignal: "- This feels settled \u2014 what one adjacent signal have I not checked?",
|
|
8305
|
+
expand: "- Is anything here compressed or referenced? Open it; do not infer its contents.",
|
|
8306
|
+
innocent: "- What is the ordinary explanation? State it before any alarming one.",
|
|
8307
|
+
label: "- For each claim: is it observed, inferred, or assumed? Say which.",
|
|
8308
|
+
gap: "- What can I not determine from what I have? Name it rather than filling it in."
|
|
8309
|
+
};
|
|
8310
|
+
function altBlock(parts) {
|
|
8311
|
+
return [DISCIPLINE_GATES_V1_ALT_HEADER, ...parts.map((p) => ALT_BULLETS[p])].join("\n");
|
|
8312
|
+
}
|
|
8313
|
+
function altGatesBlockFor(args) {
|
|
8314
|
+
const shapeAltering = args.outputMode === "text" ? ["label", "gap"] : [];
|
|
8315
|
+
const expand = args.hasTools ? ["expand"] : [];
|
|
8316
|
+
return altBlock([
|
|
8317
|
+
"observe",
|
|
8318
|
+
"extraSignal",
|
|
8319
|
+
...expand,
|
|
8320
|
+
"innocent",
|
|
8321
|
+
...shapeAltering
|
|
8322
|
+
]);
|
|
8323
|
+
}
|
|
8324
|
+
var STRATEGY_ALT_SECTION_ID = "__kgauto_strategy_eval_gates_alt__";
|
|
8325
|
+
function withAltDisciplineContract(ir) {
|
|
8326
|
+
const outputMode = resolveOutputMode({
|
|
8327
|
+
declared: ir.constraints?.outputMode,
|
|
8328
|
+
structuredOutput: ir.constraints?.structuredOutput,
|
|
8329
|
+
toolCount: ir.tools?.length ?? 0
|
|
8330
|
+
});
|
|
8331
|
+
const hasTools = (ir.tools?.length ?? 0) > 0;
|
|
8332
|
+
return {
|
|
8333
|
+
...ir,
|
|
8334
|
+
sections: [
|
|
8335
|
+
...ir.sections ?? [],
|
|
8336
|
+
{
|
|
8337
|
+
id: STRATEGY_ALT_SECTION_ID,
|
|
8338
|
+
text: altGatesBlockFor({ outputMode, hasTools })
|
|
8339
|
+
}
|
|
8340
|
+
]
|
|
8341
|
+
};
|
|
8342
|
+
}
|
|
8231
8343
|
async function runGoldenEval(opts) {
|
|
8232
8344
|
const fetchFn = opts.fetchImpl ?? fetch;
|
|
8233
8345
|
const progress = opts.onProgress ?? (() => {
|
|
@@ -8238,6 +8350,14 @@ async function runGoldenEval(opts) {
|
|
|
8238
8350
|
const minJudgeable = opts.minJudgeableCases ?? 5;
|
|
8239
8351
|
const judgeModel = opts.judgeModel ?? "claude-opus-4-8";
|
|
8240
8352
|
const notes = [];
|
|
8353
|
+
const axis = opts.axis ?? "model";
|
|
8354
|
+
const strategyId = opts.strategy;
|
|
8355
|
+
if (axis === "strategy" && !strategyId) {
|
|
8356
|
+
throw new Error("golden-eval: axis 'strategy' requires opts.strategy.");
|
|
8357
|
+
}
|
|
8358
|
+
if (axis === "model" && !opts.candidateModel) {
|
|
8359
|
+
throw new Error("golden-eval: axis 'model' requires opts.candidateModel.");
|
|
8360
|
+
}
|
|
8241
8361
|
const restHeaders = {
|
|
8242
8362
|
apikey: opts.serviceKey,
|
|
8243
8363
|
Authorization: `Bearer ${opts.serviceKey}`
|
|
@@ -8259,7 +8379,7 @@ async function runGoldenEval(opts) {
|
|
|
8259
8379
|
);
|
|
8260
8380
|
}
|
|
8261
8381
|
progress(`Loaded ${goldenRows.length} golden case(s).`);
|
|
8262
|
-
let incumbentModel = opts.incumbentModel;
|
|
8382
|
+
let incumbentModel = axis === "strategy" ? opts.strategyModel : opts.incumbentModel;
|
|
8263
8383
|
if (!incumbentModel) {
|
|
8264
8384
|
const counts = /* @__PURE__ */ new Map();
|
|
8265
8385
|
for (const row of goldenRows) {
|
|
@@ -8270,21 +8390,27 @@ async function runGoldenEval(opts) {
|
|
|
8270
8390
|
incumbentModel = top[0];
|
|
8271
8391
|
notes.push(`incumbent defaulted to most-captured model: ${incumbentModel}`);
|
|
8272
8392
|
}
|
|
8273
|
-
|
|
8393
|
+
const candidateModel = axis === "strategy" ? incumbentModel : opts.candidateModel;
|
|
8394
|
+
if (axis === "model" && incumbentModel === candidateModel) {
|
|
8274
8395
|
throw new Error("golden-eval: candidate and incumbent are the same model.");
|
|
8275
8396
|
}
|
|
8276
8397
|
const judgeProfile = tryGetProfile(judgeModel);
|
|
8277
8398
|
if (!judgeProfile) {
|
|
8278
8399
|
throw new Error(`golden-eval: judge model '${judgeModel}' is not in the roster.`);
|
|
8279
8400
|
}
|
|
8280
|
-
if (judgeProfile.family && [incumbentModel,
|
|
8401
|
+
if (judgeProfile.family && [incumbentModel, candidateModel].some(
|
|
8281
8402
|
(m) => tryGetProfile(m)?.family === judgeProfile.family
|
|
8282
8403
|
)) {
|
|
8283
8404
|
notes.push(
|
|
8284
8405
|
`WARNING: judge family (${judgeProfile.family}) overlaps a compared model \u2014 self-preference risk; consider --judge from another provider.`
|
|
8285
8406
|
);
|
|
8286
8407
|
}
|
|
8287
|
-
const
|
|
8408
|
+
const armBIr = (ir) => {
|
|
8409
|
+
if (axis !== "strategy") return ir;
|
|
8410
|
+
return strategyId === "discipline-gates-v1-alt" ? withAltDisciplineContract(ir) : withDisciplineContract(ir);
|
|
8411
|
+
};
|
|
8412
|
+
const replay = async (ir, model, captureGates = false) => {
|
|
8413
|
+
let gates;
|
|
8288
8414
|
try {
|
|
8289
8415
|
const evalIr = {
|
|
8290
8416
|
...ir,
|
|
@@ -8292,6 +8418,29 @@ async function runGoldenEval(opts) {
|
|
|
8292
8418
|
constraints: { ...ir.constraints ?? {}, forceModel: model }
|
|
8293
8419
|
};
|
|
8294
8420
|
const compiled = compile(evalIr);
|
|
8421
|
+
if (captureGates) {
|
|
8422
|
+
if (strategyId === "discipline-gates-v1-alt") {
|
|
8423
|
+
const altSection = (evalIr.sections ?? []).find(
|
|
8424
|
+
(s) => s.id === STRATEGY_ALT_SECTION_ID
|
|
8425
|
+
);
|
|
8426
|
+
gates = {
|
|
8427
|
+
fired: altSection !== void 0 && altSection.text.length > 0,
|
|
8428
|
+
rule: "discipline-gates-v1-alt",
|
|
8429
|
+
gateTokens: altSection ? countTokens(`${altSection.text}
|
|
8430
|
+
|
|
8431
|
+
`) : 0
|
|
8432
|
+
};
|
|
8433
|
+
} else {
|
|
8434
|
+
const gateRewrite = compiled.sectionRewritesApplied.find(
|
|
8435
|
+
(rw) => rw.rule.startsWith("discipline-gates-")
|
|
8436
|
+
);
|
|
8437
|
+
gates = {
|
|
8438
|
+
fired: gateRewrite !== void 0 && compiled.diagnostics.disciplineGateTokens > 0,
|
|
8439
|
+
rule: gateRewrite?.rule,
|
|
8440
|
+
gateTokens: compiled.diagnostics.disciplineGateTokens
|
|
8441
|
+
};
|
|
8442
|
+
}
|
|
8443
|
+
}
|
|
8295
8444
|
const attempt = () => execute(compiled.request, { apiKeys: opts.apiKeys, fetchImpl: opts.fetchImpl });
|
|
8296
8445
|
let started = Date.now();
|
|
8297
8446
|
let exec = await attempt();
|
|
@@ -8310,7 +8459,8 @@ async function runGoldenEval(opts) {
|
|
|
8310
8459
|
tokensIn: 0,
|
|
8311
8460
|
tokensOut: 0,
|
|
8312
8461
|
latencyMs,
|
|
8313
|
-
errorClass: exec.errorCode
|
|
8462
|
+
errorClass: exec.errorCode,
|
|
8463
|
+
...gates ? { gates } : {}
|
|
8314
8464
|
};
|
|
8315
8465
|
}
|
|
8316
8466
|
const validated = _internal.validateStructuredContract(exec, evalIr);
|
|
@@ -8322,7 +8472,8 @@ async function runGoldenEval(opts) {
|
|
|
8322
8472
|
tokensIn: exec.response.tokens.input,
|
|
8323
8473
|
tokensOut: exec.response.tokens.output,
|
|
8324
8474
|
latencyMs,
|
|
8325
|
-
contractViolation: validated.errorCode
|
|
8475
|
+
contractViolation: validated.errorCode,
|
|
8476
|
+
...gates ? { gates } : {}
|
|
8326
8477
|
};
|
|
8327
8478
|
}
|
|
8328
8479
|
return {
|
|
@@ -8332,7 +8483,8 @@ async function runGoldenEval(opts) {
|
|
|
8332
8483
|
parseError: validated.response.parseError,
|
|
8333
8484
|
tokensIn: validated.response.tokens.input,
|
|
8334
8485
|
tokensOut: validated.response.tokens.output,
|
|
8335
|
-
latencyMs
|
|
8486
|
+
latencyMs,
|
|
8487
|
+
...gates ? { gates } : {}
|
|
8336
8488
|
};
|
|
8337
8489
|
} catch (err) {
|
|
8338
8490
|
return {
|
|
@@ -8342,7 +8494,8 @@ async function runGoldenEval(opts) {
|
|
|
8342
8494
|
tokensIn: 0,
|
|
8343
8495
|
tokensOut: 0,
|
|
8344
8496
|
latencyMs: 0,
|
|
8345
|
-
errorClass: err instanceof Error ? `execute_rejected:${err.message}` : "execute_rejected"
|
|
8497
|
+
errorClass: err instanceof Error ? `execute_rejected:${err.message}` : "execute_rejected",
|
|
8498
|
+
...gates ? { gates } : {}
|
|
8346
8499
|
};
|
|
8347
8500
|
}
|
|
8348
8501
|
};
|
|
@@ -8401,12 +8554,39 @@ async function runGoldenEval(opts) {
|
|
|
8401
8554
|
};
|
|
8402
8555
|
for (const [i, row] of goldenRows.entries()) {
|
|
8403
8556
|
progress(
|
|
8404
|
-
`Case ${i + 1}/${goldenRows.length} (golden #${row.id}): replaying ${incumbentModel} + ${
|
|
8557
|
+
`Case ${i + 1}/${goldenRows.length} (golden #${row.id}): replaying ${incumbentModel} + ${candidateModel}\u2026`
|
|
8405
8558
|
);
|
|
8406
8559
|
const [inc, cand] = await Promise.all([
|
|
8407
8560
|
replay(row.ir, incumbentModel),
|
|
8408
|
-
replay(row.ir,
|
|
8561
|
+
replay(armBIr(row.ir), candidateModel, axis === "strategy")
|
|
8409
8562
|
]);
|
|
8563
|
+
if (axis === "strategy" && !cand.gates?.fired) {
|
|
8564
|
+
notes.push(
|
|
8565
|
+
`ABORT gates-did-not-fire: arm B compile produced no discipline-gate rewrite (case golden #${row.id}; archetype=${opts.archetype}; gateTokens=${cand.gates?.gateTokens ?? 0}). No verdict written \u2014 a tied-because-identical run is not evidence about the strategy.`
|
|
8566
|
+
);
|
|
8567
|
+
return {
|
|
8568
|
+
verdict: "inconclusive",
|
|
8569
|
+
axis,
|
|
8570
|
+
strategy: strategyId,
|
|
8571
|
+
appId: opts.appId,
|
|
8572
|
+
archetype: opts.archetype,
|
|
8573
|
+
incumbentModel,
|
|
8574
|
+
candidateModel,
|
|
8575
|
+
judgeModel,
|
|
8576
|
+
nCases: 0,
|
|
8577
|
+
wins: 0,
|
|
8578
|
+
ties: 0,
|
|
8579
|
+
losses: 0,
|
|
8580
|
+
floorViolations: 0,
|
|
8581
|
+
floorDetail: {},
|
|
8582
|
+
winOrTieRatio: null,
|
|
8583
|
+
latencyRatio: null,
|
|
8584
|
+
costIncumbentUsd: null,
|
|
8585
|
+
costCandidateUsd: null,
|
|
8586
|
+
cases,
|
|
8587
|
+
notes
|
|
8588
|
+
};
|
|
8589
|
+
}
|
|
8410
8590
|
if (!inc.ok || inc.contractViolation) {
|
|
8411
8591
|
const why = !inc.ok ? `incumbent replay failed (${inc.errorClass ?? "unknown"})` : `incumbent contract violation (${inc.contractViolation})`;
|
|
8412
8592
|
cases.push({
|
|
@@ -8468,6 +8648,9 @@ async function runGoldenEval(opts) {
|
|
|
8468
8648
|
cases.push({
|
|
8469
8649
|
goldenIrId: row.id,
|
|
8470
8650
|
verdict: verdict2,
|
|
8651
|
+
// Release B — the raw pair the fold discards; positional-disagreement
|
|
8652
|
+
// rate is the strategy experiment's primary metric.
|
|
8653
|
+
orderVerdicts: { run1: v1, run2: v2 },
|
|
8471
8654
|
judgeRationale: order1.rationale ?? order2.rationale,
|
|
8472
8655
|
floorViolations: [],
|
|
8473
8656
|
incumbent: incumbentLeg,
|
|
@@ -8491,7 +8674,7 @@ async function runGoldenEval(opts) {
|
|
|
8491
8674
|
);
|
|
8492
8675
|
const costCandidateUsd = sum(
|
|
8493
8676
|
evaluable.map(
|
|
8494
|
-
(c) => c.candidate ? costUsd(
|
|
8677
|
+
(c) => c.candidate ? costUsd(candidateModel, c.candidate.tokensIn, c.candidate.tokensOut) : null
|
|
8495
8678
|
)
|
|
8496
8679
|
);
|
|
8497
8680
|
const latencyFloorOk = latencyRatio === null || latencyRatio <= latencyFloorRatio;
|
|
@@ -8509,10 +8692,12 @@ async function runGoldenEval(opts) {
|
|
|
8509
8692
|
}
|
|
8510
8693
|
const result = {
|
|
8511
8694
|
verdict,
|
|
8695
|
+
axis,
|
|
8696
|
+
...strategyId ? { strategy: strategyId } : {},
|
|
8512
8697
|
appId: opts.appId,
|
|
8513
8698
|
archetype: opts.archetype,
|
|
8514
8699
|
incumbentModel,
|
|
8515
|
-
candidateModel
|
|
8700
|
+
candidateModel,
|
|
8516
8701
|
judgeModel,
|
|
8517
8702
|
nCases: evaluable.length,
|
|
8518
8703
|
wins,
|
|
@@ -8542,8 +8727,12 @@ async function runGoldenEval(opts) {
|
|
|
8542
8727
|
body: JSON.stringify({
|
|
8543
8728
|
app_id: opts.appId,
|
|
8544
8729
|
intent_archetype: opts.archetype,
|
|
8730
|
+
// Release B (migration 048) — a run row that can't say which axis
|
|
8731
|
+
// produced its verdict is a false-attribution engine.
|
|
8732
|
+
axis,
|
|
8733
|
+
strategy: strategyId ?? null,
|
|
8545
8734
|
incumbent_model: incumbentModel,
|
|
8546
|
-
candidate_model:
|
|
8735
|
+
candidate_model: candidateModel,
|
|
8547
8736
|
trigger_source: opts.triggerSource ?? "manual",
|
|
8548
8737
|
judge_model: judgeModel,
|
|
8549
8738
|
n_cases: result.nCases,
|
|
@@ -8578,8 +8767,8 @@ async function runGoldenEval(opts) {
|
|
|
8578
8767
|
return {
|
|
8579
8768
|
app_id: opts.appId,
|
|
8580
8769
|
intent_archetype: opts.archetype,
|
|
8581
|
-
family: tryGetProfile(
|
|
8582
|
-
candidate_model:
|
|
8770
|
+
family: tryGetProfile(candidateModel)?.family ?? "unknown",
|
|
8771
|
+
candidate_model: candidateModel,
|
|
8583
8772
|
current_model: incumbentModel,
|
|
8584
8773
|
prompt_hash: hashForGolden(turn),
|
|
8585
8774
|
current_response: c.incumbent?.text.slice(0, 500) ?? null,
|
|
@@ -8625,8 +8814,8 @@ async function runGoldenEval(opts) {
|
|
|
8625
8814
|
const latestOutcomeId = latest[0]?.id;
|
|
8626
8815
|
if (latestOutcomeId !== void 0) {
|
|
8627
8816
|
const costLine = costIncumbentUsd !== null && costCandidateUsd !== null ? ` Eval cost basis: incumbent $${costIncumbentUsd.toFixed(4)} vs candidate $${costCandidateUsd.toFixed(4)} across the set.` : "";
|
|
8628
|
-
const message = `Golden-set eval (run #${result.runId}): ${
|
|
8629
|
-
const suggestion = verdict === "promote-ready" ? `Candidate passed the non-inferiority rule on your real workload. To promote: node v2/scripts/promote-model.mjs --model ${
|
|
8817
|
+
const message = `Golden-set eval (run #${result.runId}): ${candidateModel} vs ${incumbentModel} on ${opts.appId}/${opts.archetype} \u2014 verdict ${verdict.toUpperCase()}. ${wins}W/${ties}T/${losses}L over ${result.nCases} real workload case(s), wins-or-ties ${(100 * (winOrTieRatio ?? 0)).toFixed(0)}% (threshold ${(100 * threshold).toFixed(0)}%), ${result.floorViolations} hard-floor violation(s), latency ratio ${latencyRatio?.toFixed(2) ?? "n/a"}.` + costLine;
|
|
8818
|
+
const suggestion = verdict === "promote-ready" ? `Candidate passed the non-inferiority rule on your real workload. To promote: node v2/scripts/promote-model.mjs --model ${candidateModel} (or ask kgauto-Cairn). Evidence: kgauto_golden_eval_runs #${result.runId} + probe_outcomes golden_run_id=${result.runId}.` : `Candidate did NOT clear the non-inferiority rule \u2014 no action needed; the incumbent stays. Evidence rows: golden_run_id=${result.runId}.`;
|
|
8630
8819
|
const advRes = await fetchFn(rest("compile_outcome_advisories"), {
|
|
8631
8820
|
method: "POST",
|
|
8632
8821
|
headers: { ...restHeaders, "Content-Type": "application/json", Prefer: "return=minimal" },
|
|
@@ -8655,6 +8844,33 @@ async function runGoldenEval(opts) {
|
|
|
8655
8844
|
}
|
|
8656
8845
|
return result;
|
|
8657
8846
|
}
|
|
8847
|
+
function classifyStrategyOutcome(r) {
|
|
8848
|
+
if (r.verdict === "inconclusive") return "inconclusive";
|
|
8849
|
+
if (r.wins > r.losses) return "wins";
|
|
8850
|
+
if (r.losses > r.wins) return "loses";
|
|
8851
|
+
return "ties";
|
|
8852
|
+
}
|
|
8853
|
+
var STRATEGY_AUTHORSHIP_LIMITATION = "Both gate wordings share one author (kgauto-Cairn); a shared blind spot is invisible to this experiment. Strong verdicts: both-lose (mechanism) and v1-loses/alt-wins (wording). An independently-authored alt would strengthen every other cell.";
|
|
8854
|
+
async function runStrategyEvalWithAttribution(opts) {
|
|
8855
|
+
const primary = await runGoldenEval({
|
|
8856
|
+
...opts,
|
|
8857
|
+
axis: "strategy",
|
|
8858
|
+
strategy: "discipline-gates-v1"
|
|
8859
|
+
});
|
|
8860
|
+
const outcome = classifyStrategyOutcome(primary);
|
|
8861
|
+
if (outcome !== "loses") {
|
|
8862
|
+
const attribution2 = outcome === "wins" ? "mechanism-works" : outcome === "ties" ? "no-lift" : "inconclusive";
|
|
8863
|
+
return { attribution: attribution2, primary, limitation: STRATEGY_AUTHORSHIP_LIMITATION };
|
|
8864
|
+
}
|
|
8865
|
+
const alt = await runGoldenEval({
|
|
8866
|
+
...opts,
|
|
8867
|
+
axis: "strategy",
|
|
8868
|
+
strategy: "discipline-gates-v1-alt"
|
|
8869
|
+
});
|
|
8870
|
+
const altOutcome = classifyStrategyOutcome(alt);
|
|
8871
|
+
const attribution = altOutcome === "wins" ? "wording-failure" : altOutcome === "loses" ? "mechanism-failure" : "wording-inconclusive";
|
|
8872
|
+
return { attribution, primary, alt, limitation: STRATEGY_AUTHORSHIP_LIMITATION };
|
|
8873
|
+
}
|
|
8658
8874
|
|
|
8659
8875
|
// src/brain-proxy.ts
|
|
8660
8876
|
var SEGMENT_TO_TABLE = {
|
|
@@ -8823,7 +9039,7 @@ function createBrainForwardRoutes(config) {
|
|
|
8823
9039
|
}
|
|
8824
9040
|
|
|
8825
9041
|
// src/version.ts
|
|
8826
|
-
var LIBRARY_VERSION = "2.0.0-alpha.
|
|
9042
|
+
var LIBRARY_VERSION = "2.0.0-alpha.75";
|
|
8827
9043
|
|
|
8828
9044
|
// src/key-health.ts
|
|
8829
9045
|
var JSON_HEADERS2 = { "Content-Type": "application/json" };
|
|
@@ -9459,6 +9675,7 @@ function compile2(ir, opts) {
|
|
|
9459
9675
|
DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
9460
9676
|
DEFAULT_PROMOTIONS_ENDPOINT,
|
|
9461
9677
|
DIALECT_VERSION,
|
|
9678
|
+
DISCIPLINE_GATES_V1_ALT_HEADER,
|
|
9462
9679
|
FamilyResolutionError,
|
|
9463
9680
|
INTENT_ARCHETYPES,
|
|
9464
9681
|
JUDGE_RUBRICS,
|
|
@@ -9468,13 +9685,17 @@ function compile2(ir, opts) {
|
|
|
9468
9685
|
MEASURED_GROUNDING_MIN_N,
|
|
9469
9686
|
PRODUCER_OWNED_RULE_CODES,
|
|
9470
9687
|
PROVIDER_ENV_KEYS,
|
|
9688
|
+
RULE_DISCIPLINE_GATES_V1,
|
|
9689
|
+
RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
9471
9690
|
RULE_SEQUENTIAL_TOOL_CLIFF,
|
|
9691
|
+
STRATEGY_AUTHORSHIP_LIMITATION,
|
|
9472
9692
|
TRANSLATOR_FLOOR,
|
|
9473
9693
|
_testResetMeasuredFailure,
|
|
9474
9694
|
_testResetPromotions,
|
|
9475
9695
|
_testWaitForMeasuredFailureRefresh,
|
|
9476
9696
|
_testWaitForPromotionsRefresh,
|
|
9477
9697
|
allProfiles,
|
|
9698
|
+
altGatesBlockFor,
|
|
9478
9699
|
applyArchetypeConvention,
|
|
9479
9700
|
applySectionRewrites,
|
|
9480
9701
|
attachCacheControlToStreamTextInput,
|
|
@@ -9489,6 +9710,7 @@ function compile2(ir, opts) {
|
|
|
9489
9710
|
buildShadowProbeRow,
|
|
9490
9711
|
call,
|
|
9491
9712
|
captureGoldenIr,
|
|
9713
|
+
classifyStrategyOutcome,
|
|
9492
9714
|
clearBrain,
|
|
9493
9715
|
combineOrderSwappedVerdicts,
|
|
9494
9716
|
compile,
|
|
@@ -9566,8 +9788,11 @@ function compile2(ir, opts) {
|
|
|
9566
9788
|
rubricFor,
|
|
9567
9789
|
runAdvisor,
|
|
9568
9790
|
runGoldenEval,
|
|
9791
|
+
runStrategyEvalWithAttribution,
|
|
9569
9792
|
setTokenizer,
|
|
9570
9793
|
shouldCaptureGolden,
|
|
9571
9794
|
tryGetProfile,
|
|
9572
|
-
wilsonLowerBound
|
|
9795
|
+
wilsonLowerBound,
|
|
9796
|
+
withAltDisciplineContract,
|
|
9797
|
+
withDisciplineContract
|
|
9573
9798
|
});
|