@warmdrift/kgauto-compiler 2.0.0-alpha.74 → 2.0.0-alpha.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-EB2YR7HL.mjs → chunk-WP22F3CX.mjs} +1 -1
- package/dist/index.d.mts +174 -8
- package/dist/index.d.ts +174 -8
- package/dist/index.js +250 -25
- package/dist/index.mjs +240 -24
- package/dist/key-health.js +1 -1
- package/dist/key-health.mjs +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
import {
|
|
17
17
|
LIBRARY_VERSION,
|
|
18
18
|
createKeyHealthRoute
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-WP22F3CX.mjs";
|
|
20
20
|
import {
|
|
21
21
|
ABSOLUTE_FLOOR,
|
|
22
22
|
ARCHETYPE_FLOOR_DEFAULT,
|
|
@@ -828,7 +828,7 @@ function passScoreTargets(ir, opts) {
|
|
|
828
828
|
}
|
|
829
829
|
const measuredGate = opts.measuredFailureGates?.get(modelId);
|
|
830
830
|
if (measuredGate) qualityGatePenalty = QUALITY_GATE_PENALTY;
|
|
831
|
-
const isPromoted = promotion?.promotedModel === modelId;
|
|
831
|
+
const isPromoted = promotion?.mode === "downswap" && promotion.promotedModel === modelId;
|
|
832
832
|
if (isPromoted && promotion.suppressQualityGate && !measuredGate) {
|
|
833
833
|
qualityGatePenalty = 0;
|
|
834
834
|
}
|
|
@@ -2341,6 +2341,16 @@ var DISCIPLINE_GATES_V1_NO_TOOLS = `Work through these gates at every judgment p
|
|
|
2341
2341
|
3. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.
|
|
2342
2342
|
4. Label each claim: mark it observed, inferred, or assumed.
|
|
2343
2343
|
5. A surfaced gap beats a guessed answer: flag what you cannot determine rather than fabricating past it.`;
|
|
2344
|
+
var RULE_DISCIPLINE_GATES_V1_STRUCTURED = "discipline-gates-v1-structured";
|
|
2345
|
+
var DISCIPLINE_GATES_V1_STRUCTURED_WITH_TOOLS = `Work through these gates at every judgment point, explicitly:
|
|
2346
|
+
1. Evidence before reasoning: cite what you observed before concluding from it.
|
|
2347
|
+
2. One extra signal: when a finding feels conclusive, check one more adjacent signal before stating it.
|
|
2348
|
+
3. Expand, don't guess: resolve a compressed or referenced item by looking it up rather than inferring its contents.
|
|
2349
|
+
4. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.`;
|
|
2350
|
+
var DISCIPLINE_GATES_V1_STRUCTURED_NO_TOOLS = `Work through these gates at every judgment point, explicitly:
|
|
2351
|
+
1. Evidence before reasoning: cite what you observed before concluding from it.
|
|
2352
|
+
2. One extra signal: when a finding feels conclusive, check one more adjacent signal before stating it.
|
|
2353
|
+
3. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.`;
|
|
2344
2354
|
var DISCIPLINE_ELIGIBLE_ARCHETYPES = /* @__PURE__ */ new Set([
|
|
2345
2355
|
"hunt",
|
|
2346
2356
|
"summarize",
|
|
@@ -2351,7 +2361,12 @@ var DISCIPLINE_ELIGIBLE_ARCHETYPES = /* @__PURE__ */ new Set([
|
|
|
2351
2361
|
function matchRule(kind, profile, archetype, ctx) {
|
|
2352
2362
|
if (kind === "discipline_contract") {
|
|
2353
2363
|
if (!DISCIPLINE_ELIGIBLE_ARCHETYPES.has(archetype)) return null;
|
|
2354
|
-
if (ctx.outputMode !== "text")
|
|
2364
|
+
if (ctx.outputMode !== "text") {
|
|
2365
|
+
return {
|
|
2366
|
+
id: RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
2367
|
+
preamble: ctx.hasTools ? DISCIPLINE_GATES_V1_STRUCTURED_WITH_TOOLS : DISCIPLINE_GATES_V1_STRUCTURED_NO_TOOLS
|
|
2368
|
+
};
|
|
2369
|
+
}
|
|
2355
2370
|
return {
|
|
2356
2371
|
id: RULE_DISCIPLINE_GATES_V1,
|
|
2357
2372
|
preamble: ctx.hasTools ? DISCIPLINE_GATES_V1_WITH_TOOLS : DISCIPLINE_GATES_V1_NO_TOOLS
|
|
@@ -2444,9 +2459,13 @@ function mapRowsToPromotions(rows) {
|
|
|
2444
2459
|
if (!isRawPromotionRow(row)) continue;
|
|
2445
2460
|
const id = coerceId(row.id);
|
|
2446
2461
|
if (id === null) continue;
|
|
2462
|
+
const mode = row.mode === "strategy" ? "strategy" : row.mode === "downswap" || row.mode === void 0 ? "downswap" : null;
|
|
2463
|
+
if (mode === null) continue;
|
|
2447
2464
|
out.push({
|
|
2448
2465
|
id,
|
|
2449
2466
|
archetype: row.intent_archetype,
|
|
2467
|
+
mode,
|
|
2468
|
+
strategy: typeof row.strategy === "string" ? row.strategy : null,
|
|
2450
2469
|
promotedModel: row.promoted_model,
|
|
2451
2470
|
incumbentModel: row.incumbent_model,
|
|
2452
2471
|
evalRunId: coerceId(row.eval_run_id ?? null),
|
|
@@ -2477,7 +2496,7 @@ function getApplicablePromotion(opts) {
|
|
|
2477
2496
|
const rt = runtime4;
|
|
2478
2497
|
if (!rt) return void 0;
|
|
2479
2498
|
const appId = opts.appId;
|
|
2480
|
-
if (!appId || !opts.archetype) return void 0;
|
|
2499
|
+
if (!appId || !opts.archetype || !opts.mode) return void 0;
|
|
2481
2500
|
let snap = snapshots4.get(appId);
|
|
2482
2501
|
if (!snap) {
|
|
2483
2502
|
snap = { data: [], expiresAt: 0, refreshing: false };
|
|
@@ -2489,7 +2508,9 @@ function getApplicablePromotion(opts) {
|
|
|
2489
2508
|
snap.refreshing = true;
|
|
2490
2509
|
void asyncRefresh4(rt, appId);
|
|
2491
2510
|
}
|
|
2492
|
-
return snap.data.find(
|
|
2511
|
+
return snap.data.find(
|
|
2512
|
+
(p) => p.archetype === opts.archetype && p.mode === opts.mode
|
|
2513
|
+
);
|
|
2493
2514
|
}
|
|
2494
2515
|
var pendingRefreshes4 = /* @__PURE__ */ new Map();
|
|
2495
2516
|
async function asyncRefresh4(rt, appId) {
|
|
@@ -2775,10 +2796,12 @@ function compile(ir, opts = {}) {
|
|
|
2775
2796
|
];
|
|
2776
2797
|
const activePromotion = getApplicablePromotion({
|
|
2777
2798
|
appId: ir.appId,
|
|
2778
|
-
archetype: ir.intent.archetype
|
|
2799
|
+
archetype: ir.intent.archetype,
|
|
2800
|
+
mode: "downswap"
|
|
2779
2801
|
});
|
|
2780
2802
|
const promotion = activePromotion ? {
|
|
2781
2803
|
id: activePromotion.id,
|
|
2804
|
+
mode: activePromotion.mode,
|
|
2782
2805
|
promotedModel: activePromotion.promotedModel,
|
|
2783
2806
|
evalRunId: activePromotion.evalRunId,
|
|
2784
2807
|
suppressQualityGate: activePromotion.suppressQualityGate
|
|
@@ -2829,6 +2852,24 @@ function compile(ir, opts = {}) {
|
|
|
2829
2852
|
structuredOutput: ir.constraints?.structuredOutput,
|
|
2830
2853
|
toolCount: ir.tools?.length ?? 0
|
|
2831
2854
|
});
|
|
2855
|
+
const strategyPromotion = getApplicablePromotion({
|
|
2856
|
+
appId: ir.appId,
|
|
2857
|
+
archetype: ir.intent.archetype,
|
|
2858
|
+
mode: "strategy"
|
|
2859
|
+
});
|
|
2860
|
+
if (strategyPromotion?.strategy === "discipline-gates-v1" && !(workingIR.sections ?? []).some((s) => s.kind === "discipline_contract")) {
|
|
2861
|
+
workingIR = {
|
|
2862
|
+
...workingIR,
|
|
2863
|
+
sections: [
|
|
2864
|
+
...workingIR.sections ?? [],
|
|
2865
|
+
{
|
|
2866
|
+
id: `__kgauto_strategy_promotion_${strategyPromotion.id}__`,
|
|
2867
|
+
kind: "discipline_contract",
|
|
2868
|
+
text: ""
|
|
2869
|
+
}
|
|
2870
|
+
]
|
|
2871
|
+
};
|
|
2872
|
+
}
|
|
2832
2873
|
const translated = applySectionRewrites({
|
|
2833
2874
|
ir: workingIR,
|
|
2834
2875
|
profile,
|
|
@@ -2837,6 +2878,16 @@ function compile(ir, opts = {}) {
|
|
|
2837
2878
|
});
|
|
2838
2879
|
workingIR = translated.rewrittenIR;
|
|
2839
2880
|
const sectionRewritesApplied = translated.rewrites;
|
|
2881
|
+
if (strategyPromotion && translated.rewrites.some(
|
|
2882
|
+
(rw) => rw.kind === "discipline_contract" && rw.sectionId === `__kgauto_strategy_promotion_${strategyPromotion.id}__`
|
|
2883
|
+
)) {
|
|
2884
|
+
accumulatedMutations.push({
|
|
2885
|
+
id: `strategy-promotion-applied-${strategyPromotion.id}`,
|
|
2886
|
+
source: "strategy_promotion",
|
|
2887
|
+
passName: "translator",
|
|
2888
|
+
description: `Strategy promotion #${strategyPromotion.id} (${strategyPromotion.strategy}) enabled the discipline gates on ${ir.appId}/${ir.intent.archetype} \u2014 measured gates-on verdict behind the 7-day rollback guard.`
|
|
2889
|
+
});
|
|
2890
|
+
}
|
|
2840
2891
|
const disciplineRewrite = sectionRewritesApplied.find(
|
|
2841
2892
|
(rw) => rw.kind === "discipline_contract"
|
|
2842
2893
|
);
|
|
@@ -5383,6 +5434,58 @@ function hashForGolden(s) {
|
|
|
5383
5434
|
}
|
|
5384
5435
|
return `g${h.toString(36)}`;
|
|
5385
5436
|
}
|
|
5437
|
+
var STRATEGY_GATES_SECTION_ID = "__kgauto_strategy_eval_gates__";
|
|
5438
|
+
function withDisciplineContract(ir) {
|
|
5439
|
+
return {
|
|
5440
|
+
...ir,
|
|
5441
|
+
sections: [
|
|
5442
|
+
...ir.sections ?? [],
|
|
5443
|
+
{ id: STRATEGY_GATES_SECTION_ID, kind: "discipline_contract", text: "" }
|
|
5444
|
+
]
|
|
5445
|
+
};
|
|
5446
|
+
}
|
|
5447
|
+
var DISCIPLINE_GATES_V1_ALT_HEADER = "Before stating any conclusion, run this check:";
|
|
5448
|
+
var ALT_BULLETS = {
|
|
5449
|
+
observe: "- What did I actually observe? Quote it before reasoning from it.",
|
|
5450
|
+
extraSignal: "- This feels settled \u2014 what one adjacent signal have I not checked?",
|
|
5451
|
+
expand: "- Is anything here compressed or referenced? Open it; do not infer its contents.",
|
|
5452
|
+
innocent: "- What is the ordinary explanation? State it before any alarming one.",
|
|
5453
|
+
label: "- For each claim: is it observed, inferred, or assumed? Say which.",
|
|
5454
|
+
gap: "- What can I not determine from what I have? Name it rather than filling it in."
|
|
5455
|
+
};
|
|
5456
|
+
function altBlock(parts) {
|
|
5457
|
+
return [DISCIPLINE_GATES_V1_ALT_HEADER, ...parts.map((p) => ALT_BULLETS[p])].join("\n");
|
|
5458
|
+
}
|
|
5459
|
+
function altGatesBlockFor(args) {
|
|
5460
|
+
const shapeAltering = args.outputMode === "text" ? ["label", "gap"] : [];
|
|
5461
|
+
const expand = args.hasTools ? ["expand"] : [];
|
|
5462
|
+
return altBlock([
|
|
5463
|
+
"observe",
|
|
5464
|
+
"extraSignal",
|
|
5465
|
+
...expand,
|
|
5466
|
+
"innocent",
|
|
5467
|
+
...shapeAltering
|
|
5468
|
+
]);
|
|
5469
|
+
}
|
|
5470
|
+
var STRATEGY_ALT_SECTION_ID = "__kgauto_strategy_eval_gates_alt__";
|
|
5471
|
+
function withAltDisciplineContract(ir) {
|
|
5472
|
+
const outputMode = resolveOutputMode({
|
|
5473
|
+
declared: ir.constraints?.outputMode,
|
|
5474
|
+
structuredOutput: ir.constraints?.structuredOutput,
|
|
5475
|
+
toolCount: ir.tools?.length ?? 0
|
|
5476
|
+
});
|
|
5477
|
+
const hasTools = (ir.tools?.length ?? 0) > 0;
|
|
5478
|
+
return {
|
|
5479
|
+
...ir,
|
|
5480
|
+
sections: [
|
|
5481
|
+
...ir.sections ?? [],
|
|
5482
|
+
{
|
|
5483
|
+
id: STRATEGY_ALT_SECTION_ID,
|
|
5484
|
+
text: altGatesBlockFor({ outputMode, hasTools })
|
|
5485
|
+
}
|
|
5486
|
+
]
|
|
5487
|
+
};
|
|
5488
|
+
}
|
|
5386
5489
|
async function runGoldenEval(opts) {
|
|
5387
5490
|
const fetchFn = opts.fetchImpl ?? fetch;
|
|
5388
5491
|
const progress = opts.onProgress ?? (() => {
|
|
@@ -5393,6 +5496,14 @@ async function runGoldenEval(opts) {
|
|
|
5393
5496
|
const minJudgeable = opts.minJudgeableCases ?? 5;
|
|
5394
5497
|
const judgeModel = opts.judgeModel ?? "claude-opus-4-8";
|
|
5395
5498
|
const notes = [];
|
|
5499
|
+
const axis = opts.axis ?? "model";
|
|
5500
|
+
const strategyId = opts.strategy;
|
|
5501
|
+
if (axis === "strategy" && !strategyId) {
|
|
5502
|
+
throw new Error("golden-eval: axis 'strategy' requires opts.strategy.");
|
|
5503
|
+
}
|
|
5504
|
+
if (axis === "model" && !opts.candidateModel) {
|
|
5505
|
+
throw new Error("golden-eval: axis 'model' requires opts.candidateModel.");
|
|
5506
|
+
}
|
|
5396
5507
|
const restHeaders = {
|
|
5397
5508
|
apikey: opts.serviceKey,
|
|
5398
5509
|
Authorization: `Bearer ${opts.serviceKey}`
|
|
@@ -5414,7 +5525,7 @@ async function runGoldenEval(opts) {
|
|
|
5414
5525
|
);
|
|
5415
5526
|
}
|
|
5416
5527
|
progress(`Loaded ${goldenRows.length} golden case(s).`);
|
|
5417
|
-
let incumbentModel = opts.incumbentModel;
|
|
5528
|
+
let incumbentModel = axis === "strategy" ? opts.strategyModel : opts.incumbentModel;
|
|
5418
5529
|
if (!incumbentModel) {
|
|
5419
5530
|
const counts = /* @__PURE__ */ new Map();
|
|
5420
5531
|
for (const row of goldenRows) {
|
|
@@ -5425,21 +5536,27 @@ async function runGoldenEval(opts) {
|
|
|
5425
5536
|
incumbentModel = top[0];
|
|
5426
5537
|
notes.push(`incumbent defaulted to most-captured model: ${incumbentModel}`);
|
|
5427
5538
|
}
|
|
5428
|
-
|
|
5539
|
+
const candidateModel = axis === "strategy" ? incumbentModel : opts.candidateModel;
|
|
5540
|
+
if (axis === "model" && incumbentModel === candidateModel) {
|
|
5429
5541
|
throw new Error("golden-eval: candidate and incumbent are the same model.");
|
|
5430
5542
|
}
|
|
5431
5543
|
const judgeProfile = tryGetProfile(judgeModel);
|
|
5432
5544
|
if (!judgeProfile) {
|
|
5433
5545
|
throw new Error(`golden-eval: judge model '${judgeModel}' is not in the roster.`);
|
|
5434
5546
|
}
|
|
5435
|
-
if (judgeProfile.family && [incumbentModel,
|
|
5547
|
+
if (judgeProfile.family && [incumbentModel, candidateModel].some(
|
|
5436
5548
|
(m) => tryGetProfile(m)?.family === judgeProfile.family
|
|
5437
5549
|
)) {
|
|
5438
5550
|
notes.push(
|
|
5439
5551
|
`WARNING: judge family (${judgeProfile.family}) overlaps a compared model \u2014 self-preference risk; consider --judge from another provider.`
|
|
5440
5552
|
);
|
|
5441
5553
|
}
|
|
5442
|
-
const
|
|
5554
|
+
const armBIr = (ir) => {
|
|
5555
|
+
if (axis !== "strategy") return ir;
|
|
5556
|
+
return strategyId === "discipline-gates-v1-alt" ? withAltDisciplineContract(ir) : withDisciplineContract(ir);
|
|
5557
|
+
};
|
|
5558
|
+
const replay = async (ir, model, captureGates = false) => {
|
|
5559
|
+
let gates;
|
|
5443
5560
|
try {
|
|
5444
5561
|
const evalIr = {
|
|
5445
5562
|
...ir,
|
|
@@ -5447,6 +5564,29 @@ async function runGoldenEval(opts) {
|
|
|
5447
5564
|
constraints: { ...ir.constraints ?? {}, forceModel: model }
|
|
5448
5565
|
};
|
|
5449
5566
|
const compiled = compile(evalIr);
|
|
5567
|
+
if (captureGates) {
|
|
5568
|
+
if (strategyId === "discipline-gates-v1-alt") {
|
|
5569
|
+
const altSection = (evalIr.sections ?? []).find(
|
|
5570
|
+
(s) => s.id === STRATEGY_ALT_SECTION_ID
|
|
5571
|
+
);
|
|
5572
|
+
gates = {
|
|
5573
|
+
fired: altSection !== void 0 && altSection.text.length > 0,
|
|
5574
|
+
rule: "discipline-gates-v1-alt",
|
|
5575
|
+
gateTokens: altSection ? countTokens(`${altSection.text}
|
|
5576
|
+
|
|
5577
|
+
`) : 0
|
|
5578
|
+
};
|
|
5579
|
+
} else {
|
|
5580
|
+
const gateRewrite = compiled.sectionRewritesApplied.find(
|
|
5581
|
+
(rw) => rw.rule.startsWith("discipline-gates-")
|
|
5582
|
+
);
|
|
5583
|
+
gates = {
|
|
5584
|
+
fired: gateRewrite !== void 0 && compiled.diagnostics.disciplineGateTokens > 0,
|
|
5585
|
+
rule: gateRewrite?.rule,
|
|
5586
|
+
gateTokens: compiled.diagnostics.disciplineGateTokens
|
|
5587
|
+
};
|
|
5588
|
+
}
|
|
5589
|
+
}
|
|
5450
5590
|
const attempt = () => execute(compiled.request, { apiKeys: opts.apiKeys, fetchImpl: opts.fetchImpl });
|
|
5451
5591
|
let started = Date.now();
|
|
5452
5592
|
let exec = await attempt();
|
|
@@ -5465,7 +5605,8 @@ async function runGoldenEval(opts) {
|
|
|
5465
5605
|
tokensIn: 0,
|
|
5466
5606
|
tokensOut: 0,
|
|
5467
5607
|
latencyMs,
|
|
5468
|
-
errorClass: exec.errorCode
|
|
5608
|
+
errorClass: exec.errorCode,
|
|
5609
|
+
...gates ? { gates } : {}
|
|
5469
5610
|
};
|
|
5470
5611
|
}
|
|
5471
5612
|
const validated = _internal.validateStructuredContract(exec, evalIr);
|
|
@@ -5477,7 +5618,8 @@ async function runGoldenEval(opts) {
|
|
|
5477
5618
|
tokensIn: exec.response.tokens.input,
|
|
5478
5619
|
tokensOut: exec.response.tokens.output,
|
|
5479
5620
|
latencyMs,
|
|
5480
|
-
contractViolation: validated.errorCode
|
|
5621
|
+
contractViolation: validated.errorCode,
|
|
5622
|
+
...gates ? { gates } : {}
|
|
5481
5623
|
};
|
|
5482
5624
|
}
|
|
5483
5625
|
return {
|
|
@@ -5487,7 +5629,8 @@ async function runGoldenEval(opts) {
|
|
|
5487
5629
|
parseError: validated.response.parseError,
|
|
5488
5630
|
tokensIn: validated.response.tokens.input,
|
|
5489
5631
|
tokensOut: validated.response.tokens.output,
|
|
5490
|
-
latencyMs
|
|
5632
|
+
latencyMs,
|
|
5633
|
+
...gates ? { gates } : {}
|
|
5491
5634
|
};
|
|
5492
5635
|
} catch (err) {
|
|
5493
5636
|
return {
|
|
@@ -5497,7 +5640,8 @@ async function runGoldenEval(opts) {
|
|
|
5497
5640
|
tokensIn: 0,
|
|
5498
5641
|
tokensOut: 0,
|
|
5499
5642
|
latencyMs: 0,
|
|
5500
|
-
errorClass: err instanceof Error ? `execute_rejected:${err.message}` : "execute_rejected"
|
|
5643
|
+
errorClass: err instanceof Error ? `execute_rejected:${err.message}` : "execute_rejected",
|
|
5644
|
+
...gates ? { gates } : {}
|
|
5501
5645
|
};
|
|
5502
5646
|
}
|
|
5503
5647
|
};
|
|
@@ -5556,12 +5700,39 @@ async function runGoldenEval(opts) {
|
|
|
5556
5700
|
};
|
|
5557
5701
|
for (const [i, row] of goldenRows.entries()) {
|
|
5558
5702
|
progress(
|
|
5559
|
-
`Case ${i + 1}/${goldenRows.length} (golden #${row.id}): replaying ${incumbentModel} + ${
|
|
5703
|
+
`Case ${i + 1}/${goldenRows.length} (golden #${row.id}): replaying ${incumbentModel} + ${candidateModel}\u2026`
|
|
5560
5704
|
);
|
|
5561
5705
|
const [inc, cand] = await Promise.all([
|
|
5562
5706
|
replay(row.ir, incumbentModel),
|
|
5563
|
-
replay(row.ir,
|
|
5707
|
+
replay(armBIr(row.ir), candidateModel, axis === "strategy")
|
|
5564
5708
|
]);
|
|
5709
|
+
if (axis === "strategy" && !cand.gates?.fired) {
|
|
5710
|
+
notes.push(
|
|
5711
|
+
`ABORT gates-did-not-fire: arm B compile produced no discipline-gate rewrite (case golden #${row.id}; archetype=${opts.archetype}; gateTokens=${cand.gates?.gateTokens ?? 0}). No verdict written \u2014 a tied-because-identical run is not evidence about the strategy.`
|
|
5712
|
+
);
|
|
5713
|
+
return {
|
|
5714
|
+
verdict: "inconclusive",
|
|
5715
|
+
axis,
|
|
5716
|
+
strategy: strategyId,
|
|
5717
|
+
appId: opts.appId,
|
|
5718
|
+
archetype: opts.archetype,
|
|
5719
|
+
incumbentModel,
|
|
5720
|
+
candidateModel,
|
|
5721
|
+
judgeModel,
|
|
5722
|
+
nCases: 0,
|
|
5723
|
+
wins: 0,
|
|
5724
|
+
ties: 0,
|
|
5725
|
+
losses: 0,
|
|
5726
|
+
floorViolations: 0,
|
|
5727
|
+
floorDetail: {},
|
|
5728
|
+
winOrTieRatio: null,
|
|
5729
|
+
latencyRatio: null,
|
|
5730
|
+
costIncumbentUsd: null,
|
|
5731
|
+
costCandidateUsd: null,
|
|
5732
|
+
cases,
|
|
5733
|
+
notes
|
|
5734
|
+
};
|
|
5735
|
+
}
|
|
5565
5736
|
if (!inc.ok || inc.contractViolation) {
|
|
5566
5737
|
const why = !inc.ok ? `incumbent replay failed (${inc.errorClass ?? "unknown"})` : `incumbent contract violation (${inc.contractViolation})`;
|
|
5567
5738
|
cases.push({
|
|
@@ -5623,6 +5794,9 @@ async function runGoldenEval(opts) {
|
|
|
5623
5794
|
cases.push({
|
|
5624
5795
|
goldenIrId: row.id,
|
|
5625
5796
|
verdict: verdict2,
|
|
5797
|
+
// Release B — the raw pair the fold discards; positional-disagreement
|
|
5798
|
+
// rate is the strategy experiment's primary metric.
|
|
5799
|
+
orderVerdicts: { run1: v1, run2: v2 },
|
|
5626
5800
|
judgeRationale: order1.rationale ?? order2.rationale,
|
|
5627
5801
|
floorViolations: [],
|
|
5628
5802
|
incumbent: incumbentLeg,
|
|
@@ -5646,7 +5820,7 @@ async function runGoldenEval(opts) {
|
|
|
5646
5820
|
);
|
|
5647
5821
|
const costCandidateUsd = sum(
|
|
5648
5822
|
evaluable.map(
|
|
5649
|
-
(c) => c.candidate ? costUsd(
|
|
5823
|
+
(c) => c.candidate ? costUsd(candidateModel, c.candidate.tokensIn, c.candidate.tokensOut) : null
|
|
5650
5824
|
)
|
|
5651
5825
|
);
|
|
5652
5826
|
const latencyFloorOk = latencyRatio === null || latencyRatio <= latencyFloorRatio;
|
|
@@ -5664,10 +5838,12 @@ async function runGoldenEval(opts) {
|
|
|
5664
5838
|
}
|
|
5665
5839
|
const result = {
|
|
5666
5840
|
verdict,
|
|
5841
|
+
axis,
|
|
5842
|
+
...strategyId ? { strategy: strategyId } : {},
|
|
5667
5843
|
appId: opts.appId,
|
|
5668
5844
|
archetype: opts.archetype,
|
|
5669
5845
|
incumbentModel,
|
|
5670
|
-
candidateModel
|
|
5846
|
+
candidateModel,
|
|
5671
5847
|
judgeModel,
|
|
5672
5848
|
nCases: evaluable.length,
|
|
5673
5849
|
wins,
|
|
@@ -5697,8 +5873,12 @@ async function runGoldenEval(opts) {
|
|
|
5697
5873
|
body: JSON.stringify({
|
|
5698
5874
|
app_id: opts.appId,
|
|
5699
5875
|
intent_archetype: opts.archetype,
|
|
5876
|
+
// Release B (migration 048) — a run row that can't say which axis
|
|
5877
|
+
// produced its verdict is a false-attribution engine.
|
|
5878
|
+
axis,
|
|
5879
|
+
strategy: strategyId ?? null,
|
|
5700
5880
|
incumbent_model: incumbentModel,
|
|
5701
|
-
candidate_model:
|
|
5881
|
+
candidate_model: candidateModel,
|
|
5702
5882
|
trigger_source: opts.triggerSource ?? "manual",
|
|
5703
5883
|
judge_model: judgeModel,
|
|
5704
5884
|
n_cases: result.nCases,
|
|
@@ -5733,8 +5913,8 @@ async function runGoldenEval(opts) {
|
|
|
5733
5913
|
return {
|
|
5734
5914
|
app_id: opts.appId,
|
|
5735
5915
|
intent_archetype: opts.archetype,
|
|
5736
|
-
family: tryGetProfile(
|
|
5737
|
-
candidate_model:
|
|
5916
|
+
family: tryGetProfile(candidateModel)?.family ?? "unknown",
|
|
5917
|
+
candidate_model: candidateModel,
|
|
5738
5918
|
current_model: incumbentModel,
|
|
5739
5919
|
prompt_hash: hashForGolden(turn),
|
|
5740
5920
|
current_response: c.incumbent?.text.slice(0, 500) ?? null,
|
|
@@ -5780,8 +5960,8 @@ async function runGoldenEval(opts) {
|
|
|
5780
5960
|
const latestOutcomeId = latest[0]?.id;
|
|
5781
5961
|
if (latestOutcomeId !== void 0) {
|
|
5782
5962
|
const costLine = costIncumbentUsd !== null && costCandidateUsd !== null ? ` Eval cost basis: incumbent $${costIncumbentUsd.toFixed(4)} vs candidate $${costCandidateUsd.toFixed(4)} across the set.` : "";
|
|
5783
|
-
const message = `Golden-set eval (run #${result.runId}): ${
|
|
5784
|
-
const suggestion = verdict === "promote-ready" ? `Candidate passed the non-inferiority rule on your real workload. To promote: node v2/scripts/promote-model.mjs --model ${
|
|
5963
|
+
const message = `Golden-set eval (run #${result.runId}): ${candidateModel} vs ${incumbentModel} on ${opts.appId}/${opts.archetype} \u2014 verdict ${verdict.toUpperCase()}. ${wins}W/${ties}T/${losses}L over ${result.nCases} real workload case(s), wins-or-ties ${(100 * (winOrTieRatio ?? 0)).toFixed(0)}% (threshold ${(100 * threshold).toFixed(0)}%), ${result.floorViolations} hard-floor violation(s), latency ratio ${latencyRatio?.toFixed(2) ?? "n/a"}.` + costLine;
|
|
5964
|
+
const suggestion = verdict === "promote-ready" ? `Candidate passed the non-inferiority rule on your real workload. To promote: node v2/scripts/promote-model.mjs --model ${candidateModel} (or ask kgauto-Cairn). Evidence: kgauto_golden_eval_runs #${result.runId} + probe_outcomes golden_run_id=${result.runId}.` : `Candidate did NOT clear the non-inferiority rule \u2014 no action needed; the incumbent stays. Evidence rows: golden_run_id=${result.runId}.`;
|
|
5785
5965
|
const advRes = await fetchFn(rest("compile_outcome_advisories"), {
|
|
5786
5966
|
method: "POST",
|
|
5787
5967
|
headers: { ...restHeaders, "Content-Type": "application/json", Prefer: "return=minimal" },
|
|
@@ -5810,6 +5990,33 @@ async function runGoldenEval(opts) {
|
|
|
5810
5990
|
}
|
|
5811
5991
|
return result;
|
|
5812
5992
|
}
|
|
5993
|
+
function classifyStrategyOutcome(r) {
|
|
5994
|
+
if (r.verdict === "inconclusive") return "inconclusive";
|
|
5995
|
+
if (r.wins > r.losses) return "wins";
|
|
5996
|
+
if (r.losses > r.wins) return "loses";
|
|
5997
|
+
return "ties";
|
|
5998
|
+
}
|
|
5999
|
+
var STRATEGY_AUTHORSHIP_LIMITATION = "Both gate wordings share one author (kgauto-Cairn); a shared blind spot is invisible to this experiment. Strong verdicts: both-lose (mechanism) and v1-loses/alt-wins (wording). An independently-authored alt would strengthen every other cell.";
|
|
6000
|
+
async function runStrategyEvalWithAttribution(opts) {
|
|
6001
|
+
const primary = await runGoldenEval({
|
|
6002
|
+
...opts,
|
|
6003
|
+
axis: "strategy",
|
|
6004
|
+
strategy: "discipline-gates-v1"
|
|
6005
|
+
});
|
|
6006
|
+
const outcome = classifyStrategyOutcome(primary);
|
|
6007
|
+
if (outcome !== "loses") {
|
|
6008
|
+
const attribution2 = outcome === "wins" ? "mechanism-works" : outcome === "ties" ? "no-lift" : "inconclusive";
|
|
6009
|
+
return { attribution: attribution2, primary, limitation: STRATEGY_AUTHORSHIP_LIMITATION };
|
|
6010
|
+
}
|
|
6011
|
+
const alt = await runGoldenEval({
|
|
6012
|
+
...opts,
|
|
6013
|
+
axis: "strategy",
|
|
6014
|
+
strategy: "discipline-gates-v1-alt"
|
|
6015
|
+
});
|
|
6016
|
+
const altOutcome = classifyStrategyOutcome(alt);
|
|
6017
|
+
const attribution = altOutcome === "wins" ? "wording-failure" : altOutcome === "loses" ? "mechanism-failure" : "wording-inconclusive";
|
|
6018
|
+
return { attribution, primary, alt, limitation: STRATEGY_AUTHORSHIP_LIMITATION };
|
|
6019
|
+
}
|
|
5813
6020
|
|
|
5814
6021
|
// src/oracle.ts
|
|
5815
6022
|
var DEFAULT_DIMENSIONS = ["correctness", "completeness", "conciseness", "format"];
|
|
@@ -6248,6 +6455,7 @@ export {
|
|
|
6248
6455
|
DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
6249
6456
|
DEFAULT_PROMOTIONS_ENDPOINT,
|
|
6250
6457
|
DIALECT_VERSION,
|
|
6458
|
+
DISCIPLINE_GATES_V1_ALT_HEADER,
|
|
6251
6459
|
FamilyResolutionError,
|
|
6252
6460
|
INTENT_ARCHETYPES,
|
|
6253
6461
|
JUDGE_RUBRICS,
|
|
@@ -6257,13 +6465,17 @@ export {
|
|
|
6257
6465
|
MEASURED_GROUNDING_MIN_N,
|
|
6258
6466
|
PRODUCER_OWNED_RULE_CODES,
|
|
6259
6467
|
PROVIDER_ENV_KEYS,
|
|
6468
|
+
RULE_DISCIPLINE_GATES_V1,
|
|
6469
|
+
RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
6260
6470
|
RULE_SEQUENTIAL_TOOL_CLIFF,
|
|
6471
|
+
STRATEGY_AUTHORSHIP_LIMITATION,
|
|
6261
6472
|
TRANSLATOR_FLOOR,
|
|
6262
6473
|
_testResetMeasuredFailure,
|
|
6263
6474
|
_testResetPromotions,
|
|
6264
6475
|
_testWaitForMeasuredFailureRefresh,
|
|
6265
6476
|
_testWaitForPromotionsRefresh,
|
|
6266
6477
|
allProfiles,
|
|
6478
|
+
altGatesBlockFor,
|
|
6267
6479
|
applyArchetypeConvention,
|
|
6268
6480
|
applySectionRewrites,
|
|
6269
6481
|
attachCacheControlToStreamTextInput,
|
|
@@ -6278,6 +6490,7 @@ export {
|
|
|
6278
6490
|
buildShadowProbeRow,
|
|
6279
6491
|
call,
|
|
6280
6492
|
captureGoldenIr,
|
|
6493
|
+
classifyStrategyOutcome,
|
|
6281
6494
|
clearBrain,
|
|
6282
6495
|
combineOrderSwappedVerdicts,
|
|
6283
6496
|
compile2 as compile,
|
|
@@ -6355,8 +6568,11 @@ export {
|
|
|
6355
6568
|
rubricFor,
|
|
6356
6569
|
runAdvisor,
|
|
6357
6570
|
runGoldenEval,
|
|
6571
|
+
runStrategyEvalWithAttribution,
|
|
6358
6572
|
setTokenizer,
|
|
6359
6573
|
shouldCaptureGolden,
|
|
6360
6574
|
tryGetProfile,
|
|
6361
|
-
wilsonLowerBound
|
|
6575
|
+
wilsonLowerBound,
|
|
6576
|
+
withAltDisciplineContract,
|
|
6577
|
+
withDisciplineContract
|
|
6362
6578
|
};
|
package/dist/key-health.js
CHANGED
|
@@ -25,7 +25,7 @@ __export(key_health_exports, {
|
|
|
25
25
|
module.exports = __toCommonJS(key_health_exports);
|
|
26
26
|
|
|
27
27
|
// src/version.ts
|
|
28
|
-
var LIBRARY_VERSION = "2.0.0-alpha.
|
|
28
|
+
var LIBRARY_VERSION = "2.0.0-alpha.75";
|
|
29
29
|
|
|
30
30
|
// src/key-health.ts
|
|
31
31
|
var JSON_HEADERS = { "Content-Type": "application/json" };
|
package/dist/key-health.mjs
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@warmdrift/kgauto-compiler",
|
|
3
|
-
"version": "2.0.0-alpha.
|
|
3
|
+
"version": "2.0.0-alpha.75",
|
|
4
4
|
"description": "Prompt compiler with executable provider knowledge for multi-model AI apps: normalized multi-provider transport with fallback chains, compile-time cliff guards, a curated model registry, and a telemetry flight recorder. Swap models without rewriting prompts.",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"module": "./dist/index.mjs",
|