@warmdrift/kgauto-compiler 2.0.0-alpha.73 → 2.0.0-alpha.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-65KZE7AC.mjs → chunk-WP22F3CX.mjs} +1 -1
- package/dist/index.d.mts +174 -8
- package/dist/index.d.ts +174 -8
- package/dist/index.js +275 -26
- package/dist/index.mjs +265 -25
- package/dist/key-health.js +1 -1
- package/dist/key-health.mjs +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
import {
|
|
17
17
|
LIBRARY_VERSION,
|
|
18
18
|
createKeyHealthRoute
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-WP22F3CX.mjs";
|
|
20
20
|
import {
|
|
21
21
|
ABSOLUTE_FLOOR,
|
|
22
22
|
ARCHETYPE_FLOOR_DEFAULT,
|
|
@@ -828,7 +828,7 @@ function passScoreTargets(ir, opts) {
|
|
|
828
828
|
}
|
|
829
829
|
const measuredGate = opts.measuredFailureGates?.get(modelId);
|
|
830
830
|
if (measuredGate) qualityGatePenalty = QUALITY_GATE_PENALTY;
|
|
831
|
-
const isPromoted = promotion?.promotedModel === modelId;
|
|
831
|
+
const isPromoted = promotion?.mode === "downswap" && promotion.promotedModel === modelId;
|
|
832
832
|
if (isPromoted && promotion.suppressQualityGate && !measuredGate) {
|
|
833
833
|
qualityGatePenalty = 0;
|
|
834
834
|
}
|
|
@@ -2341,6 +2341,16 @@ var DISCIPLINE_GATES_V1_NO_TOOLS = `Work through these gates at every judgment p
|
|
|
2341
2341
|
3. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.
|
|
2342
2342
|
4. Label each claim: mark it observed, inferred, or assumed.
|
|
2343
2343
|
5. A surfaced gap beats a guessed answer: flag what you cannot determine rather than fabricating past it.`;
|
|
2344
|
+
var RULE_DISCIPLINE_GATES_V1_STRUCTURED = "discipline-gates-v1-structured";
|
|
2345
|
+
var DISCIPLINE_GATES_V1_STRUCTURED_WITH_TOOLS = `Work through these gates at every judgment point, explicitly:
|
|
2346
|
+
1. Evidence before reasoning: cite what you observed before concluding from it.
|
|
2347
|
+
2. One extra signal: when a finding feels conclusive, check one more adjacent signal before stating it.
|
|
2348
|
+
3. Expand, don't guess: resolve a compressed or referenced item by looking it up rather than inferring its contents.
|
|
2349
|
+
4. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.`;
|
|
2350
|
+
var DISCIPLINE_GATES_V1_STRUCTURED_NO_TOOLS = `Work through these gates at every judgment point, explicitly:
|
|
2351
|
+
1. Evidence before reasoning: cite what you observed before concluding from it.
|
|
2352
|
+
2. One extra signal: when a finding feels conclusive, check one more adjacent signal before stating it.
|
|
2353
|
+
3. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.`;
|
|
2344
2354
|
var DISCIPLINE_ELIGIBLE_ARCHETYPES = /* @__PURE__ */ new Set([
|
|
2345
2355
|
"hunt",
|
|
2346
2356
|
"summarize",
|
|
@@ -2351,7 +2361,12 @@ var DISCIPLINE_ELIGIBLE_ARCHETYPES = /* @__PURE__ */ new Set([
|
|
|
2351
2361
|
function matchRule(kind, profile, archetype, ctx) {
|
|
2352
2362
|
if (kind === "discipline_contract") {
|
|
2353
2363
|
if (!DISCIPLINE_ELIGIBLE_ARCHETYPES.has(archetype)) return null;
|
|
2354
|
-
if (ctx.outputMode !== "text")
|
|
2364
|
+
if (ctx.outputMode !== "text") {
|
|
2365
|
+
return {
|
|
2366
|
+
id: RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
2367
|
+
preamble: ctx.hasTools ? DISCIPLINE_GATES_V1_STRUCTURED_WITH_TOOLS : DISCIPLINE_GATES_V1_STRUCTURED_NO_TOOLS
|
|
2368
|
+
};
|
|
2369
|
+
}
|
|
2355
2370
|
return {
|
|
2356
2371
|
id: RULE_DISCIPLINE_GATES_V1,
|
|
2357
2372
|
preamble: ctx.hasTools ? DISCIPLINE_GATES_V1_WITH_TOOLS : DISCIPLINE_GATES_V1_NO_TOOLS
|
|
@@ -2444,9 +2459,13 @@ function mapRowsToPromotions(rows) {
|
|
|
2444
2459
|
if (!isRawPromotionRow(row)) continue;
|
|
2445
2460
|
const id = coerceId(row.id);
|
|
2446
2461
|
if (id === null) continue;
|
|
2462
|
+
const mode = row.mode === "strategy" ? "strategy" : row.mode === "downswap" || row.mode === void 0 ? "downswap" : null;
|
|
2463
|
+
if (mode === null) continue;
|
|
2447
2464
|
out.push({
|
|
2448
2465
|
id,
|
|
2449
2466
|
archetype: row.intent_archetype,
|
|
2467
|
+
mode,
|
|
2468
|
+
strategy: typeof row.strategy === "string" ? row.strategy : null,
|
|
2450
2469
|
promotedModel: row.promoted_model,
|
|
2451
2470
|
incumbentModel: row.incumbent_model,
|
|
2452
2471
|
evalRunId: coerceId(row.eval_run_id ?? null),
|
|
@@ -2477,7 +2496,7 @@ function getApplicablePromotion(opts) {
|
|
|
2477
2496
|
const rt = runtime4;
|
|
2478
2497
|
if (!rt) return void 0;
|
|
2479
2498
|
const appId = opts.appId;
|
|
2480
|
-
if (!appId || !opts.archetype) return void 0;
|
|
2499
|
+
if (!appId || !opts.archetype || !opts.mode) return void 0;
|
|
2481
2500
|
let snap = snapshots4.get(appId);
|
|
2482
2501
|
if (!snap) {
|
|
2483
2502
|
snap = { data: [], expiresAt: 0, refreshing: false };
|
|
@@ -2489,7 +2508,9 @@ function getApplicablePromotion(opts) {
|
|
|
2489
2508
|
snap.refreshing = true;
|
|
2490
2509
|
void asyncRefresh4(rt, appId);
|
|
2491
2510
|
}
|
|
2492
|
-
return snap.data.find(
|
|
2511
|
+
return snap.data.find(
|
|
2512
|
+
(p) => p.archetype === opts.archetype && p.mode === opts.mode
|
|
2513
|
+
);
|
|
2493
2514
|
}
|
|
2494
2515
|
var pendingRefreshes4 = /* @__PURE__ */ new Map();
|
|
2495
2516
|
async function asyncRefresh4(rt, appId) {
|
|
@@ -2775,10 +2796,12 @@ function compile(ir, opts = {}) {
|
|
|
2775
2796
|
];
|
|
2776
2797
|
const activePromotion = getApplicablePromotion({
|
|
2777
2798
|
appId: ir.appId,
|
|
2778
|
-
archetype: ir.intent.archetype
|
|
2799
|
+
archetype: ir.intent.archetype,
|
|
2800
|
+
mode: "downswap"
|
|
2779
2801
|
});
|
|
2780
2802
|
const promotion = activePromotion ? {
|
|
2781
2803
|
id: activePromotion.id,
|
|
2804
|
+
mode: activePromotion.mode,
|
|
2782
2805
|
promotedModel: activePromotion.promotedModel,
|
|
2783
2806
|
evalRunId: activePromotion.evalRunId,
|
|
2784
2807
|
suppressQualityGate: activePromotion.suppressQualityGate
|
|
@@ -2829,6 +2852,24 @@ function compile(ir, opts = {}) {
|
|
|
2829
2852
|
structuredOutput: ir.constraints?.structuredOutput,
|
|
2830
2853
|
toolCount: ir.tools?.length ?? 0
|
|
2831
2854
|
});
|
|
2855
|
+
const strategyPromotion = getApplicablePromotion({
|
|
2856
|
+
appId: ir.appId,
|
|
2857
|
+
archetype: ir.intent.archetype,
|
|
2858
|
+
mode: "strategy"
|
|
2859
|
+
});
|
|
2860
|
+
if (strategyPromotion?.strategy === "discipline-gates-v1" && !(workingIR.sections ?? []).some((s) => s.kind === "discipline_contract")) {
|
|
2861
|
+
workingIR = {
|
|
2862
|
+
...workingIR,
|
|
2863
|
+
sections: [
|
|
2864
|
+
...workingIR.sections ?? [],
|
|
2865
|
+
{
|
|
2866
|
+
id: `__kgauto_strategy_promotion_${strategyPromotion.id}__`,
|
|
2867
|
+
kind: "discipline_contract",
|
|
2868
|
+
text: ""
|
|
2869
|
+
}
|
|
2870
|
+
]
|
|
2871
|
+
};
|
|
2872
|
+
}
|
|
2832
2873
|
const translated = applySectionRewrites({
|
|
2833
2874
|
ir: workingIR,
|
|
2834
2875
|
profile,
|
|
@@ -2837,6 +2878,16 @@ function compile(ir, opts = {}) {
|
|
|
2837
2878
|
});
|
|
2838
2879
|
workingIR = translated.rewrittenIR;
|
|
2839
2880
|
const sectionRewritesApplied = translated.rewrites;
|
|
2881
|
+
if (strategyPromotion && translated.rewrites.some(
|
|
2882
|
+
(rw) => rw.kind === "discipline_contract" && rw.sectionId === `__kgauto_strategy_promotion_${strategyPromotion.id}__`
|
|
2883
|
+
)) {
|
|
2884
|
+
accumulatedMutations.push({
|
|
2885
|
+
id: `strategy-promotion-applied-${strategyPromotion.id}`,
|
|
2886
|
+
source: "strategy_promotion",
|
|
2887
|
+
passName: "translator",
|
|
2888
|
+
description: `Strategy promotion #${strategyPromotion.id} (${strategyPromotion.strategy}) enabled the discipline gates on ${ir.appId}/${ir.intent.archetype} \u2014 measured gates-on verdict behind the 7-day rollback guard.`
|
|
2889
|
+
});
|
|
2890
|
+
}
|
|
2840
2891
|
const disciplineRewrite = sectionRewritesApplied.find(
|
|
2841
2892
|
(rw) => rw.kind === "discipline_contract"
|
|
2842
2893
|
);
|
|
@@ -4153,6 +4204,24 @@ function tryParseJson(s) {
|
|
|
4153
4204
|
return void 0;
|
|
4154
4205
|
}
|
|
4155
4206
|
}
|
|
4207
|
+
function isAuthSignatureBody(body, message) {
|
|
4208
|
+
if (body && typeof body === "object") {
|
|
4209
|
+
const err = body.error;
|
|
4210
|
+
if (err && typeof err === "object") {
|
|
4211
|
+
const e = err;
|
|
4212
|
+
if (Array.isArray(e.details)) {
|
|
4213
|
+
for (const d of e.details) {
|
|
4214
|
+
if (d && typeof d === "object" && d.reason === "API_KEY_INVALID") {
|
|
4215
|
+
return true;
|
|
4216
|
+
}
|
|
4217
|
+
}
|
|
4218
|
+
}
|
|
4219
|
+
if (e.code === "invalid_api_key") return true;
|
|
4220
|
+
}
|
|
4221
|
+
}
|
|
4222
|
+
const m = message.toLowerCase();
|
|
4223
|
+
return m.includes("api key not valid") || m.includes("invalid api key") || m.includes("invalid x-api-key") || m.includes("incorrect api key");
|
|
4224
|
+
}
|
|
4156
4225
|
function classifyHttpError(status, body) {
|
|
4157
4226
|
const message = extractErrorMessage(body) ?? `HTTP ${status}`;
|
|
4158
4227
|
if (status === 429)
|
|
@@ -4165,8 +4234,11 @@ function classifyHttpError(status, body) {
|
|
|
4165
4234
|
return { ok: false, status, errorType: "retryable", errorCode: "model_not_found", message, raw: body };
|
|
4166
4235
|
if (status === 401 || status === 403)
|
|
4167
4236
|
return { ok: false, status, errorType: "terminal", errorCode: "auth", message, raw: body };
|
|
4168
|
-
if (status === 400)
|
|
4237
|
+
if (status === 400) {
|
|
4238
|
+
if (isAuthSignatureBody(body, message))
|
|
4239
|
+
return { ok: false, status, errorType: "terminal", errorCode: "auth", message, raw: body };
|
|
4169
4240
|
return { ok: false, status, errorType: "terminal", errorCode: "invalid_request", message, raw: body };
|
|
4241
|
+
}
|
|
4170
4242
|
return { ok: false, status, errorType: "terminal", errorCode: "unknown", message, raw: body };
|
|
4171
4243
|
}
|
|
4172
4244
|
function extractErrorMessage(body) {
|
|
@@ -4431,6 +4503,9 @@ function classifyHttpError2(status, body) {
|
|
|
4431
4503
|
return { ok: false, status, errorType: "terminal", errorCode: "auth", message, raw: body };
|
|
4432
4504
|
}
|
|
4433
4505
|
if (status === 400) {
|
|
4506
|
+
if (isAuthSignatureBody(body, message)) {
|
|
4507
|
+
return { ok: false, status, errorType: "terminal", errorCode: "auth", message, raw: body };
|
|
4508
|
+
}
|
|
4434
4509
|
return { ok: false, status, errorType: "terminal", errorCode: "invalid_request", message, raw: body };
|
|
4435
4510
|
}
|
|
4436
4511
|
return { ok: false, status, errorType: "terminal", errorCode: "unknown", message, raw: body };
|
|
@@ -5359,6 +5434,58 @@ function hashForGolden(s) {
|
|
|
5359
5434
|
}
|
|
5360
5435
|
return `g${h.toString(36)}`;
|
|
5361
5436
|
}
|
|
5437
|
+
var STRATEGY_GATES_SECTION_ID = "__kgauto_strategy_eval_gates__";
|
|
5438
|
+
function withDisciplineContract(ir) {
|
|
5439
|
+
return {
|
|
5440
|
+
...ir,
|
|
5441
|
+
sections: [
|
|
5442
|
+
...ir.sections ?? [],
|
|
5443
|
+
{ id: STRATEGY_GATES_SECTION_ID, kind: "discipline_contract", text: "" }
|
|
5444
|
+
]
|
|
5445
|
+
};
|
|
5446
|
+
}
|
|
5447
|
+
var DISCIPLINE_GATES_V1_ALT_HEADER = "Before stating any conclusion, run this check:";
|
|
5448
|
+
var ALT_BULLETS = {
|
|
5449
|
+
observe: "- What did I actually observe? Quote it before reasoning from it.",
|
|
5450
|
+
extraSignal: "- This feels settled \u2014 what one adjacent signal have I not checked?",
|
|
5451
|
+
expand: "- Is anything here compressed or referenced? Open it; do not infer its contents.",
|
|
5452
|
+
innocent: "- What is the ordinary explanation? State it before any alarming one.",
|
|
5453
|
+
label: "- For each claim: is it observed, inferred, or assumed? Say which.",
|
|
5454
|
+
gap: "- What can I not determine from what I have? Name it rather than filling it in."
|
|
5455
|
+
};
|
|
5456
|
+
function altBlock(parts) {
|
|
5457
|
+
return [DISCIPLINE_GATES_V1_ALT_HEADER, ...parts.map((p) => ALT_BULLETS[p])].join("\n");
|
|
5458
|
+
}
|
|
5459
|
+
function altGatesBlockFor(args) {
|
|
5460
|
+
const shapeAltering = args.outputMode === "text" ? ["label", "gap"] : [];
|
|
5461
|
+
const expand = args.hasTools ? ["expand"] : [];
|
|
5462
|
+
return altBlock([
|
|
5463
|
+
"observe",
|
|
5464
|
+
"extraSignal",
|
|
5465
|
+
...expand,
|
|
5466
|
+
"innocent",
|
|
5467
|
+
...shapeAltering
|
|
5468
|
+
]);
|
|
5469
|
+
}
|
|
5470
|
+
var STRATEGY_ALT_SECTION_ID = "__kgauto_strategy_eval_gates_alt__";
|
|
5471
|
+
function withAltDisciplineContract(ir) {
|
|
5472
|
+
const outputMode = resolveOutputMode({
|
|
5473
|
+
declared: ir.constraints?.outputMode,
|
|
5474
|
+
structuredOutput: ir.constraints?.structuredOutput,
|
|
5475
|
+
toolCount: ir.tools?.length ?? 0
|
|
5476
|
+
});
|
|
5477
|
+
const hasTools = (ir.tools?.length ?? 0) > 0;
|
|
5478
|
+
return {
|
|
5479
|
+
...ir,
|
|
5480
|
+
sections: [
|
|
5481
|
+
...ir.sections ?? [],
|
|
5482
|
+
{
|
|
5483
|
+
id: STRATEGY_ALT_SECTION_ID,
|
|
5484
|
+
text: altGatesBlockFor({ outputMode, hasTools })
|
|
5485
|
+
}
|
|
5486
|
+
]
|
|
5487
|
+
};
|
|
5488
|
+
}
|
|
5362
5489
|
async function runGoldenEval(opts) {
|
|
5363
5490
|
const fetchFn = opts.fetchImpl ?? fetch;
|
|
5364
5491
|
const progress = opts.onProgress ?? (() => {
|
|
@@ -5369,6 +5496,14 @@ async function runGoldenEval(opts) {
|
|
|
5369
5496
|
const minJudgeable = opts.minJudgeableCases ?? 5;
|
|
5370
5497
|
const judgeModel = opts.judgeModel ?? "claude-opus-4-8";
|
|
5371
5498
|
const notes = [];
|
|
5499
|
+
const axis = opts.axis ?? "model";
|
|
5500
|
+
const strategyId = opts.strategy;
|
|
5501
|
+
if (axis === "strategy" && !strategyId) {
|
|
5502
|
+
throw new Error("golden-eval: axis 'strategy' requires opts.strategy.");
|
|
5503
|
+
}
|
|
5504
|
+
if (axis === "model" && !opts.candidateModel) {
|
|
5505
|
+
throw new Error("golden-eval: axis 'model' requires opts.candidateModel.");
|
|
5506
|
+
}
|
|
5372
5507
|
const restHeaders = {
|
|
5373
5508
|
apikey: opts.serviceKey,
|
|
5374
5509
|
Authorization: `Bearer ${opts.serviceKey}`
|
|
@@ -5390,7 +5525,7 @@ async function runGoldenEval(opts) {
|
|
|
5390
5525
|
);
|
|
5391
5526
|
}
|
|
5392
5527
|
progress(`Loaded ${goldenRows.length} golden case(s).`);
|
|
5393
|
-
let incumbentModel = opts.incumbentModel;
|
|
5528
|
+
let incumbentModel = axis === "strategy" ? opts.strategyModel : opts.incumbentModel;
|
|
5394
5529
|
if (!incumbentModel) {
|
|
5395
5530
|
const counts = /* @__PURE__ */ new Map();
|
|
5396
5531
|
for (const row of goldenRows) {
|
|
@@ -5401,21 +5536,27 @@ async function runGoldenEval(opts) {
|
|
|
5401
5536
|
incumbentModel = top[0];
|
|
5402
5537
|
notes.push(`incumbent defaulted to most-captured model: ${incumbentModel}`);
|
|
5403
5538
|
}
|
|
5404
|
-
|
|
5539
|
+
const candidateModel = axis === "strategy" ? incumbentModel : opts.candidateModel;
|
|
5540
|
+
if (axis === "model" && incumbentModel === candidateModel) {
|
|
5405
5541
|
throw new Error("golden-eval: candidate and incumbent are the same model.");
|
|
5406
5542
|
}
|
|
5407
5543
|
const judgeProfile = tryGetProfile(judgeModel);
|
|
5408
5544
|
if (!judgeProfile) {
|
|
5409
5545
|
throw new Error(`golden-eval: judge model '${judgeModel}' is not in the roster.`);
|
|
5410
5546
|
}
|
|
5411
|
-
if (judgeProfile.family && [incumbentModel,
|
|
5547
|
+
if (judgeProfile.family && [incumbentModel, candidateModel].some(
|
|
5412
5548
|
(m) => tryGetProfile(m)?.family === judgeProfile.family
|
|
5413
5549
|
)) {
|
|
5414
5550
|
notes.push(
|
|
5415
5551
|
`WARNING: judge family (${judgeProfile.family}) overlaps a compared model \u2014 self-preference risk; consider --judge from another provider.`
|
|
5416
5552
|
);
|
|
5417
5553
|
}
|
|
5418
|
-
const
|
|
5554
|
+
const armBIr = (ir) => {
|
|
5555
|
+
if (axis !== "strategy") return ir;
|
|
5556
|
+
return strategyId === "discipline-gates-v1-alt" ? withAltDisciplineContract(ir) : withDisciplineContract(ir);
|
|
5557
|
+
};
|
|
5558
|
+
const replay = async (ir, model, captureGates = false) => {
|
|
5559
|
+
let gates;
|
|
5419
5560
|
try {
|
|
5420
5561
|
const evalIr = {
|
|
5421
5562
|
...ir,
|
|
@@ -5423,6 +5564,29 @@ async function runGoldenEval(opts) {
|
|
|
5423
5564
|
constraints: { ...ir.constraints ?? {}, forceModel: model }
|
|
5424
5565
|
};
|
|
5425
5566
|
const compiled = compile(evalIr);
|
|
5567
|
+
if (captureGates) {
|
|
5568
|
+
if (strategyId === "discipline-gates-v1-alt") {
|
|
5569
|
+
const altSection = (evalIr.sections ?? []).find(
|
|
5570
|
+
(s) => s.id === STRATEGY_ALT_SECTION_ID
|
|
5571
|
+
);
|
|
5572
|
+
gates = {
|
|
5573
|
+
fired: altSection !== void 0 && altSection.text.length > 0,
|
|
5574
|
+
rule: "discipline-gates-v1-alt",
|
|
5575
|
+
gateTokens: altSection ? countTokens(`${altSection.text}
|
|
5576
|
+
|
|
5577
|
+
`) : 0
|
|
5578
|
+
};
|
|
5579
|
+
} else {
|
|
5580
|
+
const gateRewrite = compiled.sectionRewritesApplied.find(
|
|
5581
|
+
(rw) => rw.rule.startsWith("discipline-gates-")
|
|
5582
|
+
);
|
|
5583
|
+
gates = {
|
|
5584
|
+
fired: gateRewrite !== void 0 && compiled.diagnostics.disciplineGateTokens > 0,
|
|
5585
|
+
rule: gateRewrite?.rule,
|
|
5586
|
+
gateTokens: compiled.diagnostics.disciplineGateTokens
|
|
5587
|
+
};
|
|
5588
|
+
}
|
|
5589
|
+
}
|
|
5426
5590
|
const attempt = () => execute(compiled.request, { apiKeys: opts.apiKeys, fetchImpl: opts.fetchImpl });
|
|
5427
5591
|
let started = Date.now();
|
|
5428
5592
|
let exec = await attempt();
|
|
@@ -5441,7 +5605,8 @@ async function runGoldenEval(opts) {
|
|
|
5441
5605
|
tokensIn: 0,
|
|
5442
5606
|
tokensOut: 0,
|
|
5443
5607
|
latencyMs,
|
|
5444
|
-
errorClass: exec.errorCode
|
|
5608
|
+
errorClass: exec.errorCode,
|
|
5609
|
+
...gates ? { gates } : {}
|
|
5445
5610
|
};
|
|
5446
5611
|
}
|
|
5447
5612
|
const validated = _internal.validateStructuredContract(exec, evalIr);
|
|
@@ -5453,7 +5618,8 @@ async function runGoldenEval(opts) {
|
|
|
5453
5618
|
tokensIn: exec.response.tokens.input,
|
|
5454
5619
|
tokensOut: exec.response.tokens.output,
|
|
5455
5620
|
latencyMs,
|
|
5456
|
-
contractViolation: validated.errorCode
|
|
5621
|
+
contractViolation: validated.errorCode,
|
|
5622
|
+
...gates ? { gates } : {}
|
|
5457
5623
|
};
|
|
5458
5624
|
}
|
|
5459
5625
|
return {
|
|
@@ -5463,7 +5629,8 @@ async function runGoldenEval(opts) {
|
|
|
5463
5629
|
parseError: validated.response.parseError,
|
|
5464
5630
|
tokensIn: validated.response.tokens.input,
|
|
5465
5631
|
tokensOut: validated.response.tokens.output,
|
|
5466
|
-
latencyMs
|
|
5632
|
+
latencyMs,
|
|
5633
|
+
...gates ? { gates } : {}
|
|
5467
5634
|
};
|
|
5468
5635
|
} catch (err) {
|
|
5469
5636
|
return {
|
|
@@ -5473,7 +5640,8 @@ async function runGoldenEval(opts) {
|
|
|
5473
5640
|
tokensIn: 0,
|
|
5474
5641
|
tokensOut: 0,
|
|
5475
5642
|
latencyMs: 0,
|
|
5476
|
-
errorClass: err instanceof Error ? `execute_rejected:${err.message}` : "execute_rejected"
|
|
5643
|
+
errorClass: err instanceof Error ? `execute_rejected:${err.message}` : "execute_rejected",
|
|
5644
|
+
...gates ? { gates } : {}
|
|
5477
5645
|
};
|
|
5478
5646
|
}
|
|
5479
5647
|
};
|
|
@@ -5532,12 +5700,39 @@ async function runGoldenEval(opts) {
|
|
|
5532
5700
|
};
|
|
5533
5701
|
for (const [i, row] of goldenRows.entries()) {
|
|
5534
5702
|
progress(
|
|
5535
|
-
`Case ${i + 1}/${goldenRows.length} (golden #${row.id}): replaying ${incumbentModel} + ${
|
|
5703
|
+
`Case ${i + 1}/${goldenRows.length} (golden #${row.id}): replaying ${incumbentModel} + ${candidateModel}\u2026`
|
|
5536
5704
|
);
|
|
5537
5705
|
const [inc, cand] = await Promise.all([
|
|
5538
5706
|
replay(row.ir, incumbentModel),
|
|
5539
|
-
replay(row.ir,
|
|
5707
|
+
replay(armBIr(row.ir), candidateModel, axis === "strategy")
|
|
5540
5708
|
]);
|
|
5709
|
+
if (axis === "strategy" && !cand.gates?.fired) {
|
|
5710
|
+
notes.push(
|
|
5711
|
+
`ABORT gates-did-not-fire: arm B compile produced no discipline-gate rewrite (case golden #${row.id}; archetype=${opts.archetype}; gateTokens=${cand.gates?.gateTokens ?? 0}). No verdict written \u2014 a tied-because-identical run is not evidence about the strategy.`
|
|
5712
|
+
);
|
|
5713
|
+
return {
|
|
5714
|
+
verdict: "inconclusive",
|
|
5715
|
+
axis,
|
|
5716
|
+
strategy: strategyId,
|
|
5717
|
+
appId: opts.appId,
|
|
5718
|
+
archetype: opts.archetype,
|
|
5719
|
+
incumbentModel,
|
|
5720
|
+
candidateModel,
|
|
5721
|
+
judgeModel,
|
|
5722
|
+
nCases: 0,
|
|
5723
|
+
wins: 0,
|
|
5724
|
+
ties: 0,
|
|
5725
|
+
losses: 0,
|
|
5726
|
+
floorViolations: 0,
|
|
5727
|
+
floorDetail: {},
|
|
5728
|
+
winOrTieRatio: null,
|
|
5729
|
+
latencyRatio: null,
|
|
5730
|
+
costIncumbentUsd: null,
|
|
5731
|
+
costCandidateUsd: null,
|
|
5732
|
+
cases,
|
|
5733
|
+
notes
|
|
5734
|
+
};
|
|
5735
|
+
}
|
|
5541
5736
|
if (!inc.ok || inc.contractViolation) {
|
|
5542
5737
|
const why = !inc.ok ? `incumbent replay failed (${inc.errorClass ?? "unknown"})` : `incumbent contract violation (${inc.contractViolation})`;
|
|
5543
5738
|
cases.push({
|
|
@@ -5599,6 +5794,9 @@ async function runGoldenEval(opts) {
|
|
|
5599
5794
|
cases.push({
|
|
5600
5795
|
goldenIrId: row.id,
|
|
5601
5796
|
verdict: verdict2,
|
|
5797
|
+
// Release B — the raw pair the fold discards; positional-disagreement
|
|
5798
|
+
// rate is the strategy experiment's primary metric.
|
|
5799
|
+
orderVerdicts: { run1: v1, run2: v2 },
|
|
5602
5800
|
judgeRationale: order1.rationale ?? order2.rationale,
|
|
5603
5801
|
floorViolations: [],
|
|
5604
5802
|
incumbent: incumbentLeg,
|
|
@@ -5622,7 +5820,7 @@ async function runGoldenEval(opts) {
|
|
|
5622
5820
|
);
|
|
5623
5821
|
const costCandidateUsd = sum(
|
|
5624
5822
|
evaluable.map(
|
|
5625
|
-
(c) => c.candidate ? costUsd(
|
|
5823
|
+
(c) => c.candidate ? costUsd(candidateModel, c.candidate.tokensIn, c.candidate.tokensOut) : null
|
|
5626
5824
|
)
|
|
5627
5825
|
);
|
|
5628
5826
|
const latencyFloorOk = latencyRatio === null || latencyRatio <= latencyFloorRatio;
|
|
@@ -5640,10 +5838,12 @@ async function runGoldenEval(opts) {
|
|
|
5640
5838
|
}
|
|
5641
5839
|
const result = {
|
|
5642
5840
|
verdict,
|
|
5841
|
+
axis,
|
|
5842
|
+
...strategyId ? { strategy: strategyId } : {},
|
|
5643
5843
|
appId: opts.appId,
|
|
5644
5844
|
archetype: opts.archetype,
|
|
5645
5845
|
incumbentModel,
|
|
5646
|
-
candidateModel
|
|
5846
|
+
candidateModel,
|
|
5647
5847
|
judgeModel,
|
|
5648
5848
|
nCases: evaluable.length,
|
|
5649
5849
|
wins,
|
|
@@ -5673,8 +5873,12 @@ async function runGoldenEval(opts) {
|
|
|
5673
5873
|
body: JSON.stringify({
|
|
5674
5874
|
app_id: opts.appId,
|
|
5675
5875
|
intent_archetype: opts.archetype,
|
|
5876
|
+
// Release B (migration 048) — a run row that can't say which axis
|
|
5877
|
+
// produced its verdict is a false-attribution engine.
|
|
5878
|
+
axis,
|
|
5879
|
+
strategy: strategyId ?? null,
|
|
5676
5880
|
incumbent_model: incumbentModel,
|
|
5677
|
-
candidate_model:
|
|
5881
|
+
candidate_model: candidateModel,
|
|
5678
5882
|
trigger_source: opts.triggerSource ?? "manual",
|
|
5679
5883
|
judge_model: judgeModel,
|
|
5680
5884
|
n_cases: result.nCases,
|
|
@@ -5709,8 +5913,8 @@ async function runGoldenEval(opts) {
|
|
|
5709
5913
|
return {
|
|
5710
5914
|
app_id: opts.appId,
|
|
5711
5915
|
intent_archetype: opts.archetype,
|
|
5712
|
-
family: tryGetProfile(
|
|
5713
|
-
candidate_model:
|
|
5916
|
+
family: tryGetProfile(candidateModel)?.family ?? "unknown",
|
|
5917
|
+
candidate_model: candidateModel,
|
|
5714
5918
|
current_model: incumbentModel,
|
|
5715
5919
|
prompt_hash: hashForGolden(turn),
|
|
5716
5920
|
current_response: c.incumbent?.text.slice(0, 500) ?? null,
|
|
@@ -5756,8 +5960,8 @@ async function runGoldenEval(opts) {
|
|
|
5756
5960
|
const latestOutcomeId = latest[0]?.id;
|
|
5757
5961
|
if (latestOutcomeId !== void 0) {
|
|
5758
5962
|
const costLine = costIncumbentUsd !== null && costCandidateUsd !== null ? ` Eval cost basis: incumbent $${costIncumbentUsd.toFixed(4)} vs candidate $${costCandidateUsd.toFixed(4)} across the set.` : "";
|
|
5759
|
-
const message = `Golden-set eval (run #${result.runId}): ${
|
|
5760
|
-
const suggestion = verdict === "promote-ready" ? `Candidate passed the non-inferiority rule on your real workload. To promote: node v2/scripts/promote-model.mjs --model ${
|
|
5963
|
+
const message = `Golden-set eval (run #${result.runId}): ${candidateModel} vs ${incumbentModel} on ${opts.appId}/${opts.archetype} \u2014 verdict ${verdict.toUpperCase()}. ${wins}W/${ties}T/${losses}L over ${result.nCases} real workload case(s), wins-or-ties ${(100 * (winOrTieRatio ?? 0)).toFixed(0)}% (threshold ${(100 * threshold).toFixed(0)}%), ${result.floorViolations} hard-floor violation(s), latency ratio ${latencyRatio?.toFixed(2) ?? "n/a"}.` + costLine;
|
|
5964
|
+
const suggestion = verdict === "promote-ready" ? `Candidate passed the non-inferiority rule on your real workload. To promote: node v2/scripts/promote-model.mjs --model ${candidateModel} (or ask kgauto-Cairn). Evidence: kgauto_golden_eval_runs #${result.runId} + probe_outcomes golden_run_id=${result.runId}.` : `Candidate did NOT clear the non-inferiority rule \u2014 no action needed; the incumbent stays. Evidence rows: golden_run_id=${result.runId}.`;
|
|
5761
5965
|
const advRes = await fetchFn(rest("compile_outcome_advisories"), {
|
|
5762
5966
|
method: "POST",
|
|
5763
5967
|
headers: { ...restHeaders, "Content-Type": "application/json", Prefer: "return=minimal" },
|
|
@@ -5786,6 +5990,33 @@ async function runGoldenEval(opts) {
|
|
|
5786
5990
|
}
|
|
5787
5991
|
return result;
|
|
5788
5992
|
}
|
|
5993
|
+
function classifyStrategyOutcome(r) {
|
|
5994
|
+
if (r.verdict === "inconclusive") return "inconclusive";
|
|
5995
|
+
if (r.wins > r.losses) return "wins";
|
|
5996
|
+
if (r.losses > r.wins) return "loses";
|
|
5997
|
+
return "ties";
|
|
5998
|
+
}
|
|
5999
|
+
var STRATEGY_AUTHORSHIP_LIMITATION = "Both gate wordings share one author (kgauto-Cairn); a shared blind spot is invisible to this experiment. Strong verdicts: both-lose (mechanism) and v1-loses/alt-wins (wording). An independently-authored alt would strengthen every other cell.";
|
|
6000
|
+
async function runStrategyEvalWithAttribution(opts) {
|
|
6001
|
+
const primary = await runGoldenEval({
|
|
6002
|
+
...opts,
|
|
6003
|
+
axis: "strategy",
|
|
6004
|
+
strategy: "discipline-gates-v1"
|
|
6005
|
+
});
|
|
6006
|
+
const outcome = classifyStrategyOutcome(primary);
|
|
6007
|
+
if (outcome !== "loses") {
|
|
6008
|
+
const attribution2 = outcome === "wins" ? "mechanism-works" : outcome === "ties" ? "no-lift" : "inconclusive";
|
|
6009
|
+
return { attribution: attribution2, primary, limitation: STRATEGY_AUTHORSHIP_LIMITATION };
|
|
6010
|
+
}
|
|
6011
|
+
const alt = await runGoldenEval({
|
|
6012
|
+
...opts,
|
|
6013
|
+
axis: "strategy",
|
|
6014
|
+
strategy: "discipline-gates-v1-alt"
|
|
6015
|
+
});
|
|
6016
|
+
const altOutcome = classifyStrategyOutcome(alt);
|
|
6017
|
+
const attribution = altOutcome === "wins" ? "wording-failure" : altOutcome === "loses" ? "mechanism-failure" : "wording-inconclusive";
|
|
6018
|
+
return { attribution, primary, alt, limitation: STRATEGY_AUTHORSHIP_LIMITATION };
|
|
6019
|
+
}
|
|
5789
6020
|
|
|
5790
6021
|
// src/oracle.ts
|
|
5791
6022
|
var DEFAULT_DIMENSIONS = ["correctness", "completeness", "conciseness", "format"];
|
|
@@ -6224,6 +6455,7 @@ export {
|
|
|
6224
6455
|
DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
6225
6456
|
DEFAULT_PROMOTIONS_ENDPOINT,
|
|
6226
6457
|
DIALECT_VERSION,
|
|
6458
|
+
DISCIPLINE_GATES_V1_ALT_HEADER,
|
|
6227
6459
|
FamilyResolutionError,
|
|
6228
6460
|
INTENT_ARCHETYPES,
|
|
6229
6461
|
JUDGE_RUBRICS,
|
|
@@ -6233,13 +6465,17 @@ export {
|
|
|
6233
6465
|
MEASURED_GROUNDING_MIN_N,
|
|
6234
6466
|
PRODUCER_OWNED_RULE_CODES,
|
|
6235
6467
|
PROVIDER_ENV_KEYS,
|
|
6468
|
+
RULE_DISCIPLINE_GATES_V1,
|
|
6469
|
+
RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
6236
6470
|
RULE_SEQUENTIAL_TOOL_CLIFF,
|
|
6471
|
+
STRATEGY_AUTHORSHIP_LIMITATION,
|
|
6237
6472
|
TRANSLATOR_FLOOR,
|
|
6238
6473
|
_testResetMeasuredFailure,
|
|
6239
6474
|
_testResetPromotions,
|
|
6240
6475
|
_testWaitForMeasuredFailureRefresh,
|
|
6241
6476
|
_testWaitForPromotionsRefresh,
|
|
6242
6477
|
allProfiles,
|
|
6478
|
+
altGatesBlockFor,
|
|
6243
6479
|
applyArchetypeConvention,
|
|
6244
6480
|
applySectionRewrites,
|
|
6245
6481
|
attachCacheControlToStreamTextInput,
|
|
@@ -6254,6 +6490,7 @@ export {
|
|
|
6254
6490
|
buildShadowProbeRow,
|
|
6255
6491
|
call,
|
|
6256
6492
|
captureGoldenIr,
|
|
6493
|
+
classifyStrategyOutcome,
|
|
6257
6494
|
clearBrain,
|
|
6258
6495
|
combineOrderSwappedVerdicts,
|
|
6259
6496
|
compile2 as compile,
|
|
@@ -6331,8 +6568,11 @@ export {
|
|
|
6331
6568
|
rubricFor,
|
|
6332
6569
|
runAdvisor,
|
|
6333
6570
|
runGoldenEval,
|
|
6571
|
+
runStrategyEvalWithAttribution,
|
|
6334
6572
|
setTokenizer,
|
|
6335
6573
|
shouldCaptureGolden,
|
|
6336
6574
|
tryGetProfile,
|
|
6337
|
-
wilsonLowerBound
|
|
6575
|
+
wilsonLowerBound,
|
|
6576
|
+
withAltDisciplineContract,
|
|
6577
|
+
withDisciplineContract
|
|
6338
6578
|
};
|
package/dist/key-health.js
CHANGED
|
@@ -25,7 +25,7 @@ __export(key_health_exports, {
|
|
|
25
25
|
module.exports = __toCommonJS(key_health_exports);
|
|
26
26
|
|
|
27
27
|
// src/version.ts
|
|
28
|
-
var LIBRARY_VERSION = "2.0.0-alpha.
|
|
28
|
+
var LIBRARY_VERSION = "2.0.0-alpha.75";
|
|
29
29
|
|
|
30
30
|
// src/key-health.ts
|
|
31
31
|
var JSON_HEADERS = { "Content-Type": "application/json" };
|
package/dist/key-health.mjs
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@warmdrift/kgauto-compiler",
|
|
3
|
-
"version": "2.0.0-alpha.
|
|
3
|
+
"version": "2.0.0-alpha.75",
|
|
4
4
|
"description": "Prompt compiler with executable provider knowledge for multi-model AI apps: normalized multi-provider transport with fallback chains, compile-time cliff guards, a curated model registry, and a telemetry flight recorder. Swap models without rewriting prompts.",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"module": "./dist/index.mjs",
|