@warmdrift/kgauto-compiler 2.0.0-alpha.73 → 2.0.0-alpha.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-65KZE7AC.mjs → chunk-WP22F3CX.mjs} +1 -1
- package/dist/index.d.mts +174 -8
- package/dist/index.d.ts +174 -8
- package/dist/index.js +275 -26
- package/dist/index.mjs +265 -25
- package/dist/key-health.js +1 -1
- package/dist/key-health.mjs +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -30,6 +30,7 @@ __export(index_exports, {
|
|
|
30
30
|
DEFAULT_MEASURED_FAILURE_ENDPOINT: () => DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
31
31
|
DEFAULT_PROMOTIONS_ENDPOINT: () => DEFAULT_PROMOTIONS_ENDPOINT,
|
|
32
32
|
DIALECT_VERSION: () => DIALECT_VERSION,
|
|
33
|
+
DISCIPLINE_GATES_V1_ALT_HEADER: () => DISCIPLINE_GATES_V1_ALT_HEADER,
|
|
33
34
|
FamilyResolutionError: () => FamilyResolutionError,
|
|
34
35
|
INTENT_ARCHETYPES: () => INTENT_ARCHETYPES,
|
|
35
36
|
JUDGE_RUBRICS: () => JUDGE_RUBRICS,
|
|
@@ -39,13 +40,17 @@ __export(index_exports, {
|
|
|
39
40
|
MEASURED_GROUNDING_MIN_N: () => MEASURED_GROUNDING_MIN_N,
|
|
40
41
|
PRODUCER_OWNED_RULE_CODES: () => PRODUCER_OWNED_RULE_CODES,
|
|
41
42
|
PROVIDER_ENV_KEYS: () => PROVIDER_ENV_KEYS,
|
|
43
|
+
RULE_DISCIPLINE_GATES_V1: () => RULE_DISCIPLINE_GATES_V1,
|
|
44
|
+
RULE_DISCIPLINE_GATES_V1_STRUCTURED: () => RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
42
45
|
RULE_SEQUENTIAL_TOOL_CLIFF: () => RULE_SEQUENTIAL_TOOL_CLIFF,
|
|
46
|
+
STRATEGY_AUTHORSHIP_LIMITATION: () => STRATEGY_AUTHORSHIP_LIMITATION,
|
|
43
47
|
TRANSLATOR_FLOOR: () => TRANSLATOR_FLOOR,
|
|
44
48
|
_testResetMeasuredFailure: () => _testResetMeasuredFailure,
|
|
45
49
|
_testResetPromotions: () => _testResetPromotions,
|
|
46
50
|
_testWaitForMeasuredFailureRefresh: () => _testWaitForMeasuredFailureRefresh,
|
|
47
51
|
_testWaitForPromotionsRefresh: () => _testWaitForPromotionsRefresh,
|
|
48
52
|
allProfiles: () => allProfiles,
|
|
53
|
+
altGatesBlockFor: () => altGatesBlockFor,
|
|
49
54
|
applyArchetypeConvention: () => applyArchetypeConvention,
|
|
50
55
|
applySectionRewrites: () => applySectionRewrites,
|
|
51
56
|
attachCacheControlToStreamTextInput: () => attachCacheControlToStreamTextInput,
|
|
@@ -60,6 +65,7 @@ __export(index_exports, {
|
|
|
60
65
|
buildShadowProbeRow: () => buildShadowProbeRow,
|
|
61
66
|
call: () => call,
|
|
62
67
|
captureGoldenIr: () => captureGoldenIr,
|
|
68
|
+
classifyStrategyOutcome: () => classifyStrategyOutcome,
|
|
63
69
|
clearBrain: () => clearBrain,
|
|
64
70
|
combineOrderSwappedVerdicts: () => combineOrderSwappedVerdicts,
|
|
65
71
|
compile: () => compile2,
|
|
@@ -137,10 +143,13 @@ __export(index_exports, {
|
|
|
137
143
|
rubricFor: () => rubricFor,
|
|
138
144
|
runAdvisor: () => runAdvisor,
|
|
139
145
|
runGoldenEval: () => runGoldenEval,
|
|
146
|
+
runStrategyEvalWithAttribution: () => runStrategyEvalWithAttribution,
|
|
140
147
|
setTokenizer: () => setTokenizer,
|
|
141
148
|
shouldCaptureGolden: () => shouldCaptureGolden,
|
|
142
149
|
tryGetProfile: () => tryGetProfile,
|
|
143
|
-
wilsonLowerBound: () => wilsonLowerBound
|
|
150
|
+
wilsonLowerBound: () => wilsonLowerBound,
|
|
151
|
+
withAltDisciplineContract: () => withAltDisciplineContract,
|
|
152
|
+
withDisciplineContract: () => withDisciplineContract
|
|
144
153
|
});
|
|
145
154
|
module.exports = __toCommonJS(index_exports);
|
|
146
155
|
|
|
@@ -2934,7 +2943,7 @@ function passScoreTargets(ir, opts) {
|
|
|
2934
2943
|
}
|
|
2935
2944
|
const measuredGate = opts.measuredFailureGates?.get(modelId);
|
|
2936
2945
|
if (measuredGate) qualityGatePenalty = QUALITY_GATE_PENALTY;
|
|
2937
|
-
const isPromoted = promotion?.promotedModel === modelId;
|
|
2946
|
+
const isPromoted = promotion?.mode === "downswap" && promotion.promotedModel === modelId;
|
|
2938
2947
|
if (isPromoted && promotion.suppressQualityGate && !measuredGate) {
|
|
2939
2948
|
qualityGatePenalty = 0;
|
|
2940
2949
|
}
|
|
@@ -4447,6 +4456,16 @@ var DISCIPLINE_GATES_V1_NO_TOOLS = `Work through these gates at every judgment p
|
|
|
4447
4456
|
3. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.
|
|
4448
4457
|
4. Label each claim: mark it observed, inferred, or assumed.
|
|
4449
4458
|
5. A surfaced gap beats a guessed answer: flag what you cannot determine rather than fabricating past it.`;
|
|
4459
|
+
var RULE_DISCIPLINE_GATES_V1_STRUCTURED = "discipline-gates-v1-structured";
|
|
4460
|
+
var DISCIPLINE_GATES_V1_STRUCTURED_WITH_TOOLS = `Work through these gates at every judgment point, explicitly:
|
|
4461
|
+
1. Evidence before reasoning: cite what you observed before concluding from it.
|
|
4462
|
+
2. One extra signal: when a finding feels conclusive, check one more adjacent signal before stating it.
|
|
4463
|
+
3. Expand, don't guess: resolve a compressed or referenced item by looking it up rather than inferring its contents.
|
|
4464
|
+
4. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.`;
|
|
4465
|
+
var DISCIPLINE_GATES_V1_STRUCTURED_NO_TOOLS = `Work through these gates at every judgment point, explicitly:
|
|
4466
|
+
1. Evidence before reasoning: cite what you observed before concluding from it.
|
|
4467
|
+
2. One extra signal: when a finding feels conclusive, check one more adjacent signal before stating it.
|
|
4468
|
+
3. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.`;
|
|
4450
4469
|
var DISCIPLINE_ELIGIBLE_ARCHETYPES = /* @__PURE__ */ new Set([
|
|
4451
4470
|
"hunt",
|
|
4452
4471
|
"summarize",
|
|
@@ -4457,7 +4476,12 @@ var DISCIPLINE_ELIGIBLE_ARCHETYPES = /* @__PURE__ */ new Set([
|
|
|
4457
4476
|
function matchRule(kind, profile, archetype, ctx) {
|
|
4458
4477
|
if (kind === "discipline_contract") {
|
|
4459
4478
|
if (!DISCIPLINE_ELIGIBLE_ARCHETYPES.has(archetype)) return null;
|
|
4460
|
-
if (ctx.outputMode !== "text")
|
|
4479
|
+
if (ctx.outputMode !== "text") {
|
|
4480
|
+
return {
|
|
4481
|
+
id: RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
4482
|
+
preamble: ctx.hasTools ? DISCIPLINE_GATES_V1_STRUCTURED_WITH_TOOLS : DISCIPLINE_GATES_V1_STRUCTURED_NO_TOOLS
|
|
4483
|
+
};
|
|
4484
|
+
}
|
|
4461
4485
|
return {
|
|
4462
4486
|
id: RULE_DISCIPLINE_GATES_V1,
|
|
4463
4487
|
preamble: ctx.hasTools ? DISCIPLINE_GATES_V1_WITH_TOOLS : DISCIPLINE_GATES_V1_NO_TOOLS
|
|
@@ -4550,9 +4574,13 @@ function mapRowsToPromotions(rows) {
|
|
|
4550
4574
|
if (!isRawPromotionRow(row)) continue;
|
|
4551
4575
|
const id = coerceId(row.id);
|
|
4552
4576
|
if (id === null) continue;
|
|
4577
|
+
const mode = row.mode === "strategy" ? "strategy" : row.mode === "downswap" || row.mode === void 0 ? "downswap" : null;
|
|
4578
|
+
if (mode === null) continue;
|
|
4553
4579
|
out.push({
|
|
4554
4580
|
id,
|
|
4555
4581
|
archetype: row.intent_archetype,
|
|
4582
|
+
mode,
|
|
4583
|
+
strategy: typeof row.strategy === "string" ? row.strategy : null,
|
|
4556
4584
|
promotedModel: row.promoted_model,
|
|
4557
4585
|
incumbentModel: row.incumbent_model,
|
|
4558
4586
|
evalRunId: coerceId(row.eval_run_id ?? null),
|
|
@@ -4583,7 +4611,7 @@ function getApplicablePromotion(opts) {
|
|
|
4583
4611
|
const rt = runtime5;
|
|
4584
4612
|
if (!rt) return void 0;
|
|
4585
4613
|
const appId = opts.appId;
|
|
4586
|
-
if (!appId || !opts.archetype) return void 0;
|
|
4614
|
+
if (!appId || !opts.archetype || !opts.mode) return void 0;
|
|
4587
4615
|
let snap = snapshots4.get(appId);
|
|
4588
4616
|
if (!snap) {
|
|
4589
4617
|
snap = { data: [], expiresAt: 0, refreshing: false };
|
|
@@ -4595,7 +4623,9 @@ function getApplicablePromotion(opts) {
|
|
|
4595
4623
|
snap.refreshing = true;
|
|
4596
4624
|
void asyncRefresh5(rt, appId);
|
|
4597
4625
|
}
|
|
4598
|
-
return snap.data.find(
|
|
4626
|
+
return snap.data.find(
|
|
4627
|
+
(p) => p.archetype === opts.archetype && p.mode === opts.mode
|
|
4628
|
+
);
|
|
4599
4629
|
}
|
|
4600
4630
|
var pendingRefreshes4 = /* @__PURE__ */ new Map();
|
|
4601
4631
|
async function asyncRefresh5(rt, appId) {
|
|
@@ -4881,10 +4911,12 @@ function compile(ir, opts = {}) {
|
|
|
4881
4911
|
];
|
|
4882
4912
|
const activePromotion = getApplicablePromotion({
|
|
4883
4913
|
appId: ir.appId,
|
|
4884
|
-
archetype: ir.intent.archetype
|
|
4914
|
+
archetype: ir.intent.archetype,
|
|
4915
|
+
mode: "downswap"
|
|
4885
4916
|
});
|
|
4886
4917
|
const promotion = activePromotion ? {
|
|
4887
4918
|
id: activePromotion.id,
|
|
4919
|
+
mode: activePromotion.mode,
|
|
4888
4920
|
promotedModel: activePromotion.promotedModel,
|
|
4889
4921
|
evalRunId: activePromotion.evalRunId,
|
|
4890
4922
|
suppressQualityGate: activePromotion.suppressQualityGate
|
|
@@ -4935,6 +4967,24 @@ function compile(ir, opts = {}) {
|
|
|
4935
4967
|
structuredOutput: ir.constraints?.structuredOutput,
|
|
4936
4968
|
toolCount: ir.tools?.length ?? 0
|
|
4937
4969
|
});
|
|
4970
|
+
const strategyPromotion = getApplicablePromotion({
|
|
4971
|
+
appId: ir.appId,
|
|
4972
|
+
archetype: ir.intent.archetype,
|
|
4973
|
+
mode: "strategy"
|
|
4974
|
+
});
|
|
4975
|
+
if (strategyPromotion?.strategy === "discipline-gates-v1" && !(workingIR.sections ?? []).some((s) => s.kind === "discipline_contract")) {
|
|
4976
|
+
workingIR = {
|
|
4977
|
+
...workingIR,
|
|
4978
|
+
sections: [
|
|
4979
|
+
...workingIR.sections ?? [],
|
|
4980
|
+
{
|
|
4981
|
+
id: `__kgauto_strategy_promotion_${strategyPromotion.id}__`,
|
|
4982
|
+
kind: "discipline_contract",
|
|
4983
|
+
text: ""
|
|
4984
|
+
}
|
|
4985
|
+
]
|
|
4986
|
+
};
|
|
4987
|
+
}
|
|
4938
4988
|
const translated = applySectionRewrites({
|
|
4939
4989
|
ir: workingIR,
|
|
4940
4990
|
profile,
|
|
@@ -4943,6 +4993,16 @@ function compile(ir, opts = {}) {
|
|
|
4943
4993
|
});
|
|
4944
4994
|
workingIR = translated.rewrittenIR;
|
|
4945
4995
|
const sectionRewritesApplied = translated.rewrites;
|
|
4996
|
+
if (strategyPromotion && translated.rewrites.some(
|
|
4997
|
+
(rw) => rw.kind === "discipline_contract" && rw.sectionId === `__kgauto_strategy_promotion_${strategyPromotion.id}__`
|
|
4998
|
+
)) {
|
|
4999
|
+
accumulatedMutations.push({
|
|
5000
|
+
id: `strategy-promotion-applied-${strategyPromotion.id}`,
|
|
5001
|
+
source: "strategy_promotion",
|
|
5002
|
+
passName: "translator",
|
|
5003
|
+
description: `Strategy promotion #${strategyPromotion.id} (${strategyPromotion.strategy}) enabled the discipline gates on ${ir.appId}/${ir.intent.archetype} \u2014 measured gates-on verdict behind the 7-day rollback guard.`
|
|
5004
|
+
});
|
|
5005
|
+
}
|
|
4946
5006
|
const disciplineRewrite = sectionRewritesApplied.find(
|
|
4947
5007
|
(rw) => rw.kind === "discipline_contract"
|
|
4948
5008
|
);
|
|
@@ -6344,6 +6404,24 @@ function tryParseJson(s) {
|
|
|
6344
6404
|
return void 0;
|
|
6345
6405
|
}
|
|
6346
6406
|
}
|
|
6407
|
+
function isAuthSignatureBody(body, message) {
|
|
6408
|
+
if (body && typeof body === "object") {
|
|
6409
|
+
const err = body.error;
|
|
6410
|
+
if (err && typeof err === "object") {
|
|
6411
|
+
const e = err;
|
|
6412
|
+
if (Array.isArray(e.details)) {
|
|
6413
|
+
for (const d of e.details) {
|
|
6414
|
+
if (d && typeof d === "object" && d.reason === "API_KEY_INVALID") {
|
|
6415
|
+
return true;
|
|
6416
|
+
}
|
|
6417
|
+
}
|
|
6418
|
+
}
|
|
6419
|
+
if (e.code === "invalid_api_key") return true;
|
|
6420
|
+
}
|
|
6421
|
+
}
|
|
6422
|
+
const m = message.toLowerCase();
|
|
6423
|
+
return m.includes("api key not valid") || m.includes("invalid api key") || m.includes("invalid x-api-key") || m.includes("incorrect api key");
|
|
6424
|
+
}
|
|
6347
6425
|
function classifyHttpError(status, body) {
|
|
6348
6426
|
const message = extractErrorMessage(body) ?? `HTTP ${status}`;
|
|
6349
6427
|
if (status === 429)
|
|
@@ -6356,8 +6434,11 @@ function classifyHttpError(status, body) {
|
|
|
6356
6434
|
return { ok: false, status, errorType: "retryable", errorCode: "model_not_found", message, raw: body };
|
|
6357
6435
|
if (status === 401 || status === 403)
|
|
6358
6436
|
return { ok: false, status, errorType: "terminal", errorCode: "auth", message, raw: body };
|
|
6359
|
-
if (status === 400)
|
|
6437
|
+
if (status === 400) {
|
|
6438
|
+
if (isAuthSignatureBody(body, message))
|
|
6439
|
+
return { ok: false, status, errorType: "terminal", errorCode: "auth", message, raw: body };
|
|
6360
6440
|
return { ok: false, status, errorType: "terminal", errorCode: "invalid_request", message, raw: body };
|
|
6441
|
+
}
|
|
6361
6442
|
return { ok: false, status, errorType: "terminal", errorCode: "unknown", message, raw: body };
|
|
6362
6443
|
}
|
|
6363
6444
|
function extractErrorMessage(body) {
|
|
@@ -6622,6 +6703,9 @@ function classifyHttpError2(status, body) {
|
|
|
6622
6703
|
return { ok: false, status, errorType: "terminal", errorCode: "auth", message, raw: body };
|
|
6623
6704
|
}
|
|
6624
6705
|
if (status === 400) {
|
|
6706
|
+
if (isAuthSignatureBody(body, message)) {
|
|
6707
|
+
return { ok: false, status, errorType: "terminal", errorCode: "auth", message, raw: body };
|
|
6708
|
+
}
|
|
6625
6709
|
return { ok: false, status, errorType: "terminal", errorCode: "invalid_request", message, raw: body };
|
|
6626
6710
|
}
|
|
6627
6711
|
return { ok: false, status, errorType: "terminal", errorCode: "unknown", message, raw: body };
|
|
@@ -8204,6 +8288,58 @@ function hashForGolden(s) {
|
|
|
8204
8288
|
}
|
|
8205
8289
|
return `g${h.toString(36)}`;
|
|
8206
8290
|
}
|
|
8291
|
+
var STRATEGY_GATES_SECTION_ID = "__kgauto_strategy_eval_gates__";
|
|
8292
|
+
function withDisciplineContract(ir) {
|
|
8293
|
+
return {
|
|
8294
|
+
...ir,
|
|
8295
|
+
sections: [
|
|
8296
|
+
...ir.sections ?? [],
|
|
8297
|
+
{ id: STRATEGY_GATES_SECTION_ID, kind: "discipline_contract", text: "" }
|
|
8298
|
+
]
|
|
8299
|
+
};
|
|
8300
|
+
}
|
|
8301
|
+
var DISCIPLINE_GATES_V1_ALT_HEADER = "Before stating any conclusion, run this check:";
|
|
8302
|
+
var ALT_BULLETS = {
|
|
8303
|
+
observe: "- What did I actually observe? Quote it before reasoning from it.",
|
|
8304
|
+
extraSignal: "- This feels settled \u2014 what one adjacent signal have I not checked?",
|
|
8305
|
+
expand: "- Is anything here compressed or referenced? Open it; do not infer its contents.",
|
|
8306
|
+
innocent: "- What is the ordinary explanation? State it before any alarming one.",
|
|
8307
|
+
label: "- For each claim: is it observed, inferred, or assumed? Say which.",
|
|
8308
|
+
gap: "- What can I not determine from what I have? Name it rather than filling it in."
|
|
8309
|
+
};
|
|
8310
|
+
function altBlock(parts) {
|
|
8311
|
+
return [DISCIPLINE_GATES_V1_ALT_HEADER, ...parts.map((p) => ALT_BULLETS[p])].join("\n");
|
|
8312
|
+
}
|
|
8313
|
+
function altGatesBlockFor(args) {
|
|
8314
|
+
const shapeAltering = args.outputMode === "text" ? ["label", "gap"] : [];
|
|
8315
|
+
const expand = args.hasTools ? ["expand"] : [];
|
|
8316
|
+
return altBlock([
|
|
8317
|
+
"observe",
|
|
8318
|
+
"extraSignal",
|
|
8319
|
+
...expand,
|
|
8320
|
+
"innocent",
|
|
8321
|
+
...shapeAltering
|
|
8322
|
+
]);
|
|
8323
|
+
}
|
|
8324
|
+
var STRATEGY_ALT_SECTION_ID = "__kgauto_strategy_eval_gates_alt__";
|
|
8325
|
+
function withAltDisciplineContract(ir) {
|
|
8326
|
+
const outputMode = resolveOutputMode({
|
|
8327
|
+
declared: ir.constraints?.outputMode,
|
|
8328
|
+
structuredOutput: ir.constraints?.structuredOutput,
|
|
8329
|
+
toolCount: ir.tools?.length ?? 0
|
|
8330
|
+
});
|
|
8331
|
+
const hasTools = (ir.tools?.length ?? 0) > 0;
|
|
8332
|
+
return {
|
|
8333
|
+
...ir,
|
|
8334
|
+
sections: [
|
|
8335
|
+
...ir.sections ?? [],
|
|
8336
|
+
{
|
|
8337
|
+
id: STRATEGY_ALT_SECTION_ID,
|
|
8338
|
+
text: altGatesBlockFor({ outputMode, hasTools })
|
|
8339
|
+
}
|
|
8340
|
+
]
|
|
8341
|
+
};
|
|
8342
|
+
}
|
|
8207
8343
|
async function runGoldenEval(opts) {
|
|
8208
8344
|
const fetchFn = opts.fetchImpl ?? fetch;
|
|
8209
8345
|
const progress = opts.onProgress ?? (() => {
|
|
@@ -8214,6 +8350,14 @@ async function runGoldenEval(opts) {
|
|
|
8214
8350
|
const minJudgeable = opts.minJudgeableCases ?? 5;
|
|
8215
8351
|
const judgeModel = opts.judgeModel ?? "claude-opus-4-8";
|
|
8216
8352
|
const notes = [];
|
|
8353
|
+
const axis = opts.axis ?? "model";
|
|
8354
|
+
const strategyId = opts.strategy;
|
|
8355
|
+
if (axis === "strategy" && !strategyId) {
|
|
8356
|
+
throw new Error("golden-eval: axis 'strategy' requires opts.strategy.");
|
|
8357
|
+
}
|
|
8358
|
+
if (axis === "model" && !opts.candidateModel) {
|
|
8359
|
+
throw new Error("golden-eval: axis 'model' requires opts.candidateModel.");
|
|
8360
|
+
}
|
|
8217
8361
|
const restHeaders = {
|
|
8218
8362
|
apikey: opts.serviceKey,
|
|
8219
8363
|
Authorization: `Bearer ${opts.serviceKey}`
|
|
@@ -8235,7 +8379,7 @@ async function runGoldenEval(opts) {
|
|
|
8235
8379
|
);
|
|
8236
8380
|
}
|
|
8237
8381
|
progress(`Loaded ${goldenRows.length} golden case(s).`);
|
|
8238
|
-
let incumbentModel = opts.incumbentModel;
|
|
8382
|
+
let incumbentModel = axis === "strategy" ? opts.strategyModel : opts.incumbentModel;
|
|
8239
8383
|
if (!incumbentModel) {
|
|
8240
8384
|
const counts = /* @__PURE__ */ new Map();
|
|
8241
8385
|
for (const row of goldenRows) {
|
|
@@ -8246,21 +8390,27 @@ async function runGoldenEval(opts) {
|
|
|
8246
8390
|
incumbentModel = top[0];
|
|
8247
8391
|
notes.push(`incumbent defaulted to most-captured model: ${incumbentModel}`);
|
|
8248
8392
|
}
|
|
8249
|
-
|
|
8393
|
+
const candidateModel = axis === "strategy" ? incumbentModel : opts.candidateModel;
|
|
8394
|
+
if (axis === "model" && incumbentModel === candidateModel) {
|
|
8250
8395
|
throw new Error("golden-eval: candidate and incumbent are the same model.");
|
|
8251
8396
|
}
|
|
8252
8397
|
const judgeProfile = tryGetProfile(judgeModel);
|
|
8253
8398
|
if (!judgeProfile) {
|
|
8254
8399
|
throw new Error(`golden-eval: judge model '${judgeModel}' is not in the roster.`);
|
|
8255
8400
|
}
|
|
8256
|
-
if (judgeProfile.family && [incumbentModel,
|
|
8401
|
+
if (judgeProfile.family && [incumbentModel, candidateModel].some(
|
|
8257
8402
|
(m) => tryGetProfile(m)?.family === judgeProfile.family
|
|
8258
8403
|
)) {
|
|
8259
8404
|
notes.push(
|
|
8260
8405
|
`WARNING: judge family (${judgeProfile.family}) overlaps a compared model \u2014 self-preference risk; consider --judge from another provider.`
|
|
8261
8406
|
);
|
|
8262
8407
|
}
|
|
8263
|
-
const
|
|
8408
|
+
const armBIr = (ir) => {
|
|
8409
|
+
if (axis !== "strategy") return ir;
|
|
8410
|
+
return strategyId === "discipline-gates-v1-alt" ? withAltDisciplineContract(ir) : withDisciplineContract(ir);
|
|
8411
|
+
};
|
|
8412
|
+
const replay = async (ir, model, captureGates = false) => {
|
|
8413
|
+
let gates;
|
|
8264
8414
|
try {
|
|
8265
8415
|
const evalIr = {
|
|
8266
8416
|
...ir,
|
|
@@ -8268,6 +8418,29 @@ async function runGoldenEval(opts) {
|
|
|
8268
8418
|
constraints: { ...ir.constraints ?? {}, forceModel: model }
|
|
8269
8419
|
};
|
|
8270
8420
|
const compiled = compile(evalIr);
|
|
8421
|
+
if (captureGates) {
|
|
8422
|
+
if (strategyId === "discipline-gates-v1-alt") {
|
|
8423
|
+
const altSection = (evalIr.sections ?? []).find(
|
|
8424
|
+
(s) => s.id === STRATEGY_ALT_SECTION_ID
|
|
8425
|
+
);
|
|
8426
|
+
gates = {
|
|
8427
|
+
fired: altSection !== void 0 && altSection.text.length > 0,
|
|
8428
|
+
rule: "discipline-gates-v1-alt",
|
|
8429
|
+
gateTokens: altSection ? countTokens(`${altSection.text}
|
|
8430
|
+
|
|
8431
|
+
`) : 0
|
|
8432
|
+
};
|
|
8433
|
+
} else {
|
|
8434
|
+
const gateRewrite = compiled.sectionRewritesApplied.find(
|
|
8435
|
+
(rw) => rw.rule.startsWith("discipline-gates-")
|
|
8436
|
+
);
|
|
8437
|
+
gates = {
|
|
8438
|
+
fired: gateRewrite !== void 0 && compiled.diagnostics.disciplineGateTokens > 0,
|
|
8439
|
+
rule: gateRewrite?.rule,
|
|
8440
|
+
gateTokens: compiled.diagnostics.disciplineGateTokens
|
|
8441
|
+
};
|
|
8442
|
+
}
|
|
8443
|
+
}
|
|
8271
8444
|
const attempt = () => execute(compiled.request, { apiKeys: opts.apiKeys, fetchImpl: opts.fetchImpl });
|
|
8272
8445
|
let started = Date.now();
|
|
8273
8446
|
let exec = await attempt();
|
|
@@ -8286,7 +8459,8 @@ async function runGoldenEval(opts) {
|
|
|
8286
8459
|
tokensIn: 0,
|
|
8287
8460
|
tokensOut: 0,
|
|
8288
8461
|
latencyMs,
|
|
8289
|
-
errorClass: exec.errorCode
|
|
8462
|
+
errorClass: exec.errorCode,
|
|
8463
|
+
...gates ? { gates } : {}
|
|
8290
8464
|
};
|
|
8291
8465
|
}
|
|
8292
8466
|
const validated = _internal.validateStructuredContract(exec, evalIr);
|
|
@@ -8298,7 +8472,8 @@ async function runGoldenEval(opts) {
|
|
|
8298
8472
|
tokensIn: exec.response.tokens.input,
|
|
8299
8473
|
tokensOut: exec.response.tokens.output,
|
|
8300
8474
|
latencyMs,
|
|
8301
|
-
contractViolation: validated.errorCode
|
|
8475
|
+
contractViolation: validated.errorCode,
|
|
8476
|
+
...gates ? { gates } : {}
|
|
8302
8477
|
};
|
|
8303
8478
|
}
|
|
8304
8479
|
return {
|
|
@@ -8308,7 +8483,8 @@ async function runGoldenEval(opts) {
|
|
|
8308
8483
|
parseError: validated.response.parseError,
|
|
8309
8484
|
tokensIn: validated.response.tokens.input,
|
|
8310
8485
|
tokensOut: validated.response.tokens.output,
|
|
8311
|
-
latencyMs
|
|
8486
|
+
latencyMs,
|
|
8487
|
+
...gates ? { gates } : {}
|
|
8312
8488
|
};
|
|
8313
8489
|
} catch (err) {
|
|
8314
8490
|
return {
|
|
@@ -8318,7 +8494,8 @@ async function runGoldenEval(opts) {
|
|
|
8318
8494
|
tokensIn: 0,
|
|
8319
8495
|
tokensOut: 0,
|
|
8320
8496
|
latencyMs: 0,
|
|
8321
|
-
errorClass: err instanceof Error ? `execute_rejected:${err.message}` : "execute_rejected"
|
|
8497
|
+
errorClass: err instanceof Error ? `execute_rejected:${err.message}` : "execute_rejected",
|
|
8498
|
+
...gates ? { gates } : {}
|
|
8322
8499
|
};
|
|
8323
8500
|
}
|
|
8324
8501
|
};
|
|
@@ -8377,12 +8554,39 @@ async function runGoldenEval(opts) {
|
|
|
8377
8554
|
};
|
|
8378
8555
|
for (const [i, row] of goldenRows.entries()) {
|
|
8379
8556
|
progress(
|
|
8380
|
-
`Case ${i + 1}/${goldenRows.length} (golden #${row.id}): replaying ${incumbentModel} + ${
|
|
8557
|
+
`Case ${i + 1}/${goldenRows.length} (golden #${row.id}): replaying ${incumbentModel} + ${candidateModel}\u2026`
|
|
8381
8558
|
);
|
|
8382
8559
|
const [inc, cand] = await Promise.all([
|
|
8383
8560
|
replay(row.ir, incumbentModel),
|
|
8384
|
-
replay(row.ir,
|
|
8561
|
+
replay(armBIr(row.ir), candidateModel, axis === "strategy")
|
|
8385
8562
|
]);
|
|
8563
|
+
if (axis === "strategy" && !cand.gates?.fired) {
|
|
8564
|
+
notes.push(
|
|
8565
|
+
`ABORT gates-did-not-fire: arm B compile produced no discipline-gate rewrite (case golden #${row.id}; archetype=${opts.archetype}; gateTokens=${cand.gates?.gateTokens ?? 0}). No verdict written \u2014 a tied-because-identical run is not evidence about the strategy.`
|
|
8566
|
+
);
|
|
8567
|
+
return {
|
|
8568
|
+
verdict: "inconclusive",
|
|
8569
|
+
axis,
|
|
8570
|
+
strategy: strategyId,
|
|
8571
|
+
appId: opts.appId,
|
|
8572
|
+
archetype: opts.archetype,
|
|
8573
|
+
incumbentModel,
|
|
8574
|
+
candidateModel,
|
|
8575
|
+
judgeModel,
|
|
8576
|
+
nCases: 0,
|
|
8577
|
+
wins: 0,
|
|
8578
|
+
ties: 0,
|
|
8579
|
+
losses: 0,
|
|
8580
|
+
floorViolations: 0,
|
|
8581
|
+
floorDetail: {},
|
|
8582
|
+
winOrTieRatio: null,
|
|
8583
|
+
latencyRatio: null,
|
|
8584
|
+
costIncumbentUsd: null,
|
|
8585
|
+
costCandidateUsd: null,
|
|
8586
|
+
cases,
|
|
8587
|
+
notes
|
|
8588
|
+
};
|
|
8589
|
+
}
|
|
8386
8590
|
if (!inc.ok || inc.contractViolation) {
|
|
8387
8591
|
const why = !inc.ok ? `incumbent replay failed (${inc.errorClass ?? "unknown"})` : `incumbent contract violation (${inc.contractViolation})`;
|
|
8388
8592
|
cases.push({
|
|
@@ -8444,6 +8648,9 @@ async function runGoldenEval(opts) {
|
|
|
8444
8648
|
cases.push({
|
|
8445
8649
|
goldenIrId: row.id,
|
|
8446
8650
|
verdict: verdict2,
|
|
8651
|
+
// Release B — the raw pair the fold discards; positional-disagreement
|
|
8652
|
+
// rate is the strategy experiment's primary metric.
|
|
8653
|
+
orderVerdicts: { run1: v1, run2: v2 },
|
|
8447
8654
|
judgeRationale: order1.rationale ?? order2.rationale,
|
|
8448
8655
|
floorViolations: [],
|
|
8449
8656
|
incumbent: incumbentLeg,
|
|
@@ -8467,7 +8674,7 @@ async function runGoldenEval(opts) {
|
|
|
8467
8674
|
);
|
|
8468
8675
|
const costCandidateUsd = sum(
|
|
8469
8676
|
evaluable.map(
|
|
8470
|
-
(c) => c.candidate ? costUsd(
|
|
8677
|
+
(c) => c.candidate ? costUsd(candidateModel, c.candidate.tokensIn, c.candidate.tokensOut) : null
|
|
8471
8678
|
)
|
|
8472
8679
|
);
|
|
8473
8680
|
const latencyFloorOk = latencyRatio === null || latencyRatio <= latencyFloorRatio;
|
|
@@ -8485,10 +8692,12 @@ async function runGoldenEval(opts) {
|
|
|
8485
8692
|
}
|
|
8486
8693
|
const result = {
|
|
8487
8694
|
verdict,
|
|
8695
|
+
axis,
|
|
8696
|
+
...strategyId ? { strategy: strategyId } : {},
|
|
8488
8697
|
appId: opts.appId,
|
|
8489
8698
|
archetype: opts.archetype,
|
|
8490
8699
|
incumbentModel,
|
|
8491
|
-
candidateModel
|
|
8700
|
+
candidateModel,
|
|
8492
8701
|
judgeModel,
|
|
8493
8702
|
nCases: evaluable.length,
|
|
8494
8703
|
wins,
|
|
@@ -8518,8 +8727,12 @@ async function runGoldenEval(opts) {
|
|
|
8518
8727
|
body: JSON.stringify({
|
|
8519
8728
|
app_id: opts.appId,
|
|
8520
8729
|
intent_archetype: opts.archetype,
|
|
8730
|
+
// Release B (migration 048) — a run row that can't say which axis
|
|
8731
|
+
// produced its verdict is a false-attribution engine.
|
|
8732
|
+
axis,
|
|
8733
|
+
strategy: strategyId ?? null,
|
|
8521
8734
|
incumbent_model: incumbentModel,
|
|
8522
|
-
candidate_model:
|
|
8735
|
+
candidate_model: candidateModel,
|
|
8523
8736
|
trigger_source: opts.triggerSource ?? "manual",
|
|
8524
8737
|
judge_model: judgeModel,
|
|
8525
8738
|
n_cases: result.nCases,
|
|
@@ -8554,8 +8767,8 @@ async function runGoldenEval(opts) {
|
|
|
8554
8767
|
return {
|
|
8555
8768
|
app_id: opts.appId,
|
|
8556
8769
|
intent_archetype: opts.archetype,
|
|
8557
|
-
family: tryGetProfile(
|
|
8558
|
-
candidate_model:
|
|
8770
|
+
family: tryGetProfile(candidateModel)?.family ?? "unknown",
|
|
8771
|
+
candidate_model: candidateModel,
|
|
8559
8772
|
current_model: incumbentModel,
|
|
8560
8773
|
prompt_hash: hashForGolden(turn),
|
|
8561
8774
|
current_response: c.incumbent?.text.slice(0, 500) ?? null,
|
|
@@ -8601,8 +8814,8 @@ async function runGoldenEval(opts) {
|
|
|
8601
8814
|
const latestOutcomeId = latest[0]?.id;
|
|
8602
8815
|
if (latestOutcomeId !== void 0) {
|
|
8603
8816
|
const costLine = costIncumbentUsd !== null && costCandidateUsd !== null ? ` Eval cost basis: incumbent $${costIncumbentUsd.toFixed(4)} vs candidate $${costCandidateUsd.toFixed(4)} across the set.` : "";
|
|
8604
|
-
const message = `Golden-set eval (run #${result.runId}): ${
|
|
8605
|
-
const suggestion = verdict === "promote-ready" ? `Candidate passed the non-inferiority rule on your real workload. To promote: node v2/scripts/promote-model.mjs --model ${
|
|
8817
|
+
const message = `Golden-set eval (run #${result.runId}): ${candidateModel} vs ${incumbentModel} on ${opts.appId}/${opts.archetype} \u2014 verdict ${verdict.toUpperCase()}. ${wins}W/${ties}T/${losses}L over ${result.nCases} real workload case(s), wins-or-ties ${(100 * (winOrTieRatio ?? 0)).toFixed(0)}% (threshold ${(100 * threshold).toFixed(0)}%), ${result.floorViolations} hard-floor violation(s), latency ratio ${latencyRatio?.toFixed(2) ?? "n/a"}.` + costLine;
|
|
8818
|
+
const suggestion = verdict === "promote-ready" ? `Candidate passed the non-inferiority rule on your real workload. To promote: node v2/scripts/promote-model.mjs --model ${candidateModel} (or ask kgauto-Cairn). Evidence: kgauto_golden_eval_runs #${result.runId} + probe_outcomes golden_run_id=${result.runId}.` : `Candidate did NOT clear the non-inferiority rule \u2014 no action needed; the incumbent stays. Evidence rows: golden_run_id=${result.runId}.`;
|
|
8606
8819
|
const advRes = await fetchFn(rest("compile_outcome_advisories"), {
|
|
8607
8820
|
method: "POST",
|
|
8608
8821
|
headers: { ...restHeaders, "Content-Type": "application/json", Prefer: "return=minimal" },
|
|
@@ -8631,6 +8844,33 @@ async function runGoldenEval(opts) {
|
|
|
8631
8844
|
}
|
|
8632
8845
|
return result;
|
|
8633
8846
|
}
|
|
8847
|
+
function classifyStrategyOutcome(r) {
|
|
8848
|
+
if (r.verdict === "inconclusive") return "inconclusive";
|
|
8849
|
+
if (r.wins > r.losses) return "wins";
|
|
8850
|
+
if (r.losses > r.wins) return "loses";
|
|
8851
|
+
return "ties";
|
|
8852
|
+
}
|
|
8853
|
+
var STRATEGY_AUTHORSHIP_LIMITATION = "Both gate wordings share one author (kgauto-Cairn); a shared blind spot is invisible to this experiment. Strong verdicts: both-lose (mechanism) and v1-loses/alt-wins (wording). An independently-authored alt would strengthen every other cell.";
|
|
8854
|
+
async function runStrategyEvalWithAttribution(opts) {
|
|
8855
|
+
const primary = await runGoldenEval({
|
|
8856
|
+
...opts,
|
|
8857
|
+
axis: "strategy",
|
|
8858
|
+
strategy: "discipline-gates-v1"
|
|
8859
|
+
});
|
|
8860
|
+
const outcome = classifyStrategyOutcome(primary);
|
|
8861
|
+
if (outcome !== "loses") {
|
|
8862
|
+
const attribution2 = outcome === "wins" ? "mechanism-works" : outcome === "ties" ? "no-lift" : "inconclusive";
|
|
8863
|
+
return { attribution: attribution2, primary, limitation: STRATEGY_AUTHORSHIP_LIMITATION };
|
|
8864
|
+
}
|
|
8865
|
+
const alt = await runGoldenEval({
|
|
8866
|
+
...opts,
|
|
8867
|
+
axis: "strategy",
|
|
8868
|
+
strategy: "discipline-gates-v1-alt"
|
|
8869
|
+
});
|
|
8870
|
+
const altOutcome = classifyStrategyOutcome(alt);
|
|
8871
|
+
const attribution = altOutcome === "wins" ? "wording-failure" : altOutcome === "loses" ? "mechanism-failure" : "wording-inconclusive";
|
|
8872
|
+
return { attribution, primary, alt, limitation: STRATEGY_AUTHORSHIP_LIMITATION };
|
|
8873
|
+
}
|
|
8634
8874
|
|
|
8635
8875
|
// src/brain-proxy.ts
|
|
8636
8876
|
var SEGMENT_TO_TABLE = {
|
|
@@ -8799,7 +9039,7 @@ function createBrainForwardRoutes(config) {
|
|
|
8799
9039
|
}
|
|
8800
9040
|
|
|
8801
9041
|
// src/version.ts
|
|
8802
|
-
var LIBRARY_VERSION = "2.0.0-alpha.
|
|
9042
|
+
var LIBRARY_VERSION = "2.0.0-alpha.75";
|
|
8803
9043
|
|
|
8804
9044
|
// src/key-health.ts
|
|
8805
9045
|
var JSON_HEADERS2 = { "Content-Type": "application/json" };
|
|
@@ -9435,6 +9675,7 @@ function compile2(ir, opts) {
|
|
|
9435
9675
|
DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
9436
9676
|
DEFAULT_PROMOTIONS_ENDPOINT,
|
|
9437
9677
|
DIALECT_VERSION,
|
|
9678
|
+
DISCIPLINE_GATES_V1_ALT_HEADER,
|
|
9438
9679
|
FamilyResolutionError,
|
|
9439
9680
|
INTENT_ARCHETYPES,
|
|
9440
9681
|
JUDGE_RUBRICS,
|
|
@@ -9444,13 +9685,17 @@ function compile2(ir, opts) {
|
|
|
9444
9685
|
MEASURED_GROUNDING_MIN_N,
|
|
9445
9686
|
PRODUCER_OWNED_RULE_CODES,
|
|
9446
9687
|
PROVIDER_ENV_KEYS,
|
|
9688
|
+
RULE_DISCIPLINE_GATES_V1,
|
|
9689
|
+
RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
9447
9690
|
RULE_SEQUENTIAL_TOOL_CLIFF,
|
|
9691
|
+
STRATEGY_AUTHORSHIP_LIMITATION,
|
|
9448
9692
|
TRANSLATOR_FLOOR,
|
|
9449
9693
|
_testResetMeasuredFailure,
|
|
9450
9694
|
_testResetPromotions,
|
|
9451
9695
|
_testWaitForMeasuredFailureRefresh,
|
|
9452
9696
|
_testWaitForPromotionsRefresh,
|
|
9453
9697
|
allProfiles,
|
|
9698
|
+
altGatesBlockFor,
|
|
9454
9699
|
applyArchetypeConvention,
|
|
9455
9700
|
applySectionRewrites,
|
|
9456
9701
|
attachCacheControlToStreamTextInput,
|
|
@@ -9465,6 +9710,7 @@ function compile2(ir, opts) {
|
|
|
9465
9710
|
buildShadowProbeRow,
|
|
9466
9711
|
call,
|
|
9467
9712
|
captureGoldenIr,
|
|
9713
|
+
classifyStrategyOutcome,
|
|
9468
9714
|
clearBrain,
|
|
9469
9715
|
combineOrderSwappedVerdicts,
|
|
9470
9716
|
compile,
|
|
@@ -9542,8 +9788,11 @@ function compile2(ir, opts) {
|
|
|
9542
9788
|
rubricFor,
|
|
9543
9789
|
runAdvisor,
|
|
9544
9790
|
runGoldenEval,
|
|
9791
|
+
runStrategyEvalWithAttribution,
|
|
9545
9792
|
setTokenizer,
|
|
9546
9793
|
shouldCaptureGolden,
|
|
9547
9794
|
tryGetProfile,
|
|
9548
|
-
wilsonLowerBound
|
|
9795
|
+
wilsonLowerBound,
|
|
9796
|
+
withAltDisciplineContract,
|
|
9797
|
+
withDisciplineContract
|
|
9549
9798
|
});
|