@warmdrift/kgauto-compiler 2.0.0-alpha.74 → 2.0.0-alpha.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-EB2YR7HL.mjs → chunk-WZZCW6NA.mjs} +1 -1
- package/dist/index.d.mts +310 -8
- package/dist/index.d.ts +310 -8
- package/dist/index.js +463 -25
- package/dist/index.mjs +449 -24
- package/dist/key-health.js +1 -1
- package/dist/key-health.mjs +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -25,11 +25,15 @@ __export(index_exports, {
|
|
|
25
25
|
ALL_ARCHETYPES: () => ALL_ARCHETYPES,
|
|
26
26
|
ARCHETYPE_FAMILY_FITS: () => ARCHETYPE_FAMILY_FITS,
|
|
27
27
|
ARCHETYPE_FLOOR_DEFAULT: () => ARCHETYPE_FLOOR_DEFAULT,
|
|
28
|
+
COACH_CFG: () => COACH_CFG,
|
|
28
29
|
CallError: () => CallError,
|
|
30
|
+
DECOMPOSITION_TEMPLATES: () => DECOMPOSITION_TEMPLATES,
|
|
31
|
+
DECOMPOSITION_TEMPLATES_VERSION: () => DECOMPOSITION_TEMPLATES_VERSION,
|
|
29
32
|
DEFAULT_FINDINGS_ENDPOINT: () => DEFAULT_FINDINGS_ENDPOINT,
|
|
30
33
|
DEFAULT_MEASURED_FAILURE_ENDPOINT: () => DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
31
34
|
DEFAULT_PROMOTIONS_ENDPOINT: () => DEFAULT_PROMOTIONS_ENDPOINT,
|
|
32
35
|
DIALECT_VERSION: () => DIALECT_VERSION,
|
|
36
|
+
DISCIPLINE_GATES_V1_ALT_HEADER: () => DISCIPLINE_GATES_V1_ALT_HEADER,
|
|
33
37
|
FamilyResolutionError: () => FamilyResolutionError,
|
|
34
38
|
INTENT_ARCHETYPES: () => INTENT_ARCHETYPES,
|
|
35
39
|
JUDGE_RUBRICS: () => JUDGE_RUBRICS,
|
|
@@ -39,13 +43,17 @@ __export(index_exports, {
|
|
|
39
43
|
MEASURED_GROUNDING_MIN_N: () => MEASURED_GROUNDING_MIN_N,
|
|
40
44
|
PRODUCER_OWNED_RULE_CODES: () => PRODUCER_OWNED_RULE_CODES,
|
|
41
45
|
PROVIDER_ENV_KEYS: () => PROVIDER_ENV_KEYS,
|
|
46
|
+
RULE_DISCIPLINE_GATES_V1: () => RULE_DISCIPLINE_GATES_V1,
|
|
47
|
+
RULE_DISCIPLINE_GATES_V1_STRUCTURED: () => RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
42
48
|
RULE_SEQUENTIAL_TOOL_CLIFF: () => RULE_SEQUENTIAL_TOOL_CLIFF,
|
|
49
|
+
STRATEGY_AUTHORSHIP_LIMITATION: () => STRATEGY_AUTHORSHIP_LIMITATION,
|
|
43
50
|
TRANSLATOR_FLOOR: () => TRANSLATOR_FLOOR,
|
|
44
51
|
_testResetMeasuredFailure: () => _testResetMeasuredFailure,
|
|
45
52
|
_testResetPromotions: () => _testResetPromotions,
|
|
46
53
|
_testWaitForMeasuredFailureRefresh: () => _testWaitForMeasuredFailureRefresh,
|
|
47
54
|
_testWaitForPromotionsRefresh: () => _testWaitForPromotionsRefresh,
|
|
48
55
|
allProfiles: () => allProfiles,
|
|
56
|
+
altGatesBlockFor: () => altGatesBlockFor,
|
|
49
57
|
applyArchetypeConvention: () => applyArchetypeConvention,
|
|
50
58
|
applySectionRewrites: () => applySectionRewrites,
|
|
51
59
|
attachCacheControlToStreamTextInput: () => attachCacheControlToStreamTextInput,
|
|
@@ -60,6 +68,7 @@ __export(index_exports, {
|
|
|
60
68
|
buildShadowProbeRow: () => buildShadowProbeRow,
|
|
61
69
|
call: () => call,
|
|
62
70
|
captureGoldenIr: () => captureGoldenIr,
|
|
71
|
+
classifyStrategyOutcome: () => classifyStrategyOutcome,
|
|
63
72
|
clearBrain: () => clearBrain,
|
|
64
73
|
combineOrderSwappedVerdicts: () => combineOrderSwappedVerdicts,
|
|
65
74
|
compile: () => compile2,
|
|
@@ -119,6 +128,7 @@ __export(index_exports, {
|
|
|
119
128
|
parseGoldenCaptureRate: () => parseGoldenCaptureRate,
|
|
120
129
|
parseJudgeVerdict: () => parseJudgeVerdict,
|
|
121
130
|
peekBrainDeadLetter: () => peekBrainDeadLetter,
|
|
131
|
+
planDecomposition: () => planDecomposition,
|
|
122
132
|
prefetchMeasuredFailure: () => prefetchMeasuredFailure,
|
|
123
133
|
probeShadow: () => probeShadow,
|
|
124
134
|
profileToRow: () => profileToRow,
|
|
@@ -137,10 +147,13 @@ __export(index_exports, {
|
|
|
137
147
|
rubricFor: () => rubricFor,
|
|
138
148
|
runAdvisor: () => runAdvisor,
|
|
139
149
|
runGoldenEval: () => runGoldenEval,
|
|
150
|
+
runStrategyEvalWithAttribution: () => runStrategyEvalWithAttribution,
|
|
140
151
|
setTokenizer: () => setTokenizer,
|
|
141
152
|
shouldCaptureGolden: () => shouldCaptureGolden,
|
|
142
153
|
tryGetProfile: () => tryGetProfile,
|
|
143
|
-
wilsonLowerBound: () => wilsonLowerBound
|
|
154
|
+
wilsonLowerBound: () => wilsonLowerBound,
|
|
155
|
+
withAltDisciplineContract: () => withAltDisciplineContract,
|
|
156
|
+
withDisciplineContract: () => withDisciplineContract
|
|
144
157
|
});
|
|
145
158
|
module.exports = __toCommonJS(index_exports);
|
|
146
159
|
|
|
@@ -2934,7 +2947,7 @@ function passScoreTargets(ir, opts) {
|
|
|
2934
2947
|
}
|
|
2935
2948
|
const measuredGate = opts.measuredFailureGates?.get(modelId);
|
|
2936
2949
|
if (measuredGate) qualityGatePenalty = QUALITY_GATE_PENALTY;
|
|
2937
|
-
const isPromoted = promotion?.promotedModel === modelId;
|
|
2950
|
+
const isPromoted = promotion?.mode === "downswap" && promotion.promotedModel === modelId;
|
|
2938
2951
|
if (isPromoted && promotion.suppressQualityGate && !measuredGate) {
|
|
2939
2952
|
qualityGatePenalty = 0;
|
|
2940
2953
|
}
|
|
@@ -4447,6 +4460,16 @@ var DISCIPLINE_GATES_V1_NO_TOOLS = `Work through these gates at every judgment p
|
|
|
4447
4460
|
3. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.
|
|
4448
4461
|
4. Label each claim: mark it observed, inferred, or assumed.
|
|
4449
4462
|
5. A surfaced gap beats a guessed answer: flag what you cannot determine rather than fabricating past it.`;
|
|
4463
|
+
var RULE_DISCIPLINE_GATES_V1_STRUCTURED = "discipline-gates-v1-structured";
|
|
4464
|
+
var DISCIPLINE_GATES_V1_STRUCTURED_WITH_TOOLS = `Work through these gates at every judgment point, explicitly:
|
|
4465
|
+
1. Evidence before reasoning: cite what you observed before concluding from it.
|
|
4466
|
+
2. One extra signal: when a finding feels conclusive, check one more adjacent signal before stating it.
|
|
4467
|
+
3. Expand, don't guess: resolve a compressed or referenced item by looking it up rather than inferring its contents.
|
|
4468
|
+
4. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.`;
|
|
4469
|
+
var DISCIPLINE_GATES_V1_STRUCTURED_NO_TOOLS = `Work through these gates at every judgment point, explicitly:
|
|
4470
|
+
1. Evidence before reasoning: cite what you observed before concluding from it.
|
|
4471
|
+
2. One extra signal: when a finding feels conclusive, check one more adjacent signal before stating it.
|
|
4472
|
+
3. Innocent explanation first: state the most plausible benign reading before alleging the alarming one.`;
|
|
4450
4473
|
var DISCIPLINE_ELIGIBLE_ARCHETYPES = /* @__PURE__ */ new Set([
|
|
4451
4474
|
"hunt",
|
|
4452
4475
|
"summarize",
|
|
@@ -4457,7 +4480,12 @@ var DISCIPLINE_ELIGIBLE_ARCHETYPES = /* @__PURE__ */ new Set([
|
|
|
4457
4480
|
function matchRule(kind, profile, archetype, ctx) {
|
|
4458
4481
|
if (kind === "discipline_contract") {
|
|
4459
4482
|
if (!DISCIPLINE_ELIGIBLE_ARCHETYPES.has(archetype)) return null;
|
|
4460
|
-
if (ctx.outputMode !== "text")
|
|
4483
|
+
if (ctx.outputMode !== "text") {
|
|
4484
|
+
return {
|
|
4485
|
+
id: RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
4486
|
+
preamble: ctx.hasTools ? DISCIPLINE_GATES_V1_STRUCTURED_WITH_TOOLS : DISCIPLINE_GATES_V1_STRUCTURED_NO_TOOLS
|
|
4487
|
+
};
|
|
4488
|
+
}
|
|
4461
4489
|
return {
|
|
4462
4490
|
id: RULE_DISCIPLINE_GATES_V1,
|
|
4463
4491
|
preamble: ctx.hasTools ? DISCIPLINE_GATES_V1_WITH_TOOLS : DISCIPLINE_GATES_V1_NO_TOOLS
|
|
@@ -4550,9 +4578,13 @@ function mapRowsToPromotions(rows) {
|
|
|
4550
4578
|
if (!isRawPromotionRow(row)) continue;
|
|
4551
4579
|
const id = coerceId(row.id);
|
|
4552
4580
|
if (id === null) continue;
|
|
4581
|
+
const mode = row.mode === "strategy" ? "strategy" : row.mode === "downswap" || row.mode === void 0 ? "downswap" : null;
|
|
4582
|
+
if (mode === null) continue;
|
|
4553
4583
|
out.push({
|
|
4554
4584
|
id,
|
|
4555
4585
|
archetype: row.intent_archetype,
|
|
4586
|
+
mode,
|
|
4587
|
+
strategy: typeof row.strategy === "string" ? row.strategy : null,
|
|
4556
4588
|
promotedModel: row.promoted_model,
|
|
4557
4589
|
incumbentModel: row.incumbent_model,
|
|
4558
4590
|
evalRunId: coerceId(row.eval_run_id ?? null),
|
|
@@ -4583,7 +4615,7 @@ function getApplicablePromotion(opts) {
|
|
|
4583
4615
|
const rt = runtime5;
|
|
4584
4616
|
if (!rt) return void 0;
|
|
4585
4617
|
const appId = opts.appId;
|
|
4586
|
-
if (!appId || !opts.archetype) return void 0;
|
|
4618
|
+
if (!appId || !opts.archetype || !opts.mode) return void 0;
|
|
4587
4619
|
let snap = snapshots4.get(appId);
|
|
4588
4620
|
if (!snap) {
|
|
4589
4621
|
snap = { data: [], expiresAt: 0, refreshing: false };
|
|
@@ -4595,7 +4627,9 @@ function getApplicablePromotion(opts) {
|
|
|
4595
4627
|
snap.refreshing = true;
|
|
4596
4628
|
void asyncRefresh5(rt, appId);
|
|
4597
4629
|
}
|
|
4598
|
-
return snap.data.find(
|
|
4630
|
+
return snap.data.find(
|
|
4631
|
+
(p) => p.archetype === opts.archetype && p.mode === opts.mode
|
|
4632
|
+
);
|
|
4599
4633
|
}
|
|
4600
4634
|
var pendingRefreshes4 = /* @__PURE__ */ new Map();
|
|
4601
4635
|
async function asyncRefresh5(rt, appId) {
|
|
@@ -4881,10 +4915,12 @@ function compile(ir, opts = {}) {
|
|
|
4881
4915
|
];
|
|
4882
4916
|
const activePromotion = getApplicablePromotion({
|
|
4883
4917
|
appId: ir.appId,
|
|
4884
|
-
archetype: ir.intent.archetype
|
|
4918
|
+
archetype: ir.intent.archetype,
|
|
4919
|
+
mode: "downswap"
|
|
4885
4920
|
});
|
|
4886
4921
|
const promotion = activePromotion ? {
|
|
4887
4922
|
id: activePromotion.id,
|
|
4923
|
+
mode: activePromotion.mode,
|
|
4888
4924
|
promotedModel: activePromotion.promotedModel,
|
|
4889
4925
|
evalRunId: activePromotion.evalRunId,
|
|
4890
4926
|
suppressQualityGate: activePromotion.suppressQualityGate
|
|
@@ -4935,6 +4971,24 @@ function compile(ir, opts = {}) {
|
|
|
4935
4971
|
structuredOutput: ir.constraints?.structuredOutput,
|
|
4936
4972
|
toolCount: ir.tools?.length ?? 0
|
|
4937
4973
|
});
|
|
4974
|
+
const strategyPromotion = getApplicablePromotion({
|
|
4975
|
+
appId: ir.appId,
|
|
4976
|
+
archetype: ir.intent.archetype,
|
|
4977
|
+
mode: "strategy"
|
|
4978
|
+
});
|
|
4979
|
+
if (strategyPromotion?.strategy === "discipline-gates-v1" && !(workingIR.sections ?? []).some((s) => s.kind === "discipline_contract")) {
|
|
4980
|
+
workingIR = {
|
|
4981
|
+
...workingIR,
|
|
4982
|
+
sections: [
|
|
4983
|
+
...workingIR.sections ?? [],
|
|
4984
|
+
{
|
|
4985
|
+
id: `__kgauto_strategy_promotion_${strategyPromotion.id}__`,
|
|
4986
|
+
kind: "discipline_contract",
|
|
4987
|
+
text: ""
|
|
4988
|
+
}
|
|
4989
|
+
]
|
|
4990
|
+
};
|
|
4991
|
+
}
|
|
4938
4992
|
const translated = applySectionRewrites({
|
|
4939
4993
|
ir: workingIR,
|
|
4940
4994
|
profile,
|
|
@@ -4943,6 +4997,16 @@ function compile(ir, opts = {}) {
|
|
|
4943
4997
|
});
|
|
4944
4998
|
workingIR = translated.rewrittenIR;
|
|
4945
4999
|
const sectionRewritesApplied = translated.rewrites;
|
|
5000
|
+
if (strategyPromotion && translated.rewrites.some(
|
|
5001
|
+
(rw) => rw.kind === "discipline_contract" && rw.sectionId === `__kgauto_strategy_promotion_${strategyPromotion.id}__`
|
|
5002
|
+
)) {
|
|
5003
|
+
accumulatedMutations.push({
|
|
5004
|
+
id: `strategy-promotion-applied-${strategyPromotion.id}`,
|
|
5005
|
+
source: "strategy_promotion",
|
|
5006
|
+
passName: "translator",
|
|
5007
|
+
description: `Strategy promotion #${strategyPromotion.id} (${strategyPromotion.strategy}) enabled the discipline gates on ${ir.appId}/${ir.intent.archetype} \u2014 measured gates-on verdict behind the 7-day rollback guard.`
|
|
5008
|
+
});
|
|
5009
|
+
}
|
|
4946
5010
|
const disciplineRewrite = sectionRewritesApplied.find(
|
|
4947
5011
|
(rw) => rw.kind === "discipline_contract"
|
|
4948
5012
|
);
|
|
@@ -8228,6 +8292,58 @@ function hashForGolden(s) {
|
|
|
8228
8292
|
}
|
|
8229
8293
|
return `g${h.toString(36)}`;
|
|
8230
8294
|
}
|
|
8295
|
+
var STRATEGY_GATES_SECTION_ID = "__kgauto_strategy_eval_gates__";
|
|
8296
|
+
function withDisciplineContract(ir) {
|
|
8297
|
+
return {
|
|
8298
|
+
...ir,
|
|
8299
|
+
sections: [
|
|
8300
|
+
...ir.sections ?? [],
|
|
8301
|
+
{ id: STRATEGY_GATES_SECTION_ID, kind: "discipline_contract", text: "" }
|
|
8302
|
+
]
|
|
8303
|
+
};
|
|
8304
|
+
}
|
|
8305
|
+
var DISCIPLINE_GATES_V1_ALT_HEADER = "Before stating any conclusion, run this check:";
|
|
8306
|
+
var ALT_BULLETS = {
|
|
8307
|
+
observe: "- What did I actually observe? Quote it before reasoning from it.",
|
|
8308
|
+
extraSignal: "- This feels settled \u2014 what one adjacent signal have I not checked?",
|
|
8309
|
+
expand: "- Is anything here compressed or referenced? Open it; do not infer its contents.",
|
|
8310
|
+
innocent: "- What is the ordinary explanation? State it before any alarming one.",
|
|
8311
|
+
label: "- For each claim: is it observed, inferred, or assumed? Say which.",
|
|
8312
|
+
gap: "- What can I not determine from what I have? Name it rather than filling it in."
|
|
8313
|
+
};
|
|
8314
|
+
function altBlock(parts) {
|
|
8315
|
+
return [DISCIPLINE_GATES_V1_ALT_HEADER, ...parts.map((p) => ALT_BULLETS[p])].join("\n");
|
|
8316
|
+
}
|
|
8317
|
+
function altGatesBlockFor(args) {
|
|
8318
|
+
const shapeAltering = args.outputMode === "text" ? ["label", "gap"] : [];
|
|
8319
|
+
const expand = args.hasTools ? ["expand"] : [];
|
|
8320
|
+
return altBlock([
|
|
8321
|
+
"observe",
|
|
8322
|
+
"extraSignal",
|
|
8323
|
+
...expand,
|
|
8324
|
+
"innocent",
|
|
8325
|
+
...shapeAltering
|
|
8326
|
+
]);
|
|
8327
|
+
}
|
|
8328
|
+
var STRATEGY_ALT_SECTION_ID = "__kgauto_strategy_eval_gates_alt__";
|
|
8329
|
+
function withAltDisciplineContract(ir) {
|
|
8330
|
+
const outputMode = resolveOutputMode({
|
|
8331
|
+
declared: ir.constraints?.outputMode,
|
|
8332
|
+
structuredOutput: ir.constraints?.structuredOutput,
|
|
8333
|
+
toolCount: ir.tools?.length ?? 0
|
|
8334
|
+
});
|
|
8335
|
+
const hasTools = (ir.tools?.length ?? 0) > 0;
|
|
8336
|
+
return {
|
|
8337
|
+
...ir,
|
|
8338
|
+
sections: [
|
|
8339
|
+
...ir.sections ?? [],
|
|
8340
|
+
{
|
|
8341
|
+
id: STRATEGY_ALT_SECTION_ID,
|
|
8342
|
+
text: altGatesBlockFor({ outputMode, hasTools })
|
|
8343
|
+
}
|
|
8344
|
+
]
|
|
8345
|
+
};
|
|
8346
|
+
}
|
|
8231
8347
|
async function runGoldenEval(opts) {
|
|
8232
8348
|
const fetchFn = opts.fetchImpl ?? fetch;
|
|
8233
8349
|
const progress = opts.onProgress ?? (() => {
|
|
@@ -8238,6 +8354,14 @@ async function runGoldenEval(opts) {
|
|
|
8238
8354
|
const minJudgeable = opts.minJudgeableCases ?? 5;
|
|
8239
8355
|
const judgeModel = opts.judgeModel ?? "claude-opus-4-8";
|
|
8240
8356
|
const notes = [];
|
|
8357
|
+
const axis = opts.axis ?? "model";
|
|
8358
|
+
const strategyId = opts.strategy;
|
|
8359
|
+
if (axis === "strategy" && !strategyId) {
|
|
8360
|
+
throw new Error("golden-eval: axis 'strategy' requires opts.strategy.");
|
|
8361
|
+
}
|
|
8362
|
+
if (axis === "model" && !opts.candidateModel) {
|
|
8363
|
+
throw new Error("golden-eval: axis 'model' requires opts.candidateModel.");
|
|
8364
|
+
}
|
|
8241
8365
|
const restHeaders = {
|
|
8242
8366
|
apikey: opts.serviceKey,
|
|
8243
8367
|
Authorization: `Bearer ${opts.serviceKey}`
|
|
@@ -8259,7 +8383,7 @@ async function runGoldenEval(opts) {
|
|
|
8259
8383
|
);
|
|
8260
8384
|
}
|
|
8261
8385
|
progress(`Loaded ${goldenRows.length} golden case(s).`);
|
|
8262
|
-
let incumbentModel = opts.incumbentModel;
|
|
8386
|
+
let incumbentModel = axis === "strategy" ? opts.strategyModel : opts.incumbentModel;
|
|
8263
8387
|
if (!incumbentModel) {
|
|
8264
8388
|
const counts = /* @__PURE__ */ new Map();
|
|
8265
8389
|
for (const row of goldenRows) {
|
|
@@ -8270,21 +8394,27 @@ async function runGoldenEval(opts) {
|
|
|
8270
8394
|
incumbentModel = top[0];
|
|
8271
8395
|
notes.push(`incumbent defaulted to most-captured model: ${incumbentModel}`);
|
|
8272
8396
|
}
|
|
8273
|
-
|
|
8397
|
+
const candidateModel = axis === "strategy" ? incumbentModel : opts.candidateModel;
|
|
8398
|
+
if (axis === "model" && incumbentModel === candidateModel) {
|
|
8274
8399
|
throw new Error("golden-eval: candidate and incumbent are the same model.");
|
|
8275
8400
|
}
|
|
8276
8401
|
const judgeProfile = tryGetProfile(judgeModel);
|
|
8277
8402
|
if (!judgeProfile) {
|
|
8278
8403
|
throw new Error(`golden-eval: judge model '${judgeModel}' is not in the roster.`);
|
|
8279
8404
|
}
|
|
8280
|
-
if (judgeProfile.family && [incumbentModel,
|
|
8405
|
+
if (judgeProfile.family && [incumbentModel, candidateModel].some(
|
|
8281
8406
|
(m) => tryGetProfile(m)?.family === judgeProfile.family
|
|
8282
8407
|
)) {
|
|
8283
8408
|
notes.push(
|
|
8284
8409
|
`WARNING: judge family (${judgeProfile.family}) overlaps a compared model \u2014 self-preference risk; consider --judge from another provider.`
|
|
8285
8410
|
);
|
|
8286
8411
|
}
|
|
8287
|
-
const
|
|
8412
|
+
const armBIr = (ir) => {
|
|
8413
|
+
if (axis !== "strategy") return ir;
|
|
8414
|
+
return strategyId === "discipline-gates-v1-alt" ? withAltDisciplineContract(ir) : withDisciplineContract(ir);
|
|
8415
|
+
};
|
|
8416
|
+
const replay = async (ir, model, captureGates = false) => {
|
|
8417
|
+
let gates;
|
|
8288
8418
|
try {
|
|
8289
8419
|
const evalIr = {
|
|
8290
8420
|
...ir,
|
|
@@ -8292,6 +8422,29 @@ async function runGoldenEval(opts) {
|
|
|
8292
8422
|
constraints: { ...ir.constraints ?? {}, forceModel: model }
|
|
8293
8423
|
};
|
|
8294
8424
|
const compiled = compile(evalIr);
|
|
8425
|
+
if (captureGates) {
|
|
8426
|
+
if (strategyId === "discipline-gates-v1-alt") {
|
|
8427
|
+
const altSection = (evalIr.sections ?? []).find(
|
|
8428
|
+
(s) => s.id === STRATEGY_ALT_SECTION_ID
|
|
8429
|
+
);
|
|
8430
|
+
gates = {
|
|
8431
|
+
fired: altSection !== void 0 && altSection.text.length > 0,
|
|
8432
|
+
rule: "discipline-gates-v1-alt",
|
|
8433
|
+
gateTokens: altSection ? countTokens(`${altSection.text}
|
|
8434
|
+
|
|
8435
|
+
`) : 0
|
|
8436
|
+
};
|
|
8437
|
+
} else {
|
|
8438
|
+
const gateRewrite = compiled.sectionRewritesApplied.find(
|
|
8439
|
+
(rw) => rw.rule.startsWith("discipline-gates-")
|
|
8440
|
+
);
|
|
8441
|
+
gates = {
|
|
8442
|
+
fired: gateRewrite !== void 0 && compiled.diagnostics.disciplineGateTokens > 0,
|
|
8443
|
+
rule: gateRewrite?.rule,
|
|
8444
|
+
gateTokens: compiled.diagnostics.disciplineGateTokens
|
|
8445
|
+
};
|
|
8446
|
+
}
|
|
8447
|
+
}
|
|
8295
8448
|
const attempt = () => execute(compiled.request, { apiKeys: opts.apiKeys, fetchImpl: opts.fetchImpl });
|
|
8296
8449
|
let started = Date.now();
|
|
8297
8450
|
let exec = await attempt();
|
|
@@ -8310,7 +8463,8 @@ async function runGoldenEval(opts) {
|
|
|
8310
8463
|
tokensIn: 0,
|
|
8311
8464
|
tokensOut: 0,
|
|
8312
8465
|
latencyMs,
|
|
8313
|
-
errorClass: exec.errorCode
|
|
8466
|
+
errorClass: exec.errorCode,
|
|
8467
|
+
...gates ? { gates } : {}
|
|
8314
8468
|
};
|
|
8315
8469
|
}
|
|
8316
8470
|
const validated = _internal.validateStructuredContract(exec, evalIr);
|
|
@@ -8322,7 +8476,8 @@ async function runGoldenEval(opts) {
|
|
|
8322
8476
|
tokensIn: exec.response.tokens.input,
|
|
8323
8477
|
tokensOut: exec.response.tokens.output,
|
|
8324
8478
|
latencyMs,
|
|
8325
|
-
contractViolation: validated.errorCode
|
|
8479
|
+
contractViolation: validated.errorCode,
|
|
8480
|
+
...gates ? { gates } : {}
|
|
8326
8481
|
};
|
|
8327
8482
|
}
|
|
8328
8483
|
return {
|
|
@@ -8332,7 +8487,8 @@ async function runGoldenEval(opts) {
|
|
|
8332
8487
|
parseError: validated.response.parseError,
|
|
8333
8488
|
tokensIn: validated.response.tokens.input,
|
|
8334
8489
|
tokensOut: validated.response.tokens.output,
|
|
8335
|
-
latencyMs
|
|
8490
|
+
latencyMs,
|
|
8491
|
+
...gates ? { gates } : {}
|
|
8336
8492
|
};
|
|
8337
8493
|
} catch (err) {
|
|
8338
8494
|
return {
|
|
@@ -8342,7 +8498,8 @@ async function runGoldenEval(opts) {
|
|
|
8342
8498
|
tokensIn: 0,
|
|
8343
8499
|
tokensOut: 0,
|
|
8344
8500
|
latencyMs: 0,
|
|
8345
|
-
errorClass: err instanceof Error ? `execute_rejected:${err.message}` : "execute_rejected"
|
|
8501
|
+
errorClass: err instanceof Error ? `execute_rejected:${err.message}` : "execute_rejected",
|
|
8502
|
+
...gates ? { gates } : {}
|
|
8346
8503
|
};
|
|
8347
8504
|
}
|
|
8348
8505
|
};
|
|
@@ -8401,12 +8558,39 @@ async function runGoldenEval(opts) {
|
|
|
8401
8558
|
};
|
|
8402
8559
|
for (const [i, row] of goldenRows.entries()) {
|
|
8403
8560
|
progress(
|
|
8404
|
-
`Case ${i + 1}/${goldenRows.length} (golden #${row.id}): replaying ${incumbentModel} + ${
|
|
8561
|
+
`Case ${i + 1}/${goldenRows.length} (golden #${row.id}): replaying ${incumbentModel} + ${candidateModel}\u2026`
|
|
8405
8562
|
);
|
|
8406
8563
|
const [inc, cand] = await Promise.all([
|
|
8407
8564
|
replay(row.ir, incumbentModel),
|
|
8408
|
-
replay(row.ir,
|
|
8565
|
+
replay(armBIr(row.ir), candidateModel, axis === "strategy")
|
|
8409
8566
|
]);
|
|
8567
|
+
if (axis === "strategy" && !cand.gates?.fired) {
|
|
8568
|
+
notes.push(
|
|
8569
|
+
`ABORT gates-did-not-fire: arm B compile produced no discipline-gate rewrite (case golden #${row.id}; archetype=${opts.archetype}; gateTokens=${cand.gates?.gateTokens ?? 0}). No verdict written \u2014 a tied-because-identical run is not evidence about the strategy.`
|
|
8570
|
+
);
|
|
8571
|
+
return {
|
|
8572
|
+
verdict: "inconclusive",
|
|
8573
|
+
axis,
|
|
8574
|
+
strategy: strategyId,
|
|
8575
|
+
appId: opts.appId,
|
|
8576
|
+
archetype: opts.archetype,
|
|
8577
|
+
incumbentModel,
|
|
8578
|
+
candidateModel,
|
|
8579
|
+
judgeModel,
|
|
8580
|
+
nCases: 0,
|
|
8581
|
+
wins: 0,
|
|
8582
|
+
ties: 0,
|
|
8583
|
+
losses: 0,
|
|
8584
|
+
floorViolations: 0,
|
|
8585
|
+
floorDetail: {},
|
|
8586
|
+
winOrTieRatio: null,
|
|
8587
|
+
latencyRatio: null,
|
|
8588
|
+
costIncumbentUsd: null,
|
|
8589
|
+
costCandidateUsd: null,
|
|
8590
|
+
cases,
|
|
8591
|
+
notes
|
|
8592
|
+
};
|
|
8593
|
+
}
|
|
8410
8594
|
if (!inc.ok || inc.contractViolation) {
|
|
8411
8595
|
const why = !inc.ok ? `incumbent replay failed (${inc.errorClass ?? "unknown"})` : `incumbent contract violation (${inc.contractViolation})`;
|
|
8412
8596
|
cases.push({
|
|
@@ -8468,6 +8652,9 @@ async function runGoldenEval(opts) {
|
|
|
8468
8652
|
cases.push({
|
|
8469
8653
|
goldenIrId: row.id,
|
|
8470
8654
|
verdict: verdict2,
|
|
8655
|
+
// Release B — the raw pair the fold discards; positional-disagreement
|
|
8656
|
+
// rate is the strategy experiment's primary metric.
|
|
8657
|
+
orderVerdicts: { run1: v1, run2: v2 },
|
|
8471
8658
|
judgeRationale: order1.rationale ?? order2.rationale,
|
|
8472
8659
|
floorViolations: [],
|
|
8473
8660
|
incumbent: incumbentLeg,
|
|
@@ -8491,7 +8678,7 @@ async function runGoldenEval(opts) {
|
|
|
8491
8678
|
);
|
|
8492
8679
|
const costCandidateUsd = sum(
|
|
8493
8680
|
evaluable.map(
|
|
8494
|
-
(c) => c.candidate ? costUsd(
|
|
8681
|
+
(c) => c.candidate ? costUsd(candidateModel, c.candidate.tokensIn, c.candidate.tokensOut) : null
|
|
8495
8682
|
)
|
|
8496
8683
|
);
|
|
8497
8684
|
const latencyFloorOk = latencyRatio === null || latencyRatio <= latencyFloorRatio;
|
|
@@ -8509,10 +8696,12 @@ async function runGoldenEval(opts) {
|
|
|
8509
8696
|
}
|
|
8510
8697
|
const result = {
|
|
8511
8698
|
verdict,
|
|
8699
|
+
axis,
|
|
8700
|
+
...strategyId ? { strategy: strategyId } : {},
|
|
8512
8701
|
appId: opts.appId,
|
|
8513
8702
|
archetype: opts.archetype,
|
|
8514
8703
|
incumbentModel,
|
|
8515
|
-
candidateModel
|
|
8704
|
+
candidateModel,
|
|
8516
8705
|
judgeModel,
|
|
8517
8706
|
nCases: evaluable.length,
|
|
8518
8707
|
wins,
|
|
@@ -8542,8 +8731,12 @@ async function runGoldenEval(opts) {
|
|
|
8542
8731
|
body: JSON.stringify({
|
|
8543
8732
|
app_id: opts.appId,
|
|
8544
8733
|
intent_archetype: opts.archetype,
|
|
8734
|
+
// Release B (migration 048) — a run row that can't say which axis
|
|
8735
|
+
// produced its verdict is a false-attribution engine.
|
|
8736
|
+
axis,
|
|
8737
|
+
strategy: strategyId ?? null,
|
|
8545
8738
|
incumbent_model: incumbentModel,
|
|
8546
|
-
candidate_model:
|
|
8739
|
+
candidate_model: candidateModel,
|
|
8547
8740
|
trigger_source: opts.triggerSource ?? "manual",
|
|
8548
8741
|
judge_model: judgeModel,
|
|
8549
8742
|
n_cases: result.nCases,
|
|
@@ -8578,8 +8771,8 @@ async function runGoldenEval(opts) {
|
|
|
8578
8771
|
return {
|
|
8579
8772
|
app_id: opts.appId,
|
|
8580
8773
|
intent_archetype: opts.archetype,
|
|
8581
|
-
family: tryGetProfile(
|
|
8582
|
-
candidate_model:
|
|
8774
|
+
family: tryGetProfile(candidateModel)?.family ?? "unknown",
|
|
8775
|
+
candidate_model: candidateModel,
|
|
8583
8776
|
current_model: incumbentModel,
|
|
8584
8777
|
prompt_hash: hashForGolden(turn),
|
|
8585
8778
|
current_response: c.incumbent?.text.slice(0, 500) ?? null,
|
|
@@ -8625,8 +8818,8 @@ async function runGoldenEval(opts) {
|
|
|
8625
8818
|
const latestOutcomeId = latest[0]?.id;
|
|
8626
8819
|
if (latestOutcomeId !== void 0) {
|
|
8627
8820
|
const costLine = costIncumbentUsd !== null && costCandidateUsd !== null ? ` Eval cost basis: incumbent $${costIncumbentUsd.toFixed(4)} vs candidate $${costCandidateUsd.toFixed(4)} across the set.` : "";
|
|
8628
|
-
const message = `Golden-set eval (run #${result.runId}): ${
|
|
8629
|
-
const suggestion = verdict === "promote-ready" ? `Candidate passed the non-inferiority rule on your real workload. To promote: node v2/scripts/promote-model.mjs --model ${
|
|
8821
|
+
const message = `Golden-set eval (run #${result.runId}): ${candidateModel} vs ${incumbentModel} on ${opts.appId}/${opts.archetype} \u2014 verdict ${verdict.toUpperCase()}. ${wins}W/${ties}T/${losses}L over ${result.nCases} real workload case(s), wins-or-ties ${(100 * (winOrTieRatio ?? 0)).toFixed(0)}% (threshold ${(100 * threshold).toFixed(0)}%), ${result.floorViolations} hard-floor violation(s), latency ratio ${latencyRatio?.toFixed(2) ?? "n/a"}.` + costLine;
|
|
8822
|
+
const suggestion = verdict === "promote-ready" ? `Candidate passed the non-inferiority rule on your real workload. To promote: node v2/scripts/promote-model.mjs --model ${candidateModel} (or ask kgauto-Cairn). Evidence: kgauto_golden_eval_runs #${result.runId} + probe_outcomes golden_run_id=${result.runId}.` : `Candidate did NOT clear the non-inferiority rule \u2014 no action needed; the incumbent stays. Evidence rows: golden_run_id=${result.runId}.`;
|
|
8630
8823
|
const advRes = await fetchFn(rest("compile_outcome_advisories"), {
|
|
8631
8824
|
method: "POST",
|
|
8632
8825
|
headers: { ...restHeaders, "Content-Type": "application/json", Prefer: "return=minimal" },
|
|
@@ -8655,6 +8848,33 @@ async function runGoldenEval(opts) {
|
|
|
8655
8848
|
}
|
|
8656
8849
|
return result;
|
|
8657
8850
|
}
|
|
8851
|
+
function classifyStrategyOutcome(r) {
|
|
8852
|
+
if (r.verdict === "inconclusive") return "inconclusive";
|
|
8853
|
+
if (r.wins > r.losses) return "wins";
|
|
8854
|
+
if (r.losses > r.wins) return "loses";
|
|
8855
|
+
return "ties";
|
|
8856
|
+
}
|
|
8857
|
+
var STRATEGY_AUTHORSHIP_LIMITATION = "Both gate wordings share one author (kgauto-Cairn); a shared blind spot is invisible to this experiment. Strong verdicts: both-lose (mechanism) and v1-loses/alt-wins (wording). An independently-authored alt would strengthen every other cell.";
|
|
8858
|
+
async function runStrategyEvalWithAttribution(opts) {
|
|
8859
|
+
const primary = await runGoldenEval({
|
|
8860
|
+
...opts,
|
|
8861
|
+
axis: "strategy",
|
|
8862
|
+
strategy: "discipline-gates-v1"
|
|
8863
|
+
});
|
|
8864
|
+
const outcome = classifyStrategyOutcome(primary);
|
|
8865
|
+
if (outcome !== "loses") {
|
|
8866
|
+
const attribution2 = outcome === "wins" ? "mechanism-works" : outcome === "ties" ? "no-lift" : "inconclusive";
|
|
8867
|
+
return { attribution: attribution2, primary, limitation: STRATEGY_AUTHORSHIP_LIMITATION };
|
|
8868
|
+
}
|
|
8869
|
+
const alt = await runGoldenEval({
|
|
8870
|
+
...opts,
|
|
8871
|
+
axis: "strategy",
|
|
8872
|
+
strategy: "discipline-gates-v1-alt"
|
|
8873
|
+
});
|
|
8874
|
+
const altOutcome = classifyStrategyOutcome(alt);
|
|
8875
|
+
const attribution = altOutcome === "wins" ? "wording-failure" : altOutcome === "loses" ? "mechanism-failure" : "wording-inconclusive";
|
|
8876
|
+
return { attribution, primary, alt, limitation: STRATEGY_AUTHORSHIP_LIMITATION };
|
|
8877
|
+
}
|
|
8658
8878
|
|
|
8659
8879
|
// src/brain-proxy.ts
|
|
8660
8880
|
var SEGMENT_TO_TABLE = {
|
|
@@ -8823,7 +9043,7 @@ function createBrainForwardRoutes(config) {
|
|
|
8823
9043
|
}
|
|
8824
9044
|
|
|
8825
9045
|
// src/version.ts
|
|
8826
|
-
var LIBRARY_VERSION = "2.0.0-alpha.
|
|
9046
|
+
var LIBRARY_VERSION = "2.0.0-alpha.76";
|
|
8827
9047
|
|
|
8828
9048
|
// src/key-health.ts
|
|
8829
9049
|
var JSON_HEADERS2 = { "Content-Type": "application/json" };
|
|
@@ -9441,6 +9661,211 @@ async function markExclusionFindingHandled(opts) {
|
|
|
9441
9661
|
return { ok: true };
|
|
9442
9662
|
}
|
|
9443
9663
|
|
|
9664
|
+
// src/decomposition.ts
|
|
9665
|
+
var DECOMPOSITION_TEMPLATES_VERSION = "decomposition-templates-v1";
|
|
9666
|
+
var DECOMPOSITION_TEMPLATES = {
|
|
9667
|
+
summarize: {
|
|
9668
|
+
archetype: "summarize",
|
|
9669
|
+
version: DECOMPOSITION_TEMPLATES_VERSION,
|
|
9670
|
+
rationale: "A long-input summarize is mostly reading. A cheap extractor reads the full payload and emits compressed notes; the incumbent composes the summary from the notes \u2014 frontier tokens are spent only on the part that needs frontier judgment.",
|
|
9671
|
+
steps: [
|
|
9672
|
+
{
|
|
9673
|
+
role: "chunk-extract",
|
|
9674
|
+
archetype: "extract",
|
|
9675
|
+
tier: "delegate",
|
|
9676
|
+
inputShare: 1,
|
|
9677
|
+
emitsShareOfInput: 0.15,
|
|
9678
|
+
outputShare: 0
|
|
9679
|
+
},
|
|
9680
|
+
{
|
|
9681
|
+
role: "compose",
|
|
9682
|
+
archetype: "summarize",
|
|
9683
|
+
tier: "anchor",
|
|
9684
|
+
inputShare: 0.15,
|
|
9685
|
+
// reads the notes, not the raw payload
|
|
9686
|
+
emitsShareOfInput: 0,
|
|
9687
|
+
outputShare: 1
|
|
9688
|
+
}
|
|
9689
|
+
]
|
|
9690
|
+
},
|
|
9691
|
+
hunt: {
|
|
9692
|
+
archetype: "hunt",
|
|
9693
|
+
version: DECOMPOSITION_TEMPLATES_VERSION,
|
|
9694
|
+
rationale: "Hunt decomposes into breadth (search sweeps), mechanical harvesting (extraction), and judgment (dedupe + compose). The sweeps and the harvest are grunt work; the composition anchors on the incumbent.",
|
|
9695
|
+
steps: [
|
|
9696
|
+
{
|
|
9697
|
+
role: "search-sweep",
|
|
9698
|
+
archetype: "hunt",
|
|
9699
|
+
tier: "delegate",
|
|
9700
|
+
inputShare: 0.5,
|
|
9701
|
+
emitsShareOfInput: 0.2,
|
|
9702
|
+
outputShare: 0
|
|
9703
|
+
},
|
|
9704
|
+
{
|
|
9705
|
+
role: "harvest",
|
|
9706
|
+
archetype: "extract",
|
|
9707
|
+
tier: "delegate",
|
|
9708
|
+
inputShare: 0.35,
|
|
9709
|
+
emitsShareOfInput: 0.1,
|
|
9710
|
+
outputShare: 0
|
|
9711
|
+
},
|
|
9712
|
+
{
|
|
9713
|
+
role: "dedupe-compose",
|
|
9714
|
+
archetype: "judge",
|
|
9715
|
+
tier: "anchor",
|
|
9716
|
+
inputShare: 0.3,
|
|
9717
|
+
// sweep notes + harvest notes
|
|
9718
|
+
emitsShareOfInput: 0,
|
|
9719
|
+
outputShare: 1
|
|
9720
|
+
}
|
|
9721
|
+
]
|
|
9722
|
+
},
|
|
9723
|
+
plan: {
|
|
9724
|
+
archetype: "plan",
|
|
9725
|
+
version: DECOMPOSITION_TEMPLATES_VERSION,
|
|
9726
|
+
rationale: "Planning splits into context-gathering (mechanical reading) and the plan itself (judgment). The gatherer reads the corpus and briefs; the incumbent plans from the brief.",
|
|
9727
|
+
steps: [
|
|
9728
|
+
{
|
|
9729
|
+
role: "gather-brief",
|
|
9730
|
+
archetype: "extract",
|
|
9731
|
+
tier: "delegate",
|
|
9732
|
+
inputShare: 1,
|
|
9733
|
+
emitsShareOfInput: 0.2,
|
|
9734
|
+
outputShare: 0
|
|
9735
|
+
},
|
|
9736
|
+
{
|
|
9737
|
+
role: "draft-plan",
|
|
9738
|
+
archetype: "plan",
|
|
9739
|
+
tier: "anchor",
|
|
9740
|
+
inputShare: 0.2,
|
|
9741
|
+
emitsShareOfInput: 0,
|
|
9742
|
+
outputShare: 1
|
|
9743
|
+
}
|
|
9744
|
+
]
|
|
9745
|
+
}
|
|
9746
|
+
};
|
|
9747
|
+
var COACH_CFG = {
|
|
9748
|
+
/** Same felt-utility floor as promotions (alpha.67 family). */
|
|
9749
|
+
minMonthlySavingUsd: 5,
|
|
9750
|
+
/** Executor must clear this archetypePerf on the step's archetype —
|
|
9751
|
+
* same floor as the translator/advisor (ARCHETYPE_FLOOR_DEFAULT). */
|
|
9752
|
+
executorPerfFloor: 6,
|
|
9753
|
+
daysPerMonth: 30
|
|
9754
|
+
};
|
|
9755
|
+
function planDecomposition(args) {
|
|
9756
|
+
const { stats, incumbentPricing, pickExecutor } = args;
|
|
9757
|
+
const cfg = { ...COACH_CFG, ...args.cfg ?? {} };
|
|
9758
|
+
const template = DECOMPOSITION_TEMPLATES[stats.archetype];
|
|
9759
|
+
if (!template) return void 0;
|
|
9760
|
+
const perCall = (pricing, tokensIn, tokensOut) => tokensIn / 1e6 * pricing.costInputPer1m + tokensOut / 1e6 * pricing.costOutputPer1m;
|
|
9761
|
+
const monolithCostPerCallUsd = perCall(
|
|
9762
|
+
incumbentPricing,
|
|
9763
|
+
stats.avgTokensIn,
|
|
9764
|
+
stats.avgTokensOut
|
|
9765
|
+
);
|
|
9766
|
+
const assumptions = [
|
|
9767
|
+
`per-step token shares are ${template.version} JUDGMENT numbers \u2014 no fan-out traffic exists to measure them from yet; they graduate per surface when branch traffic lands`
|
|
9768
|
+
];
|
|
9769
|
+
const steps = [];
|
|
9770
|
+
let splitCostPerCallUsd = 0;
|
|
9771
|
+
for (const step of template.steps) {
|
|
9772
|
+
const stepTokensIn = stats.avgTokensIn * step.inputShare;
|
|
9773
|
+
const stepTokensOut = stats.avgTokensIn * step.emitsShareOfInput + stats.avgTokensOut * step.outputShare;
|
|
9774
|
+
if (step.tier === "anchor") {
|
|
9775
|
+
const cost2 = perCall(incumbentPricing, stepTokensIn, stepTokensOut);
|
|
9776
|
+
splitCostPerCallUsd += cost2;
|
|
9777
|
+
steps.push({
|
|
9778
|
+
...step,
|
|
9779
|
+
executorModel: stats.incumbentModel,
|
|
9780
|
+
executorGrounding: "measured",
|
|
9781
|
+
// the incumbent IS the measured baseline
|
|
9782
|
+
executorPerfScore: null,
|
|
9783
|
+
projectedCostPerCallUsd: cost2
|
|
9784
|
+
});
|
|
9785
|
+
continue;
|
|
9786
|
+
}
|
|
9787
|
+
const candidate = pickExecutor(step.archetype);
|
|
9788
|
+
if (candidate && candidate.modelId === stats.incumbentModel) {
|
|
9789
|
+
const cost2 = perCall(incumbentPricing, stepTokensIn, stepTokensOut);
|
|
9790
|
+
splitCostPerCallUsd += cost2;
|
|
9791
|
+
steps.push({
|
|
9792
|
+
...step,
|
|
9793
|
+
tier: "anchor",
|
|
9794
|
+
executorModel: stats.incumbentModel,
|
|
9795
|
+
executorGrounding: candidate.grounding,
|
|
9796
|
+
executorPerfScore: candidate.perfScore,
|
|
9797
|
+
projectedCostPerCallUsd: cost2
|
|
9798
|
+
});
|
|
9799
|
+
assumptions.push(
|
|
9800
|
+
`step '${step.role}': the cheapest qualified executor IS the incumbent \u2014 nothing to delegate to`
|
|
9801
|
+
);
|
|
9802
|
+
continue;
|
|
9803
|
+
}
|
|
9804
|
+
if (!candidate || candidate.perfScore < cfg.executorPerfFloor) {
|
|
9805
|
+
const cost2 = perCall(incumbentPricing, stepTokensIn, stepTokensOut);
|
|
9806
|
+
splitCostPerCallUsd += cost2;
|
|
9807
|
+
steps.push({
|
|
9808
|
+
...step,
|
|
9809
|
+
tier: "anchor",
|
|
9810
|
+
executorModel: stats.incumbentModel,
|
|
9811
|
+
executorGrounding: "measured",
|
|
9812
|
+
executorPerfScore: candidate?.perfScore ?? null,
|
|
9813
|
+
projectedCostPerCallUsd: cost2
|
|
9814
|
+
});
|
|
9815
|
+
assumptions.push(
|
|
9816
|
+
`step '${step.role}' (${step.archetype}): no executor clears the perf floor ${cfg.executorPerfFloor} \u2014 kept on the incumbent`
|
|
9817
|
+
);
|
|
9818
|
+
continue;
|
|
9819
|
+
}
|
|
9820
|
+
const cost = perCall(candidate, stepTokensIn, stepTokensOut);
|
|
9821
|
+
splitCostPerCallUsd += cost;
|
|
9822
|
+
steps.push({
|
|
9823
|
+
...step,
|
|
9824
|
+
executorModel: candidate.modelId,
|
|
9825
|
+
executorGrounding: candidate.grounding,
|
|
9826
|
+
executorPerfScore: candidate.perfScore,
|
|
9827
|
+
projectedCostPerCallUsd: cost
|
|
9828
|
+
});
|
|
9829
|
+
if (candidate.grounding === "judgment") {
|
|
9830
|
+
assumptions.push(
|
|
9831
|
+
`step '${step.role}': ${candidate.modelId} perf ${candidate.perfScore}/10 on ${step.archetype} is JUDGMENT-grounded (no measured portfolio outcomes for the tuple yet)`
|
|
9832
|
+
);
|
|
9833
|
+
}
|
|
9834
|
+
}
|
|
9835
|
+
const savingPerCallUsd = monolithCostPerCallUsd - splitCostPerCallUsd;
|
|
9836
|
+
const monthlyCalls = stats.nCalls / stats.windowDays * cfg.daysPerMonth;
|
|
9837
|
+
const projectedMonthlySavingUsd = savingPerCallUsd * monthlyCalls;
|
|
9838
|
+
const breakEvenMonthlyCalls = savingPerCallUsd > 0 ? cfg.minMonthlySavingUsd / savingPerCallUsd : null;
|
|
9839
|
+
let verdict;
|
|
9840
|
+
let verdictReason;
|
|
9841
|
+
if (savingPerCallUsd <= 0) {
|
|
9842
|
+
verdict = "keep-monolith";
|
|
9843
|
+
verdictReason = "the split costs MORE per call than the monolith at current pricing \u2014 no volume makes it pay";
|
|
9844
|
+
} else if (projectedMonthlySavingUsd < cfg.minMonthlySavingUsd) {
|
|
9845
|
+
verdict = "keep-monolith";
|
|
9846
|
+
verdictReason = `the split pays $${projectedMonthlySavingUsd.toFixed(2)}/mo at your ~${Math.round(monthlyCalls)} calls/mo \u2014 below the $${cfg.minMonthlySavingUsd} felt-utility floor. Break-even is ~${Math.ceil(breakEvenMonthlyCalls ?? 0)} calls/mo; revisit when volume gets there. Decomposition adds moving parts, and a saving you can't feel doesn't buy them`;
|
|
9847
|
+
} else {
|
|
9848
|
+
verdict = "split-pays";
|
|
9849
|
+
verdictReason = `projected $${projectedMonthlySavingUsd.toFixed(2)}/mo saving at ~${Math.round(monthlyCalls)} calls/mo (monolith $${monolithCostPerCallUsd.toFixed(4)}/call \u2192 split $${splitCostPerCallUsd.toFixed(4)}/call)`;
|
|
9850
|
+
}
|
|
9851
|
+
return {
|
|
9852
|
+
appId: stats.appId,
|
|
9853
|
+
archetype: String(stats.archetype),
|
|
9854
|
+
incumbentModel: stats.incumbentModel,
|
|
9855
|
+
template,
|
|
9856
|
+
steps,
|
|
9857
|
+
monolithCostPerCallUsd,
|
|
9858
|
+
splitCostPerCallUsd,
|
|
9859
|
+
savingPerCallUsd,
|
|
9860
|
+
monthlyCalls,
|
|
9861
|
+
projectedMonthlySavingUsd,
|
|
9862
|
+
breakEvenMonthlyCalls,
|
|
9863
|
+
verdict,
|
|
9864
|
+
verdictReason,
|
|
9865
|
+
assumptions
|
|
9866
|
+
};
|
|
9867
|
+
}
|
|
9868
|
+
|
|
9444
9869
|
// src/index.ts
|
|
9445
9870
|
function compile2(ir, opts) {
|
|
9446
9871
|
const result = compile(ir, opts);
|
|
@@ -9454,11 +9879,15 @@ function compile2(ir, opts) {
|
|
|
9454
9879
|
ALL_ARCHETYPES,
|
|
9455
9880
|
ARCHETYPE_FAMILY_FITS,
|
|
9456
9881
|
ARCHETYPE_FLOOR_DEFAULT,
|
|
9882
|
+
COACH_CFG,
|
|
9457
9883
|
CallError,
|
|
9884
|
+
DECOMPOSITION_TEMPLATES,
|
|
9885
|
+
DECOMPOSITION_TEMPLATES_VERSION,
|
|
9458
9886
|
DEFAULT_FINDINGS_ENDPOINT,
|
|
9459
9887
|
DEFAULT_MEASURED_FAILURE_ENDPOINT,
|
|
9460
9888
|
DEFAULT_PROMOTIONS_ENDPOINT,
|
|
9461
9889
|
DIALECT_VERSION,
|
|
9890
|
+
DISCIPLINE_GATES_V1_ALT_HEADER,
|
|
9462
9891
|
FamilyResolutionError,
|
|
9463
9892
|
INTENT_ARCHETYPES,
|
|
9464
9893
|
JUDGE_RUBRICS,
|
|
@@ -9468,13 +9897,17 @@ function compile2(ir, opts) {
|
|
|
9468
9897
|
MEASURED_GROUNDING_MIN_N,
|
|
9469
9898
|
PRODUCER_OWNED_RULE_CODES,
|
|
9470
9899
|
PROVIDER_ENV_KEYS,
|
|
9900
|
+
RULE_DISCIPLINE_GATES_V1,
|
|
9901
|
+
RULE_DISCIPLINE_GATES_V1_STRUCTURED,
|
|
9471
9902
|
RULE_SEQUENTIAL_TOOL_CLIFF,
|
|
9903
|
+
STRATEGY_AUTHORSHIP_LIMITATION,
|
|
9472
9904
|
TRANSLATOR_FLOOR,
|
|
9473
9905
|
_testResetMeasuredFailure,
|
|
9474
9906
|
_testResetPromotions,
|
|
9475
9907
|
_testWaitForMeasuredFailureRefresh,
|
|
9476
9908
|
_testWaitForPromotionsRefresh,
|
|
9477
9909
|
allProfiles,
|
|
9910
|
+
altGatesBlockFor,
|
|
9478
9911
|
applyArchetypeConvention,
|
|
9479
9912
|
applySectionRewrites,
|
|
9480
9913
|
attachCacheControlToStreamTextInput,
|
|
@@ -9489,6 +9922,7 @@ function compile2(ir, opts) {
|
|
|
9489
9922
|
buildShadowProbeRow,
|
|
9490
9923
|
call,
|
|
9491
9924
|
captureGoldenIr,
|
|
9925
|
+
classifyStrategyOutcome,
|
|
9492
9926
|
clearBrain,
|
|
9493
9927
|
combineOrderSwappedVerdicts,
|
|
9494
9928
|
compile,
|
|
@@ -9548,6 +9982,7 @@ function compile2(ir, opts) {
|
|
|
9548
9982
|
parseGoldenCaptureRate,
|
|
9549
9983
|
parseJudgeVerdict,
|
|
9550
9984
|
peekBrainDeadLetter,
|
|
9985
|
+
planDecomposition,
|
|
9551
9986
|
prefetchMeasuredFailure,
|
|
9552
9987
|
probeShadow,
|
|
9553
9988
|
profileToRow,
|
|
@@ -9566,8 +10001,11 @@ function compile2(ir, opts) {
|
|
|
9566
10001
|
rubricFor,
|
|
9567
10002
|
runAdvisor,
|
|
9568
10003
|
runGoldenEval,
|
|
10004
|
+
runStrategyEvalWithAttribution,
|
|
9569
10005
|
setTokenizer,
|
|
9570
10006
|
shouldCaptureGolden,
|
|
9571
10007
|
tryGetProfile,
|
|
9572
|
-
wilsonLowerBound
|
|
10008
|
+
wilsonLowerBound,
|
|
10009
|
+
withAltDisciplineContract,
|
|
10010
|
+
withDisciplineContract
|
|
9573
10011
|
});
|